From 34c68663b6c1865b702247dc48d99781b8bd748e Mon Sep 17 00:00:00 2001 From: hanrui1sensetime <83800577+hanrui1sensetime@users.noreply.github.com> Date: Mon, 13 Mar 2023 11:31:39 +0800 Subject: [PATCH] [Sync] Sync Java API to master (#1856) * sync rotated detector java api to master * sync mmseg score output to master * sync java docs for demo * sync java docs for master --- .github/scripts/test_java_demo.py | 7 ++ csrc/mmdeploy/apis/java/CMakeLists.txt | 1 + .../apis/java/mmdeploy/Classifier.java | 46 ++++++++- .../mmdeploy/apis/java/mmdeploy/DataType.java | 4 + .../mmdeploy/apis/java/mmdeploy/Detector.java | 52 +++++++++- .../apis/java/mmdeploy/InstanceMask.java | 11 ++- csrc/mmdeploy/apis/java/mmdeploy/Mat.java | 18 +++- .../apis/java/mmdeploy/PixelFormat.java | 4 + csrc/mmdeploy/apis/java/mmdeploy/PointF.java | 10 +- .../apis/java/mmdeploy/PoseDetector.java | 46 ++++++++- csrc/mmdeploy/apis/java/mmdeploy/Rect.java | 16 ++- .../mmdeploy/apis/java/mmdeploy/Restorer.java | 43 +++++++- .../apis/java/mmdeploy/RotatedDetector.java | 97 +++++++++++++++++++ .../apis/java/mmdeploy/Segmentor.java | 58 ++++++++++- .../apis/java/mmdeploy/TextDetector.java | 46 ++++++++- .../apis/java/mmdeploy/TextRecognizer.java | 52 +++++++++- csrc/mmdeploy/apis/java/native/CMakeLists.txt | 1 + .../java/native/mmdeploy_RotatedDetector.cpp | 61 ++++++++++++ .../java/native/mmdeploy_RotatedDetector.h | 36 +++++++ .../apis/java/native/mmdeploy_Segmentor.cpp | 30 ++++-- demo/java/ImageClassification.java | 6 ++ demo/java/ImageRestorer.java | 7 ++ demo/java/ImageSegmentation.java | 7 ++ demo/java/ObjectDetection.java | 8 ++ demo/java/Ocr.java | 15 ++- demo/java/PoseDetection.java | 17 +++- demo/java/RotatedDetection.java | 83 ++++++++++++++++ demo/java/Utils.java | 13 +++ 28 files changed, 750 insertions(+), 45 deletions(-) create mode 100644 csrc/mmdeploy/apis/java/mmdeploy/RotatedDetector.java create mode 100644 csrc/mmdeploy/apis/java/native/mmdeploy_RotatedDetector.cpp create mode 100644 csrc/mmdeploy/apis/java/native/mmdeploy_RotatedDetector.h create mode 100644 demo/java/RotatedDetection.java diff --git a/.github/scripts/test_java_demo.py b/.github/scripts/test_java_demo.py index 432cfd4f7..a86f349c9 100644 --- a/.github/scripts/test_java_demo.py +++ b/.github/scripts/test_java_demo.py @@ -47,6 +47,13 @@ PARAMS = [ 'configs': [ 'https://media.githubusercontent.com/media/hanrui1sensetime/mmdeploy-javaapi-testdata/master/litehrnet.tar' # noqa: E501 ] + }, + { + 'task': + 'RotatedDetection', + 'configs': [ + 'https://media.githubusercontent.com/media/hanrui1sensetime/mmdeploy-javaapi-testdata/master/gliding-vertex.tar' # noqa: E501 + ] } ] diff --git a/csrc/mmdeploy/apis/java/CMakeLists.txt b/csrc/mmdeploy/apis/java/CMakeLists.txt index 07725613a..eaf96f8c1 100644 --- a/csrc/mmdeploy/apis/java/CMakeLists.txt +++ b/csrc/mmdeploy/apis/java/CMakeLists.txt @@ -23,5 +23,6 @@ add_jar(${PROJECT_NAME} SOURCES mmdeploy/TextRecognizer.java mmdeploy/Restorer.java mmdeploy/PoseDetector.java + mmdeploy/RotatedDetector.java OUTPUT_NAME mmdeploy OUTPUT_DIR ${CMAKE_LIBRARY_OUTPUT_DIRECTORY}) diff --git a/csrc/mmdeploy/apis/java/mmdeploy/Classifier.java b/csrc/mmdeploy/apis/java/mmdeploy/Classifier.java index 3919881fc..9767cb4d3 100644 --- a/csrc/mmdeploy/apis/java/mmdeploy/Classifier.java +++ b/csrc/mmdeploy/apis/java/mmdeploy/Classifier.java @@ -1,5 +1,6 @@ package mmdeploy; +/** @description: the Java API class of Classifier. */ public class Classifier { static { System.loadLibrary("mmdeploy_java"); @@ -7,22 +8,49 @@ public class Classifier { private final long handle; + /** @description: Single classification result of a picture. */ public static class Result { + + /** Class id. */ public int label_id; + + /** Class score. */ public float score; + + /** Initializes a new instance of the Result class. + * @param label_id: class id. + * @param score: class score. + */ public Result(int label_id, float score) { this.label_id = label_id; this.score = score; } } - public Classifier(String modelPath, String deviceName, int deviceId) { + /** Initializes a new instance of the Classifier class. + * @param modelPath: model path. + * @param deviceName: device name. + * @param deviceId: device ID. + * @exception Exception: create Classifier failed exception. + */ + public Classifier(String modelPath, String deviceName, int deviceId) throws Exception{ handle = create(modelPath, deviceName, deviceId); + if (handle == -1) { + throw new Exception("Create Classifier failed!"); + } } - public Result[][] apply(Mat[] images) { + /** Get label information of each image in a batch. + * @param images: input mats. + * @return: results of each input mat. + * @exception Exception: apply Classifier failed exception. + */ + public Result[][] apply(Mat[] images) throws Exception{ int[] counts = new int[images.length]; Result[] results = apply(handle, images, counts); + if (results == null) { + throw new Exception("Apply Classifier failed!"); + } Result[][] rets = new Result[images.length][]; int offset = 0; for (int i = 0; i < images.length; ++i) { @@ -36,12 +64,22 @@ public class Classifier { return rets; } - public Result[] apply(Mat image) { + /** Get label information of one image. + * @param image: input mat. + * @return: result of input mat. + * @exception Exception: apply Classifier failed exception. + */ + public Result[] apply(Mat image) throws Exception{ int[] counts = new int[1]; Mat[] images = new Mat[]{image}; - return apply(handle, images, counts); + Result[] results = apply(handle, images, counts); + if (results == null) { + throw new Exception("Apply Classifier failed!"); + } + return results; } + /** Release the instance of Classifier. */ public void release() { destroy(handle); } diff --git a/csrc/mmdeploy/apis/java/mmdeploy/DataType.java b/csrc/mmdeploy/apis/java/mmdeploy/DataType.java index ea52d0134..4a0ce4173 100644 --- a/csrc/mmdeploy/apis/java/mmdeploy/DataType.java +++ b/csrc/mmdeploy/apis/java/mmdeploy/DataType.java @@ -1,5 +1,6 @@ package mmdeploy; +/** @description: DataType. */ public enum DataType { FLOAT(0), HALF(1), @@ -7,6 +8,9 @@ public enum DataType { INT32(3); final int value; + /** Initializes a new instance of the DataType class. + * @param value: the value. + */ DataType(int value) { this.value = value; } diff --git a/csrc/mmdeploy/apis/java/mmdeploy/Detector.java b/csrc/mmdeploy/apis/java/mmdeploy/Detector.java index 0692d3cf4..ce85403fc 100644 --- a/csrc/mmdeploy/apis/java/mmdeploy/Detector.java +++ b/csrc/mmdeploy/apis/java/mmdeploy/Detector.java @@ -1,5 +1,6 @@ package mmdeploy; +/** @description: the Java API class of Detector. */ public class Detector { static { System.loadLibrary("mmdeploy_java"); @@ -7,11 +8,27 @@ public class Detector { private final long handle; + /** @description: Single detection result of a picture. */ public static class Result { + + /** Bbox class id. */ public int label_id; + + /** Bbox score. */ public float score; + + /** Bbox coordinates. */ public Rect bbox; + + /** Bbox mask. */ public InstanceMask mask; + + /** Initializes a new instance of the Result class. + * @param label_id: bbox class id. + * @param score: bbox score. + * @param bbox: bbox coordinates. + * @param mask: bbox mask. + */ public Result(int label_id, float score, Rect bbox, InstanceMask mask) { this.label_id = label_id; this.score = score; @@ -20,13 +37,30 @@ public class Detector { } } - public Detector(String modelPath, String deviceName, int deviceId) { + /** Initializes a new instance of the Detector class. + * @param modelPath: model path. + * @param deviceName: device name. + * @param deviceId: device ID. + * @exception Exception: create Detector failed exception. + */ + public Detector(String modelPath, String deviceName, int deviceId) throws Exception { handle = create(modelPath, deviceName, deviceId); + if (handle == -1) { + throw new Exception("Create Detector failed!"); + } } - public Result[][] apply(Mat[] images) { + /** Get information of each image in a batch. + * @param images: input mats. + * @return: results of each input mat. + * @exception Exception: apply Detector failed exception. + */ + public Result[][] apply(Mat[] images) throws Exception { int[] counts = new int[images.length]; Result[] results = apply(handle, images, counts); + if (results == null) { + throw new Exception("Apply Detector failed!"); + } Result[][] rets = new Result[images.length][]; int offset = 0; for (int i = 0; i < images.length; ++i) { @@ -40,12 +74,22 @@ public class Detector { return rets; } - public Result[] apply(Mat image) { + /** Get information of one image. + * @param image: input mat. + * @return: result of input mat. + * @exception Exception: apply Detector failed exception. + */ + public Result[] apply(Mat image) throws Exception{ int[] counts = new int[1]; Mat[] images = new Mat[]{image}; - return apply(handle, images, counts); + Result[] results = apply(handle, images, counts); + if (results == null) { + throw new Exception("Apply Detector failed!"); + } + return results; } + /** Release the instance of Detector. */ public void release() { destroy(handle); } diff --git a/csrc/mmdeploy/apis/java/mmdeploy/InstanceMask.java b/csrc/mmdeploy/apis/java/mmdeploy/InstanceMask.java index 9958da6e8..55645ba37 100644 --- a/csrc/mmdeploy/apis/java/mmdeploy/InstanceMask.java +++ b/csrc/mmdeploy/apis/java/mmdeploy/InstanceMask.java @@ -1,10 +1,19 @@ package mmdeploy; +/** @description: InstanceMask. */ public class InstanceMask { + + /** Mask shape. */ public int[] shape; + + /** Mask data. */ public char[] data; - + /** Initialize a new instance of the InstanceMask class. + * @param height: height. + * @param width: width. + * @param data: mask data. + */ public InstanceMask(int height, int width, char[] data) { shape = new int[]{height, width}; this.data = data; diff --git a/csrc/mmdeploy/apis/java/mmdeploy/Mat.java b/csrc/mmdeploy/apis/java/mmdeploy/Mat.java index 1bfe6c5d3..09fd647e0 100644 --- a/csrc/mmdeploy/apis/java/mmdeploy/Mat.java +++ b/csrc/mmdeploy/apis/java/mmdeploy/Mat.java @@ -1,12 +1,28 @@ package mmdeploy; +/** @description: Mat. */ public class Mat { + + /** Shape. */ public int[] shape; + + /** Pixel format. */ public int format; + + /** Data type. */ public int type; + + /** Mat data. */ public byte[] data; - + /** Initialize a new instance of the Mat class. + * @param height: height. + * @param width: width. + * @param channel: channel. + * @param format: pixel format. + * @param type: data type. + * @param data: mat data. + */ public Mat(int height, int width, int channel, PixelFormat format, DataType type, byte[] data) { shape = new int[]{height, width, channel}; diff --git a/csrc/mmdeploy/apis/java/mmdeploy/PixelFormat.java b/csrc/mmdeploy/apis/java/mmdeploy/PixelFormat.java index bb1167ada..0bf3324bb 100644 --- a/csrc/mmdeploy/apis/java/mmdeploy/PixelFormat.java +++ b/csrc/mmdeploy/apis/java/mmdeploy/PixelFormat.java @@ -1,5 +1,6 @@ package mmdeploy; +/** @description: PixelFormat. */ public enum PixelFormat { BGR(0), RGB(1), @@ -9,6 +10,9 @@ public enum PixelFormat { BGRA(5); final int value; + /** Initialize a new instance of the PixelFormat class. + * @param value: the value. + */ PixelFormat(int value) { this.value = value; } diff --git a/csrc/mmdeploy/apis/java/mmdeploy/PointF.java b/csrc/mmdeploy/apis/java/mmdeploy/PointF.java index 564e5e9b9..215755a38 100644 --- a/csrc/mmdeploy/apis/java/mmdeploy/PointF.java +++ b/csrc/mmdeploy/apis/java/mmdeploy/PointF.java @@ -1,10 +1,18 @@ package mmdeploy; +/** @description: the PointF class. */ public class PointF { + + /** x coordinate. */ public float x; + + /** y coordinate. */ public float y; - + /** Initialize a new instance of the PointF class. + * @param x: x coordinate. + * @param y: y coordinate. + */ public PointF(float x, float y) { this.x = x; this.y = y; diff --git a/csrc/mmdeploy/apis/java/mmdeploy/PoseDetector.java b/csrc/mmdeploy/apis/java/mmdeploy/PoseDetector.java index c8f8f3e09..03cdd66a0 100644 --- a/csrc/mmdeploy/apis/java/mmdeploy/PoseDetector.java +++ b/csrc/mmdeploy/apis/java/mmdeploy/PoseDetector.java @@ -1,5 +1,6 @@ package mmdeploy; +/** @description: the Java API class of PoseDetector. */ public class PoseDetector { static { System.loadLibrary("mmdeploy_java"); @@ -7,21 +8,48 @@ public class PoseDetector { private final long handle; + /** @description: Single pose estimation result of a picture. */ public static class Result { + + /** Points. */ public PointF[] point; + + /** Scores of points */ public float[] score; + + /** Initializes a new instance of the Result class. + * @param point: points. + * @param score: scores of points. + */ public Result(PointF[] point, float [] score) { this.point = point; this.score = score; } } - public PoseDetector(String modelPath, String deviceName, int deviceId) { + /** Initializes a new instance of the PoseDetector class. + * @param modelPath: model path. + * @param deviceName: device name. + * @param deviceId: device ID. + * @exception Exception: create PoseDetector failed exception. + */ + public PoseDetector(String modelPath, String deviceName, int deviceId) throws Exception{ handle = create(modelPath, deviceName, deviceId); + if (handle == -1) { + throw new Exception("Create PoseDetector failed!"); + } } - public Result[][] apply(Mat[] images) { + /** Get information of each image in a batch. + * @param images: input mats. + * @return: results of each input mat. + * @exception Exception: apply PoseDetector failed exception. + */ + public Result[][] apply(Mat[] images) throws Exception{ Result[] results = apply(handle, images); + if (results == null) { + throw new Exception("Apply PoseDetector failed!"); + } Result[][] rets = new Result[images.length][]; int offset = 0; for (int i = 0; i < images.length; ++i) { @@ -33,11 +61,21 @@ public class PoseDetector { return rets; } - public Result[] apply(Mat image) { + /** Get information of one image. + * @param image: input mat. + * @return: result of input mat. + * @exception Exception: apply PoseDetector failed exception. + */ + public Result[] apply(Mat image) throws Exception{ Mat[] images = new Mat[]{image}; - return apply(handle, images); + Result[] results = apply(handle, images); + if (results == null) { + throw new Exception("Apply PoseDetector failed!"); + } + return results; } + /** Release the instance of PoseDetector. */ public void release() { destroy(handle); } diff --git a/csrc/mmdeploy/apis/java/mmdeploy/Rect.java b/csrc/mmdeploy/apis/java/mmdeploy/Rect.java index 28ce70760..e3556f941 100644 --- a/csrc/mmdeploy/apis/java/mmdeploy/Rect.java +++ b/csrc/mmdeploy/apis/java/mmdeploy/Rect.java @@ -1,12 +1,26 @@ package mmdeploy; +/** @description: the Rect class. */ public class Rect { + + /** left coordinate. */ public float left; + + /** top coordinate. */ public float top; + + /** right coordinate. */ public float right; + + /** bottom coordinate. */ public float bottom; - + /** Initialize a new instance of the Rect class. + * @param left: left coordinate. + * @param top: top coordinate. + * @param right: right coordinate. + * @param bottom: bottom coordinate. + */ public Rect(float left, float top, float right, float bottom) { this.left = left; this.top = top; diff --git a/csrc/mmdeploy/apis/java/mmdeploy/Restorer.java b/csrc/mmdeploy/apis/java/mmdeploy/Restorer.java index 0485a5642..a8cd80dc5 100644 --- a/csrc/mmdeploy/apis/java/mmdeploy/Restorer.java +++ b/csrc/mmdeploy/apis/java/mmdeploy/Restorer.java @@ -1,5 +1,6 @@ package mmdeploy; +/** @description: the Java API class of Restorer. */ public class Restorer { static { System.loadLibrary("mmdeploy_java"); @@ -7,19 +8,43 @@ public class Restorer { private final long handle; + /** @description: Single image restore result of a picture. */ public static class Result { + + /** Result mat. */ public Mat res; + + /** Initializes a new instance of the Result class. + * @param res: result mat. + */ public Result(Mat res) { this.res = res; } } - public Restorer(String modelPath, String deviceName, int deviceId) { + /** Initializes a new instance of the Restorer class. + * @param modelPath: model path. + * @param deviceName: device name. + * @param deviceId: device ID. + * @exception Exception: create Restorer failed exception. + */ + public Restorer(String modelPath, String deviceName, int deviceId) throws Exception { handle = create(modelPath, deviceName, deviceId); + if (handle == -1) { + throw new Exception("Create Restorer failed!"); + } } - public Result[][] apply(Mat[] images) { + /** Get information of each image in a batch. + * @param images: input mats. + * @exception Exception: apply Restorer failed exception. + * @return: results of each input mat. + */ + public Result[][] apply(Mat[] images) throws Exception { Result[] results = apply(handle, images); + if (results == null) { + throw new Exception("Apply Restorer failed!"); + } Result[][] rets = new Result[images.length][]; int offset = 0; for (int i = 0; i < images.length; ++i) { @@ -31,11 +56,21 @@ public class Restorer { return rets; } - public Result[] apply(Mat image) { + /** Get information of one image. + * @param image: input mat. + * @exception Exception: apply Restorer failed exception. + * @return: result of input mat. + */ + public Result[] apply(Mat image) throws Exception{ Mat[] images = new Mat[]{image}; - return apply(handle, images); + Result[] results = apply(handle, images); + if (results == null) { + throw new Exception("Apply Restorer failed!"); + } + return results; } + /** Release the instance of Restorer. */ public void release() { destroy(handle); } diff --git a/csrc/mmdeploy/apis/java/mmdeploy/RotatedDetector.java b/csrc/mmdeploy/apis/java/mmdeploy/RotatedDetector.java new file mode 100644 index 000000000..16891a591 --- /dev/null +++ b/csrc/mmdeploy/apis/java/mmdeploy/RotatedDetector.java @@ -0,0 +1,97 @@ +package mmdeploy; + +/** @description: the Java API class of RotatedDetector. */ +public class RotatedDetector { + static { + System.loadLibrary("mmdeploy_java"); + } + + private final long handle; + + /** @description: Single rotated detection result of a picture. */ + public static class Result { + + /** Label ID. */ + public int label_id; + + /** Score. */ + public float score; + + /** Rotated bbox. */ + public float[] rbbox; + + /** Initializes a new instance of the Result class. + * @param label_id: label ID. + * @param score: score. + * @param rbbox: rotated bbox. + */ + public Result(int label_id, float score, float[] rbbox) { + this.label_id = label_id; + this.score = score; + this.rbbox = rbbox; + } + } + + /** Initializes a new instance of the RotatedDetector class. + * @param modelPath: model path. + * @param deviceName: device name. + * @param deviceId: device ID. + * @exception Exception: create RotatedDetector failed exception. + */ + public RotatedDetector(String modelPath, String deviceName, int deviceId) throws Exception{ + handle = create(modelPath, deviceName, deviceId); + if (handle == -1) { + throw new Exception("Create RotatedDetector failed!"); + } + } + + /** Get information of each image in a batch. + * @param images: input mats. + * @exception Exception: apply RotatedDetector failed exception. + * @return: results of each input mat. + */ + public Result[][] apply(Mat[] images) throws Exception{ + int[] counts = new int[images.length]; + Result[] results = apply(handle, images, counts); + if (results == null) { + throw new Exception("Apply RotatedDetector failed!"); + } + Result[][] rets = new Result[images.length][]; + int offset = 0; + for (int i = 0; i < images.length; ++i) { + Result[] row = new Result[counts[i]]; + if (counts[i] >= 0) { + System.arraycopy(results, offset, row, 0, counts[i]); + } + offset += counts[i]; + rets[i] = row; + } + return rets; + } + + /** Get information of one image. + * @param image: input mat. + * @exception Exception: apply RotatedDetector failed exception. + * @return: result of input mat. + */ + public Result[] apply(Mat image) throws Exception{ + int[] counts = new int[1]; + Mat[] images = new Mat[]{image}; + Result[] results = apply(handle, images, counts); + if (results == null) { + throw new Exception("Apply RotatedDetector failed!"); + } + return results; + } + + /** Release the instance of Rotated Detector. */ + public void release() { + destroy(handle); + } + + private native long create(String modelPath, String deviceName, int deviceId); + + private native void destroy(long handle); + + private native Result[] apply(long handle, Mat[] images, int[] count); +} diff --git a/csrc/mmdeploy/apis/java/mmdeploy/Segmentor.java b/csrc/mmdeploy/apis/java/mmdeploy/Segmentor.java index e5f593efd..b854f6aa7 100644 --- a/csrc/mmdeploy/apis/java/mmdeploy/Segmentor.java +++ b/csrc/mmdeploy/apis/java/mmdeploy/Segmentor.java @@ -7,25 +7,63 @@ public class Segmentor { private final long handle; + /** @description: Single image segmentation result of a picture. */ public static class Result { + + /** Height. */ public int height; + + /** Width. */ public int width; + + /** Number of classes. */ public int classes; + + /** Segmentation mask. */ public int[] mask; - public Result(int height, int width, int classes, int [] mask) { + + /** Segmentation score. */ + public float[] score; + + /** Initializes a new instance of the Result class. + * @param height: height. + * @param width: width. + * @param classes: number of classes. + * @param mask: segmentation mask. + * @param score: segmentation score. + */ + public Result(int height, int width, int classes, int [] mask, float [] score) { this.height = height; this.width = width; this.classes = classes; this.mask = mask; + this.score = score; } } - public Segmentor(String modelPath, String deviceName, int deviceId) { + /** Initializes a new instance of the Segmentor class. + * @param modelPath: model path. + * @param deviceName: device name. + * @param deviceId: device ID. + * @exception Exception: create Segmentator failed exception. + */ + public Segmentor(String modelPath, String deviceName, int deviceId) throws Exception{ handle = create(modelPath, deviceName, deviceId); + if (handle == -1) { + throw new Exception("Create Segmentor failed!"); + } } - public Result[][] apply(Mat[] images) { + /** Get information of each image in a batch. + * @param images: input mats. + * @exception Exception: apply Segmentor failed exception. + * @return: results of each input mat. + */ + public Result[][] apply(Mat[] images) throws Exception{ Result[] results = apply(handle, images); + if (results == null) { + throw new Exception("Apply Segmentor failed!"); + } Result[][] rets = new Result[images.length][]; int offset = 0; for (int i = 0; i < images.length; ++i) { @@ -37,11 +75,21 @@ public class Segmentor { return rets; } - public Result[] apply(Mat image) { + /** Get information of one image. + * @param image: input mat. + * @exception Exception: apply Segmentor failed exception. + * @return: result of input mat. + */ + public Result[] apply(Mat image) throws Exception{ Mat[] images = new Mat[]{image}; - return apply(handle, images); + Result[] results = apply(handle, images); + if (results == null) { + throw new Exception("Apply Segmentor failed!"); + } + return results; } + /** Release the instance of Segmentor. */ public void release() { destroy(handle); } diff --git a/csrc/mmdeploy/apis/java/mmdeploy/TextDetector.java b/csrc/mmdeploy/apis/java/mmdeploy/TextDetector.java index 3a1862a3f..689c13a82 100644 --- a/csrc/mmdeploy/apis/java/mmdeploy/TextDetector.java +++ b/csrc/mmdeploy/apis/java/mmdeploy/TextDetector.java @@ -1,5 +1,6 @@ package mmdeploy; +/** @description: the Java API class of TextDetector. */ public class TextDetector { static { System.loadLibrary("mmdeploy_java"); @@ -7,22 +8,49 @@ public class TextDetector { private final long handle; + /** @description: Single text detection result of a picture. */ public static class Result { + + /** Bbox. */ public PointF[] bbox; + + /** Score. */ public float score; + + /** Initializes a new instance of the Result class. + * @param bbox: bbox. + * @param score: score. + */ public Result(PointF[] bbox, float score) { this.bbox = bbox; this.score = score; } } - public TextDetector(String modelPath, String deviceName, int deviceId) { + /** Initializes a new instance of the TextDetector class. + * @param modelPath: model path. + * @param deviceName: device name. + * @param deviceId: device ID. + * @exception Exception: create TextDetector failed exception. + */ + public TextDetector(String modelPath, String deviceName, int deviceId) throws Exception{ handle = create(modelPath, deviceName, deviceId); + if (handle == -1) { + throw new Exception("Create TextDetector failed!"); + } } - public Result[][] apply(Mat[] images) { + /** Get information of each image in a batch. + * @param images: input mats. + * @exception Exception: apply TextDetector failed exception. + * @return: results of each input mat. + */ + public Result[][] apply(Mat[] images) throws Exception{ int[] counts = new int[images.length]; Result[] results = apply(handle, images, counts); + if (results == null) { + throw new Exception("Apply TextDetector failed!"); + } Result[][] rets = new Result[images.length][]; int offset = 0; for (int i = 0; i < images.length; ++i) { @@ -36,12 +64,22 @@ public class TextDetector { return rets; } - public Result[] apply(Mat image) { + /** Get information of one image. + * @param image: input mat. + * @exception Exception: apply TextDetector failed exception. + * @return: result of input mat. + */ + public Result[] apply(Mat image) throws Exception{ int[] counts = new int[1]; Mat[] images = new Mat[]{image}; - return apply(handle, images, counts); + Result[] results = apply(handle, images, counts); + if (results == null) { + throw new Exception("Apply TextDetector failed!"); + } + return results; } + /** Release the instance of TextDetector. */ public void release() { destroy(handle); } diff --git a/csrc/mmdeploy/apis/java/mmdeploy/TextRecognizer.java b/csrc/mmdeploy/apis/java/mmdeploy/TextRecognizer.java index dd5f662ff..fff67972f 100644 --- a/csrc/mmdeploy/apis/java/mmdeploy/TextRecognizer.java +++ b/csrc/mmdeploy/apis/java/mmdeploy/TextRecognizer.java @@ -1,5 +1,6 @@ package mmdeploy; +/** @description: the Java API class of TextRecognizer. */ public class TextRecognizer { static { System.loadLibrary("mmdeploy_java"); @@ -7,21 +8,48 @@ public class TextRecognizer { private final long handle; + /** @description: Single text recognition result of a picture. */ public static class Result { + + /** Text. */ public byte [] text; + + /** Score. */ public float [] score; + + /** Initializes a new instance of the Result class. + * @param text: text. + * @param score: score. + */ public Result(byte [] text, float [] score) { this.text = text; this.score = score; } } - public TextRecognizer(String modelPath, String deviceName, int deviceId) { + /** Initializes a new instance of the TextRecognizer class. + * @param modelPath: model path. + * @param deviceName: device name. + * @param deviceId: device ID. + * @exception Exception: create TextRecognizer failed exception. + */ + public TextRecognizer(String modelPath, String deviceName, int deviceId) throws Exception{ handle = create(modelPath, deviceName, deviceId); + if (handle == -1) { + throw new Exception("Create TextRecognizer failed!"); + } } - public Result[][] apply(Mat[] images) { + /** Get information of each image in a batch. + * @param images: input mats. + * @exception Exception: apply TextRecognizer failed exception. + * @return: results of each input mat. + */ + public Result[][] apply(Mat[] images) throws Exception{ Result[] results = apply(handle, images); + if (results == null) { + throw new Exception("Apply TextRecognizer failed!"); + } Result[][] rets = new Result[images.length][]; int offset = 0; for (int i = 0; i < images.length; ++i) { @@ -33,16 +61,32 @@ public class TextRecognizer { return rets; } - public Result[] apply(Mat image) { + /** Get information of one image. + * @param image: input mat. + * @exception Exception: apply TextDetector failed exception. + * @return: result of input mat. + */ + public Result[] apply(Mat image) throws Exception{ Mat[] images = new Mat[]{image}; - return apply(handle, images); + Result[] results = apply(handle, images); + if (results == null) { + throw new Exception("Apply TextRecognizer failed!"); + } + return results; } + /** Get information of one image from bboxes. + * @param image: input mat. + * @param bbox: bboxes information. + * @param bbox_count: numter of bboxes + * @return: result of input mat. + */ public Result[] applyBbox(Mat image, TextDetector.Result[] bbox, int[] bbox_count) { Mat[] images = new Mat[]{image}; return applyBbox(handle, images, bbox, bbox_count); } + /** Release the instance of TextRecognizer. */ public void release() { destroy(handle); } diff --git a/csrc/mmdeploy/apis/java/native/CMakeLists.txt b/csrc/mmdeploy/apis/java/native/CMakeLists.txt index aad606d74..43770aaa1 100644 --- a/csrc/mmdeploy/apis/java/native/CMakeLists.txt +++ b/csrc/mmdeploy/apis/java/native/CMakeLists.txt @@ -13,6 +13,7 @@ mmdeploy_add_library(${PROJECT_NAME} SHARED EXCLUDE mmdeploy_Segmentor.cpp mmdeploy_Restorer.cpp mmdeploy_PoseDetector.cpp + mmdeploy_RotatedDetector.cpp mmdeploy_TextDetector.cpp mmdeploy_TextRecognizer.cpp) diff --git a/csrc/mmdeploy/apis/java/native/mmdeploy_RotatedDetector.cpp b/csrc/mmdeploy/apis/java/native/mmdeploy_RotatedDetector.cpp new file mode 100644 index 000000000..3872e7e15 --- /dev/null +++ b/csrc/mmdeploy/apis/java/native/mmdeploy_RotatedDetector.cpp @@ -0,0 +1,61 @@ +#include "mmdeploy_RotatedDetector.h" + +#include + +#include "mmdeploy/apis/c/mmdeploy/rotated_detector.h" +#include "mmdeploy/apis/java/native/common.h" +#include "mmdeploy/core/logger.h" + +jlong Java_mmdeploy_RotatedDetector_create(JNIEnv *env, jobject, jstring modelPath, + jstring deviceName, jint device_id) { + auto model_path = env->GetStringUTFChars(modelPath, nullptr); + auto device_name = env->GetStringUTFChars(deviceName, nullptr); + mmdeploy_rotated_detector_t rotated_detector{}; + auto ec = mmdeploy_rotated_detector_create_by_path(model_path, device_name, (int)device_id, + &rotated_detector); + env->ReleaseStringUTFChars(modelPath, model_path); + env->ReleaseStringUTFChars(deviceName, device_name); + if (ec) { + MMDEPLOY_ERROR("failed to create rotated detector, code = {}", ec); + return -1; + } + return (jlong)rotated_detector; +} + +void Java_mmdeploy_RotatedDetector_destroy(JNIEnv *, jobject, jlong handle) { + MMDEPLOY_DEBUG("Java_mmdeploy_RotatedDetector_destroy"); + mmdeploy_rotated_detector_destroy((mmdeploy_rotated_detector_t)handle); +} + +jobjectArray Java_mmdeploy_RotatedDetector_apply(JNIEnv *env, jobject thiz, jlong handle, + jobjectArray images, jintArray counts) { + return With(env, images, [&](const mmdeploy_mat_t imgs[], int size) -> jobjectArray { + mmdeploy_rotated_detection_t *results{}; + int *result_count{}; + auto ec = mmdeploy_rotated_detector_apply((mmdeploy_rotated_detector_t)handle, imgs, size, + &results, &result_count); + if (ec) { + MMDEPLOY_ERROR("failed to apply rotated detector, code = {}", ec); + return NULL; + } + auto result_cls = env->FindClass("mmdeploy/RotatedDetector$Result"); + auto result_ctor = env->GetMethodID(result_cls, "", "(IF[F)V"); + auto total = std::accumulate(result_count, result_count + size, 0); + auto array = env->NewObjectArray(total, result_cls, nullptr); + + for (int i = 0; i < total; ++i) { + jfloatArray rbbox = env->NewFloatArray(5); + env->SetFloatArrayRegion(rbbox, 0, 5, (jfloat *)results[i].rbbox); + auto res = env->NewObject(result_cls, result_ctor, (jint)results[i].label_id, + (jfloat)results[i].score, rbbox); + env->SetObjectArrayElement(array, i, res); + } + auto counts_array = env->GetIntArrayElements(counts, nullptr); + for (int i = 0; i < size; ++i) { + counts_array[i] = result_count[i]; + } + env->ReleaseIntArrayElements(counts, counts_array, 0); + mmdeploy_rotated_detector_release_result(results, result_count); + return array; + }); +} diff --git a/csrc/mmdeploy/apis/java/native/mmdeploy_RotatedDetector.h b/csrc/mmdeploy/apis/java/native/mmdeploy_RotatedDetector.h new file mode 100644 index 000000000..6de527ec4 --- /dev/null +++ b/csrc/mmdeploy/apis/java/native/mmdeploy_RotatedDetector.h @@ -0,0 +1,36 @@ +/* DO NOT EDIT THIS FILE - it is machine generated */ +#include +/* Header for class mmdeploy_RotatedDetector */ + +#ifndef _Included_mmdeploy_RotatedDetector +#define _Included_mmdeploy_RotatedDetector +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: mmdeploy_RotatedDetector + * Method: create + * Signature: (Ljava/lang/String;Ljava/lang/String;I)J + */ +JNIEXPORT jlong JNICALL Java_mmdeploy_RotatedDetector_create(JNIEnv *, jobject, jstring, jstring, + jint); + +/* + * Class: mmdeploy_RotatedDetector + * Method: destroy + * Signature: (J)V + */ +JNIEXPORT void JNICALL Java_mmdeploy_RotatedDetector_destroy(JNIEnv *, jobject, jlong); + +/* + * Class: mmdeploy_RotatedDetector + * Method: apply + * Signature: (J[Lmmdeploy/Mat;[I)[Lmmdeploy/RotatedDetector/Result; + */ +JNIEXPORT jobjectArray JNICALL Java_mmdeploy_RotatedDetector_apply(JNIEnv *, jobject, jlong, + jobjectArray, jintArray); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/csrc/mmdeploy/apis/java/native/mmdeploy_Segmentor.cpp b/csrc/mmdeploy/apis/java/native/mmdeploy_Segmentor.cpp index f3c466cd0..12df31a49 100644 --- a/csrc/mmdeploy/apis/java/native/mmdeploy_Segmentor.cpp +++ b/csrc/mmdeploy/apis/java/native/mmdeploy_Segmentor.cpp @@ -16,6 +16,7 @@ jlong Java_mmdeploy_Segmentor_create(JNIEnv *env, jobject, jstring modelPath, js env->ReleaseStringUTFChars(deviceName, device_name); if (ec) { MMDEPLOY_ERROR("failed to create segmentor, code = {}", ec); + return -1; } return (jlong)segmentor; } @@ -27,23 +28,40 @@ void Java_mmdeploy_Segmentor_destroy(JNIEnv *, jobject, jlong handle) { jobjectArray Java_mmdeploy_Segmentor_apply(JNIEnv *env, jobject thiz, jlong handle, jobjectArray images) { - return With(env, images, [&](const mmdeploy_mat_t imgs[], int size) { + return With(env, images, [&](const mmdeploy_mat_t imgs[], int size) -> jobjectArray { mmdeploy_segmentation_t *results{}; auto ec = mmdeploy_segmentor_apply((mmdeploy_segmentor_t)handle, imgs, size, &results); if (ec) { MMDEPLOY_ERROR("failed to apply segmentor, code = {}", ec); + return NULL; } auto result_cls = env->FindClass("mmdeploy/Segmentor$Result"); - auto result_ctor = env->GetMethodID(result_cls, "", "(III[I)V"); + auto result_ctor = env->GetMethodID(result_cls, "", "(III[I[F)V"); auto array = env->NewObjectArray(size, result_cls, nullptr); - + jintArray jmask; + jfloatArray jscore; for (int i = 0; i < size; ++i) { int *mask = results[i].mask; - jintArray jmask = env->NewIntArray(results[i].height * results[i].width); - env->SetIntArrayRegion(jmask, 0, results[i].width * results[i].height, (const jint *)mask); + float *score = results[i].score; + if (results[i].mask) { + jmask = env->NewIntArray(results[i].height * results[i].width); + env->SetIntArrayRegion(jmask, 0, results[i].width * results[i].height, (const jint *)mask); + } else { + jmask = env->NewIntArray(0); + env->SetIntArrayRegion(jmask, 0, 0, nullptr); + } + if (results[i].score) { + jscore = env->NewFloatArray(results[i].classes * results[i].height * results[i].width); + env->SetFloatArrayRegion(jscore, 0, + results[i].classes * results[i].height * results[i].width, + (const jfloat *)score); + } else { + jscore = env->NewFloatArray(0); + env->SetFloatArrayRegion(jscore, 0, 0, nullptr); + } auto res = env->NewObject(result_cls, result_ctor, (jint)results[i].height, - (jint)results[i].width, (jint)results[i].classes, jmask); + (jint)results[i].width, (jint)results[i].classes, jmask, jscore); env->SetObjectArrayElement(array, i, res); } mmdeploy_segmentor_release_result(results, size); diff --git a/demo/java/ImageClassification.java b/demo/java/ImageClassification.java index 6016f53b7..b8ff53aa5 100644 --- a/demo/java/ImageClassification.java +++ b/demo/java/ImageClassification.java @@ -9,8 +9,14 @@ import java.awt.image.DataBufferByte; import java.io.File; import java.io.IOException; +/** @description: this is a class for ImageClassification java demo. */ public class ImageClassification { + /** The main function for ImageClassification Java demo. + * @param deviceName: the device name of the demo. + * @param modelPath: the image classification model path. + * @param imagePath: the image path. + */ public static void main(String[] args) { // Parse arguments if (args.length != 3) { diff --git a/demo/java/ImageRestorer.java b/demo/java/ImageRestorer.java index 957a494b0..719169db7 100644 --- a/demo/java/ImageRestorer.java +++ b/demo/java/ImageRestorer.java @@ -9,8 +9,14 @@ import java.awt.image.DataBufferByte; import java.io.File; import java.io.IOException; +/** @description: this is a class for ImageRestorer java demo. */ public class ImageRestorer { + /** The main function for ImageRestorer Java demo. + * @param deviceName: the device name of the demo. + * @param modelPath: the image restorer model path. + * @param imagePath: the image path. + */ public static void main(String[] args) { // Parse arguments if (args.length != 3) { @@ -26,6 +32,7 @@ public class ImageRestorer { try { restorer = new Restorer(modelPath, deviceName, 0); + // load image Mat img = Utils.loadImage(imagePath); diff --git a/demo/java/ImageSegmentation.java b/demo/java/ImageSegmentation.java index 86953fb0e..7a64b9a11 100644 --- a/demo/java/ImageSegmentation.java +++ b/demo/java/ImageSegmentation.java @@ -9,8 +9,14 @@ import java.awt.image.DataBufferByte; import java.io.File; import java.io.IOException; +/** @description: this is a class for ImageSegmentation java demo. */ public class ImageSegmentation { + /** The main function for ImageSegmentation Java demo. + * @param deviceName: the device name of the demo. + * @param modelPath: the image segmentation model path. + * @param imagePath: the image path. + */ public static void main(String[] args) { // Parse arguments if (args.length != 3) { @@ -26,6 +32,7 @@ public class ImageSegmentation { try { segmentor = new Segmentor(modelPath, deviceName, 0); + // load image Mat img = Utils.loadImage(imagePath); diff --git a/demo/java/ObjectDetection.java b/demo/java/ObjectDetection.java index 01eee41b4..3f68c2575 100644 --- a/demo/java/ObjectDetection.java +++ b/demo/java/ObjectDetection.java @@ -12,8 +12,14 @@ import java.awt.Graphics; import java.io.File; import java.io.IOException; +/** @description: this is a class for ObjectDetection java demo. */ public class ObjectDetection { + /** The main function for ObjectDetection Java demo. + * @param deviceName: the device name of the demo. + * @param modelPath: the object detection model path. + * @param imagePath: the image path. + */ public static void main(String[] args) { // Parse arguments if (args.length != 3) { @@ -28,12 +34,14 @@ public class ObjectDetection { Detector detector = null; try { detector = new Detector(modelPath, deviceName, 0); + // load image BufferedImage srcImg = ImageIO.read(new File(imagePath)); Mat img = Utils.bufferedImage2Mat(srcImg); // apply detector Detector.Result[] result = detector.apply(img); + // print results Graphics ghandle = srcImg.createGraphics(); for (int i = 0; i < result.length; i++) { diff --git a/demo/java/Ocr.java b/demo/java/Ocr.java index 1edc04d77..8d9a71852 100644 --- a/demo/java/Ocr.java +++ b/demo/java/Ocr.java @@ -10,12 +10,19 @@ import java.awt.image.DataBufferByte; import java.io.File; import java.io.IOException; +/** @description: this is a class for Ocr java demo. */ public class Ocr { + /** The main function for Ocr Java demo. + * @param deviceName: the device name of the demo. + * @param detModelPath: the text detection model path. + * @param recModelPath: the text recognition model path. + * @param imagePath: the image path. + */ public static void main(String[] args) { // Parse arguments if (args.length != 4) { - System.out.println("usage:\njava TextDetection deviceName detModelPath recModelPath imagePath"); + System.out.println("usage:\njava Ocr deviceName detModelPath recModelPath imagePath"); return; } String deviceName = args[0]; @@ -30,13 +37,19 @@ public class Ocr { try { text_detector = new TextDetector(detModelPath, deviceName, 0); text_recognizer = new TextRecognizer(recModelPath, deviceName, 0); + // load image Mat img = Utils.loadImage(imagePath); // apply text detector TextDetector.Result[] detResult = text_detector.apply(img); + int [] detResultCount = {detResult.length}; TextRecognizer.Result[] recResult = text_recognizer.applyBbox(img, detResult, detResultCount); + if (recResult == null) { + System.out.println("Apply TextRecognizer failed."); + System.exit(1); + } // print results for (int i = 0; i < detResultCount[0]; ++i) { System.out.printf("box[%d]: %s\n", i, new String(recResult[i].text)); diff --git a/demo/java/PoseDetection.java b/demo/java/PoseDetection.java index e2c381ac3..6c6bba4f7 100644 --- a/demo/java/PoseDetection.java +++ b/demo/java/PoseDetection.java @@ -9,8 +9,14 @@ import java.awt.image.DataBufferByte; import java.io.File; import java.io.IOException; +/** @description: this is a class for PoseDetection java demo. */ public class PoseDetection { + /** The main function for PoseDetection Java demo. + * @param deviceName: the device name of the demo. + * @param modelPath: the pose detection model path. + * @param imagePath: the image path. + */ public static void main(String[] args) { // Parse arguments if (args.length != 3) { @@ -22,15 +28,16 @@ public class PoseDetection { String imagePath = args[2]; // create pose estimator - PoseDetector pose_estimator = null; + PoseDetector poseEstimator = null; try { - pose_estimator = new PoseDetector(modelPath, deviceName, 0); + poseEstimator = new PoseDetector(modelPath, deviceName, 0); + // load image Mat img = Utils.loadImage(imagePath); // apply pose estimator - PoseDetector.Result[] result = pose_estimator.apply(img); + PoseDetector.Result[] result = poseEstimator.apply(img); // print results for (PoseDetector.Result value : result) { @@ -42,8 +49,8 @@ public class PoseDetection { System.out.println("exception: " + e.getMessage()); } finally { // release pose estimator - if (pose_estimator != null) { - pose_estimator.release(); + if (poseEstimator != null) { + poseEstimator.release(); } } } diff --git a/demo/java/RotatedDetection.java b/demo/java/RotatedDetection.java new file mode 100644 index 000000000..ed27966cb --- /dev/null +++ b/demo/java/RotatedDetection.java @@ -0,0 +1,83 @@ +import mmdeploy.RotatedDetector; +import mmdeploy.PixelFormat; +import mmdeploy.DataType; +import mmdeploy.Mat; + +import javax.imageio.ImageIO; + +import java.awt.Color; +import java.awt.image.BufferedImage; +import java.awt.image.DataBufferByte; +import java.awt.Graphics; +import java.io.File; +import java.io.IOException; +import java.lang.Math; + +/** @description: this is a class for RotatedDetection java demo. */ +public class RotatedDetection { + + /** The main function for RotatedDetection Java demo. + * @param deviceName: the device name of the demo. + * @param modelPath: the rotated detection model path. + * @param imagePath: the image path. + */ + public static void main(String[] args) { + // Parse arguments + if (args.length != 3) { + System.out.println("usage:\njava RotatedDetection deviceName modelPath imagePath"); + return; + } + String deviceName = args[0]; + String modelPath = args[1]; + String imagePath = args[2]; + + // create rotated detector + RotatedDetector rotatedDetector = null; + try { + rotatedDetector = new RotatedDetector(modelPath, deviceName, 0); + + // load image + BufferedImage srcImg = ImageIO.read(new File(imagePath)); + Mat img = Utils.bufferedImage2Mat(srcImg); + + // apply rotated detector + RotatedDetector.Result[] result = rotatedDetector.apply(img); + + // print results + Graphics ghandle = srcImg.createGraphics(); + for (int i = 0; i < result.length; i++) { + RotatedDetector.Result value = result[i]; + float cx, cy, w, h, angle; + cx = value.rbbox[0]; + cy = value.rbbox[1]; + w = value.rbbox[2]; + h = value.rbbox[3]; + angle = value.rbbox[4]; + float wx = w / 2 * (float)Math.cos(angle); + float wy = w / 2 * (float)Math.sin(angle); + float hx = -h / 2 * (float)Math.sin(angle); + float hy = h / 2 * (float)Math.cos(angle); + System.out.printf("box %d, score %.2f, point1: (%.2f, %.2f), point2: (%.2f, %.2f), point3: (%.2f, %.2f), point4: (%.2f, %.2f)\n", + i, value.score, cx - wx - hx, cy - wy - hy, cx + wx - hx, cy + wy - hy, cx + wx + hx, cy + wy + hy, cx - wx + hx, cy - wy + hy); + + // skip rotated detections less than specified score threshold + if (value.score < 0.1) { + continue; + } + ghandle.setColor(new Color(0, 255, 0)); + int[] polygonX = new int[] {(int)(cx - wx - hx), (int)(cx + wx - hx), (int)(cx + wx + hx), (int)(cx - wx + hx)}; + int[] polygonY = new int[] {(int)(cy - wy - hy), (int)(cy + wy - hy), (int)(cy + wy + hy), (int)(cy - wy + hy)}; + ghandle.drawPolygon(polygonX, polygonY, 4); + } + ghandle.dispose(); + ImageIO.write(srcImg, "png", new File("output_rotated_detection.png")); + } catch (Exception e) { + System.out.println("exception: " + e.getMessage()); + } finally { + // release rotated detector + if (rotatedDetector != null) { + rotatedDetector.release(); + } + } + } +} diff --git a/demo/java/Utils.java b/demo/java/Utils.java index daf0607f1..6936b749f 100644 --- a/demo/java/Utils.java +++ b/demo/java/Utils.java @@ -7,12 +7,25 @@ import java.awt.image.BufferedImage; import java.awt.image.DataBufferByte; import java.io.File; import java.io.IOException; +import java.lang.*; +/** @description: this is a util class for java demo. */ public class Utils { + + /** This function loads the image by path. + * @param path: the image path. + * @return: the image with Mat format. + * @exception IOException: throws an IO exception when load failed. + */ public static Mat loadImage(String path) throws IOException { BufferedImage img = ImageIO.read(new File(path)); return bufferedImage2Mat(img); } + + /** This function changes bufferedImage to Mat. + * @param img: the bufferedImage. + * @return: the image with Mat format. + */ public static Mat bufferedImage2Mat(BufferedImage img) { byte[] data = ((DataBufferByte) img.getData().getDataBuffer()).getData(); return new Mat(img.getHeight(), img.getWidth(), img.getColorModel().getNumComponents(),