diff --git a/deploy/shitu_android_demo/app/app.iml b/deploy/shitu_android_demo/app/app.iml new file mode 100644 index 000000000..988639175 --- /dev/null +++ b/deploy/shitu_android_demo/app/app.iml @@ -0,0 +1,183 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/deploy/shitu_android_demo/app/build.gradle b/deploy/shitu_android_demo/app/build.gradle new file mode 100644 index 000000000..163832e93 --- /dev/null +++ b/deploy/shitu_android_demo/app/build.gradle @@ -0,0 +1,93 @@ +import java.security.MessageDigest + +apply plugin: 'com.android.application' + +android { + compileSdkVersion 30 + defaultConfig { + applicationId "com.baidu.paddle.lite.demo.pp_shitu" + minSdkVersion 15 + targetSdkVersion 30 + versionCode 1 + versionName "1.0" + testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner" + externalNativeBuild { + cmake { + arguments '-DANDROID_PLATFORM=android-23', '-DANDROID_STL=c++_shared', "-DANDROID_TOOLCHAIN=" + abiFilters 'arm64-v8a' + cppFlags "-std=c++11" + } + } + } + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + } + } + externalNativeBuild { + cmake { + path "src/main/cpp/CMakeLists.txt" + version "3.18.1" + } + } +} + +dependencies { + implementation fileTree(include: ['*.jar'], dir: 'libs') + implementation 'com.android.support:appcompat-v7:28.0.0' + implementation 'com.android.support.constraint:constraint-layout:1.1.3' + implementation 'com.android.support:design:28.0.0' + testImplementation 'junit:junit:4.12' + androidTestImplementation 'com.android.support.test:runner:1.0.2' + androidTestImplementation 'com.android.support.test.espresso:espresso-core:3.0.2' +} +tasks.withType(JavaCompile) { + options.encoding = "UTF-8" +} +def archives = [ + [ + 'src' : 'https://paddlelite-demo.bj.bcebos.com/libs/android/paddle_lite_libs_v2_10_gpu.tar.gz', + 'dest': 'PaddleLite' + ], + [ + 'src' : 'https://paddlelite-demo.bj.bcebos.com/libs/android/opencv-4.2.0-android-sdk.tar.gz', + 'dest': 'OpenCV' + ], + [ + 'src' : 'https://paddlelite-demo.bj.bcebos.com/demo/PP_shitu/models/ppshitu_lite_models_v1.0.tar.gz', + 'dest' : 'src/main/assets/models' + ] +] + +task downloadAndExtractArchives(type: DefaultTask) { + doFirst { + println "Downloading and extracting archives including libs and models" + } + doLast { + // Prepare cache folder for archives + String cachePath = "cache" + if (!file("${cachePath}").exists()) { + mkdir "${cachePath}" + } + archives.eachWithIndex { archive, index -> + MessageDigest messageDigest = MessageDigest.getInstance('MD5') + messageDigest.update(archive.src.bytes) + String cacheName = new BigInteger(1, messageDigest.digest()).toString(32) + // Download the target archive if not exists + boolean copyFiles = !file("${archive.dest}").exists() + if (!file("${cachePath}/${cacheName}.tar.gz").exists()) { + ant.get(src: archive.src, dest: file("${cachePath}/${cacheName}.tar.gz")) + copyFiles = true; // force to copy files from the latest archive files + } + // Extract the target archive if its dest path does not exists + if (copyFiles) { + copy { + from tarTree("${cachePath}/${cacheName}.tar.gz") + into "${archive.dest}" + } + } + } + } +} +preBuild.dependsOn downloadAndExtractArchives diff --git a/deploy/shitu_android_demo/app/proguard-rules.pro b/deploy/shitu_android_demo/app/proguard-rules.pro new file mode 100644 index 000000000..f1b424510 --- /dev/null +++ b/deploy/shitu_android_demo/app/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile diff --git a/deploy/shitu_android_demo/app/src/androidTest/java/com/baidu/paddle/lite/demo/pp_shitu/ExampleInstrumentedTest.java b/deploy/shitu_android_demo/app/src/androidTest/java/com/baidu/paddle/lite/demo/pp_shitu/ExampleInstrumentedTest.java new file mode 100644 index 000000000..2d628caff --- /dev/null +++ b/deploy/shitu_android_demo/app/src/androidTest/java/com/baidu/paddle/lite/demo/pp_shitu/ExampleInstrumentedTest.java @@ -0,0 +1,26 @@ +package com.baidu.paddle.lite.demo.pp_shitu; + +import android.content.Context; +import android.support.test.InstrumentationRegistry; +import android.support.test.runner.AndroidJUnit4; + +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.junit.Assert.*; + +/** + * Instrumented test, which will execute on an Android device. + * + * @see Testing documentation + */ +@RunWith(AndroidJUnit4.class) +public class ExampleInstrumentedTest { + @Test + public void useAppContext() { + // Context of the app under test. + Context appContext = InstrumentationRegistry.getTargetContext(); + + assertEquals("com.baidu.paddle.lite.demo", appContext.getPackageName()); + } +} diff --git a/deploy/shitu_android_demo/app/src/main/AndroidManifest.xml b/deploy/shitu_android_demo/app/src/main/AndroidManifest.xml new file mode 100644 index 000000000..d35c5729d --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/AndroidManifest.xml @@ -0,0 +1,34 @@ + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/deploy/shitu_android_demo/app/src/main/assets/images/demo.jpg b/deploy/shitu_android_demo/app/src/main/assets/images/demo.jpg new file mode 100644 index 000000000..2ef10aae5 Binary files /dev/null and b/deploy/shitu_android_demo/app/src/main/assets/images/demo.jpg differ diff --git a/deploy/shitu_android_demo/app/src/main/assets/index/README.md b/deploy/shitu_android_demo/app/src/main/assets/index/README.md new file mode 100644 index 000000000..7e530c42c --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/assets/index/README.md @@ -0,0 +1 @@ +put `*.index` and `*.txt` here. such as `original.index` and `original.txt` diff --git a/deploy/shitu_android_demo/app/src/main/assets/models/README.md b/deploy/shitu_android_demo/app/src/main/assets/models/README.md new file mode 100644 index 000000000..b318305b1 --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/assets/models/README.md @@ -0,0 +1 @@ +put `*.nb` inference model file there. such as `general_PPLCNetV2_base_quant_v1.0_lite.nb` and `mainbody_PPLCNet_x2_5_640_quant_v1.0_lite.nb` diff --git a/deploy/shitu_android_demo/app/src/main/cpp/CMakeLists.txt b/deploy/shitu_android_demo/app/src/main/cpp/CMakeLists.txt new file mode 100644 index 000000000..1dad50c94 --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/cpp/CMakeLists.txt @@ -0,0 +1,105 @@ +# For more information about using CMake with Android Studio, read the +# documentation: https://d.android.com/studio/projects/add-native-code.html + +# Sets the minimum version of CMake required to build the native library. +#project(ShituDemo) + +cmake_minimum_required(VERSION 3.4.1) + +# Creates and names a library, sets it as either STATIC or SHARED, and provides +# the relative paths to its source code. You can define multiple libraries, and +# CMake builds them for you. Gradle automatically packages shared libraries with +# your APK. + +set(PaddleLite_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../PaddleLite") +include_directories(${PaddleLite_DIR}/cxx/include) + +set(OpenCV_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../OpenCV/sdk/native/jni") +find_package(OpenCV REQUIRED) +message(STATUS "OpenCV libraries: ${OpenCV_LIBS}") +include_directories(${OpenCV_INCLUDE_DIRS}) + +#set(PaddleLite_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../PaddleLite") + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../../PaddleLite/src/main/cpp/include/faiss) +set(target faiss) + +set(CMAKE_CXX_FLAGS + "${CMAKE_CXX_FLAGS} -ffast-math -Ofast -Os -DNDEBUG -fexceptions -fomit-frame-pointer -fno-asynchronous-unwind-tables -fno-unwind-tables" +) +set(CMAKE_CXX_FLAGS + "${CMAKE_CXX_FLAGS} -fvisibility=hidden -fvisibility-inlines-hidden -fdata-sections -ffunction-sections" +) +set(CMAKE_SHARED_LINKER_FLAGS + "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gc-sections -Wl,-z,nocopyreloc") + +add_library( + # Sets the name of the library. + Native + # Sets the library as a shared library. + SHARED + # Provides a relative path to your source file(s). + Native.cc Pipeline.cc Utils.cc ObjectDetector.cc FeatureExtractor.cc VectorSearch.cc) + +find_library( + # Sets the name of the path variable. + log-lib + # Specifies the name of the NDK library that you want CMake to locate. + log) + +add_library( + # Sets the name of the library. + paddle_light_api_shared + # Sets the library as a shared library. + SHARED + # Provides a relative path to your source file(s). + IMPORTED) + +add_library( + # Sets the name of the library. + faiss + STATIC + IMPORTED) + +set_target_properties( + # Specifies the target library. + paddle_light_api_shared + # Specifies the parameter you want to define. + PROPERTIES + IMPORTED_LOCATION + ${PaddleLite_DIR}/cxx/libs/${ANDROID_ABI}/libpaddle_light_api_shared.so + # Provides the path to the library you want to import. +) + +# if there libfaiss.a not exist, will download it automatically +IF(NOT ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/main/jniLibs/arm64-v8a/libfaiss.a) + message(STATUS "Downloading ${OCI_LIB_ZIP_NAME} to ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/main/jniLibs/arm64-v8a/") + FILE(DOWNLOAD https://paddle-imagenet-models-name.bj.bcebos.com/demos/lib/libfaiss.a + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/main/jniLibs/arm64-v8a/libfaiss.a + TIMEOUT ${DOWNLOAD_OCI_LIB_TIMEOUT} + STATUS ERR + SHOW_PROGRESS) +ENDIF() + +set_target_properties( + faiss + PROPERTIES + IMPORTED_LOCATION + ${CMAKE_CURRENT_SOURCE_DIR}/../../../src/main/jniLibs/arm64-v8a/libfaiss.a +) + +# Specifies libraries CMake should link to your target library. You can link +# multiple libraries, such as libraries you define in this build script, +# prebuilt third-party libraries, or system libraries. + +target_link_libraries( + # Specifies the target library. + Native + paddle_light_api_shared + jnigraphics + ${OpenCV_LIBS} + GLESv2 + EGL + ${log-lib} + faiss +) diff --git a/deploy/shitu_android_demo/app/src/main/cpp/FeatureExtractor.cc b/deploy/shitu_android_demo/app/src/main/cpp/FeatureExtractor.cc new file mode 100644 index 000000000..090e0d675 --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/cpp/FeatureExtractor.cc @@ -0,0 +1,98 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "FeatureExtractor.h" // NOLINT +#include // NOLINT + +void FeatureExtract::RunRecModel(const cv::Mat &img, double &cost_time, + std::vector &feature) { + // Read img + cv::Mat img_fp; + ResizeImage(img, img_fp); + NormalizeImage(&img_fp, this->mean_, this->std_, this->scale_); + std::vector input(1 * 3 * img_fp.rows * img_fp.cols, 0.0f); + Permute(&img_fp, input.data()); + auto pre_cost0 = GetCurrentUS(); + + // Prepare input data from image + std::unique_ptr input_tensor( + std::move(this->predictor_->GetInput(0))); + input_tensor->Resize({1, 3, this->size, this->size}); + auto *data0 = input_tensor->mutable_data(); + + for (int i = 0; i < input.size(); ++i) { + data0[i] = input[i]; + } + auto start = std::chrono::system_clock::now(); + // Run predictor + this->predictor_->Run(); + + // Get output and post process + std::unique_ptr output_tensor( + std::move(this->predictor_->GetOutput(0))); + auto end = std::chrono::system_clock::now(); + auto duration = + std::chrono::duration_cast(end - start); + cost_time = double(duration.count()) * + std::chrono::microseconds::period::num / + std::chrono::microseconds::period::den; + + // do postprocess + int output_size = 1; + for (auto dim : output_tensor->shape()) { + output_size *= dim; + } + feature.resize(output_size); + output_tensor->CopyToCpu(feature.data()); + + // postprocess include sqrt or binarize. + FeatureNorm(feature); +} + +void FeatureExtract::FeatureNorm(std::vector &feature) { + float feature_sqrt = std::sqrt(std::inner_product( + feature.begin(), feature.end(), feature.begin(), 0.0f)); + for (int i = 0; i < feature.size(); ++i) { + feature[i] /= feature_sqrt; + } +} + +void FeatureExtract::Permute(const cv::Mat *im, float *data) { + int rh = im->rows; + int rw = im->cols; + int rc = im->channels(); + for (int i = 0; i < rc; ++i) { + cv::extractChannel(*im, cv::Mat(rh, rw, CV_32FC1, data + i * rh * rw), i); + } +} + +void FeatureExtract::ResizeImage(const cv::Mat &img, cv::Mat &resize_img) { + cv::resize(img, resize_img, cv::Size(this->size, this->size)); +} + +void FeatureExtract::NormalizeImage(cv::Mat *im, const std::vector &mean, + const std::vector &std, + float scale) { + (*im).convertTo(*im, CV_32FC3, scale); + for (int h = 0; h < im->rows; h++) { + for (int w = 0; w < im->cols; w++) { + im->at(h, w)[0] = + (im->at(h, w)[0] - mean[0]) / std[0]; + im->at(h, w)[1] = + (im->at(h, w)[1] - mean[1]) / std[1]; + im->at(h, w)[2] = + (im->at(h, w)[2] - mean[2]) / std[2]; + } + } +} diff --git a/deploy/shitu_android_demo/app/src/main/cpp/FeatureExtractor.h b/deploy/shitu_android_demo/app/src/main/cpp/FeatureExtractor.h new file mode 100644 index 000000000..254407583 --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/cpp/FeatureExtractor.h @@ -0,0 +1,62 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "Utils.h" // NOLINT +#include "numeric" +#include "paddle_api.h" // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT + +using namespace paddle::lite_api; // NOLINT +using namespace std; // NOLINT + +class FeatureExtract { +public: // NOLINT + explicit FeatureExtract(std::string model_path, std::vector input_shape, + int cpu_nums, std::string cpu_power) { + MobileConfig config; + config.set_threads(cpu_nums); + config.set_power_mode(ParsePowerMode(cpu_power)); + config.set_model_from_file(model_path); + this->predictor_ = CreatePaddlePredictor(config); + } + + void RunRecModel(const cv::Mat &img, double &cost_time, + std::vector &feature); // NOLINT + void FeatureNorm(std::vector &feature); + + void ResizeImage(const cv::Mat &img, cv::Mat &resize_img); + + void Permute(const cv::Mat *im, float *data); + + void NormalizeImage(cv::Mat *im, const std::vector &mean, + const std::vector &std, float scale); + +private: // NOLINT + std::shared_ptr predictor_; + std::vector mean_ = {0.485f, 0.456f, 0.406f}; + std::vector std_ = {0.229f, 0.224f, 0.225f}; + double scale_ = 0.00392157; // 1/255.0 + int size = 224; +}; diff --git a/deploy/shitu_android_demo/app/src/main/cpp/Native.cc b/deploy/shitu_android_demo/app/src/main/cpp/Native.cc new file mode 100644 index 000000000..eea19b02a --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/cpp/Native.cc @@ -0,0 +1,212 @@ +// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "Native.h" +#include "Pipeline.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif +/* + * Class: Java_com_baidu_paddle_lite_demo_pp_1shitu_Native + * Method: nativeInit + * Signature: + * (Ljava/lang/String;Ljava/lang/String;ILjava/lang/String;II[F[FF)J + */ +JNIEXPORT jlong JNICALL +Java_com_baidu_paddle_lite_demo_pp_1shitu_Native_nativeInit( + JNIEnv *env, jclass thiz, jstring jDetModelDir, jstring jRecModelDir, + jstring jLabelPath, jstring jIndexPath, jlongArray jDetInputShape, + jlongArray jRecInputShape, jint cpuThreadNum, jint WarmUp, jint Repeats, + jint topk, jboolean jaddGallery, jstring cpu_power) { + std::string det_model_path = jstring_to_cpp_string(env, jDetModelDir); + std::string rec_model_path = jstring_to_cpp_string(env, jRecModelDir); + std::string label_path = jstring_to_cpp_string(env, jLabelPath); + std::string index_path = jstring_to_cpp_string(env, jIndexPath); + bool add_gallery = jaddGallery; + const std::string cpu_mode = jstring_to_cpp_string(env, cpu_power); + std::vector det_input_shape = + jlongarray_to_int64_vector(env, jDetInputShape); + std::vector rec_input_shape = + jlongarray_to_int64_vector(env, jRecInputShape); + std::vector det_input_shape_int; + std::vector rec_input_shape_int; + for (auto &tmp : det_input_shape) + det_input_shape_int.emplace_back(static_cast(tmp)); + for (auto &tmp : rec_input_shape) + rec_input_shape_int.emplace_back(static_cast(tmp)); + return reinterpret_cast( + new PipeLine(det_model_path, rec_model_path, label_path, index_path, + det_input_shape_int, rec_input_shape_int, cpuThreadNum, + WarmUp, Repeats, topk, add_gallery, cpu_mode)); +} + +/* + * Class: Java_com_baidu_paddle_lite_demo_pp_1shitu_Native + * Method: nativeRelease + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL +Java_com_baidu_paddle_lite_demo_pp_1shitu_Native_nativeRelease(JNIEnv *env, + jclass thiz, + jlong ctx) { + if (ctx == 0) { + return JNI_FALSE; + } + auto *pipeline = reinterpret_cast(ctx); + delete pipeline; + return JNI_TRUE; +} + +JNIEXPORT jboolean JNICALL +Java_com_baidu_paddle_lite_demo_pp_1shitu_Native_nativesetAddGallery( + JNIEnv *env, jclass thiz, jlong ctx, jboolean flag) { + if (ctx == 0) { + return JNI_FALSE; + } + auto *pipeline = reinterpret_cast(ctx); + pipeline->set_add_gallery(flag); + return JNI_TRUE; +} + +JNIEXPORT jboolean JNICALL +Java_com_baidu_paddle_lite_demo_pp_1shitu_Native_nativeclearGallery(JNIEnv *env, + jclass thiz, + jlong ctx) { + if (ctx == 0) { + return JNI_FALSE; + } + auto *pipeline = reinterpret_cast(ctx); + pipeline->ClearFeature(); + return JNI_TRUE; +} + +/* + * Class: Java_com_baidu_paddle_lite_demo_pp_1shitu_Native + * Method: nativeProcess + * Signature: (JIIIILjava/lang/String;)Z + */ +JNIEXPORT jstring JNICALL +Java_com_baidu_paddle_lite_demo_pp_1shitu_Native_nativeProcess( + JNIEnv *env, jclass thiz, jlong ctx, jobject jARGB8888ImageBitmap, + jstring jlabel_name) { + if (ctx == 0) { + return JNI_FALSE; + } + + // Convert the android bitmap(ARGB8888) to the OpenCV RGBA image. Actually, + // the data layout of AGRB8888 is R, G, B, A, it's the same as CV RGBA image, + // so it is unnecessary to do the conversion of color format, check + // https://developer.android.com/reference/android/graphics/Bitmap.Config#ARGB_8888 + // to get the more details about Bitmap.Config.ARGB8888 + auto t = GetCurrentTime(); + void *bitmapPixels; + AndroidBitmapInfo bitmapInfo; + if (AndroidBitmap_getInfo(env, jARGB8888ImageBitmap, &bitmapInfo) < 0) { + LOGE("Invoke AndroidBitmap_getInfo() failed!"); + return JNI_FALSE; + } + if (bitmapInfo.format != ANDROID_BITMAP_FORMAT_RGBA_8888) { + LOGE("Only Bitmap.Config.ARGB8888 color format is supported!"); + return JNI_FALSE; + } + if (AndroidBitmap_lockPixels(env, jARGB8888ImageBitmap, &bitmapPixels) < 0) { + LOGE("Invoke AndroidBitmap_lockPixels() failed!"); + return JNI_FALSE; + } + cv::Mat bmpImage(bitmapInfo.height, bitmapInfo.width, CV_8UC4, bitmapPixels); + cv::Mat rgbaImage; + std::string label_name = jstring_to_cpp_string(env, jlabel_name); + bmpImage.copyTo(rgbaImage); + if (AndroidBitmap_unlockPixels(env, jARGB8888ImageBitmap) < 0) { + LOGE("Invoke AndroidBitmap_unlockPixels() failed!"); + return JNI_FALSE; + } + LOGD("Read from bitmap costs %f ms", GetElapsedTime(t)); + + auto *pipeline = reinterpret_cast(ctx); + + std::vector input_mat; + std::vector out_object; + cv::Mat rgb_input; + cv::cvtColor(rgbaImage, rgb_input, cv::COLOR_RGBA2RGB); + input_mat.emplace_back(rgb_input); + std::string res_str = pipeline->run(input_mat, out_object, 1, label_name); + bool modified = res_str.empty(); + if (!modified) { + cv::Mat res_img; + cv::cvtColor(input_mat[0], res_img, cv::COLOR_RGB2RGBA); + // Convert the OpenCV RGBA image to the android bitmap(ARGB8888) + if (res_img.type() != CV_8UC4) { + LOGE("Only CV_8UC4 color format is supported!"); + return JNI_FALSE; + } + t = GetCurrentTime(); + if (AndroidBitmap_lockPixels(env, jARGB8888ImageBitmap, &bitmapPixels) < + 0) { + LOGE("Invoke AndroidBitmap_lockPixels() failed!"); + return JNI_FALSE; + } + cv::Mat bmpImage(bitmapInfo.height, bitmapInfo.width, CV_8UC4, + bitmapPixels); + res_img.copyTo(bmpImage); + if (AndroidBitmap_unlockPixels(env, jARGB8888ImageBitmap) < 0) { + LOGE("Invoke AndroidBitmap_unlockPixels() failed!"); + return JNI_FALSE; + } + LOGD("Write to bitmap costs %f ms", GetElapsedTime(t)); + } + return cpp_string_to_jstring(env, res_str); +} + +#ifdef __cplusplus +} +#endif + +extern "C" JNIEXPORT jboolean JNICALL +Java_com_baidu_paddle_lite_demo_pp_1shitu_Native_nativesaveIndex( + JNIEnv *env, jclass clazz, jlong ctx, jstring jsave_file_name) { + // TODO: implement nativesaveIndex() + if (ctx == 0) { + return JNI_FALSE; + } + auto *pipeline = reinterpret_cast(ctx); + std::string save_file_name = jstring_to_cpp_string(env, jsave_file_name); + pipeline->SaveIndex(save_file_name); + return JNI_TRUE; +} +extern "C" JNIEXPORT jboolean JNICALL +Java_com_baidu_paddle_lite_demo_pp_1shitu_Native_nativeloadIndex( + JNIEnv *env, jclass clazz, jlong ctx, jstring jload_file_name) { + // TODO: implement nativeloadIndex() + if (ctx == 0) { + return JNI_FALSE; + } + auto *pipeline = reinterpret_cast(ctx); + std::string load_file_name = jstring_to_cpp_string(env, jload_file_name); + bool load_flag = pipeline->LoadIndex(load_file_name); + return JNI_TRUE && load_flag; +} +extern "C" JNIEXPORT jstring JNICALL +Java_com_baidu_paddle_lite_demo_pp_1shitu_Native_nativegetClassname( + JNIEnv *env, jclass clazz, jlong ctx) { + // TODO: implement nativegetClassname() + if (ctx == 0) { + return JNI_FALSE; + } + auto *pipeline = reinterpret_cast(ctx); + std::string class_name_content = pipeline->GetLabelList(); + return cpp_string_to_jstring(env, class_name_content); +} \ No newline at end of file diff --git a/deploy/shitu_android_demo/app/src/main/cpp/Native.h b/deploy/shitu_android_demo/app/src/main/cpp/Native.h new file mode 100644 index 000000000..14693766a --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/cpp/Native.h @@ -0,0 +1,147 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +inline std::string jstring_to_cpp_string(JNIEnv *env, jstring jstr) { + // In java, a unicode char will be encoded using 2 bytes (utf16). + // so jstring will contain characters utf16. std::string in c++ is + // essentially a string of bytes, not characters, so if we want to + // pass jstring from JNI to c++, we have convert utf16 to bytes. + if (!jstr) { + return ""; + } + const jclass stringClass = env->GetObjectClass(jstr); + const jmethodID getBytes = + env->GetMethodID(stringClass, "getBytes", "(Ljava/lang/String;)[B"); + const jbyteArray stringJbytes = (jbyteArray)env->CallObjectMethod( + jstr, getBytes, env->NewStringUTF("UTF-8")); + + size_t length = (size_t)env->GetArrayLength(stringJbytes); + jbyte *pBytes = env->GetByteArrayElements(stringJbytes, NULL); + + std::string ret = std::string(reinterpret_cast(pBytes), length); + env->ReleaseByteArrayElements(stringJbytes, pBytes, JNI_ABORT); + + env->DeleteLocalRef(stringJbytes); + env->DeleteLocalRef(stringClass); + return ret; +} + +inline jstring cpp_string_to_jstring(JNIEnv *env, std::string str) { + auto *data = str.c_str(); + jclass strClass = env->FindClass("java/lang/String"); + jmethodID strClassInitMethodID = + env->GetMethodID(strClass, "", "([BLjava/lang/String;)V"); + + jbyteArray bytes = env->NewByteArray(strlen(data)); + env->SetByteArrayRegion(bytes, 0, strlen(data), + reinterpret_cast(data)); + + jstring encoding = env->NewStringUTF("UTF-8"); + jstring res = (jstring)( + env->NewObject(strClass, strClassInitMethodID, bytes, encoding)); + + env->DeleteLocalRef(strClass); + env->DeleteLocalRef(encoding); + env->DeleteLocalRef(bytes); + + return res; +} + +inline jobject cpp_string_to_jobect(JNIEnv *env, std::string str) { + auto *data = str.c_str(); + jclass strClass = env->FindClass("java/lang/String"); + jmethodID strClassInitMethodID = + env->GetMethodID(strClass, "", "([BLjava/lang/String;)V"); + + jbyteArray bytes = env->NewByteArray(strlen(data)); + env->SetByteArrayRegion(bytes, 0, strlen(data), + reinterpret_cast(data)); + + jstring encoding = env->NewStringUTF("UTF-8"); + jobject res = env->NewObject(strClass, strClassInitMethodID, bytes, encoding); + + env->DeleteLocalRef(strClass); + env->DeleteLocalRef(encoding); + env->DeleteLocalRef(bytes); + + return res; +} + +inline jfloatArray cpp_array_to_jfloatarray(JNIEnv *env, const float *buf, + int64_t len) { + jfloatArray result = env->NewFloatArray(len); + env->SetFloatArrayRegion(result, 0, len, buf); + return result; +} + +inline jintArray cpp_array_to_jintarray(JNIEnv *env, const int *buf, + int64_t len) { + jintArray result = env->NewIntArray(len); + env->SetIntArrayRegion(result, 0, len, buf); + return result; +} + +inline jbyteArray cpp_array_to_jbytearray(JNIEnv *env, const int8_t *buf, + int64_t len) { + jbyteArray result = env->NewByteArray(len); + env->SetByteArrayRegion(result, 0, len, buf); + return result; +} + +inline jobjectArray +cpp_array_to_jobjectarray(JNIEnv *env, const std::string *buf, int64_t len) { + jclass jclz = env->FindClass("java/lang/String"); + jobjectArray result = env->NewObjectArray(len, jclz, NULL); + for (int i = 0; i < len; i++) { + jobject job = cpp_string_to_jobect(env, buf[i]); + env->SetObjectArrayElement(result, i, job); + } + return result; +} + +inline jlongArray int64_vector_to_jlongarray(JNIEnv *env, + const std::vector &vec) { + jlongArray result = env->NewLongArray(vec.size()); + jlong *buf = new jlong[vec.size()]; + for (size_t i = 0; i < vec.size(); ++i) { + buf[i] = (jlong)vec[i]; + } + env->SetLongArrayRegion(result, 0, vec.size(), buf); + delete[] buf; + return result; +} + +inline std::vector jlongarray_to_int64_vector(JNIEnv *env, + jlongArray data) { + int data_size = env->GetArrayLength(data); + jlong *data_ptr = env->GetLongArrayElements(data, nullptr); + std::vector data_vec(data_ptr, data_ptr + data_size); + env->ReleaseLongArrayElements(data, data_ptr, 0); + return data_vec; +} + +inline std::vector jfloatarray_to_float_vector(JNIEnv *env, + jfloatArray data) { + int data_size = env->GetArrayLength(data); + jfloat *data_ptr = env->GetFloatArrayElements(data, nullptr); + std::vector data_vec(data_ptr, data_ptr + data_size); + env->ReleaseFloatArrayElements(data, data_ptr, 0); + return data_vec; +} diff --git a/deploy/shitu_android_demo/app/src/main/cpp/ObjectDetector.cc b/deploy/shitu_android_demo/app/src/main/cpp/ObjectDetector.cc new file mode 100644 index 000000000..bf7619c15 --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/cpp/ObjectDetector.cc @@ -0,0 +1,324 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ObjectDetector.h" // NOLINT +#include // NOLINT +#include // NOLINT +#include +#include // NOLINT + +// PicoDet decode +ObjectResult DisPred2Bbox(const float *dfl_det, int label, float score, int x, + int y, int stride, std::vector im_shape, + int reg_max) { + float ct_x = (x + 0.5) * stride; + float ct_y = (y + 0.5) * stride; + std::vector dis_pred; + dis_pred.resize(4); + for (int i = 0; i < 4; i++) { + float dis = 0; + float *dis_after_sm = new float[reg_max + 1]; + activation_function_softmax(dfl_det + i * (reg_max + 1), dis_after_sm, + reg_max + 1); + for (int j = 0; j < reg_max + 1; j++) { + dis += j * dis_after_sm[j]; + } + dis *= stride; + dis_pred[i] = dis; + delete[] dis_after_sm; + } + int xmin = static_cast(std::max(ct_x - dis_pred[0], .0f)); + int ymin = static_cast(std::max(ct_y - dis_pred[1], .0f)); + int xmax = static_cast(std::min(ct_x + dis_pred[2], im_shape[0])); + int ymax = static_cast(std::min(ct_y + dis_pred[3], im_shape[1])); + ObjectResult result_item; + result_item.rect = {xmin, ymin, xmax, ymax}; + result_item.class_id = label; + result_item.confidence = score; + return result_item; +} + +void PicoDetPostProcess(std::vector *results, + std::vector outs, + std::vector fpn_stride, + std::vector im_shape, + std::vector scale_factor, float score_threshold, + float nms_threshold, int num_class, int reg_max) { + std::vector> bbox_results; + bbox_results.resize(num_class); + int in_h = im_shape[0], in_w = im_shape[1]; + for (int i = 0; i < fpn_stride.size(); ++i) { + int feature_h = ceil(in_h * 1.f / fpn_stride[i]); + int feature_w = ceil(in_w * 1.f / fpn_stride[i]); + for (int idx = 0; idx < feature_h * feature_w; idx++) { + const float *scores = outs[i] + (idx * num_class); + int row = idx / feature_w; + int col = idx % feature_w; + float score = 0; + int cur_label = 0; + for (int label = 0; label < num_class; label++) { + if (scores[label] > score) { + score = scores[label]; + cur_label = label; + } + } + if (score > score_threshold) { + const float *bbox_pred = + outs[i + fpn_stride.size()] + (idx * 4 * (reg_max + 1)); + bbox_results[cur_label].push_back( + DisPred2Bbox(bbox_pred, cur_label, score, col, row, fpn_stride[i], + im_shape, reg_max)); + } + } + } + for (int i = 0; i < bbox_results.size(); i++) { + nms(&bbox_results[i], nms_threshold); + for (auto box : bbox_results[i]) { + box.rect[0] = box.rect[0] / scale_factor[1]; + box.rect[2] = box.rect[2] / scale_factor[1]; + box.rect[1] = box.rect[1] / scale_factor[0]; + box.rect[3] = box.rect[3] / scale_factor[0]; + results->push_back(box); + } + } +} + +// ***************************** member Function *************************** +// Load Model and create model predictor +void ObjectDetector::LoadModel(const std::string &model_file, int num_theads, + std::string cpu_power) { + MobileConfig config; + config.set_threads(num_theads); + config.set_model_from_file(model_file); + config.set_power_mode(ParsePowerMode(cpu_power)); + if (access(model_file.c_str(), 0) != 0) { + LOGD("File not exist!"); + } + predictor_ = CreatePaddlePredictor(config); +} + +void ObjectDetector::Preprocess(const cv::Mat &ori_im) { + // Clone the image : keep the original mat for postprocess + cv::Mat im = ori_im.clone(); + for (auto &op_process : ops_) { + op_process(&im, &inputs_, pre_param_); + } +} + +void ObjectDetector::Postprocess(const std::vector mats, + std::vector *result, + std::vector bbox_num, + bool is_rbox = false) { + result->clear(); + int start_idx = 0; + for (int im_id = 0; im_id < mats.size(); im_id++) { + cv::Mat raw_mat = mats[im_id]; + int rh = 1; + int rw = 1; + if (arch_ == "Face") { + rh = raw_mat.rows; + rw = raw_mat.cols; + } + for (int j = start_idx; j < start_idx + bbox_num[im_id]; j++) { + if (is_rbox) { + // Class id + int class_id = static_cast(round(output_data_[0 + j * 10])); + // Confidence score + float score = output_data_[1 + j * 10]; + int x1 = (output_data_[2 + j * 10] * rw); + int y1 = (output_data_[3 + j * 10] * rh); + int x2 = (output_data_[4 + j * 10] * rw); + int y2 = (output_data_[5 + j * 10] * rh); + int x3 = (output_data_[6 + j * 10] * rw); + int y3 = (output_data_[7 + j * 10] * rh); + int x4 = (output_data_[8 + j * 10] * rw); + int y4 = (output_data_[9 + j * 10] * rh); + ObjectResult result_item; + result_item.rect = {x1, y1, x2, y2, x3, y3, x4, y4}; + result_item.class_id = class_id; + result_item.confidence = score; + result->push_back(result_item); + } else { + // Class id + int class_id = static_cast(round(output_data_[0 + j * 6])); + // Confidence score + float score = output_data_[1 + j * 6]; + int xmin = (output_data_[2 + j * 6] * rw); + int ymin = (output_data_[3 + j * 6] * rh); + int xmax = (output_data_[4 + j * 6] * rw); + int ymax = (output_data_[5 + j * 6] * rh); + int wd = xmax - xmin; + int hd = ymax - ymin; + + ObjectResult result_item; + result_item.rect = {xmin, ymin, xmax, ymax}; + result_item.class_id = class_id; + result_item.confidence = score; + result->push_back(result_item); + } + } + start_idx += bbox_num[im_id]; + } +} + +void ObjectDetector::Predict(const std::vector &imgs, const int warmup, + const int repeats, + std::vector *result, + std::vector *bbox_num, + std::vector *times) { + auto preprocess_start = std::chrono::steady_clock::now(); + int batch_size = imgs.size(); + + // in_data_batch + std::vector in_data_all; + std::vector im_shape_all(batch_size * 2); + std::vector scale_factor_all(batch_size * 2); + // Preprocess image + for (int bs_idx = 0; bs_idx < batch_size; bs_idx++) { + cv::Mat im = imgs.at(bs_idx); + Preprocess(im); + im_shape_all[bs_idx * 2] = inputs_.im_shape_[0]; + im_shape_all[bs_idx * 2 + 1] = inputs_.im_shape_[1]; + + scale_factor_all[bs_idx * 2] = inputs_.scale_factor_[0]; + scale_factor_all[bs_idx * 2 + 1] = inputs_.scale_factor_[1]; + + // TODO: reduce cost time + in_data_all.insert(in_data_all.end(), inputs_.im_data_.begin(), + inputs_.im_data_.end()); + } + auto preprocess_end = std::chrono::steady_clock::now(); + std::vector output_data_list_; + // Prepare input tensor + + auto input_names = predictor_->GetInputNames(); + for (const auto &tensor_name : input_names) { + auto in_tensor = predictor_->GetInputByName(tensor_name); + if (tensor_name == "image") { + int rh = inputs_.in_net_shape_[0]; + int rw = inputs_.in_net_shape_[1]; + in_tensor->Resize({batch_size, 3, rh, rw}); + auto *inptr = in_tensor->mutable_data(); + std::copy_n(in_data_all.data(), in_data_all.size(), inptr); + } else if (tensor_name == "im_shape") { + in_tensor->Resize({batch_size, 2}); + auto *inptr = in_tensor->mutable_data(); + std::copy_n(im_shape_all.data(), im_shape_all.size(), inptr); + } else if (tensor_name == "scale_factor") { + in_tensor->Resize({batch_size, 2}); + auto *inptr = in_tensor->mutable_data(); + std::copy_n(scale_factor_all.data(), scale_factor_all.size(), inptr); + } + } + + // Run predictor + // warmup + for (int i = 0; i < warmup; i++) { + predictor_->Run(); + // Get output tensor + auto output_names = predictor_->GetOutputNames(); + if (arch_ == "PicoDet") { + for (int j = 0; j < output_names.size(); j++) { + auto output_tensor = predictor_->GetTensor(output_names[j]); + const float *outptr = output_tensor->data(); + std::vector output_shape = output_tensor->shape(); + output_data_list_.push_back(outptr); + } + } else { + auto out_tensor = predictor_->GetTensor(output_names[0]); + auto out_bbox_num = predictor_->GetTensor(output_names[1]); + } + } + + bool is_rbox = false; + auto inference_start = std::chrono::steady_clock::now(); + for (int i = 0; i < repeats; i++) { + predictor_->Run(); + } + auto inference_end = std::chrono::steady_clock::now(); + auto postprocess_start = std::chrono::steady_clock::now(); + + // Get output tensor + output_data_list_.clear(); + int num_class = 1; + int reg_max = 7; + auto output_names = predictor_->GetOutputNames(); + // TODO: Unified model output. + if (arch_ == "PicoDet") { + for (int i = 0; i < output_names.size(); i++) { + auto output_tensor = predictor_->GetTensor(output_names[i]); + const float *outptr = output_tensor->data(); + std::vector output_shape = output_tensor->shape(); + if (i == 0) { + num_class = output_shape[2]; + } + if (i == fpn_stride_.size()) { + reg_max = output_shape[2] / 4 - 1; + } + output_data_list_.push_back(outptr); + } + } else { + auto output_tensor = predictor_->GetTensor(output_names[0]); + auto output_shape = output_tensor->shape(); + auto out_bbox_num = predictor_->GetTensor(output_names[1]); + auto out_bbox_num_shape = out_bbox_num->shape(); + // Calculate output length + int output_size = 1; + for (int j = 0; j < output_shape.size(); ++j) { + output_size *= output_shape[j]; + } + is_rbox = output_shape[output_shape.size() - 1] % 10 == 0; + + if (output_size < 6) { + std::cerr << "[WARNING] No object detected." << std::endl; + } + output_data_.resize(output_size); + std::copy_n(output_tensor->mutable_data(), output_size, + output_data_.data()); + + int out_bbox_num_size = 1; + for (int j = 0; j < out_bbox_num_shape.size(); ++j) { + out_bbox_num_size *= out_bbox_num_shape[j]; + } + out_bbox_num_data_.resize(out_bbox_num_size); + std::copy_n(out_bbox_num->mutable_data(), out_bbox_num_size, + out_bbox_num_data_.data()); + } + // Postprocessing result + + result->clear(); + if (arch_ == "PicoDet") { + PicoDetPostProcess(result, output_data_list_, fpn_stride_, + inputs_.im_shape_, inputs_.scale_factor_, + score_threshold_, nms_threshold_, num_class, reg_max); + bbox_num->push_back(result->size()); + } else { + Postprocess(imgs, result, out_bbox_num_data_, is_rbox); + bbox_num->clear(); + for (int k = 0; k < out_bbox_num_data_.size(); k++) { + int tmp = out_bbox_num_data_[k]; + bbox_num->push_back(tmp); + } + } + auto postprocess_end = std::chrono::steady_clock::now(); + + std::chrono::duration preprocess_diff = + preprocess_end - preprocess_start; + times->push_back(double(preprocess_diff.count() * 1000)); + std::chrono::duration inference_diff = inference_end - inference_start; + times->push_back(double(inference_diff.count() / repeats * 1000)); + std::chrono::duration postprocess_diff = + postprocess_end - postprocess_start; + times->push_back(double(postprocess_diff.count() * 1000)); +} diff --git a/deploy/shitu_android_demo/app/src/main/cpp/ObjectDetector.h b/deploy/shitu_android_demo/app/src/main/cpp/ObjectDetector.h new file mode 100644 index 000000000..6add13c07 --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/cpp/ObjectDetector.h @@ -0,0 +1,254 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "Utils.h" // NOLINT +#include "paddle_api.h" // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT + +using namespace paddle::lite_api; // NOLINT + +struct ObjectPreprocessParam { + // Normalisze + std::vector mean; + std::vector std; + bool is_scale; + // resize + int interp; + bool keep_ratio; + std::vector target_size; + // Pad + int stride; + // TopDownEvalAffine + std::vector trainsize; +}; + +using PreprocessFunc = std::function; + +// PreProcess Function +inline void InitInfo(cv::Mat *im, ImageBlob *data, ObjectPreprocessParam item) { + data->im_shape_ = {static_cast(im->rows), + static_cast(im->cols)}; + data->scale_factor_ = {1., 1.}; + data->in_net_shape_ = {static_cast(im->rows), + static_cast(im->cols)}; +} + +inline void NormalizeImage(cv::Mat *im, ImageBlob *data, + ObjectPreprocessParam item) { + std::vector mean; + std::vector scale; + bool is_scale; + for (auto tmp : item.mean) { + mean.emplace_back(tmp); + } + for (auto tmp : item.std) { + scale.emplace_back(tmp); + } + is_scale = item.is_scale; + double e = 1.0; + if (is_scale) { + e *= 1. / 255.0; + } + (*im).convertTo(*im, CV_32FC3, e); + for (int h = 0; h < im->rows; h++) { + for (int w = 0; w < im->cols; w++) { + im->at(h, w)[0] = + (im->at(h, w)[0] - mean[0]) / scale[0]; + im->at(h, w)[1] = + (im->at(h, w)[1] - mean[1]) / scale[1]; + im->at(h, w)[2] = + (im->at(h, w)[2] - mean[2]) / scale[2]; + } + } +} + +inline void Permute(cv::Mat *im, ImageBlob *data, ObjectPreprocessParam item) { + (*im).convertTo(*im, CV_32FC3); + int rh = im->rows; + int rw = im->cols; + int rc = im->channels(); + (data->im_data_).resize(rc * rh * rw); + float *base = (data->im_data_).data(); + for (int i = 0; i < rc; ++i) { + cv::extractChannel(*im, cv::Mat(rh, rw, CV_32FC1, base + i * rh * rw), i); + } +} + +inline void Resize(cv::Mat *im, ImageBlob *data, ObjectPreprocessParam item) { + std::vector target_size; + int interp = item.interp; + bool keep_ratio = item.keep_ratio; + for (auto tmp : item.target_size) { + target_size.emplace_back(tmp); + } + std::pair resize_scale; + int origin_w = im->cols; + int origin_h = im->rows; + if (keep_ratio) { + int im_size_max = std::max(origin_w, origin_h); + int im_size_min = std::min(origin_w, origin_h); + int target_size_max = + *std::max_element(target_size.begin(), target_size.end()); + int target_size_min = + *std::min_element(target_size.begin(), target_size.end()); + float scale_min = + static_cast(target_size_min) / static_cast(im_size_min); + float scale_max = + static_cast(target_size_max) / static_cast(im_size_max); + float scale_ratio = std::min(scale_min, scale_max); + resize_scale = {scale_ratio, scale_ratio}; + } else { + resize_scale.first = + static_cast(target_size[1]) / static_cast(origin_w); + resize_scale.second = + static_cast(target_size[0]) / static_cast(origin_h); + } + data->im_shape_ = {static_cast(im->cols * resize_scale.first), + static_cast(im->rows * resize_scale.second)}; + data->in_net_shape_ = {static_cast(im->cols * resize_scale.first), + static_cast(im->rows * resize_scale.second)}; + cv::resize(*im, *im, cv::Size(), resize_scale.first, resize_scale.second, + interp); + data->im_shape_ = { + static_cast(im->rows), static_cast(im->cols), + }; + data->scale_factor_ = { + resize_scale.second, resize_scale.first, + }; +} + +inline void PadStride(cv::Mat *im, ImageBlob *data, + ObjectPreprocessParam item) { + int stride = item.stride; + if (stride <= 0) { + return; + } + int rc = im->channels(); + int rh = im->rows; + int rw = im->cols; + int nh = (rh / stride) * stride + (rh % stride != 0) * stride; + int nw = (rw / stride) * stride + (rw % stride != 0) * stride; + cv::copyMakeBorder(*im, *im, 0, nh - rh, 0, nw - rw, cv::BORDER_CONSTANT, + cv::Scalar(0)); + data->in_net_shape_ = { + static_cast(im->rows), static_cast(im->cols), + }; +} + +inline void TopDownEvalAffine(cv::Mat *im, ImageBlob *data, + ObjectPreprocessParam item) { + int interp = 1; + std::vector trainsize; + for (auto tmp : item.trainsize) { + trainsize.emplace_back(tmp); + } + cv::resize(*im, *im, cv::Size(trainsize[0], trainsize[1]), 0, 0, interp); + // todo: Simd::ResizeBilinear(); + data->in_net_shape_ = { + static_cast(trainsize[1]), static_cast(trainsize[0]), + }; +} + +class ObjectDetector { +public: // NOLINT + explicit ObjectDetector(const std::string &model_dir, + std::vector det_input_shape, + const int &cpu_threads, std::string cpu_power, + const int &batch_size = 1) { + // global + fpn_stride_ = std::vector({8, 16, 32, 64}); + // Init preprocess param + // Normalisze + pre_param_.mean = std::vector({0.485, 0.456, 0.406}); + pre_param_.std = std::vector({0.229, 0.224, 0.225}); + pre_param_.is_scale = true; + // resize + pre_param_.interp = 2; + pre_param_.keep_ratio = false; + pre_param_.target_size = + std::vector({det_input_shape[2], det_input_shape[3]}); + // Pad + pre_param_.stride = 0; + // TopDownEvalAffine + pre_param_.trainsize = + std::vector({det_input_shape[2], det_input_shape[3]}); + // op + preprocess_op_func_ = std::vector( + {"DetResize", "DetNormalizeImage", "DetPermute"}); + // init postprocess param + arch_ = "PicoDet"; + nms_threshold_ = 0.5f; + score_threshold_ = 0.3f; + op_map_["InitInfo"] = (PreprocessFunc)InitInfo; + op_map_["DetNormalizeImage"] = (PreprocessFunc)NormalizeImage; + op_map_["DetPermute"] = (PreprocessFunc)Permute; + op_map_["DetResize"] = (PreprocessFunc)Resize; + op_map_["DetPadStride"] = (PreprocessFunc)PadStride; + op_map_["DetTopDownEvalAffine"] = (PreprocessFunc)TopDownEvalAffine; + for (auto op_name : preprocess_op_func_) { + ops_.emplace_back(op_map_[op_name]); + } + LoadModel(model_dir, cpu_threads, cpu_power); + } + + // Load Paddle inference model + void LoadModel(const std::string &model_file, int num_theads, + std::string cpu_power); + + // Run predictor + void Predict(const std::vector &imgs, const int warmup = 0, + const int repeats = 1, + std::vector *result = nullptr, + std::vector *bbox_num = nullptr, + std::vector *times = nullptr); + +private: // NOLINT + // Preprocess image and copy data to input buffer + void Preprocess(const cv::Mat &image_mat); + + // Postprocess result + void Postprocess(const std::vector mats, + std::vector *result, std::vector bbox_num, + bool is_rbox); + + std::shared_ptr predictor_; + + ObjectPreprocessParam pre_param_; + std::vector ops_; + std::vector preprocess_op_func_; + + ImageBlob inputs_; + std::vector output_data_; + std::vector out_bbox_num_data_; + float nms_threshold_; + std::unordered_map op_map_; + std::string arch_; + float score_threshold_; + + std::vector fpn_stride_; +}; diff --git a/deploy/shitu_android_demo/app/src/main/cpp/Pipeline.cc b/deploy/shitu_android_demo/app/src/main/cpp/Pipeline.cc new file mode 100644 index 000000000..f96e57394 --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/cpp/Pipeline.cc @@ -0,0 +1,305 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "Pipeline.h" +#include "FeatureExtractor.h" +#include "ObjectDetector.h" +#include "VectorSearch.h" +#include +#include +#include +#include +#include + +void PrintResult(std::vector &det_result, + const std::shared_ptr &vector_search, + SearchResult &search_result) { + for (int i = 0; i < std::min((int)det_result.size(), 1); ++i) { + int t = i; + LOGD("\tresult%d: bbox[%d, %d, %d, %d], score: %f, label: %s\n", i, + det_result[t].rect[0], det_result[t].rect[1], det_result[t].rect[2], + det_result[t].rect[3], det_result[t].confidence, + vector_search->GetLabel(search_result.I[search_result.return_k * t]) + .c_str()); + } +} + +void VisualResult(cv::Mat &img, std::vector &results) { // NOLINT + for (int i = 0; i < 1; i++) { + int w = results[i].rect[2] - results[i].rect[0]; + int h = results[i].rect[3] - results[i].rect[1]; + cv::Rect roi = cv::Rect(results[i].rect[0], results[i].rect[1], w, h); + cv::rectangle(img, roi, cv::Scalar(41, 50, 255), 3); + } +} + +PipeLine::PipeLine(std::string det_model_path, std::string rec_model_path, + std::string label_path, std::string index_path, + std::vector det_input_shape, + std::vector rec_input_shape, int cpu_num_threads, + int warm_up, int repeats, int topk, bool add_gallery, + std::string cpu_power) { + det_model_path_ = det_model_path; + rec_model_path_ = rec_model_path; + label_path_ = label_path; + index_path_ = index_path; + det_input_shape_ = det_input_shape; + rec_input_shape_ = rec_input_shape; + cpu_num_threads_ = cpu_num_threads; + add_gallery_flag = add_gallery; + + max_det_num_ = topk; + cpu_pow_ = cpu_power; + det_model_path_ = + det_model_path_ + "/mainbody_PPLCNet_x2_5_640_quant_v1.0_lite.nb"; + rec_model_path_ = + rec_model_path_ + "/general_PPLCNetV2_base_quant_v1.0_lite.nb"; + + // create object detector + det_ = std::make_shared(det_model_path_, det_input_shape_, + cpu_num_threads_, cpu_pow_); + + // create rec model + rec_ = std::make_shared(rec_model_path_, rec_input_shape_, + cpu_num_threads_, cpu_pow_); + // create vector search + searcher_ = std::make_shared(label_path_, index_path_, 5, 0.5); +} + +std::string PipeLine::run(std::vector &batch_imgs, // NOLINT + std::vector &det_result, // NOLINT + int batch_size, const std::string &label_name) { + std::fill(times_.begin(), times_.end(), 0); + + // if (!this->add_gallery_flag) + // { + DetPredictImage(batch_imgs, &det_result, batch_size, det_, + max_det_num_); // det_result获取[l,d,r,u] + // } + // add the whole image for recognition to improve recall + + ObjectResult result_whole_img = { + {0, 0, batch_imgs[0].cols, batch_imgs[0].rows}, 0, 1.0}; + det_result.push_back(result_whole_img); // 加入整图的坐标，提升召回率 + // get rec result + for (int j = 0; j < det_result.size(); ++j) { + double rec_time = 0.0; // .rect:vector = {l, d, r, u} + vector feature; + int w = det_result[j].rect[2] - det_result[j].rect[0]; + int h = det_result[j].rect[3] - det_result[j].rect[1]; + cv::Rect rect(det_result[j].rect[0], det_result[j].rect[1], w, h); + cv::Mat crop_img = batch_imgs[0](rect); + rec_->RunRecModel(crop_img, rec_time, feature); + if (this->add_gallery_flag) { + this->searcher_->AddFeature(feature.data(), label_name); + } else { + features.insert(features.end(), feature.begin(), + feature.end()); //每次插入一个512的向量 + } + } + if (this->add_gallery_flag) { + VisualResult(batch_imgs[0], det_result); + det_result.clear(); + features.clear(); + indices.clear(); + std::string res = std::to_string(times_[1] + times_[4]) + "\n"; + return res; + } + // do vectore search + SearchResult search_result = searcher_->Search( + features.data(), + det_result + .size()); // 一次搜索多个向量(展平在features里)，共det_result.size()个 + // for (int j = 0; j < det_result.size(); ++j) + for (int j = 0; j < 1; ++j) // 对于每个检测框，只把 + { + det_result[j].confidence = + search_result.return_k * j < search_result.D.size() + ? search_result.D[search_result.return_k * j] + : 0.0f; + for (int k = 0; k < this->max_index_num_; ++k) { + std::size_t tidx = + min((std::size_t)(search_result.return_k * j + k), + search_result.D.size() - 1); + + std::string _class_name = searcher_->GetLabel(search_result.I[tidx]); + int _index = (int)(search_result.I[tidx]); + float _dist = search_result.D[tidx]; + if (_dist > 1e5 || _dist < -1e5) { + _dist = 0.0; + } + + det_result[j].rec_result.push_back({_class_name, _index, _dist}); + } + } + // sort(det_result.begin(), det_result.end(), [](const ObjectResult &a, + // const ObjectResult &b){ + // if (a.rec_result.empty() and b.rec_result.empty()) + // { + // return 0; + // } + // else if (a.rec_result.empty() and !b.rec_result.empty()) + // { + // return 0; + // } + // else if (!a.rec_result.empty() and b.rec_result.empty()) + // { + // return 1; + // } + // else + // { + // return (int)(a.rec_result[0].score > b.rec_result[0].score); + // } + // }); + NMSBoxes(det_result, searcher_->GetThreshold(), this->rec_nms_thresold_, + indices); + VisualResult(batch_imgs[0], det_result); + LOGD("================== result summary ========================="); + PrintResult(det_result, searcher_, search_result); + + // results + std::string res; + res += std::to_string(times_[1] + times_[4]) + "\n"; + for (int i = 0; i < 1; i++) { + res.append(det_result[i].rec_result[0].class_name + ", " + + std::to_string((int)(det_result[i].rec_result[0].score * 1000) * + 1.0 / 1000) + + "\n"); + } + det_result.clear(); + features.clear(); + indices.clear(); + return res; +} + +void PipeLine::DetPredictImage(const std::vector batch_imgs, + std::vector *im_result, + const int batch_size_det, + std::shared_ptr det, + const int max_det_num) { + int steps = ceil(float(batch_imgs.size()) / batch_size_det); + for (int idx = 0; idx < steps; idx++) { + int left_image_cnt = (int)batch_imgs.size() - idx * batch_size_det; + if (left_image_cnt > batch_size_det) { + left_image_cnt = batch_size_det; + } + // Store all detected result + std::vector result; + std::vector bbox_num; + std::vector det_times; + + // bool is_rbox = false; + det->Predict(batch_imgs, 0, 1, &result, &bbox_num, &det_times); + int item_start_idx = 0; + for (int i = 0; i < left_image_cnt; i++) { + cv::Mat im = batch_imgs[i]; + int detect_num = 0; + for (int j = 0; j < min(bbox_num[i], max_det_num); j++) { + ObjectResult item = result[item_start_idx + j]; + if (item.class_id == -1) { + continue; + } + detect_num += 1; + im_result->push_back(item); + } + item_start_idx = item_start_idx + bbox_num[i]; + } + times_[0] += det_times[0]; + times_[1] += det_times[1]; + times_[2] += det_times[2]; + } +} + +template +static inline bool SortScorePairDescend(const std::pair &pair1, + const std::pair &pair2) { + return pair1.first > pair2.first; +} + +inline void +GetMaxScoreIndex(const std::vector &det_result, + const float threshold, + std::vector> &score_index_vec) { + // Generate index score pairs. + for (size_t i = 0; i < det_result.size(); ++i) { + if (det_result[i].confidence > threshold) { + score_index_vec.push_back(std::make_pair(det_result[i].confidence, i)); + } + } + + // Sort the score pair according to the scores in descending order + std::stable_sort(score_index_vec.begin(), score_index_vec.end(), + SortScorePairDescend); +} + +float RectOverlap(const ObjectResult &a, const ObjectResult &b) { + float Aa = (a.rect[2] - a.rect[0] + 1) * (a.rect[3] - a.rect[1] + 1); + float Ab = (b.rect[2] - b.rect[0] + 1) * (b.rect[3] - b.rect[1] + 1); + + int iou_w = max(min(a.rect[2], b.rect[2]) - max(a.rect[0], b.rect[0]) + 1, 0); + int iou_h = max(min(a.rect[3], b.rect[3]) - max(a.rect[1], b.rect[1]) + 1, 0); + float Aab = iou_w * iou_h; + return Aab / (Aa + Ab - Aab); +} + +void PipeLine::NMSBoxes(const std::vector &det_result, + const float score_threshold, const float nms_threshold, + std::vector &indices) { + // Get top_k scores (with corresponding indices). + std::vector> score_index_vec; + GetMaxScoreIndex(det_result, score_threshold, score_index_vec); + + // Do nms + indices.clear(); + for (size_t i = 0; i < score_index_vec.size(); ++i) { + const int idx = score_index_vec[i].second; + bool keep = true; + for (int k = 0; k < (int)indices.size() && keep; ++k) { + const int kept_idx = indices[k]; + float overlap = RectOverlap(det_result[idx], det_result[kept_idx]); + keep = overlap <= nms_threshold; + } + if (keep) + indices.push_back(idx); + } +} + +void PipeLine::set_add_gallery(const bool &flag) { + this->add_gallery_flag = flag; +} + +void PipeLine::ClearFeature() { this->searcher_->ClearFeature(); } + +void PipeLine::SaveIndex(const string &save_file_name) { + this->searcher_->SaveIndex(save_file_name); +} + +bool PipeLine::LoadIndex(const string &save_file_name) { + return this->searcher_->LoadFromSaveFileName(save_file_name); +} + +string PipeLine::GetLabelList() { + std::vector class_name_list = this->searcher_->GetLabelList(); + string ret; + ret += "共"; + ret += std::to_string(class_name_list.size()); + ret += "类"; + ret += "\n"; + ret += "====================\n"; + for (const auto &str : class_name_list) { + ret += str; + ret += "\n"; + } + return ret; +} diff --git a/deploy/shitu_android_demo/app/src/main/cpp/Pipeline.h b/deploy/shitu_android_demo/app/src/main/cpp/Pipeline.h new file mode 100644 index 000000000..9940a6b1c --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/cpp/Pipeline.h @@ -0,0 +1,90 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "FeatureExtractor.h" // NOLINT +#include "ObjectDetector.h" // NOLINT +#include "Utils.h" +#include "VectorSearch.h" +#include "paddle_api.h" // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT + +class PipeLine { +public: // NOLINT + explicit PipeLine(std::string det_model_path, std::string rec_model_path, + std::string label_path, std::string index_path, + std::vector det_input_shape, + std::vector rec_input_shape, int cpu_num_threads, + int warm_up, int repeats, int topk, bool add_gallery, + std::string cpu_power); + + std::string run(std::vector &batch_imgs, // NOLINT + std::vector &det_result, // NOLINT + int batch_size, const std::string &label_name = ""); + + void set_add_gallery(const bool &flag); + + void ClearFeature(); + + void SaveIndex(const string &save_file_name); + + bool LoadIndex(const string &load_file_name); + + string GetLabelList(); + +private: // NOLINT + std::string det_model_path_; + std::string rec_model_path_; + std::string label_path_; + std::string index_path_; + std::vector det_input_shape_; + std::vector rec_input_shape_; + int cpu_num_threads_; + bool add_gallery_flag; + std::string cpu_pow_; + // 实例化检测类 + std::shared_ptr det_; + + // 实例化特征提取(rec)类 + std::shared_ptr rec_; + + // 实例化特征检索类 + std::shared_ptr searcher_; + + int max_det_num_ = 3; + int max_index_num_ = 5; + float rec_nms_thresold_ = 0.05f; + std::vector features; + std::vector indices; + std::vector times_{0, 0, 0, 0, 0, 0, 0}; + + void DetPredictImage(const std::vector batch_imgs, + std::vector *im_result, + const int batch_size_det, + std::shared_ptr det, + const int max_det_num = 3); + + void NMSBoxes(const std::vector &det_result, + const float score_threshold, const float nms_threshold, + std::vector &indices); +}; diff --git a/deploy/shitu_android_demo/app/src/main/cpp/README.md b/deploy/shitu_android_demo/app/src/main/cpp/README.md new file mode 100644 index 000000000..9fa46c767 --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/cpp/README.md @@ -0,0 +1,3 @@ +download [faiss1.5.3.tar.gz](https://paddle-inference-dist.bj.bcebos.com/faiss1.5.3.tar.gz) and unzip it, put `include` directory here, file structure like below: + +main/cpp/include/faiss/*.h diff --git a/deploy/shitu_android_demo/app/src/main/cpp/Utils.cc b/deploy/shitu_android_demo/app/src/main/cpp/Utils.cc new file mode 100644 index 000000000..e4a6f5712 --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/cpp/Utils.cc @@ -0,0 +1,183 @@ +// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "Utils.h" // NOLINT +#include // NOLINT +#include // NOLINT + +int64_t ShapeProduction(const std::vector &shape) { + int64_t res = 1; + for (auto i : shape) + res *= i; + return res; +} + +void NHWC3ToNC3HW(const float *src, float *dst, const float *mean, + const float *std, int width, int height) { + int size = height * width; + float32x4_t vmean0 = vdupq_n_f32(mean ? mean[0] : 0.0f); + float32x4_t vmean1 = vdupq_n_f32(mean ? mean[1] : 0.0f); + float32x4_t vmean2 = vdupq_n_f32(mean ? mean[2] : 0.0f); + float32x4_t vscale0 = vdupq_n_f32(std ? (1.0f / std[0]) : 1.0f); + float32x4_t vscale1 = vdupq_n_f32(std ? (1.0f / std[1]) : 1.0f); + float32x4_t vscale2 = vdupq_n_f32(std ? (1.0f / std[2]) : 1.0f); + float *dst_c0 = dst; + float *dst_c1 = dst + size; + float *dst_c2 = dst + size * 2; + int i = 0; + for (; i < size - 3; i += 4) { + float32x4x3_t vin3 = vld3q_f32(src); + float32x4_t vsub0 = vsubq_f32(vin3.val[0], vmean0); + float32x4_t vsub1 = vsubq_f32(vin3.val[1], vmean1); + float32x4_t vsub2 = vsubq_f32(vin3.val[2], vmean2); + float32x4_t vs0 = vmulq_f32(vsub0, vscale0); + float32x4_t vs1 = vmulq_f32(vsub1, vscale1); + float32x4_t vs2 = vmulq_f32(vsub2, vscale2); + vst1q_f32(dst_c0, vs0); + vst1q_f32(dst_c1, vs1); + vst1q_f32(dst_c2, vs2); + src += 12; + dst_c0 += 4; + dst_c1 += 4; + dst_c2 += 4; + } + for (; i < size; i++) { + *(dst_c0++) = (*(src++) - mean[0]) / std[0]; + *(dst_c1++) = (*(src++) - mean[1]) / std[1]; + *(dst_c2++) = (*(src++) - mean[2]) / std[2]; + } +} + +void NHWC1ToNC1HW(const float *src, float *dst, const float *mean, + const float *std, int width, int height) { + int size = height * width; + float32x4_t vmean = vdupq_n_f32(mean ? mean[0] : 0.0f); + float32x4_t vscale = vdupq_n_f32(std ? (1.0f / std[0]) : 1.0f); + int i = 0; + for (; i < size - 3; i += 4) { + float32x4_t vin = vld1q_f32(src); + float32x4_t vsub = vsubq_f32(vin, vmean); + float32x4_t vs = vmulq_f32(vsub, vscale); + vst1q_f32(dst, vs); + src += 4; + dst += 4; + } + for (; i < size; i++) { + *(dst++) = (*(src++) - mean[0]) / std[0]; + } +} + +void nms(std::vector *input_boxes, float nms_threshold, + bool rec_nms) { + if (!rec_nms) { + std::sort(input_boxes->begin(), input_boxes->end(), + [](ObjectResult a, ObjectResult b) { + return a.confidence > b.confidence; + }); + } else { + std::sort(input_boxes->begin(), input_boxes->end(), + [](ObjectResult a, ObjectResult b) { + return a.rec_result[0].score > b.rec_result[0].score; + }); + } + std::vector vArea(input_boxes->size()); + for (int i = 0; i < input_boxes->size(); ++i) { + vArea[i] = (input_boxes->at(i).rect[2] - input_boxes->at(i).rect[0] + 1) * + (input_boxes->at(i).rect[3] - input_boxes->at(i).rect[1] + 1); + } + for (int i = 0; i < input_boxes->size(); ++i) { + for (int j = i + 1; j < input_boxes->size();) { + float xx1 = + std::max(input_boxes->at(i).rect[0], input_boxes->at(j).rect[0]); + float yy1 = + std::max(input_boxes->at(i).rect[1], input_boxes->at(j).rect[1]); + float xx2 = + std::min(input_boxes->at(i).rect[2], input_boxes->at(j).rect[2]); + float yy2 = + std::min(input_boxes->at(i).rect[3], input_boxes->at(j).rect[3]); + float w = std::max(0.f, xx2 - xx1 + 1); + float h = std::max(0.f, yy2 - yy1 + 1); + float inter = w * h; + float ovr = inter / (vArea[i] + vArea[j] - inter); + if (ovr >= nms_threshold) { + input_boxes->erase(input_boxes->begin() + j); + vArea.erase(vArea.begin() + j); + } else { + j++; + } + } + } +} + +// fill tensor with mean and scale +// and trans layout: nhwc -> nchw, neon speed up +void neon_mean_scale(const float *din, float *dout, int size, float *mean, + float *scale) { + float32x4_t vmean0 = vdupq_n_f32(mean[0]); + float32x4_t vmean1 = vdupq_n_f32(mean[1]); + float32x4_t vmean2 = vdupq_n_f32(mean[2]); + float32x4_t vscale0 = vdupq_n_f32(1.f / scale[0]); + float32x4_t vscale1 = vdupq_n_f32(1.f / scale[1]); + float32x4_t vscale2 = vdupq_n_f32(1.f / scale[2]); + + float *dout_c0 = dout; + float *dout_c1 = dout + size; + float *dout_c2 = dout + size * 2; + + int i = 0; + for (; i < size - 3; i += 4) { + float32x4x3_t vin3 = vld3q_f32(din); + float32x4_t vsub0 = vsubq_f32(vin3.val[0], vmean0); + float32x4_t vsub1 = vsubq_f32(vin3.val[1], vmean1); + float32x4_t vsub2 = vsubq_f32(vin3.val[2], vmean2); + float32x4_t vs0 = vmulq_f32(vsub0, vscale0); + float32x4_t vs1 = vmulq_f32(vsub1, vscale1); + float32x4_t vs2 = vmulq_f32(vsub2, vscale2); + vst1q_f32(dout_c0, vs0); + vst1q_f32(dout_c1, vs1); + vst1q_f32(dout_c2, vs2); + + din += 12; + dout_c0 += 4; + dout_c1 += 4; + dout_c2 += 4; + } + for (; i < size; i++) { + *(dout_c0++) = (*(din++) - mean[0]) / scale[0]; + *(dout_c0++) = (*(din++) - mean[1]) / scale[1]; + *(dout_c0++) = (*(din++) - mean[2]) / scale[2]; + } +} + +float fast_exp(float x) { + union { + uint32_t i; + float f; + } v{}; + v.i = (1 << 23) * (1.4426950409 * x + 126.93490512f); + return v.f; +} + +void activation_function_softmax(const float *src, float *dst, int length) { + const float alpha = *std::max_element(src, src + length); + float denominator{0.f}; + + for (int i = 0; i < length; ++i) { + dst[i] = fast_exp(src[i] - alpha); + denominator += dst[i]; + } + for (int i = 0; i < length; ++i) { + dst[i] /= denominator; + } +} diff --git a/deploy/shitu_android_demo/app/src/main/cpp/Utils.h b/deploy/shitu_android_demo/app/src/main/cpp/Utils.h new file mode 100644 index 000000000..ef9a0bcb3 --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/cpp/Utils.h @@ -0,0 +1,143 @@ +// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle_api.h" // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT +#include // NOLINT + +#define TAG "JNI" +#define LOGD(...) __android_log_print(ANDROID_LOG_DEBUG, TAG, __VA_ARGS__) +#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__) +#define LOGW(...) __android_log_print(ANDROID_LOG_WARN, TAG, __VA_ARGS__) +#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, TAG, __VA_ARGS__) +#define LOGF(...) __android_log_print(ANDROID_LOG_FATAL, TAG, __VA_ARGS__) + +#define MIN(a, b) (((a) < (b)) ? (a) : (b)) +#define MAX(a, b) (((a) > (b)) ? (a) : (b)) + +int64_t ShapeProduction(const std::vector &shape); + +template +bool ReadFile(const std::string &path, std::vector *data) { + std::ifstream file(path, std::ifstream::binary); + if (file) { + file.seekg(0, file.end); + int size = file.tellg(); + LOGD("file size=%lld\n", size); + data->resize(size / sizeof(T)); + file.seekg(0, file.beg); + file.read(reinterpret_cast(data->data()), size); + file.close(); + return true; + } else { + LOGE("Can't read file from %s\n", path.c_str()); + } + return false; +} + +template +bool WriteFile(const std::string &path, const std::vector &data) { + std::ofstream file{path, std::ios::binary}; + if (!file.is_open()) { + LOGE("Can't write file to %s\n", path.c_str()); + return false; + } + file.write(reinterpret_cast(data.data()), + data.size() * sizeof(T)); + file.close(); + return true; +} + +inline int64_t GetCurrentTime() { + struct timeval time; + gettimeofday(&time, NULL); + return 1000000LL * (int64_t)time.tv_sec + (int64_t)time.tv_usec; +} + +inline double GetElapsedTime(int64_t time) { + return (GetCurrentTime() - time) / 1000.0f; +} + +inline paddle::lite_api::PowerMode ParsePowerMode(std::string mode) { + if (mode == "LITE_POWER_HIGH") { + return paddle::lite_api::LITE_POWER_HIGH; + } else if (mode == "LITE_POWER_LOW") { + return paddle::lite_api::LITE_POWER_LOW; + } else if (mode == "LITE_POWER_FULL") { + return paddle::lite_api::LITE_POWER_FULL; + } else if (mode == "LITE_POWER_RAND_HIGH") { + return paddle::lite_api::LITE_POWER_RAND_HIGH; + } else if (mode == "LITE_POWER_RAND_LOW") { + return paddle::lite_api::LITE_POWER_RAND_LOW; + } + return paddle::lite_api::LITE_POWER_NO_BIND; +} + +void NHWC3ToNC3HW(const float *src, float *dst, const float *mean, + const float *std, int width, int height); + +void NHWC1ToNC1HW(const float *src, float *dst, const float *mean, + const float *std, int width, int height); + +// Recognise Result +struct RectResult { + std::string class_name; + int class_id; + float score; +}; + +// Object Detection Result +struct ObjectResult { + // Rectangle coordinates of detected object: left, right, top, down + std::vector rect; + // Class id of detected object + int class_id; + // Confidence of detected object + float confidence; + // RecModel result + std::vector rec_result; +}; + +// Object for storing all preprocessed data +class ImageBlob { +public: // NOLINT + // image width and height + std::vector im_shape_; + // Buffer for image data after preprocessing + std::vector im_data_; + // in net data shape(after pad) + std::vector in_net_shape_; + // Evaluation image width and height + // std::vector eval_im_size_f_; + // Scale factor for image size to origin image size + std::vector scale_factor_; +}; + +void nms(std::vector *input_boxes, float nms_threshold, + bool rec_nms = false); + +void neon_mean_scale(const float *din, float *dout, int size, float *mean, + float *scale); + +void activation_function_softmax(const float *src, float *dst, int length); + +inline double GetCurrentUS() { + struct timeval time; + gettimeofday(&time, NULL); + return 1e+6 * time.tv_sec + time.tv_usec; +} diff --git a/deploy/shitu_android_demo/app/src/main/cpp/VectorSearch.cc b/deploy/shitu_android_demo/app/src/main/cpp/VectorSearch.cc new file mode 100644 index 000000000..ae8aba192 --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/cpp/VectorSearch.cc @@ -0,0 +1,166 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "VectorSearch.h" +#include "Utils.h" +#include +#include +#include +#include +#include +#include + +void VectorSearch::LoadIndexFile() { + //"/storage/emulated/0/Android/data/com.baidu.paddle.lite.demo.pp_shitu/files/index/vector.index" + std::string file_path = this->index_path; + const char *fname = file_path.c_str(); + this->index = faiss::read_index(fname, 0); +} + +// load id_map.txt +void VectorSearch::LoadIdMap() { + std::string file_path = this->label_path; + std::ifstream in(file_path); + std::string line; + std::vector m_vec; + if (in) { + while (getline(in, line)) { + std::regex ws_re("\\s+"); + std::vector v( + std::sregex_token_iterator(line.begin(), line.end(), ws_re, -1), + std::sregex_token_iterator()); + if (v.size() != 2) { + std::cout << "The number of element for each line in : " << file_path + << "must be 2, exit the program..." << std::endl; + exit(1); + } else + this->id_map.insert(std::pair( + std::stol(v[0], nullptr, 10), v[1])); + } + } +} + +// doing search +const SearchResult &VectorSearch::Search(float *feature, int query_number) { + this->D.resize(this->return_k * query_number); + this->I.resize(this->return_k * query_number); + this->index->search(query_number, feature, return_k, D.data(), I.data()); + this->sr.return_k = this->return_k; + this->sr.D = this->D; + this->sr.I = this->I; + return this->sr; +} + +std::string VectorSearch::GetLabel(faiss::Index::idx_t ind) { + if (this->id_map.count(ind)) { + return this->id_map[ind]; + } else { + return "None"; + } +} + +int VectorSearch::AddFeature(float *feature, const std::string &label) { + this->index->add(1, feature); + int id = (int)(this->id_map.size()); + if (!label.empty()) { + this->id_map.insert(std::pair(id, label)); + } else { + this->id_map.insert( + std::pair(id, std::to_string(id))); + } + return (int)(this->index->ntotal); +} + +void VectorSearch::SaveIndex(const std::string &save_file_name) { + // save_file_name 为无后缀的文件名字，如 vector、vector_new 等 + std::string file_path_index, file_path_labelmap; + if (save_file_name.empty()) { + file_path_index = this->index_path; + file_path_labelmap = this->label_path; + } else { + int begin_pos = (int)this->index_path.find_last_of('/') + 1; + int end_pos = (int)this->index_path.find_last_of('.'); + int replace_len = end_pos - begin_pos; + file_path_index = + this->index_path.replace(begin_pos, replace_len, save_file_name); + + begin_pos = (int)this->label_path.find_last_of('/') + 1; + end_pos = (int)this->label_path.find_last_of('.'); + replace_len = end_pos - begin_pos; + file_path_labelmap = + this->label_path.replace(begin_pos, replace_len, save_file_name); + } + // save index + faiss::write_index(this->index, file_path_index.c_str()); + LOGD("index file saved at [%s]", file_path_index.c_str()); + + // save label_map + std::ofstream out(file_path_labelmap); + std::map::iterator iter; + for (iter = this->id_map.begin(); iter != this->id_map.end(); iter++) { + std::string content = std::to_string(iter->first) + " " + iter->second; + out.write(content.c_str(), (int)content.size()); + out << std::endl; + } + out.close(); +} + +void VectorSearch::ClearFeature() { + this->index->reset(); + this->id_map.clear(); + LOGD("=========================features cleard"); +} + +const float &VectorSearch::GetThreshold() const { return this->score_thres; } + +bool file_exist(const std::string &file_name) { + return access(file_name.c_str(), F_OK) != -1; +} + +bool VectorSearch::LoadFromSaveFileName(const std::string &load_file_name) { + std::string origin_label_path = GetLabelPath(); + int begin_pos = (int)origin_label_path.find_last_of('/') + 1; + int end_pos = (int)origin_label_path.find_last_of('.'); + int replace_len = end_pos - begin_pos; + std::string new_label_path = + origin_label_path.replace(begin_pos, replace_len, load_file_name); + + std::string origin_index_path = GetIndexPath(); + begin_pos = (int)origin_index_path.find_last_of('/') + 1; + end_pos = (int)origin_index_path.find_last_of('.'); + replace_len = end_pos - begin_pos; + std::string new_index_path = + origin_index_path.replace(begin_pos, replace_len, load_file_name); + + if (!file_exist(new_label_path) || !file_exist(new_index_path)) { + return false; + } + this->label_path = new_label_path; + this->id_map.clear(); + LoadIdMap(); + + this->index_path = new_index_path; + LoadIndexFile(); + return true; +} + +std::vector VectorSearch::GetLabelList() const { + std::vector tmp; + for (const auto &it : this->id_map) { + tmp.emplace_back(it.second); + } + std::sort(tmp.begin(), tmp.end()); + tmp.erase(unique(tmp.begin(), tmp.end()), tmp.end()); + return tmp; +} diff --git a/deploy/shitu_android_demo/app/src/main/cpp/VectorSearch.h b/deploy/shitu_android_demo/app/src/main/cpp/VectorSearch.h new file mode 100644 index 000000000..ee86c7f35 --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/cpp/VectorSearch.h @@ -0,0 +1,90 @@ +// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#ifdef WIN32 +#define OS_PATH_SEP "\\" +#else +#define OS_PATH_SEP "/" +#endif + +#include "include/faiss/Index.h" +#include "include/faiss/index_io.h" +#include +#include +#include + +struct SearchResult { + std::vector I; + std::vector D; + int return_k; +}; + +class VectorSearch { +public: + explicit VectorSearch(const std::string &label_path, + const std::string &index_path, const int &return_k = 5, + const float &score_thres = 0.5) { + // // IndexProcess + this->label_path = label_path; + this->index_path = index_path; + this->return_k = return_k; + this->score_thres = score_thres; + + LoadIdMap(); + LoadIndexFile(); + this->I.resize(this->return_k * this->max_query_number); + this->D.resize(this->return_k * this->max_query_number); + printf("faiss index load success!\n"); + }; + + void LoadIdMap(); + + bool LoadFromSaveFileName(const std::string &load_file_name); + + void LoadIndexFile(); + + int AddFeature(float *feature, const std::string &label = ""); + + const SearchResult &Search(float *feature, int query_number); + + const int GetIndexLength() { return this->index->ntotal; } + + void SaveIndex(const std::string &save_path = ""); + + std::string GetIndexPath() { return this->index_path; } + + std::string GetLabelPath() { return this->label_path; } + + std::string GetLabel(faiss::Index::idx_t ind); + + void ClearFeature(); + + const float &GetThreshold() const; + + std::vector GetLabelList() const; + +private: + std::string index_path; + std::string label_path; + int return_k = 5; + float score_thres = 0.5; + + std::map id_map; + faiss::Index *index; + int max_query_number = 6; + std::vector D; + std::vector I; + SearchResult sr; +}; diff --git a/deploy/shitu_android_demo/app/src/main/java/com/baidu/paddle/lite/demo/common/AppCompatPreferenceActivity.java b/deploy/shitu_android_demo/app/src/main/java/com/baidu/paddle/lite/demo/common/AppCompatPreferenceActivity.java new file mode 100644 index 000000000..51d716b45 --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/java/com/baidu/paddle/lite/demo/common/AppCompatPreferenceActivity.java @@ -0,0 +1,127 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.baidu.paddle.lite.demo.common; + +import android.content.res.Configuration; +import android.os.Bundle; +import android.preference.PreferenceActivity; +import android.support.annotation.LayoutRes; +import android.support.annotation.Nullable; +import android.support.v7.app.ActionBar; +import android.support.v7.app.AppCompatDelegate; +import android.support.v7.widget.Toolbar; +import android.view.MenuInflater; +import android.view.View; +import android.view.ViewGroup; + +/** + * A {@link PreferenceActivity} which implements and proxies the necessary calls + * to be used with AppCompat. + *

+ * This technique can be used with an {@link android.app.Activity} class, not just + * {@link PreferenceActivity}. + */ +public abstract class AppCompatPreferenceActivity extends PreferenceActivity { + private AppCompatDelegate mDelegate; + + @Override + protected void onCreate(Bundle savedInstanceState) { + getDelegate().installViewFactory(); + getDelegate().onCreate(savedInstanceState); + super.onCreate(savedInstanceState); + } + + @Override + protected void onPostCreate(Bundle savedInstanceState) { + super.onPostCreate(savedInstanceState); + getDelegate().onPostCreate(savedInstanceState); + } + + public ActionBar getSupportActionBar() { + return getDelegate().getSupportActionBar(); + } + + public void setSupportActionBar(@Nullable Toolbar toolbar) { + getDelegate().setSupportActionBar(toolbar); + } + + @Override + public MenuInflater getMenuInflater() { + return getDelegate().getMenuInflater(); + } + + @Override + public void setContentView(@LayoutRes int layoutResID) { + getDelegate().setContentView(layoutResID); + } + + @Override + public void setContentView(View view) { + getDelegate().setContentView(view); + } + + @Override + public void setContentView(View view, ViewGroup.LayoutParams params) { + getDelegate().setContentView(view, params); + } + + @Override + public void addContentView(View view, ViewGroup.LayoutParams params) { + getDelegate().addContentView(view, params); + } + + @Override + protected void onPostResume() { + super.onPostResume(); + getDelegate().onPostResume(); + } + + @Override + protected void onTitleChanged(CharSequence title, int color) { + super.onTitleChanged(title, color); + getDelegate().setTitle(title); + } + + @Override + public void onConfigurationChanged(Configuration newConfig) { + super.onConfigurationChanged(newConfig); + getDelegate().onConfigurationChanged(newConfig); + } + + @Override + protected void onStop() { + super.onStop(); + getDelegate().onStop(); + } + + @Override + protected void onDestroy() { + super.onDestroy(); + getDelegate().onDestroy(); + } + + public void invalidateOptionsMenu() { + getDelegate().invalidateOptionsMenu(); + } + + private AppCompatDelegate getDelegate() { + if (mDelegate == null) { + mDelegate = AppCompatDelegate.create(this, null); + } + return mDelegate; + } +} diff --git a/deploy/shitu_android_demo/app/src/main/java/com/baidu/paddle/lite/demo/common/Utils.java b/deploy/shitu_android_demo/app/src/main/java/com/baidu/paddle/lite/demo/common/Utils.java new file mode 100644 index 000000000..21964f95f --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/java/com/baidu/paddle/lite/demo/common/Utils.java @@ -0,0 +1,259 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.baidu.paddle.lite.demo.common; + +import android.content.Context; +import android.content.res.AssetManager; +import android.content.pm.PackageManager; +import android.content.res.Resources; +import android.hardware.Camera; +import android.opengl.GLES20; +import android.os.Environment; +import android.util.Log; +import android.view.Surface; +import android.view.WindowManager; + +import java.io.*; +import java.util.List; + +public class Utils { + private static final String TAG = Utils.class.getSimpleName(); + + public static void RecursiveCreateDirectories(String fileDir) { + String[] fileDirs = fileDir.split("\\/"); + String topPath = ""; + for (int i = 0; i < fileDirs.length; i++) { + topPath += "/" + fileDirs[i]; + File file = new File(topPath); + if (file.exists()) { + continue; + } else { + file.mkdir(); + } + } + } + + public static void copyFileFromAssets(Context appCtx, String srcPath, String dstPath) { + if (srcPath.isEmpty() || dstPath.isEmpty()) { + return; + } + String dstDir = dstPath.substring(0, dstPath.lastIndexOf('/')); + if (dstDir.length() > 0) { + RecursiveCreateDirectories(dstDir); + } + InputStream is = null; + OutputStream os = null; + if (new File(dstPath).exists()) + { + return ; + } + try { + is = new BufferedInputStream(appCtx.getAssets().open(srcPath)); + os = new BufferedOutputStream(new FileOutputStream(new File(dstPath))); + byte[] buffer = new byte[1024]; + int length = 0; + while ((length = is.read(buffer)) != -1) { + os.write(buffer, 0, length); + } + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + os.close(); + is.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + public static void copyDirectoryFromAssets(Context appCtx, String srcDir, String dstDir) { + if (srcDir.isEmpty() || dstDir.isEmpty()) { + return; + } + try { + if (!new File(dstDir).exists()) { + new File(dstDir).mkdirs(); + } + for (String fileName : appCtx.getAssets().list(srcDir)) { + String srcSubPath = srcDir + File.separator + fileName; + String dstSubPath = dstDir + File.separator + fileName; + if (new File(srcSubPath).isDirectory()) { + copyDirectoryFromAssets(appCtx, srcSubPath, dstSubPath); + } else { + copyFileFromAssets(appCtx, srcSubPath, dstSubPath); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + public static float[] parseFloatsFromString(String string, String delimiter) { + String[] pieces = string.trim().toLowerCase().split(delimiter); + float[] floats = new float[pieces.length]; + for (int i = 0; i < pieces.length; i++) { + floats[i] = Float.parseFloat(pieces[i].trim()); + } + return floats; + } + + public static long[] parseLongsFromString(String string, String delimiter) { + String[] pieces = string.trim().toLowerCase().split(delimiter); + long[] longs = new long[pieces.length]; + for (int i = 0; i < pieces.length; i++) { + longs[i] = Long.parseLong(pieces[i].trim()); + } + return longs; + } + + public static String getSDCardDirectory() { + return Environment.getExternalStorageDirectory().getAbsolutePath(); + } + + public static String getDCIMDirectory() { + return Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DCIM).getAbsolutePath(); + } + + public static Camera.Size getOptimalPreviewSize(List sizes, int w, int h) { + final double ASPECT_TOLERANCE = 0.1; + double targetRatio = (double) w / h; + if (sizes == null) return null; + + Camera.Size optimalSize = null; + double minDiff = Double.MAX_VALUE; + + int targetHeight = h; + + // Try to find an size match aspect ratio and size + for (Camera.Size size : sizes) { + double ratio = (double) size.width / size.height; + if (Math.abs(ratio - targetRatio) > ASPECT_TOLERANCE) continue; + if (Math.abs(size.height - targetHeight) < minDiff) { + optimalSize = size; + minDiff = Math.abs(size.height - targetHeight); + } + } + + // Cannot find the one match the aspect ratio, ignore the requirement + if (optimalSize == null) { + minDiff = Double.MAX_VALUE; + for (Camera.Size size : sizes) { + if (Math.abs(size.height - targetHeight) < minDiff) { + optimalSize = size; + minDiff = Math.abs(size.height - targetHeight); + } + } + } + return optimalSize; + } + + public static int getScreenWidth() { + return Resources.getSystem().getDisplayMetrics().widthPixels; + } + + public static int getScreenHeight() { + return Resources.getSystem().getDisplayMetrics().heightPixels; + } + + public static int getCameraDisplayOrientation(Context context, int cameraId) { + android.hardware.Camera.CameraInfo info = new android.hardware.Camera.CameraInfo(); + android.hardware.Camera.getCameraInfo(cameraId, info); + WindowManager wm = (WindowManager) context.getSystemService(Context.WINDOW_SERVICE); + int rotation = wm.getDefaultDisplay().getRotation(); + int degrees = 0; + switch (rotation) { + case Surface.ROTATION_0: + degrees = 0; + break; + case Surface.ROTATION_90: + degrees = 90; + break; + case Surface.ROTATION_180: + degrees = 180; + break; + case Surface.ROTATION_270: + degrees = 270; + break; + } + int result; + if (info.facing == Camera.CameraInfo.CAMERA_FACING_FRONT) { + result = (info.orientation + degrees) % 360; + result = (360 - result) % 360; // compensate the mirror + } else { + // back-facing + result = (info.orientation - degrees + 360) % 360; + } + return result; + } + + public static int createShaderProgram(String vss, String fss) { + int vshader = GLES20.glCreateShader(GLES20.GL_VERTEX_SHADER); + GLES20.glShaderSource(vshader, vss); + GLES20.glCompileShader(vshader); + int[] status = new int[1]; + GLES20.glGetShaderiv(vshader, GLES20.GL_COMPILE_STATUS, status, 0); + if (status[0] == 0) { + Log.e(TAG, GLES20.glGetShaderInfoLog(vshader)); + GLES20.glDeleteShader(vshader); + vshader = 0; + return 0; + } + + int fshader = GLES20.glCreateShader(GLES20.GL_FRAGMENT_SHADER); + GLES20.glShaderSource(fshader, fss); + GLES20.glCompileShader(fshader); + GLES20.glGetShaderiv(fshader, GLES20.GL_COMPILE_STATUS, status, 0); + if (status[0] == 0) { + Log.e(TAG, GLES20.glGetShaderInfoLog(fshader)); + GLES20.glDeleteShader(vshader); + GLES20.glDeleteShader(fshader); + fshader = 0; + return 0; + } + + int program = GLES20.glCreateProgram(); + GLES20.glAttachShader(program, vshader); + GLES20.glAttachShader(program, fshader); + GLES20.glLinkProgram(program); + GLES20.glDeleteShader(vshader); + GLES20.glDeleteShader(fshader); + GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, status, 0); + if (status[0] == 0) { + Log.e(TAG, GLES20.glGetProgramInfoLog(program)); + program = 0; + return 0; + } + GLES20.glValidateProgram(program); + GLES20.glGetProgramiv(program, GLES20.GL_VALIDATE_STATUS, status, 0); + if (status[0] == 0) { + Log.e(TAG, GLES20.glGetProgramInfoLog(program)); + GLES20.glDeleteProgram(program); + program = 0; + return 0; + } + + return program; + } + + public static boolean isSupportedNPU() { + String hardware = android.os.Build.HARDWARE; + return hardware.equalsIgnoreCase("kirin810") || hardware.equalsIgnoreCase("kirin990"); + } +} diff --git a/deploy/shitu_android_demo/app/src/main/java/com/baidu/paddle/lite/demo/pp_shitu/MainActivity.java b/deploy/shitu_android_demo/app/src/main/java/com/baidu/paddle/lite/demo/pp_shitu/MainActivity.java new file mode 100644 index 000000000..4def33e82 --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/java/com/baidu/paddle/lite/demo/pp_shitu/MainActivity.java @@ -0,0 +1,701 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.baidu.paddle.lite.demo.pp_shitu; + +import android.Manifest; +import android.annotation.SuppressLint; +import android.app.AlertDialog; +import android.app.ProgressDialog; +import android.content.ContentResolver; +import android.content.Context; +import android.content.Intent; +import android.content.SharedPreferences; +import android.content.pm.ApplicationInfo; +import android.content.pm.PackageManager; +import android.database.Cursor; +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.graphics.Typeface; +import android.net.Uri; +import android.os.Bundle; +import android.os.Handler; +import android.os.HandlerThread; +import android.os.Message; +import android.preference.PreferenceManager; +import android.provider.MediaStore; +import android.support.annotation.NonNull; +import android.support.v4.app.ActivityCompat; +import android.support.v4.content.ContextCompat; +import android.support.v7.app.AppCompatActivity; +import android.text.SpannableString; +import android.text.Spanned; +import android.text.style.StyleSpan; +import android.util.Log; +import android.view.Menu; +import android.view.MenuInflater; +import android.view.MenuItem; +import android.view.View; +import android.widget.Button; +import android.widget.EditText; +import android.widget.ImageView; +import android.widget.TextView; +import android.widget.Toast; + +import com.baidu.paddle.lite.demo.common.Utils; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; + +public class MainActivity extends AppCompatActivity { + private static final String TAG = MainActivity.class.getSimpleName(); + public static final int OPEN_QUERY_PHOTO_REQUEST_CODE = 0; + public static final int TAKE_QUERY_PHOTO_REQUEST_CODE = 1; + public static final int OPEN_GALLERY_PHOTO_REQUEST_CODE = 4; + public static final int TAKE_GALLERY_PHOTO_REQUEST_CODE = 5; + public static final int CLEAR_FEATURE_REQUEST_CODE = 6; + public static final int REQUEST_LOAD_MODEL = 0; + public static final int REQUEST_RUN_MODEL = 1; + public static final int RESPONSE_LOAD_MODEL_SUCCESSED = 0; + public static final int RESPONSE_LOAD_MODEL_FAILED = 1; + public static final int RESPONSE_RUN_MODEL_SUCCESSED = 2; + public static final int RESPONSE_RUN_MODEL_FAILED = 3; + + protected ProgressDialog pbLoadModel = null; + protected ProgressDialog pbRunModel = null; + protected Handler receiver = null; // Receive messages from worker thread + protected Handler sender = null; // Send command to worker thread + protected HandlerThread worker = null; // Worker thread to load&run model + + // UI components of image classification + protected ImageView ivInputImage; + protected TextView tvTop1Result; + protected TextView tvInferenceTime; + protected TextView tvSimilarity; + protected TextView tvIndexName; + protected TextView tv_description; + + //protected Switch mSwitch; + + // Model settings of image classification + protected String modelPath = ""; + protected String labelPath = ""; + protected String indexPath = ""; + + protected String imagePath = ""; + protected String DetModelPath = ""; + protected String RecModelPath = ""; + protected int cpuThreadNum = 1; + protected EditText label_name; + protected Button label_botton; + protected Button cancel_botton; + protected boolean add_gallery = false; + protected int topk = 3; + protected String cpuMode = ""; + protected long[] detinputShape = new long[]{}; + protected long[] recinputShape = new long[]{}; + protected boolean useGpu = false; + protected Native predictor = new Native(); + + ImageView mImage_add_query; + ImageView mImage_take_query; + ImageView mImage_add_gallery; + ImageView mImage_take_gallery; + ImageView mImage_save; + ImageView mShow_index; + View help; + View reset; + + @SuppressLint("HandlerLeak") + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.activity_main); + + // Clear all setting items to avoid app crashing due to the incorrect settings + SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(this); + SharedPreferences.Editor editor = sharedPreferences.edit(); + editor.clear(); + editor.apply(); + + // Prepare the worker thread for mode loading and inference + receiver = new Handler() { + @Override + public void handleMessage(Message msg) { + switch (msg.what) { + case RESPONSE_LOAD_MODEL_SUCCESSED: + pbLoadModel.dismiss(); + onLoadModelSuccessed(); + break; + case RESPONSE_LOAD_MODEL_FAILED: + pbLoadModel.dismiss(); + Toast.makeText(MainActivity.this, "Load model failed!", Toast.LENGTH_SHORT).show(); + onLoadModelFailed(); + break; + case RESPONSE_RUN_MODEL_SUCCESSED: + pbRunModel.dismiss(); + onRunModelSuccessed(); + break; + case RESPONSE_RUN_MODEL_FAILED: + pbRunModel.dismiss(); + Toast.makeText(MainActivity.this, "Run model failed!", Toast.LENGTH_SHORT).show(); + onRunModelFailed(); + break; + default: + break; + } + } + }; + worker = new HandlerThread("Predictor Worker"); + worker.start(); + sender = new Handler(worker.getLooper()) { + public void handleMessage(Message msg) { + switch (msg.what) { + case REQUEST_LOAD_MODEL: + // Load model and reload test image + if (onLoadModel()) { + receiver.sendEmptyMessage(RESPONSE_LOAD_MODEL_SUCCESSED); + } else { + receiver.sendEmptyMessage(RESPONSE_LOAD_MODEL_FAILED); + } + break; + case REQUEST_RUN_MODEL: + // Run model if model is loaded + if (onRunModel()) { + receiver.sendEmptyMessage(RESPONSE_RUN_MODEL_SUCCESSED); + } else { + receiver.sendEmptyMessage(RESPONSE_RUN_MODEL_FAILED); + } + break; + default: + break; + } + } + }; + + // Setup the UI components + ivInputImage = findViewById(R.id.iv_input_image); + tvTop1Result = findViewById(R.id.tv_top1_result); + tvInferenceTime = findViewById(R.id.tv_inference_time); + tvSimilarity = findViewById(R.id.similairy); + tvIndexName = findViewById(R.id.index_name); + tv_description = findViewById(R.id.description); + + // 启动时隐藏输入label的输入框和确定按钮 + label_name = findViewById(R.id.label_name); + label_name.setVisibility(View.INVISIBLE); + label_botton = findViewById(R.id.label_botton); + label_botton.setVisibility(View.INVISIBLE); + cancel_botton = findViewById(R.id.cancel_botton); + cancel_botton.setVisibility(View.INVISIBLE); + + mImage_add_query = (ImageView) findViewById(R.id.add_query); + mImage_take_query = (ImageView) findViewById(R.id.take_query); + mImage_add_gallery = (ImageView) findViewById(R.id.add_gallery); + mImage_take_gallery = (ImageView) findViewById(R.id.take_gallery); + mImage_save = (ImageView) findViewById(R.id.save); + mShow_index = (ImageView) findViewById(R.id.show_index); + + File fileindex = new File(getExternalFilesDir(null) + "/" + "index/latest.index"); + if (fileindex.exists()) { + this.indexPath = "index/latest.index"; + } else { + this.indexPath = "index/original.index"; + } + File filelabel = new File(getExternalFilesDir(null) + "/" + "index/latest.txt"); + if (filelabel.exists()) { + this.labelPath = "index/latest.txt"; + } else { + this.labelPath = "index/original.txt"; + } + mImage_add_query.setOnClickListener(v -> { + if (requestAllPermissions()) { + label_name.setVisibility(View.INVISIBLE); + label_botton.setVisibility(View.INVISIBLE); + cancel_botton.setVisibility(View.INVISIBLE); + openQueryPhoto(); + } else { + Toast.makeText(MainActivity.this, "请开启相机和读写文件权限", Toast.LENGTH_SHORT).show(); + } + }); + mImage_take_query.setOnClickListener(v -> { + if (requestAllPermissions()) { + label_name.setVisibility(View.INVISIBLE); + label_botton.setVisibility(View.INVISIBLE); + cancel_botton.setVisibility(View.INVISIBLE); + takeQueryPhoto(); + } else { + Toast.makeText(MainActivity.this, "请开启相机和读写文件权限", Toast.LENGTH_SHORT).show(); + } + }); + mImage_add_gallery.setOnClickListener(v -> { + if (requestAllPermissions()) { + label_name.setText(""); + label_name.setVisibility(View.INVISIBLE); + label_botton.setVisibility(View.INVISIBLE); + cancel_botton.setVisibility(View.INVISIBLE); + openGalleryPhoto(); + } else { + Toast.makeText(MainActivity.this, "请开启相机和读写文件权限", Toast.LENGTH_SHORT).show(); + } + }); + mImage_take_gallery.setOnClickListener(v -> { + if (requestAllPermissions()) { + label_name.setVisibility(View.INVISIBLE); + label_botton.setVisibility(View.INVISIBLE); + cancel_botton.setVisibility(View.INVISIBLE); + takeGalleryPhoto(); + } else { + Toast.makeText(MainActivity.this, "请开启相机和读写文件权限", Toast.LENGTH_SHORT).show(); + } + }); + mImage_save.setOnClickListener(v -> { + if (requestAllPermissions()) { + label_name.setVisibility(View.INVISIBLE); + label_botton.setVisibility(View.INVISIBLE); + cancel_botton.setVisibility(View.INVISIBLE); + saveIndex(); + } else { + Toast.makeText(MainActivity.this, "请开启相机和读写文件权限", Toast.LENGTH_SHORT).show(); + } + }); + + mShow_index.setOnClickListener(view -> { +// String fullpath = getExternalFilesDir(null) + "/" + labelPath; + if (requestAllPermissions()) { + String labelfile_content = predictor.getClassname(); + AlertDialog alertDialog = new AlertDialog.Builder(MainActivity.this) + //标题 + .setTitle(labelPath) + //内容 + .setMessage(labelfile_content) + //图标 + .setIcon(R.mipmap.ic_launcher) + .setPositiveButton("确认", null) + .create(); + alertDialog.show(); + } else { + Toast.makeText(MainActivity.this, "请开启相机和读写文件权限", Toast.LENGTH_SHORT).show(); + } + }); + } + + + @Override + protected void onResume() { + super.onResume(); + SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(this); + boolean settingsChanged; + String model_path = sharedPreferences.getString(getString(R.string.MODEL_PATH_KEY), + getString(R.string.MODEL_PATH_DEFAULT)); + String label_path = sharedPreferences.getString(getString(R.string.LABEL_PATH_KEY), + getString(R.string.LABEL_PATH_DEFAULT)); + String index_path = sharedPreferences.getString(getString(R.string.INDEX_PATH_KEY), + getString(R.string.INDEX_PATH_DEFAULT)); + File fileindex = new File(getExternalFilesDir(null) + "/" + "index/latest.index"); + if (fileindex.exists()) { + this.indexPath = "index/latest.index"; + } else { + this.indexPath = "index/original.index"; + } + File filelabel = new File(getExternalFilesDir(null) + "/" + "index/latest.txt"); + if (filelabel.exists()) { + this.labelPath = "index/latest.txt"; + } else { + this.labelPath = "index/original.txt"; + } + + String image_path = "images/demo.jpg"; + settingsChanged = !model_path.equalsIgnoreCase(modelPath); + settingsChanged |= !image_path.equalsIgnoreCase(imagePath); + int cpu_thread_num = Integer.parseInt("4"); + settingsChanged |= cpu_thread_num != cpuThreadNum; + long[] det_input_shape = + Utils.parseLongsFromString("1,3,640,640", ","); + long[] rec_input_shape = + Utils.parseLongsFromString("1,3,224,224", ","); + String cpu_power_mode = "LITE_POWER_HIGH"; + settingsChanged |= !cpu_power_mode.equalsIgnoreCase(cpuMode); + int top_k = Integer.parseInt("3"); + settingsChanged |= top_k != topk; + settingsChanged |= det_input_shape.length != detinputShape.length; + settingsChanged |= rec_input_shape.length != recinputShape.length; + if (!settingsChanged) { + for (int i = 0; i < det_input_shape.length; i++) { + settingsChanged |= det_input_shape[i] != detinputShape[i]; + } + for (int i = 0; i < rec_input_shape.length; i++) { + settingsChanged |= rec_input_shape[i] != recinputShape[i]; + } + } + if (settingsChanged || useGpu) { + modelPath = model_path; + imagePath = image_path; + cpuThreadNum = cpu_thread_num; + detinputShape = det_input_shape; + recinputShape = rec_input_shape; + DetModelPath = modelPath; + RecModelPath = modelPath; + topk = top_k; + cpuMode = cpu_power_mode; + loadModel(); + } + } + + public void loadModel() { + pbLoadModel = ProgressDialog.show(this, "", "Loading model...", false, false); + sender.sendEmptyMessage(REQUEST_LOAD_MODEL); + } + + public void runModel() { + pbRunModel = ProgressDialog.show(this, "", "Running model...", false, false); + sender.sendEmptyMessage(REQUEST_RUN_MODEL); + } + + + public boolean onLoadModel() { + Context context = getBaseContext(); + ApplicationInfo info = context.getApplicationInfo(); + int debug_mode = info.flags & ApplicationInfo.FLAG_DEBUGGABLE; + // push model to sdcard + String realDetModelDir = getExternalFilesDir(null) + "/" + DetModelPath; + if (debug_mode != 0) { + Utils.copyDirectoryFromAssets(this, DetModelPath, realDetModelDir); + } + String realRecModelDir = getExternalFilesDir(null) + "/" + RecModelPath; + if (debug_mode != 0) { + Utils.copyDirectoryFromAssets(this, RecModelPath, realRecModelDir); + } + // push label to sdcard + String realLabelPath = getExternalFilesDir(null) + "/" + labelPath; + if (debug_mode != 0) { + Utils.copyFileFromAssets(this, labelPath, realLabelPath); + } + String realIndexDir = getExternalFilesDir(null) + "/" + indexPath; + if (debug_mode != 0) { + Utils.copyFileFromAssets(this, indexPath, realIndexDir); + } + return predictor.init(realDetModelDir, realRecModelDir, realLabelPath, realIndexDir, + detinputShape, recinputShape, cpuThreadNum, 0, 1, topk, add_gallery, cpuMode); + } + + public boolean onRunModel() { + return predictor.isLoaded() && predictor.process(); + } + + public void onLoadModelSuccessed() { + // Load test image from path and run model + try { + if (imagePath.isEmpty()) { + return; + } + Bitmap image; + // Read test image file from custom path if the first character of mode path is '/', otherwise read test + // image file from assets + if (imagePath.charAt(0) != '/') { + InputStream imageStream = getAssets().open(imagePath); + image = BitmapFactory.decodeStream(imageStream); + } else { + if (!new File(imagePath).exists()) { + return; + } + image = BitmapFactory.decodeFile(imagePath); + } + if (image != null && predictor.isLoaded()) { + predictor.setInputImage(image); + runModel(); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + public void onLoadModelFailed() { + } + + @SuppressLint("SetTextI18n") + public void onRunModelSuccessed() { + // Obtain results and update UI + tvInferenceTime.setText("预测时间：" + predictor.inferenceTime() + " ms"); + Bitmap inputImage = predictor.inputImage(); + if (inputImage != null) { + ivInputImage.setImageBitmap(inputImage); + } + String res = predictor.top1Result(); + if (res.contains(",")) { + String[] res_split = res.split(","); + tvTop1Result.setText("类别：" + res_split[0].trim()); + res_split[1] = res_split[1].trim().substring(0, 5); + tvSimilarity.setText("相似度：" + res_split[1]); + tvIndexName.setText("检索库名称：" + indexPath.split("/")[1].split("\\.")[0]); + } + } + + public void onRunModelFailed() { + } + + public void onImageChanged(Bitmap image) { + // Rerun model if users pick test image from gallery or camera + if (image != null && predictor.isLoaded()) { + label_name.setVisibility(View.INVISIBLE); + label_botton.setVisibility(View.INVISIBLE); + cancel_botton.setVisibility(View.INVISIBLE); + predictor.setAddGallery(0); + predictor.setInputImage(image); + runModel(); + tv_description.setText("【待识别图片】"); + } + } + + @SuppressLint("SetTextI18n") + public void onAddGallery(Bitmap image) { + if (image != null && predictor.isLoaded()) { + ivInputImage.setImageBitmap(image); + tv_description.setText("【待加库图片】"); + predictor.setAddGallery(1); + predictor.setInputImage(image); + runModel(); + label_name.setVisibility(View.VISIBLE); + label_name.setHint("image label name"); + label_name.setText(""); + label_botton.setVisibility(View.VISIBLE); + cancel_botton.setVisibility(View.VISIBLE); + tvTop1Result.setText("类别："); + tvSimilarity.setText("相似度："); + tvInferenceTime.setText("预测时间：" + predictor.inferenceTime() + " ms"); + cancel_botton.setOnClickListener(view -> { + predictor.setAddGallery(0); + label_name.setVisibility(View.INVISIBLE); + label_botton.setVisibility(View.INVISIBLE); + cancel_botton.setVisibility(View.INVISIBLE); + tvTop1Result.setText("类别："); + tvSimilarity.setText("相似度："); + tvInferenceTime.setText("预测时间："); + }); + label_botton.setOnClickListener(view -> { + predictor.setAddGallery(2); + predictor.setLabelName(label_name.getText().toString()); + runModel(); + label_name.setVisibility(View.INVISIBLE); + label_botton.setVisibility(View.INVISIBLE); + cancel_botton.setVisibility(View.INVISIBLE); + label_name.setText(""); + tvTop1Result.setText("类别："); + tvSimilarity.setText("相似度："); + tvInferenceTime.setText("预测时间：" + predictor.inferenceTime() + " ms"); + Toast.makeText(MainActivity.this, "已添加至检索库", Toast.LENGTH_SHORT).show(); + }); + } + } + + + public void onSettingsClicked() { + startActivity(new Intent(MainActivity.this, SettingsActivity.class)); + } + + @Override + public boolean onCreateOptionsMenu(Menu menu) { + MenuInflater inflater = getMenuInflater(); + inflater.inflate(R.menu.menu_action_options, menu); + help = findViewById(R.id.help); + reset = findViewById(R.id.reset); + return true; + } + + + public boolean onPrepareOptionsMenu(Menu menu) { + boolean isLoaded = predictor.isLoaded(); + return super.onPrepareOptionsMenu(menu); + } + + @SuppressLint("NonConstantResourceId") + @Override + public boolean onOptionsItemSelected(MenuItem item) { + switch (item.getItemId()) { + case android.R.id.home: + finish(); + break; + case R.id.reset: + File filelabel = new File(getExternalFilesDir(null) + "/" + labelPath); + if (filelabel.exists()) { + filelabel.delete(); + } + File fileindex = new File(getExternalFilesDir(null) + "/" + indexPath); + if (fileindex.exists()) { + fileindex.delete(); + } + labelPath = getString(R.string.LABEL_PATH_DEFAULT); + indexPath = getString(R.string.INDEX_PATH_DEFAULT); + predictor.loadIndex("original"); + tvIndexName.setText("检索库名称：" + indexPath.split("/")[1].split("\\.")[0]); + Toast.makeText(this, "检索库已初始化为original", Toast.LENGTH_SHORT).show(); + break; + case R.id.help: + if (requestAllPermissions()) { + // Make sure we have SDCard r&w permissions to load model from SDCard + String help_content = "【功能说明】\n本APP基于PaddleClas图像分类开发套件中的通用图像识别系统PP-ShiTu开发，支持对拍照/本地上传的图片进行识别。\n【默认检索库说明】\n · 默认内置检索库名为：original，主要包含常见饮料类别共计196种，可通过 [类别查询] 查看已有类别信息。\n" + + " · 可根据实际需求，通过拍照/上传本地图像补充检索库，以提高识别准确率或增加可识别类别。\n注意：修改检索库后需进行保存，否则重启APP后将重置为初始库original。\n"; + SpannableString span = new SpannableString(help_content); + int idx = help_content.indexOf("【功能说明】"); + span.setSpan(new StyleSpan(Typeface.BOLD), idx, idx + "【功能说明】".length(), Spanned.SPAN_EXCLUSIVE_EXCLUSIVE); + idx = help_content.indexOf("【默认检索库说明】"); + span.setSpan(new StyleSpan(Typeface.BOLD), idx, idx + "【默认检索库说明】".length(), Spanned.SPAN_EXCLUSIVE_EXCLUSIVE); + idx = help_content.indexOf("注意："); + span.setSpan(new StyleSpan(Typeface.BOLD), idx, idx + "注意：".length(), Spanned.SPAN_EXCLUSIVE_EXCLUSIVE); + AlertDialog alertDialog = new AlertDialog.Builder(MainActivity.this) + //标题 + .setTitle("使用说明") + //内容 + .setMessage(span) + //图标 + .setIcon(R.mipmap.ic_launcher) + .setPositiveButton("确认", null) + .create(); + alertDialog.show(); + } + break; + case R.id.settings: + if (requestAllPermissions()) { + // Make sure we have SDCard r&w permissions to load model from SDCard + onSettingsClicked(); + } + break; + } + return super.onOptionsItemSelected(item); + } + + + @Override + public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, + @NonNull int[] grantResults) { + super.onRequestPermissionsResult(requestCode, permissions, grantResults); + if (grantResults[0] != PackageManager.PERMISSION_GRANTED || grantResults[1] != PackageManager.PERMISSION_GRANTED) { + Toast.makeText(this, "Permission Denied", Toast.LENGTH_SHORT).show(); + } + } + + private boolean requestAllPermissions() { + if (ContextCompat.checkSelfPermission(this, Manifest.permission.WRITE_EXTERNAL_STORAGE) != PackageManager.PERMISSION_GRANTED + || ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) { + ActivityCompat.requestPermissions(this, new String[]{Manifest.permission.WRITE_EXTERNAL_STORAGE, Manifest.permission.CAMERA}, 0); + return false; + } + return true; + } + + private void openQueryPhoto() { + Intent intent = new Intent(Intent.ACTION_PICK, null); // 选择数据 + intent.setDataAndType(MediaStore.Images.Media.EXTERNAL_CONTENT_URI, "image/*"); + startActivityForResult(intent, OPEN_QUERY_PHOTO_REQUEST_CODE); + } + + @SuppressLint("QueryPermissionsNeeded") + private void takeQueryPhoto() { + Intent takePhotoIntent = new Intent(MediaStore.ACTION_IMAGE_CAPTURE); + startActivityForResult(takePhotoIntent, TAKE_QUERY_PHOTO_REQUEST_CODE); + } + + private void openGalleryPhoto() { + // 增加现有图片到库中---主逻辑代码 + Intent intent = new Intent(Intent.ACTION_PICK, null); // 选择数据 + intent.setDataAndType(MediaStore.Images.Media.EXTERNAL_CONTENT_URI, "image/*"); + startActivityForResult(intent, OPEN_GALLERY_PHOTO_REQUEST_CODE); + } + + @SuppressLint("QueryPermissionsNeeded") + private void takeGalleryPhoto() { + // 直接拍一张图片到库中---主逻辑代码 + Intent intent = new Intent(MediaStore.ACTION_IMAGE_CAPTURE); + startActivityForResult(intent, TAKE_GALLERY_PHOTO_REQUEST_CODE); + } + + private void clearIndex() { + // 清空index主逻辑代码 + predictor.clearFeature(); + } + + @SuppressLint("SetTextI18n") + private void saveIndex() { + label_name.setText("latest"); + indexPath = "index/latest.index"; + labelPath = "index/latest.txt"; + predictor.saveIndex(label_name.getText().toString()); + tvIndexName.setText("检索库名称：" + indexPath.split("/")[1].split("\\.")[0]); + Toast.makeText(MainActivity.this, "检索库保存并更新为latest", Toast.LENGTH_SHORT).show(); + } + + + @Override + protected void onActivityResult(int requestCode, int resultCode, Intent data) { + super.onActivityResult(requestCode, resultCode, data); + if (resultCode == RESULT_OK && data != null) { + switch (requestCode) { + case OPEN_QUERY_PHOTO_REQUEST_CODE: + try { + ContentResolver resolver = getContentResolver(); + Uri uri = data.getData(); + Bitmap image = MediaStore.Images.Media.getBitmap(resolver, uri); + String[] proj = {MediaStore.Images.Media.DATA}; + Cursor cursor = managedQuery(uri, proj, null, null, null); + cursor.moveToFirst(); + onImageChanged(image); + } catch (IOException e) { + Log.e(TAG, e.toString()); + } + break; + case OPEN_GALLERY_PHOTO_REQUEST_CODE: + try { + + ContentResolver resolver = getContentResolver(); + Uri uri = data.getData(); + Bitmap image = MediaStore.Images.Media.getBitmap(resolver, uri); + String[] proj = {MediaStore.Images.Media.DATA}; + Cursor cursor = managedQuery(uri, proj, null, null, null); + cursor.moveToFirst(); + onAddGallery(image); + } catch (IOException e) { + Log.e(TAG, e.toString()); + } + break; + case TAKE_GALLERY_PHOTO_REQUEST_CODE: + Bundle gextras = data.getExtras(); + Bitmap gimage = (Bitmap) gextras.get("data"); + onAddGallery(gimage); + break; + case TAKE_QUERY_PHOTO_REQUEST_CODE: + Bundle extras = data.getExtras(); + Bitmap image = (Bitmap) extras.get("data"); + onImageChanged(image); + break; + case CLEAR_FEATURE_REQUEST_CODE: + clearIndex(); + break; + default: + break; + } + } + } + + @Override + protected void onDestroy() { + if (predictor != null) { + predictor.release(); + } + worker.quit(); + super.onDestroy(); + } +} diff --git a/deploy/shitu_android_demo/app/src/main/java/com/baidu/paddle/lite/demo/pp_shitu/Native.java b/deploy/shitu_android_demo/app/src/main/java/com/baidu/paddle/lite/demo/pp_shitu/Native.java new file mode 100644 index 000000000..736c1d5a3 --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/java/com/baidu/paddle/lite/demo/pp_shitu/Native.java @@ -0,0 +1,186 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.baidu.paddle.lite.demo.pp_shitu; + +import android.graphics.Bitmap; +import android.util.Log; + +import java.util.Arrays; + +public class Native { + static { + System.loadLibrary("Native"); + } + + protected Bitmap inputImage = null; + protected long[] detinputShape = new long[]{1, 3, 640, 640}; + protected long[] recinputShape = new long[]{1, 3, 224, 224}; + protected float inferenceTime = 0; + protected String top1Result = ""; + protected int topk = 1; + protected boolean addGallery = false; + protected String label_name = ""; + protected boolean clearFeature = false; + private long ctx = 0; + + public boolean init(String DetModelDir, + String RecModelDir, + String labelPath, + String IndexDir, + long[] DetInputShape, + long[] RecInputShape, + int cpuThreadNum, + int WarmUp, + int Repeat, + int topk, + boolean add_gallery, + String cpuMode) { + if (DetInputShape.length != 4) { + Log.i("Paddle-lite", "Size of input shape should be: 4"); + return false; + } + if (DetInputShape[0] != 1) { + Log.i("Paddle-lite", "Only one batch is supported in the image classification demo, you can use any batch size in " + + "your Apps!"); + return false; + } + if (DetInputShape[1] != 1 && DetInputShape[1] != 3) { + Log.i("Paddle-lite", "Only one/three channels are supported in the image classification demo, you can use any " + + "channel size in your Apps!"); + return false; + } + this.detinputShape = DetInputShape; + this.recinputShape = RecInputShape; + this.topk = topk; + this.addGallery = add_gallery; + ctx = nativeInit( + DetModelDir, + RecModelDir, + labelPath, + IndexDir, + DetInputShape, + RecInputShape, + cpuThreadNum, + WarmUp, + Repeat, + topk, + add_gallery, + cpuMode); + return ctx != 0; + } + + public boolean release() { + if (ctx == 0) { + return false; + } + return nativeRelease(ctx); + } + + public void setAddGallery(int flag) { + nativesetAddGallery(ctx, flag); + } + + public void saveIndex(String save_file_name) { + nativesaveIndex(ctx, save_file_name); + } + + public boolean loadIndex(String load_file_name) { + return nativeloadIndex(ctx, load_file_name); + } + + public void clearFeature() { + boolean ret = nativeclearGallery(ctx); + } + + public void setLabelName(String label_name) { + this.label_name = label_name; + } + + public String getClassname() { + return nativegetClassname(ctx); + } + + public boolean process() { + if (ctx == 0) { + return false; + } + // ARGB8888 bitmap is only supported in native, other color formats can be added by yourself. + String[] res = nativeProcess(ctx, this.inputImage, label_name).split("\n"); + if (res.length >= 1) { + if (!Arrays.toString(res).contains("success")) { + inferenceTime = Float.parseFloat(res[0]); + if (res.length >= 2) { + top1Result = res[1]; + } else { + top1Result = ""; + } + } + } + return (res.length > 0); + } + + public boolean isLoaded() { + return ctx != 0; + } + + public void setInputImage(Bitmap image) { + if (image == null) { + return; + } + this.inputImage = image.copy(Bitmap.Config.ARGB_8888, true); + } + + public float inferenceTime() { + inferenceTime = (float) (Math.round(inferenceTime * 100)) / 100; + return inferenceTime; + } + + public Bitmap inputImage() { + return inputImage; + } + + public String top1Result() { + return top1Result; + } + + public static native long nativeInit(String DetModelDir, + String RecModelDir, + String labelPath, + String IndexDir, + long[] DetInputShape, + long[] RecInputShape, + int cpuThreadNum, + int WarmUp, + int Repeat, + int topk, + boolean addGallery, + String cpuMode); + + public static native boolean nativeRelease(long ctx); + + public static native boolean nativesetAddGallery(long ctx, int flag); + + public static native boolean nativeclearGallery(long ctx); + + public static native String nativeProcess(long ctx, Bitmap ARGB888ImageBitmap, String label_name); + + public static native boolean nativesaveIndex(long ctx, String save_file_name); + + public static native boolean nativeloadIndex(long ctx, String load_file_name); + + public static native String nativegetClassname(long ctx); +} diff --git a/deploy/shitu_android_demo/app/src/main/java/com/baidu/paddle/lite/demo/pp_shitu/SettingsActivity.java b/deploy/shitu_android_demo/app/src/main/java/com/baidu/paddle/lite/demo/pp_shitu/SettingsActivity.java new file mode 100644 index 000000000..d53e1f3da --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/java/com/baidu/paddle/lite/demo/pp_shitu/SettingsActivity.java @@ -0,0 +1,150 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.baidu.paddle.lite.demo.pp_shitu; + +import android.app.AlertDialog; +import android.content.SharedPreferences; +import android.os.Bundle; +import android.preference.ListPreference; +import android.support.v7.app.ActionBar; +import android.view.View; +import android.widget.Button; +import android.widget.ListView; +import android.widget.Toast; + +import com.baidu.paddle.lite.demo.common.AppCompatPreferenceActivity; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +public class SettingsActivity extends AppCompatPreferenceActivity implements SharedPreferences.OnSharedPreferenceChangeListener { + ListPreference lpChoosePreInstalledModel = null; + ListPreference lpLabelPath = null; + ListPreference lpIndexPath = null; + + + List preInstalledModelPaths = null; + List preInstalledLabelPaths = null; + List preInstalledIndexDirs = null; + List preInstalledImagePaths = null; + + + @Override + public void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + addPreferencesFromResource(R.xml.settings); + ActionBar supportActionBar = getSupportActionBar(); + if (supportActionBar != null) { + supportActionBar.setDisplayHomeAsUpEnabled(true); + } + + + // Initialized pre-installed models + preInstalledModelPaths = new ArrayList<>(); + preInstalledLabelPaths = new ArrayList<>(); + preInstalledIndexDirs = new ArrayList<>(); + preInstalledImagePaths = new ArrayList<>(); + + // Add mobilenet_v1_for_cpu + preInstalledModelPaths.add(getString(R.string.MODEL_PATH_DEFAULT)); + preInstalledLabelPaths.add(getString(R.string.LABEL_PATH_DEFAULT)); + preInstalledIndexDirs.add(getString(R.string.INDEX_PATH_DEFAULT)); + preInstalledImagePaths.add(getString(R.string.IMAGE_PATH_DEFAULT)); + + // Setup UI components + lpChoosePreInstalledModel = + (ListPreference) findPreference(getString(R.string.CHOOSE_PRE_INSTALLED_MODEL_KEY)); + String[] preInstalledModelNames = new String[preInstalledModelPaths.size()]; + for (int i = 0; i < preInstalledModelPaths.size(); i++) { + preInstalledModelNames[i] = + preInstalledModelPaths.get(i).substring(preInstalledModelPaths.get(i).lastIndexOf("/") + 1); + } + lpChoosePreInstalledModel.setEntries(preInstalledModelNames); + lpChoosePreInstalledModel.setEntryValues(preInstalledModelPaths.toArray(new String[preInstalledModelPaths.size()])); + + lpLabelPath = (ListPreference) findPreference(getString(R.string.LABEL_PATH_KEY)); + String label_dir = getExternalFilesDir(null) + "/index/"; + File dir = new File(label_dir); + String[] files = dir.list(); + ArrayList files_ = new ArrayList<>(); + for (int i = 0; i < Objects.requireNonNull(files).length; i++) { + if (!files[i].endsWith(".txt")) { + continue; + } + files_.add("index/" + files[i]); + files[i] = label_dir + files[i]; + } + lpLabelPath.setEntries(files_.toArray(new String[files_.size()])); + lpLabelPath.setEntryValues(files_.toArray(new String[files_.size()])); + + lpIndexPath = (ListPreference) findPreference(getString(R.string.INDEX_PATH_KEY)); + String index_dir = getExternalFilesDir(null) + "/index/"; + dir = new File(index_dir); + files = dir.list(); + files_ = new ArrayList<>(); + for (int i = 0; i < Objects.requireNonNull(files).length; i++) { + if (!files[i].endsWith(".index")) { + continue; + } + files_.add("index/" + files[i]); + files[i] = index_dir + files[i]; + } + lpIndexPath.setEntries(files_.toArray(new String[files_.size()])); + lpIndexPath.setEntryValues(files_.toArray(new String[files_.size()])); + } + + private void reloadPreferenceAndUpdateUI() { + SharedPreferences sharedPreferences = getPreferenceScreen().getSharedPreferences(); + String modelPath = "models"; + int modelIdx = lpChoosePreInstalledModel.findIndexOfValue(modelPath); + if (modelIdx >= 0 && modelIdx < preInstalledModelPaths.size()) { + lpChoosePreInstalledModel.setSummary(modelPath); + } + + String labelPath = sharedPreferences.getString(getString(R.string.LABEL_PATH_KEY), + getString(R.string.LABEL_PATH_DEFAULT)); + String indexPath = sharedPreferences.getString(getString(R.string.INDEX_PATH_KEY), + getString(R.string.INDEX_PATH_DEFAULT)); + + lpLabelPath.setSummary(labelPath); + lpIndexPath.setSummary(indexPath); + } + + @Override + protected void onResume() { + super.onResume(); + getPreferenceScreen().getSharedPreferences().registerOnSharedPreferenceChangeListener(this); + reloadPreferenceAndUpdateUI(); + } + + @Override + protected void onPause() { + super.onPause(); + getPreferenceScreen().getSharedPreferences().unregisterOnSharedPreferenceChangeListener(this); + } + + @Override + public void onSharedPreferenceChanged(SharedPreferences sharedPreferences, String key) { + reloadPreferenceAndUpdateUI(); + } +} diff --git a/deploy/shitu_android_demo/app/src/main/jniLibs/arm64-v8a/README.md b/deploy/shitu_android_demo/app/src/main/jniLibs/arm64-v8a/README.md new file mode 100644 index 000000000..6d06ad89d --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/jniLibs/arm64-v8a/README.md @@ -0,0 +1 @@ +put `libfaiss.a` in this folder diff --git a/deploy/shitu_android_demo/app/src/main/res/drawable-v24/ic_launcher_foreground.xml b/deploy/shitu_android_demo/app/src/main/res/drawable-v24/ic_launcher_foreground.xml new file mode 100644 index 000000000..1f6bb2906 --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/res/drawable-v24/ic_launcher_foreground.xml @@ -0,0 +1,34 @@ + + + + + + + + + + + diff --git a/deploy/shitu_android_demo/app/src/main/res/drawable-xxhdpi-v4/btn_switch_default.png b/deploy/shitu_android_demo/app/src/main/res/drawable-xxhdpi-v4/btn_switch_default.png new file mode 100644 index 000000000..b9e66c7f6 Binary files /dev/null and b/deploy/shitu_android_demo/app/src/main/res/drawable-xxhdpi-v4/btn_switch_default.png differ diff --git a/deploy/shitu_android_demo/app/src/main/res/drawable-xxhdpi-v4/btn_switch_pressed.png b/deploy/shitu_android_demo/app/src/main/res/drawable-xxhdpi-v4/btn_switch_pressed.png new file mode 100644 index 000000000..9544133bd Binary files /dev/null and b/deploy/shitu_android_demo/app/src/main/res/drawable-xxhdpi-v4/btn_switch_pressed.png differ diff --git a/deploy/shitu_android_demo/app/src/main/res/drawable/app_icon_crop.png b/deploy/shitu_android_demo/app/src/main/res/drawable/app_icon_crop.png new file mode 100644 index 000000000..c25d20521 Binary files /dev/null and b/deploy/shitu_android_demo/app/src/main/res/drawable/app_icon_crop.png differ diff --git a/deploy/shitu_android_demo/app/src/main/res/drawable/btn_settings.xml b/deploy/shitu_android_demo/app/src/main/res/drawable/btn_settings.xml new file mode 100644 index 000000000..917897b99 --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/res/drawable/btn_settings.xml @@ -0,0 +1,6 @@ + + + + + + diff --git a/deploy/shitu_android_demo/app/src/main/res/drawable/btn_settings_default.xml b/deploy/shitu_android_demo/app/src/main/res/drawable/btn_settings_default.xml new file mode 100644 index 000000000..e19589a97 --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/res/drawable/btn_settings_default.xml @@ -0,0 +1,13 @@ + + + + diff --git a/deploy/shitu_android_demo/app/src/main/res/drawable/btn_settings_pressed.xml b/deploy/shitu_android_demo/app/src/main/res/drawable/btn_settings_pressed.xml new file mode 100644 index 000000000..c4af2a042 --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/res/drawable/btn_settings_pressed.xml @@ -0,0 +1,13 @@ + + + + diff --git a/deploy/shitu_android_demo/app/src/main/res/drawable/btn_shutter.xml b/deploy/shitu_android_demo/app/src/main/res/drawable/btn_shutter.xml new file mode 100644 index 000000000..4f9826d3a --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/res/drawable/btn_shutter.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/deploy/shitu_android_demo/app/src/main/res/drawable/btn_shutter_default.xml b/deploy/shitu_android_demo/app/src/main/res/drawable/btn_shutter_default.xml new file mode 100644 index 000000000..234ca014a --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/res/drawable/btn_shutter_default.xml @@ -0,0 +1,17 @@ + + + + + diff --git a/deploy/shitu_android_demo/app/src/main/res/drawable/btn_shutter_pressed.xml b/deploy/shitu_android_demo/app/src/main/res/drawable/btn_shutter_pressed.xml new file mode 100644 index 000000000..accc7aced --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/res/drawable/btn_shutter_pressed.xml @@ -0,0 +1,17 @@ + + + + + diff --git a/deploy/shitu_android_demo/app/src/main/res/drawable/btn_switch.xml b/deploy/shitu_android_demo/app/src/main/res/drawable/btn_switch.xml new file mode 100644 index 000000000..691e8c2e9 --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/res/drawable/btn_switch.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/deploy/shitu_android_demo/app/src/main/res/drawable/ic_launcher_background.xml b/deploy/shitu_android_demo/app/src/main/res/drawable/ic_launcher_background.xml new file mode 100644 index 000000000..0d025f9bf --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/res/drawable/ic_launcher_background.xml @@ -0,0 +1,170 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/deploy/shitu_android_demo/app/src/main/res/drawable/reset.png b/deploy/shitu_android_demo/app/src/main/res/drawable/reset.png new file mode 100644 index 000000000..361111dff Binary files /dev/null and b/deploy/shitu_android_demo/app/src/main/res/drawable/reset.png differ diff --git a/deploy/shitu_android_demo/app/src/main/res/drawable/shezhi.png b/deploy/shitu_android_demo/app/src/main/res/drawable/shezhi.png new file mode 100644 index 000000000..574fbe9fa Binary files /dev/null and b/deploy/shitu_android_demo/app/src/main/res/drawable/shezhi.png differ diff --git a/deploy/shitu_android_demo/app/src/main/res/drawable/shuoming.png b/deploy/shitu_android_demo/app/src/main/res/drawable/shuoming.png new file mode 100644 index 000000000..22112793a Binary files /dev/null and b/deploy/shitu_android_demo/app/src/main/res/drawable/shuoming.png differ diff --git a/deploy/shitu_android_demo/app/src/main/res/drawable/white.png b/deploy/shitu_android_demo/app/src/main/res/drawable/white.png new file mode 100644 index 000000000..23f514fac Binary files /dev/null and b/deploy/shitu_android_demo/app/src/main/res/drawable/white.png differ diff --git a/deploy/shitu_android_demo/app/src/main/res/layout-land/activity_main_old.xml b/deploy/shitu_android_demo/app/src/main/res/layout-land/activity_main_old.xml new file mode 100644 index 000000000..871e4f0c0 --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/res/layout-land/activity_main_old.xml @@ -0,0 +1,99 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/deploy/shitu_android_demo/app/src/main/res/layout/activity_main.xml b/deploy/shitu_android_demo/app/src/main/res/layout/activity_main.xml new file mode 100644 index 000000000..1e632b253 --- /dev/null +++ b/deploy/shitu_android_demo/app/src/main/res/layout/activity_main.xml @@ -0,0 +1,257 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +