From e54d607f24d4bb4df8fd841e2d9aa46ffa135d99 Mon Sep 17 00:00:00 2001 From: "q.yao" Date: Fri, 17 Dec 2021 13:45:32 +0800 Subject: [PATCH] [SDK] Add OpenVINO SDK support (#257) * Add OpenVINO SDK support * fix reshape * remove resize input --- csrc/net/CMakeLists.txt | 5 + csrc/net/openvino/CMakeLists.txt | 26 +++ csrc/net/openvino/openvino_net.cpp | 287 +++++++++++++++++++++++++++++ csrc/net/openvino/openvino_net.h | 36 ++++ 4 files changed, 354 insertions(+) create mode 100644 csrc/net/openvino/CMakeLists.txt create mode 100644 csrc/net/openvino/openvino_net.cpp create mode 100644 csrc/net/openvino/openvino_net.h diff --git a/csrc/net/CMakeLists.txt b/csrc/net/CMakeLists.txt index 13733bb96..8c6ccc2b2 100644 --- a/csrc/net/CMakeLists.txt +++ b/csrc/net/CMakeLists.txt @@ -15,10 +15,15 @@ endif () if ("ort" IN_LIST MMDEPLOY_TARGET_BACKENDS) add_subdirectory(ort) endif () + if ("ncnn" IN_LIST MMDEPLOY_TARGET_BACKENDS) add_subdirectory(ncnn) endif () +if ("openvino" IN_LIST MMDEPLOY_TARGET_BACKENDS) + add_subdirectory(openvino) +endif () + set_targets(${PROJECT_NAME} NET_MODULE_OBJ NET_MODULE_STATIC NET_MODULE_SHARED) build_object_target(${NET_MODULE_OBJ} net_module.cpp) target_link_libraries(${NET_MODULE_OBJ} PRIVATE mmdeploy::core::static) diff --git a/csrc/net/openvino/CMakeLists.txt b/csrc/net/openvino/CMakeLists.txt new file mode 100644 index 000000000..16e82737e --- /dev/null +++ b/csrc/net/openvino/CMakeLists.txt @@ -0,0 +1,26 @@ +# Copyright (c) OpenMMLab. All rights reserved. +cmake_minimum_required(VERSION 3.14) +project(mmdeploy_openvino_net) + +if ("cpu" IN_LIST MMDEPLOY_TARGET_DEVICES) + include(${CMAKE_SOURCE_DIR}/cmake/common.cmake) + + set_targets(${PROJECT_NAME} OPENVINO_NET_OBJ OPENVINO_NET_STATIC OPENVINO_NET_SHARED) + + find_package(InferenceEngine REQUIRED) + + build_object_target(${OPENVINO_NET_OBJ} openvino_net.cpp) + target_link_libraries(${OPENVINO_NET_OBJ} PRIVATE + mmdeploy::core::static + ${InferenceEngine_LIBRARIES}) + + build_static_target(${OPENVINO_NET_STATIC} ${OPENVINO_NET_OBJ} "PUBLIC") + add_library(mmdeploy::openvino_net::static ALIAS ${OPENVINO_NET_STATIC}) + + build_shared_target(${OPENVINO_NET_SHARED} ${OPENVINO_NET_OBJ} "PRIVATE") + add_library(mmdeploy::openvino_net ALIAS ${OPENVINO_NET_SHARED}) + + export_module(${OPENVINO_NET_STATIC} ${OPENVINO_NET_SHARED} ${OPENVINO_NET_OBJ}) +else () + message(ERROR "'openvino_net' is NOT supported in target devices: ${MMDEPLOY_TARGET_DEVICES}") +endif () diff --git a/csrc/net/openvino/openvino_net.cpp b/csrc/net/openvino/openvino_net.cpp new file mode 100644 index 000000000..af20899a6 --- /dev/null +++ b/csrc/net/openvino/openvino_net.cpp @@ -0,0 +1,287 @@ +// Copyright (c) OpenMMLab. All rights reserved. +#include "openvino_net.h" + +#include + +#if __GNUC__ >= 8 +#include +namespace fs = std::filesystem; +#else +#include +namespace fs = std::experimental::filesystem; +#endif +#include + +#include "core/logger.h" +#include "core/model.h" +#include "core/utils/formatter.h" + +namespace mmdeploy { + +template +Result> openvino_try(T* v) { + if (v) { + return success(v); + } + return Status(eFail); +} + +static Result ConvertElementType(InferenceEngine::Precision prec) { + auto type = InferenceEngine::Precision::ePrecision(prec); + switch (type) { + case InferenceEngine::Precision::ePrecision::FP32: + return DataType::kFLOAT; + case InferenceEngine::Precision::ePrecision::FP16: + return DataType::kHALF; + case InferenceEngine::Precision::ePrecision::I8: + return DataType::kINT8; + case InferenceEngine::Precision::ePrecision::I32: + return DataType::kINT32; + case InferenceEngine::Precision::ePrecision::I64: + return DataType::kINT64; + default: + ERROR("unsupported InferenceEngine Precision: {}", static_cast(type)); + return Status(eNotSupported); + } +} + +static Result ConvertPrecision(DataType type) { + switch (type) { + case DataType::kFLOAT: + return InferenceEngine::Precision::ePrecision::FP32; + case DataType::kHALF: + return InferenceEngine::Precision::ePrecision::FP16; + case DataType::kINT8: + return InferenceEngine::Precision::ePrecision::I8; + case DataType::kINT32: + return InferenceEngine::Precision::ePrecision::I32; + case DataType::kINT64: + return InferenceEngine::Precision::ePrecision::I64; + default: + ERROR("unsupported DataType: {}", static_cast(type)); + return Status(eNotSupported); + } +} + +static Result ConvertDeviceName(const Device& device) { + if (device.is_host()) { + return "CPU"; + } + return Status(eNotSupported); +} + +Result OpenVINONet::Init(const Value& args) { + auto& context = args["context"]; + device_ = context["device"].get(); + stream_ = context["stream"].get(); + + if (!device_.is_host()) { + return Status(eNotSupported); + } + + auto name = args["name"].get(); + auto model = context["model"].get(); + OUTCOME_TRY(auto config, model.GetModelConfig(name)); + + // TODO: read network with stream + // save xml and bin to temp file + auto tmp_dir = fs::temp_directory_path(); + std::string tmp_xml = (tmp_dir / fs::path("tmp.xml")).string(); + std::string tmp_bin = (tmp_dir / fs::path("tmp.bin")).string(); + OUTCOME_TRY(auto raw_xml, model.ReadFile(config.net)); + OUTCOME_TRY(auto raw_bin, model.ReadFile(config.weights)); + + try { + std::ofstream xml_out(tmp_xml); + xml_out << raw_xml; + xml_out.close(); + std::ofstream bin_out(tmp_bin); + bin_out << raw_bin; + bin_out.close(); + } catch (const std::exception& e) { + ERROR("unhandled exception when creating tmp xml/bin: {}", e.what()); + return Status(eFail); + } + + try { + // create cnnnetwork + core_ = InferenceEngine::Core(); + network_ = core_.ReadNetwork(tmp_xml, tmp_bin); + + // set input tensor + InferenceEngine::InputsDataMap input_info = network_.getInputsInfo(); + for (auto& item : input_info) { + auto input_data = item.second; + const auto& input_name = input_data->name(); + OUTCOME_TRY(auto data_type, ConvertElementType(input_data->getPrecision())); + const auto& size_vector = input_data->getTensorDesc().getDims(); + TensorShape shape{size_vector.begin(), size_vector.end()}; + input_tensors_.emplace_back(TensorDesc{ + .device = device_, .data_type = data_type, .shape = shape, .name = input_name}); + } + + // set output tensor + InferenceEngine::OutputsDataMap output_info = network_.getOutputsInfo(); + for (auto& item : output_info) { + auto output_data = item.second; + const auto& output_name = output_data->getName(); + OUTCOME_TRY(auto data_type, ConvertElementType(output_data->getPrecision())); + const auto& size_vector = output_data->getDims(); + TensorShape shape{size_vector.begin(), size_vector.end()}; + output_tensors_.emplace_back(TensorDesc{ + .device = device_, .data_type = data_type, .shape = shape, .name = output_name}); + } + + // create request + net_config_ = + std::map{{InferenceEngine::PluginConfigParams::KEY_PERF_COUNT, + InferenceEngine::PluginConfigParams::YES}}; + OUTCOME_TRY(auto device_str, ConvertDeviceName(device_)); + auto executable_network = core_.LoadNetwork(network_, device_str, net_config_); + request_ = executable_network.CreateInferRequest(); + + } catch (const std::exception& e) { + ERROR("unhandled exception when creating OpenVINO: {}", e.what()); + return Status(eFail); + } + return success(); +} + +Result OpenVINONet::ForwardAsync(Event* event) { return Status(eNotSupported); } + +Result OpenVINONet::Deinit() { return success(); } + +Result> OpenVINONet::GetInputTensors() { return input_tensors_; } + +Result> OpenVINONet::GetOutputTensors() { return output_tensors_; } + +Result OpenVINONet::Reshape(Span input_shapes) { + for (size_t i = 0; i < input_shapes.size(); ++i) { + input_tensors_[i].Reshape(input_shapes[i]); + } + return success(); +} + +static Result SetBlob(InferenceEngine::InferRequest& request, Tensor& tensor) { + const auto& input_name = tensor.desc().name; + + const auto& desc = tensor.desc(); + const auto& shape = desc.shape; + InferenceEngine::SizeVector size_vector{shape.begin(), shape.end()}; + OUTCOME_TRY(auto prec, ConvertPrecision(desc.data_type)); + InferenceEngine::TensorDesc ie_desc(prec, size_vector, InferenceEngine::Layout::NCHW); + + // TODO: find a better way instead of switch case + switch (desc.data_type) { + case DataType::kFLOAT: + request.SetBlob(input_name, + InferenceEngine::make_shared_blob(ie_desc, tensor.data())); + break; + case DataType::kINT8: + request.SetBlob(input_name, + InferenceEngine::make_shared_blob(ie_desc, tensor.data())); + break; + case DataType::kINT32: + request.SetBlob(input_name, + InferenceEngine::make_shared_blob(ie_desc, tensor.data())); + break; + case DataType::kINT64: + request.SetBlob(input_name, + InferenceEngine::make_shared_blob(ie_desc, tensor.data())); + break; + default: + ERROR("unsupported DataType: {}", static_cast(desc.data_type)); + return Status(eNotSupported); + } + return success(); +} + +static Result GetBlob(InferenceEngine::InferRequest& request, Tensor& tensor, + Stream& stream) { + const auto& desc = tensor.desc(); + const auto& output_name = desc.name; + const auto device = desc.device; + const auto data_type = desc.data_type; + const auto& output = request.GetBlob(output_name); + const auto& size_vector = output->getTensorDesc().getDims(); + TensorShape shape{size_vector.begin(), size_vector.end()}; + + InferenceEngine::MemoryBlob::CPtr moutput = + InferenceEngine::as(output); + auto moutputHolder = moutput->rmap(); + std::shared_ptr data(const_cast(moutputHolder.as()), [](void*) {}); + + Tensor blob_tensor = { + TensorDesc{.device = device, .data_type = data_type, .shape = shape, .name = output_name}, + data}; + if (!std::equal(blob_tensor.shape().begin(), blob_tensor.shape().end(), tensor.shape().begin())) + tensor.Reshape(shape); + OUTCOME_TRY(tensor.CopyFrom(blob_tensor, stream)); + + return success(); +} + +Result OpenVINONet::Forward() { + OUTCOME_TRY(stream_.Wait()); + + // reshape network if shape does not match + bool need_reshape = false; + auto input_shapes = network_.getInputShapes(); + for (auto& tensor : input_tensors_) { + const auto& input_name = tensor.desc().name; + const auto& tensor_shape = tensor.desc().shape; + auto& size_vector = input_shapes[input_name]; + bool shape_changed = !std::equal(size_vector.begin(), size_vector.end(), tensor_shape.begin(), + [](size_t a, int64_t b) { return a == size_t(b); }); + need_reshape |= shape_changed; + if (shape_changed) + size_vector = InferenceEngine::SizeVector{tensor_shape.begin(), tensor_shape.end()}; + } + + if (need_reshape) { + network_.reshape(input_shapes); + OUTCOME_TRY(auto device_str, ConvertDeviceName(device_)); + auto executable_network = core_.LoadNetwork(network_, device_str, net_config_); + request_ = executable_network.CreateInferRequest(); + } + + // fill input into request + for (auto& tensor : input_tensors_) { + OUTCOME_TRY(SetBlob(request_, tensor)); + } + + request_.StartAsync(); + request_.Wait(InferenceEngine::InferRequest::WaitMode::RESULT_READY); + + // read output from request + for (auto& tensor : output_tensors_) { + OUTCOME_TRY(GetBlob(request_, tensor, stream_)); + } + OUTCOME_TRY(stream_.Wait()); + + return success(); +} + +class OpenVINONetCreator : public Creator { + public: + const char* GetName() const override { return "openvino"; } + int GetVersion() const override { return 0; } + std::unique_ptr Create(const Value& args) override { + try { + auto p = std::make_unique(); + if (auto r = p->Init(args)) { + return p; + } else { + ERROR("error creating OpenVINONet: {}", r.error().message().c_str()); + return nullptr; + } + } catch (const std::exception& e) { + ERROR("unhandled exception when creating OpenVINONet: {}", e.what()); + return nullptr; + } + } +}; + +REGISTER_MODULE(Net, OpenVINONetCreator); + +} // namespace mmdeploy diff --git a/csrc/net/openvino/openvino_net.h b/csrc/net/openvino/openvino_net.h new file mode 100644 index 000000000..08c4ec285 --- /dev/null +++ b/csrc/net/openvino/openvino_net.h @@ -0,0 +1,36 @@ +// Copyright (c) OpenMMLab. All rights reserved. + +#ifndef MMDEPLOY_SRC_NET_OPENVINO_OPENVINO_NET_H_ +#define MMDEPLOY_SRC_NET_OPENVINO_OPENVINO_NET_H_ + +#include "core/net.h" +#include "inference_engine.hpp" + +namespace mmdeploy { + +class OpenVINONet : public Net { + public: + ~OpenVINONet() override = default; + Result Init(const Value& cfg) override; + Result Deinit() override; + Result> GetInputTensors() override; + Result> GetOutputTensors() override; + Result Reshape(Span input_shapes) override; + Result Forward() override; + Result ForwardAsync(Event* event) override; + + private: + InferenceEngine::Core core_; + InferenceEngine::CNNNetwork network_; + InferenceEngine::InferRequest request_; + std::map net_config_; + std::vector input_tensors_; + std::vector output_tensors_; + std::string device_str_; + Device device_; + Stream stream_; +}; + +} // namespace mmdeploy + +#endif // MMDEPLOY_SRC_NET_OPENVINO_OPENVINO_NET_H_