[SDK] Add OpenVINO SDK support (#257)

* Add OpenVINO SDK support

* fix reshape

* remove resize input

csrc/net/CMakeLists.txt

@@ -15,10 +15,15 @@ endif ()
 if ("ort" IN_LIST MMDEPLOY_TARGET_BACKENDS)
     add_subdirectory(ort)
 endif ()
+
 if ("ncnn" IN_LIST MMDEPLOY_TARGET_BACKENDS)
     add_subdirectory(ncnn)
 endif ()
 
+if ("openvino" IN_LIST MMDEPLOY_TARGET_BACKENDS)
+    add_subdirectory(openvino)
+endif ()
+
 set_targets(${PROJECT_NAME} NET_MODULE_OBJ NET_MODULE_STATIC NET_MODULE_SHARED)
 build_object_target(${NET_MODULE_OBJ} net_module.cpp)
 target_link_libraries(${NET_MODULE_OBJ} PRIVATE mmdeploy::core::static)

csrc/net/openvino/CMakeLists.txt

@@ -0,0 +1,26 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+cmake_minimum_required(VERSION 3.14)
+project(mmdeploy_openvino_net)
+
+if ("cpu" IN_LIST MMDEPLOY_TARGET_DEVICES)
+    include(${CMAKE_SOURCE_DIR}/cmake/common.cmake)
+
+    set_targets(${PROJECT_NAME} OPENVINO_NET_OBJ OPENVINO_NET_STATIC OPENVINO_NET_SHARED)
+
+    find_package(InferenceEngine REQUIRED)
+
+    build_object_target(${OPENVINO_NET_OBJ} openvino_net.cpp)
+    target_link_libraries(${OPENVINO_NET_OBJ} PRIVATE
+            mmdeploy::core::static
+            ${InferenceEngine_LIBRARIES})
+
+    build_static_target(${OPENVINO_NET_STATIC} ${OPENVINO_NET_OBJ} "PUBLIC")
+    add_library(mmdeploy::openvino_net::static ALIAS ${OPENVINO_NET_STATIC})
+
+    build_shared_target(${OPENVINO_NET_SHARED} ${OPENVINO_NET_OBJ} "PRIVATE")
+    add_library(mmdeploy::openvino_net ALIAS ${OPENVINO_NET_SHARED})
+
+    export_module(${OPENVINO_NET_STATIC} ${OPENVINO_NET_SHARED} ${OPENVINO_NET_OBJ})
+else ()
+    message(ERROR "'openvino_net' is NOT supported in target devices: ${MMDEPLOY_TARGET_DEVICES}")
+endif ()

csrc/net/openvino/openvino_net.cpp

@@ -0,0 +1,287 @@
+// Copyright (c) OpenMMLab. All rights reserved.
+#include "openvino_net.h"
+
+#include <stdio.h>
+
+#if __GNUC__ >= 8
+#include <filesystem>
+namespace fs = std::filesystem;
+#else
+#include <experimental/filesystem>
+namespace fs = std::experimental::filesystem;
+#endif
+#include <fstream>
+
+#include "core/logger.h"
+#include "core/model.h"
+#include "core/utils/formatter.h"
+
+namespace mmdeploy {
+
+template <typename T>
+Result<std::unique_ptr<T>> openvino_try(T* v) {
+  if (v) {
+    return success(v);
+  }
+  return Status(eFail);
+}
+
+static Result<DataType> ConvertElementType(InferenceEngine::Precision prec) {
+  auto type = InferenceEngine::Precision::ePrecision(prec);
+  switch (type) {
+    case InferenceEngine::Precision::ePrecision::FP32:
+      return DataType::kFLOAT;
+    case InferenceEngine::Precision::ePrecision::FP16:
+      return DataType::kHALF;
+    case InferenceEngine::Precision::ePrecision::I8:
+      return DataType::kINT8;
+    case InferenceEngine::Precision::ePrecision::I32:
+      return DataType::kINT32;
+    case InferenceEngine::Precision::ePrecision::I64:
+      return DataType::kINT64;
+    default:
+      ERROR("unsupported InferenceEngine Precision: {}", static_cast<int>(type));
+      return Status(eNotSupported);
+  }
+}
+
+static Result<InferenceEngine::Precision::ePrecision> ConvertPrecision(DataType type) {
+  switch (type) {
+    case DataType::kFLOAT:
+      return InferenceEngine::Precision::ePrecision::FP32;
+    case DataType::kHALF:
+      return InferenceEngine::Precision::ePrecision::FP16;
+    case DataType::kINT8:
+      return InferenceEngine::Precision::ePrecision::I8;
+    case DataType::kINT32:
+      return InferenceEngine::Precision::ePrecision::I32;
+    case DataType::kINT64:
+      return InferenceEngine::Precision::ePrecision::I64;
+    default:
+      ERROR("unsupported DataType: {}", static_cast<int>(type));
+      return Status(eNotSupported);
+  }
+}
+
+static Result<std::string> ConvertDeviceName(const Device& device) {
+  if (device.is_host()) {
+    return "CPU";
+  }
+  return Status(eNotSupported);
+}
+
+Result<void> OpenVINONet::Init(const Value& args) {
+  auto& context = args["context"];
+  device_ = context["device"].get<Device>();
+  stream_ = context["stream"].get<Stream>();
+
+  if (!device_.is_host()) {
+    return Status(eNotSupported);
+  }
+
+  auto name = args["name"].get<std::string>();
+  auto model = context["model"].get<Model>();
+  OUTCOME_TRY(auto config, model.GetModelConfig(name));
+
+  // TODO: read network with stream
+  // save xml and bin to temp file
+  auto tmp_dir = fs::temp_directory_path();
+  std::string tmp_xml = (tmp_dir / fs::path("tmp.xml")).string();
+  std::string tmp_bin = (tmp_dir / fs::path("tmp.bin")).string();
+  OUTCOME_TRY(auto raw_xml, model.ReadFile(config.net));
+  OUTCOME_TRY(auto raw_bin, model.ReadFile(config.weights));
+
+  try {
+    std::ofstream xml_out(tmp_xml);
+    xml_out << raw_xml;
+    xml_out.close();
+    std::ofstream bin_out(tmp_bin);
+    bin_out << raw_bin;
+    bin_out.close();
+  } catch (const std::exception& e) {
+    ERROR("unhandled exception when creating tmp xml/bin: {}", e.what());
+    return Status(eFail);
+  }
+
+  try {
+    // create cnnnetwork
+    core_ = InferenceEngine::Core();
+    network_ = core_.ReadNetwork(tmp_xml, tmp_bin);
+
+    // set input tensor
+    InferenceEngine::InputsDataMap input_info = network_.getInputsInfo();
+    for (auto& item : input_info) {
+      auto input_data = item.second;
+      const auto& input_name = input_data->name();
+      OUTCOME_TRY(auto data_type, ConvertElementType(input_data->getPrecision()));
+      const auto& size_vector = input_data->getTensorDesc().getDims();
+      TensorShape shape{size_vector.begin(), size_vector.end()};
+      input_tensors_.emplace_back(TensorDesc{
+          .device = device_, .data_type = data_type, .shape = shape, .name = input_name});
+    }
+
+    // set output tensor
+    InferenceEngine::OutputsDataMap output_info = network_.getOutputsInfo();
+    for (auto& item : output_info) {
+      auto output_data = item.second;
+      const auto& output_name = output_data->getName();
+      OUTCOME_TRY(auto data_type, ConvertElementType(output_data->getPrecision()));
+      const auto& size_vector = output_data->getDims();
+      TensorShape shape{size_vector.begin(), size_vector.end()};
+      output_tensors_.emplace_back(TensorDesc{
+          .device = device_, .data_type = data_type, .shape = shape, .name = output_name});
+    }
+
+    // create request
+    net_config_ =
+        std::map<std::string, std::string>{{InferenceEngine::PluginConfigParams::KEY_PERF_COUNT,
+                                            InferenceEngine::PluginConfigParams::YES}};
+    OUTCOME_TRY(auto device_str, ConvertDeviceName(device_));
+    auto executable_network = core_.LoadNetwork(network_, device_str, net_config_);
+    request_ = executable_network.CreateInferRequest();
+
+  } catch (const std::exception& e) {
+    ERROR("unhandled exception when creating OpenVINO: {}", e.what());
+    return Status(eFail);
+  }
+  return success();
+}
+
+Result<void> OpenVINONet::ForwardAsync(Event* event) { return Status(eNotSupported); }
+
+Result<void> OpenVINONet::Deinit() { return success(); }
+
+Result<Span<Tensor>> OpenVINONet::GetInputTensors() { return input_tensors_; }
+
+Result<Span<Tensor>> OpenVINONet::GetOutputTensors() { return output_tensors_; }
+
+Result<void> OpenVINONet::Reshape(Span<TensorShape> input_shapes) {
+  for (size_t i = 0; i < input_shapes.size(); ++i) {
+    input_tensors_[i].Reshape(input_shapes[i]);
+  }
+  return success();
+}
+
+static Result<void> SetBlob(InferenceEngine::InferRequest& request, Tensor& tensor) {
+  const auto& input_name = tensor.desc().name;
+
+  const auto& desc = tensor.desc();
+  const auto& shape = desc.shape;
+  InferenceEngine::SizeVector size_vector{shape.begin(), shape.end()};
+  OUTCOME_TRY(auto prec, ConvertPrecision(desc.data_type));
+  InferenceEngine::TensorDesc ie_desc(prec, size_vector, InferenceEngine::Layout::NCHW);
+
+  // TODO: find a better way instead of switch case
+  switch (desc.data_type) {
+    case DataType::kFLOAT:
+      request.SetBlob(input_name,
+                      InferenceEngine::make_shared_blob<float>(ie_desc, tensor.data<float>()));
+      break;
+    case DataType::kINT8:
+      request.SetBlob(input_name,
+                      InferenceEngine::make_shared_blob<int8_t>(ie_desc, tensor.data<int8_t>()));
+      break;
+    case DataType::kINT32:
+      request.SetBlob(input_name,
+                      InferenceEngine::make_shared_blob<int32_t>(ie_desc, tensor.data<int32_t>()));
+      break;
+    case DataType::kINT64:
+      request.SetBlob(input_name,
+                      InferenceEngine::make_shared_blob<int64_t>(ie_desc, tensor.data<int64_t>()));
+      break;
+    default:
+      ERROR("unsupported DataType: {}", static_cast<int>(desc.data_type));
+      return Status(eNotSupported);
+  }
+  return success();
+}
+
+static Result<void> GetBlob(InferenceEngine::InferRequest& request, Tensor& tensor,
+                            Stream& stream) {
+  const auto& desc = tensor.desc();
+  const auto& output_name = desc.name;
+  const auto device = desc.device;
+  const auto data_type = desc.data_type;
+  const auto& output = request.GetBlob(output_name);
+  const auto& size_vector = output->getTensorDesc().getDims();
+  TensorShape shape{size_vector.begin(), size_vector.end()};
+
+  InferenceEngine::MemoryBlob::CPtr moutput =
+      InferenceEngine::as<InferenceEngine::MemoryBlob>(output);
+  auto moutputHolder = moutput->rmap();
+  std::shared_ptr<void> data(const_cast<void*>(moutputHolder.as<const void*>()), [](void*) {});
+
+  Tensor blob_tensor = {
+      TensorDesc{.device = device, .data_type = data_type, .shape = shape, .name = output_name},
+      data};
+  if (!std::equal(blob_tensor.shape().begin(), blob_tensor.shape().end(), tensor.shape().begin()))
+    tensor.Reshape(shape);
+  OUTCOME_TRY(tensor.CopyFrom(blob_tensor, stream));
+
+  return success();
+}
+
+Result<void> OpenVINONet::Forward() {
+  OUTCOME_TRY(stream_.Wait());
+
+  // reshape network if shape does not match
+  bool need_reshape = false;
+  auto input_shapes = network_.getInputShapes();
+  for (auto& tensor : input_tensors_) {
+    const auto& input_name = tensor.desc().name;
+    const auto& tensor_shape = tensor.desc().shape;
+    auto& size_vector = input_shapes[input_name];
+    bool shape_changed = !std::equal(size_vector.begin(), size_vector.end(), tensor_shape.begin(),
+                                     [](size_t a, int64_t b) { return a == size_t(b); });
+    need_reshape |= shape_changed;
+    if (shape_changed)
+      size_vector = InferenceEngine::SizeVector{tensor_shape.begin(), tensor_shape.end()};
+  }
+
+  if (need_reshape) {
+    network_.reshape(input_shapes);
+    OUTCOME_TRY(auto device_str, ConvertDeviceName(device_));
+    auto executable_network = core_.LoadNetwork(network_, device_str, net_config_);
+    request_ = executable_network.CreateInferRequest();
+  }
+
+  // fill input into request
+  for (auto& tensor : input_tensors_) {
+    OUTCOME_TRY(SetBlob(request_, tensor));
+  }
+
+  request_.StartAsync();
+  request_.Wait(InferenceEngine::InferRequest::WaitMode::RESULT_READY);
+
+  // read output from request
+  for (auto& tensor : output_tensors_) {
+    OUTCOME_TRY(GetBlob(request_, tensor, stream_));
+  }
+  OUTCOME_TRY(stream_.Wait());
+
+  return success();
+}
+
+class OpenVINONetCreator : public Creator<Net> {
+ public:
+  const char* GetName() const override { return "openvino"; }
+  int GetVersion() const override { return 0; }
+  std::unique_ptr<Net> Create(const Value& args) override {
+    try {
+      auto p = std::make_unique<OpenVINONet>();
+      if (auto r = p->Init(args)) {
+        return p;
+      } else {
+        ERROR("error creating OpenVINONet: {}", r.error().message().c_str());
+        return nullptr;
+      }
+    } catch (const std::exception& e) {
+      ERROR("unhandled exception when creating OpenVINONet: {}", e.what());
+      return nullptr;
+    }
+  }
+};
+
+REGISTER_MODULE(Net, OpenVINONetCreator);
+
+}  // namespace mmdeploy

csrc/net/openvino/openvino_net.h

@@ -0,0 +1,36 @@
+// Copyright (c) OpenMMLab. All rights reserved.
+
+#ifndef MMDEPLOY_SRC_NET_OPENVINO_OPENVINO_NET_H_
+#define MMDEPLOY_SRC_NET_OPENVINO_OPENVINO_NET_H_
+
+#include "core/net.h"
+#include "inference_engine.hpp"
+
+namespace mmdeploy {
+
+class OpenVINONet : public Net {
+ public:
+  ~OpenVINONet() override = default;
+  Result<void> Init(const Value& cfg) override;
+  Result<void> Deinit() override;
+  Result<Span<Tensor>> GetInputTensors() override;
+  Result<Span<Tensor>> GetOutputTensors() override;
+  Result<void> Reshape(Span<TensorShape> input_shapes) override;
+  Result<void> Forward() override;
+  Result<void> ForwardAsync(Event* event) override;
+
+ private:
+  InferenceEngine::Core core_;
+  InferenceEngine::CNNNetwork network_;
+  InferenceEngine::InferRequest request_;
+  std::map<std::string, std::string> net_config_;
+  std::vector<Tensor> input_tensors_;
+  std::vector<Tensor> output_tensors_;
+  std::string device_str_;
+  Device device_;
+  Stream stream_;
+};
+
+}  // namespace mmdeploy
+
+#endif  // MMDEPLOY_SRC_NET_OPENVINO_OPENVINO_NET_H_