// Copyright (c) OpenMMLab. All rights reserved. #include "openvino_net.h" #include #if __GNUC__ >= 8 #include namespace fs = std::filesystem; #else #include namespace fs = std::experimental::filesystem; #endif #include #include "core/logger.h" #include "core/model.h" #include "core/utils/formatter.h" namespace mmdeploy { template Result> openvino_try(T* v) { if (v) { return success(v); } return Status(eFail); } static Result ConvertElementType(InferenceEngine::Precision prec) { auto type = InferenceEngine::Precision::ePrecision(prec); switch (type) { case InferenceEngine::Precision::ePrecision::FP32: return DataType::kFLOAT; case InferenceEngine::Precision::ePrecision::FP16: return DataType::kHALF; case InferenceEngine::Precision::ePrecision::I8: return DataType::kINT8; case InferenceEngine::Precision::ePrecision::I32: return DataType::kINT32; case InferenceEngine::Precision::ePrecision::I64: return DataType::kINT64; default: ERROR("unsupported InferenceEngine Precision: {}", static_cast(type)); return Status(eNotSupported); } } static Result ConvertPrecision(DataType type) { switch (type) { case DataType::kFLOAT: return InferenceEngine::Precision::ePrecision::FP32; case DataType::kHALF: return InferenceEngine::Precision::ePrecision::FP16; case DataType::kINT8: return InferenceEngine::Precision::ePrecision::I8; case DataType::kINT32: return InferenceEngine::Precision::ePrecision::I32; case DataType::kINT64: return InferenceEngine::Precision::ePrecision::I64; default: ERROR("unsupported DataType: {}", static_cast(type)); return Status(eNotSupported); } } static Result ConvertDeviceName(const Device& device) { if (device.is_host()) { return "CPU"; } return Status(eNotSupported); } Result OpenVINONet::Init(const Value& args) { auto& context = args["context"]; device_ = context["device"].get(); stream_ = context["stream"].get(); if (!device_.is_host()) { return Status(eNotSupported); } auto name = args["name"].get(); auto model = context["model"].get(); OUTCOME_TRY(auto config, model.GetModelConfig(name)); // TODO: read network with stream // save xml and bin to temp file auto tmp_dir = fs::temp_directory_path(); std::string tmp_xml = (tmp_dir / fs::path("tmp.xml")).string(); std::string tmp_bin = (tmp_dir / fs::path("tmp.bin")).string(); OUTCOME_TRY(auto raw_xml, model.ReadFile(config.net)); OUTCOME_TRY(auto raw_bin, model.ReadFile(config.weights)); try { std::ofstream xml_out(tmp_xml); xml_out << raw_xml; xml_out.close(); std::ofstream bin_out(tmp_bin); bin_out << raw_bin; bin_out.close(); } catch (const std::exception& e) { ERROR("unhandled exception when creating tmp xml/bin: {}", e.what()); return Status(eFail); } try { // create cnnnetwork core_ = InferenceEngine::Core(); network_ = core_.ReadNetwork(tmp_xml, tmp_bin); // set input tensor InferenceEngine::InputsDataMap input_info = network_.getInputsInfo(); for (auto& item : input_info) { auto input_data = item.second; const auto& input_name = input_data->name(); OUTCOME_TRY(auto data_type, ConvertElementType(input_data->getPrecision())); const auto& size_vector = input_data->getTensorDesc().getDims(); TensorShape shape{size_vector.begin(), size_vector.end()}; input_tensors_.emplace_back(TensorDesc{ .device = device_, .data_type = data_type, .shape = shape, .name = input_name}); } // set output tensor InferenceEngine::OutputsDataMap output_info = network_.getOutputsInfo(); for (auto& item : output_info) { auto output_data = item.second; const auto& output_name = output_data->getName(); OUTCOME_TRY(auto data_type, ConvertElementType(output_data->getPrecision())); const auto& size_vector = output_data->getDims(); TensorShape shape{size_vector.begin(), size_vector.end()}; output_tensors_.emplace_back(TensorDesc{ .device = device_, .data_type = data_type, .shape = shape, .name = output_name}); } // create request net_config_ = std::map{{InferenceEngine::PluginConfigParams::KEY_PERF_COUNT, InferenceEngine::PluginConfigParams::YES}}; OUTCOME_TRY(auto device_str, ConvertDeviceName(device_)); auto executable_network = core_.LoadNetwork(network_, device_str, net_config_); request_ = executable_network.CreateInferRequest(); } catch (const std::exception& e) { ERROR("unhandled exception when creating OpenVINO: {}", e.what()); return Status(eFail); } return success(); } Result OpenVINONet::ForwardAsync(Event* event) { return Status(eNotSupported); } Result OpenVINONet::Deinit() { return success(); } Result> OpenVINONet::GetInputTensors() { return input_tensors_; } Result> OpenVINONet::GetOutputTensors() { return output_tensors_; } Result OpenVINONet::Reshape(Span input_shapes) { for (size_t i = 0; i < input_shapes.size(); ++i) { input_tensors_[i].Reshape(input_shapes[i]); } return success(); } static Result SetBlob(InferenceEngine::InferRequest& request, Tensor& tensor) { const auto& input_name = tensor.desc().name; const auto& desc = tensor.desc(); const auto& shape = desc.shape; InferenceEngine::SizeVector size_vector{shape.begin(), shape.end()}; OUTCOME_TRY(auto prec, ConvertPrecision(desc.data_type)); InferenceEngine::TensorDesc ie_desc(prec, size_vector, InferenceEngine::Layout::NCHW); // TODO: find a better way instead of switch case switch (desc.data_type) { case DataType::kFLOAT: request.SetBlob(input_name, InferenceEngine::make_shared_blob(ie_desc, tensor.data())); break; case DataType::kINT8: request.SetBlob(input_name, InferenceEngine::make_shared_blob(ie_desc, tensor.data())); break; case DataType::kINT32: request.SetBlob(input_name, InferenceEngine::make_shared_blob(ie_desc, tensor.data())); break; case DataType::kINT64: request.SetBlob(input_name, InferenceEngine::make_shared_blob(ie_desc, tensor.data())); break; default: ERROR("unsupported DataType: {}", static_cast(desc.data_type)); return Status(eNotSupported); } return success(); } static Result GetBlob(InferenceEngine::InferRequest& request, Tensor& tensor, Stream& stream) { const auto& desc = tensor.desc(); const auto& output_name = desc.name; const auto device = desc.device; const auto data_type = desc.data_type; const auto& output = request.GetBlob(output_name); const auto& size_vector = output->getTensorDesc().getDims(); TensorShape shape{size_vector.begin(), size_vector.end()}; InferenceEngine::MemoryBlob::CPtr moutput = InferenceEngine::as(output); auto moutputHolder = moutput->rmap(); std::shared_ptr data(const_cast(moutputHolder.as()), [](void*) {}); Tensor blob_tensor = { TensorDesc{.device = device, .data_type = data_type, .shape = shape, .name = output_name}, data}; if (!std::equal(blob_tensor.shape().begin(), blob_tensor.shape().end(), tensor.shape().begin())) tensor.Reshape(shape); OUTCOME_TRY(tensor.CopyFrom(blob_tensor, stream)); return success(); } Result OpenVINONet::Forward() { OUTCOME_TRY(stream_.Wait()); // reshape network if shape does not match bool need_reshape = false; auto input_shapes = network_.getInputShapes(); for (auto& tensor : input_tensors_) { const auto& input_name = tensor.desc().name; const auto& tensor_shape = tensor.desc().shape; auto& size_vector = input_shapes[input_name]; bool shape_changed = !std::equal(size_vector.begin(), size_vector.end(), tensor_shape.begin(), [](size_t a, int64_t b) { return a == size_t(b); }); need_reshape |= shape_changed; if (shape_changed) size_vector = InferenceEngine::SizeVector{tensor_shape.begin(), tensor_shape.end()}; } if (need_reshape) { network_.reshape(input_shapes); OUTCOME_TRY(auto device_str, ConvertDeviceName(device_)); auto executable_network = core_.LoadNetwork(network_, device_str, net_config_); request_ = executable_network.CreateInferRequest(); } // fill input into request for (auto& tensor : input_tensors_) { OUTCOME_TRY(SetBlob(request_, tensor)); } request_.StartAsync(); request_.Wait(InferenceEngine::InferRequest::WaitMode::RESULT_READY); // read output from request for (auto& tensor : output_tensors_) { OUTCOME_TRY(GetBlob(request_, tensor, stream_)); } OUTCOME_TRY(stream_.Wait()); return success(); } class OpenVINONetCreator : public Creator { public: const char* GetName() const override { return "openvino"; } int GetVersion() const override { return 0; } std::unique_ptr Create(const Value& args) override { try { auto p = std::make_unique(); if (auto r = p->Init(args)) { return p; } else { ERROR("error creating OpenVINONet: {}", r.error().message().c_str()); return nullptr; } } catch (const std::exception& e) { ERROR("unhandled exception when creating OpenVINONet: {}", e.what()); return nullptr; } } }; REGISTER_MODULE(Net, OpenVINONetCreator); } // namespace mmdeploy