mirrors
/
mmdeploy
mirror of https://github.com/open-mmlab/mmdeploy.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'support-snpe-fix' into support-snpe

pull/722/head^2^2
tpoisonooo 2022-07-11 13:11:18 +08:00
parent 1ad3534814 5666c88b9f
commit a15bd81dbc
19 changed files with 439 additions and 350 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
csrc/mmdeploy/backend_ops/snpe/client/end2end.dlc 120000
View File

@ -0,0 +1 @@
/home/PJLAB/konghuanjun/GitProjects/mmdeploy/resnet18/end2end.dlc

1
csrc/mmdeploy/backend_ops/snpe/client/end2end.onnx 120000
View File

@ -0,0 +1 @@
/home/PJLAB/konghuanjun/GitProjects/mmdeploy/resnet18/end2end.onnx

82
csrc/mmdeploy/backend_ops/snpe/client/inference_client.py 100644
View File

@ -0,0 +1,82 @@
# Copyright 2015 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""The Python implementation of the GRPC helloworld.Greeter client."""
from __future__ import print_function
import logging
import grpc
import inference_pb2
import inference_pb2_grpc
import os
import cv2
import numpy as np
def build_dummy_tensor():
img = cv2.imread('/home/PJLAB/konghuanjun/Downloads/snpe-1.55.0.2958/models/alexnet/data/chairs.jpg')
m = cv2.resize(img, (600, 400))
data = (m.astype(np.float32) - 127.5) / 127.5
print(data.shape)
tensor = inference_pb2.Tensor(data=data.tobytes(), shape=list(data.shape), name='data_0', dtype='float32')
return tensor
def run():
# NOTE(gRPC Python Team): .close() is possible on a channel and should be
# used in circumstances in which the with statement does not fit the needs
# of the code.
filename = 'end2end.dlc'
filesize = os.stat(filename).st_size
weights = bytes()
# with open(filename, 'rb') as f:
# weights = f.read(filesize)
# if len(weights) >= (2 << 29):
# print('model size too big')
# https://github.com/grpc/grpc/blob/v1.46.x/include/grpc/impl/codegen/grpc_types.h
# https://grpc.io/docs/guides/performance/
with grpc.insecure_channel('10.1.80.67:50051',
options=(
('grpc.GRPC_ARG_KEEPALIVE_TIME_MS', 2000),
('grpc.max_send_message_length', 2<<29),
('grpc.keepalive_permit_without_calls', 1))) as channel:
print("channel type {}".format(type(channel)))
# with grpc.insecure_channel('[0:0:fe80::3455:bf2a]:50051') as channel:
stub = inference_pb2_grpc.InferenceStub(channel)
response = stub.Echo(inference_pb2.Empty())
print("Response echo {}".format(response))
model = inference_pb2.Model(name= filename, weights=weights, device=1)
print("Sending model to init, please wait...")
response = stub.Init(model)
print("Response init {}".format(response))
response = stub.OutputNames(inference_pb2.Empty())
print("Response outputnames {}".format(response))
tensor = build_dummy_tensor()
tensorList = inference_pb2.TensorList(datas = [tensor])
for x in range(1):
response = stub.Inference(tensorList)
if response.status == 0:
prob = np.frombuffer(response.datas[0].data, dtype=np.float32)
print("prob argmax: {} max: {}".format(prob.argmax(), prob.max()))
else:
print(response.info)
if __name__ == '__main__':
logging.basicConfig()
run()

91
csrc/mmdeploy/backend_ops/snpe/client/inference_pb2.py 100644
View File

@ -0,0 +1,91 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: inference.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0finference.proto\x12\x08mmdeploy\"\x91\x01\n\x05Model\x12\x11\n\x04name\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x07weights\x18\x02 \x01(\x0c\x12+\n\x06\x64\x65vice\x18\x03 \x01(\x0e\x32\x16.mmdeploy.Model.DeviceH\x01\x88\x01\x01\"#\n\x06\x44\x65vice\x12\x07\n\x03\x43PU\x10\x00\x12\x07\n\x03GPU\x10\x01\x12\x07\n\x03\x44SP\x10\x02\x42\x07\n\x05_nameB\t\n\x07_device\"\x07\n\x05\x45mpty\"Q\n\x06Tensor\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\x05\x64type\x18\x02 \x01(\tH\x00\x88\x01\x01\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\r\n\x05shape\x18\x04 \x03(\x05\x42\x08\n\x06_dtype\"-\n\nTensorList\x12\x1f\n\x05\x64\x61tas\x18\x01 \x03(\x0b\x32\x10.mmdeploy.Tensor\"F\n\x05Reply\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x0c\n\x04info\x18\x02 \x01(\t\x12\x1f\n\x05\x64\x61tas\x18\x03 \x03(\x0b\x32\x10.mmdeploy.Tensor\"\x16\n\x05Names\x12\r\n\x05names\x18\x01 \x03(\t2\xfb\x01\n\tInference\x12*\n\x04\x45\x63ho\x12\x0f.mmdeploy.Empty\x1a\x0f.mmdeploy.Reply\"\x00\x12*\n\x04Init\x12\x0f.mmdeploy.Model\x1a\x0f.mmdeploy.Reply\"\x00\x12\x31\n\x0bOutputNames\x12\x0f.mmdeploy.Empty\x1a\x0f.mmdeploy.Names\"\x00\x12\x34\n\tInference\x12\x14.mmdeploy.TensorList\x1a\x0f.mmdeploy.Reply\"\x00\x12-\n\x07\x44\x65stroy\x12\x0f.mmdeploy.Empty\x1a\x0f.mmdeploy.Reply\"\x00\x42%\n\rmmdeploy.snpeB\x0bSNPEWrapperP\x01\xa2\x02\x04SNPEb\x06proto3')
_MODEL = DESCRIPTOR.message_types_by_name['Model']
_EMPTY = DESCRIPTOR.message_types_by_name['Empty']
_TENSOR = DESCRIPTOR.message_types_by_name['Tensor']
_TENSORLIST = DESCRIPTOR.message_types_by_name['TensorList']
_REPLY = DESCRIPTOR.message_types_by_name['Reply']
_NAMES = DESCRIPTOR.message_types_by_name['Names']
_MODEL_DEVICE = _MODEL.enum_types_by_name['Device']
Model = _reflection.GeneratedProtocolMessageType('Model', (_message.Message,), {
'DESCRIPTOR' : _MODEL,
'__module__' : 'inference_pb2'
# @@protoc_insertion_point(class_scope:mmdeploy.Model)
})
_sym_db.RegisterMessage(Model)
Empty = _reflection.GeneratedProtocolMessageType('Empty', (_message.Message,), {
'DESCRIPTOR' : _EMPTY,
'__module__' : 'inference_pb2'
# @@protoc_insertion_point(class_scope:mmdeploy.Empty)
})
_sym_db.RegisterMessage(Empty)
Tensor = _reflection.GeneratedProtocolMessageType('Tensor', (_message.Message,), {
'DESCRIPTOR' : _TENSOR,
'__module__' : 'inference_pb2'
# @@protoc_insertion_point(class_scope:mmdeploy.Tensor)
})
_sym_db.RegisterMessage(Tensor)
TensorList = _reflection.GeneratedProtocolMessageType('TensorList', (_message.Message,), {
'DESCRIPTOR' : _TENSORLIST,
'__module__' : 'inference_pb2'
# @@protoc_insertion_point(class_scope:mmdeploy.TensorList)
})
_sym_db.RegisterMessage(TensorList)
Reply = _reflection.GeneratedProtocolMessageType('Reply', (_message.Message,), {
'DESCRIPTOR' : _REPLY,
'__module__' : 'inference_pb2'
# @@protoc_insertion_point(class_scope:mmdeploy.Reply)
})
_sym_db.RegisterMessage(Reply)
Names = _reflection.GeneratedProtocolMessageType('Names', (_message.Message,), {
'DESCRIPTOR' : _NAMES,
'__module__' : 'inference_pb2'
# @@protoc_insertion_point(class_scope:mmdeploy.Names)
})
_sym_db.RegisterMessage(Names)
_INFERENCE = DESCRIPTOR.services_by_name['Inference']
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
DESCRIPTOR._serialized_options = b'\n\rmmdeploy.snpeB\013SNPEWrapperP\001\242\002\004SNPE'
_MODEL._serialized_start=30
_MODEL._serialized_end=175
_MODEL_DEVICE._serialized_start=120
_MODEL_DEVICE._serialized_end=155
_EMPTY._serialized_start=177
_EMPTY._serialized_end=184
_TENSOR._serialized_start=186
_TENSOR._serialized_end=267
_TENSORLIST._serialized_start=269
_TENSORLIST._serialized_end=314
_REPLY._serialized_start=316
_REPLY._serialized_end=386
_NAMES._serialized_start=388
_NAMES._serialized_end=410
_INFERENCE._serialized_start=413
_INFERENCE._serialized_end=664
# @@protoc_insertion_point(module_scope)

205
csrc/mmdeploy/backend_ops/snpe/client/inference_pb2_grpc.py 100644
View File

@ -0,0 +1,205 @@
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
"""Client and server classes corresponding to protobuf-defined services."""
import grpc
import inference_pb2 as inference__pb2
class InferenceStub(object):
"""The inference service definition.
"""
def __init__(self, channel):
"""Constructor.
Args:
channel: A grpc.Channel.
"""
self.Echo = channel.unary_unary(
'/mmdeploy.Inference/Echo',
request_serializer=inference__pb2.Empty.SerializeToString,
response_deserializer=inference__pb2.Reply.FromString,
)
self.Init = channel.unary_unary(
'/mmdeploy.Inference/Init',
request_serializer=inference__pb2.Model.SerializeToString,
response_deserializer=inference__pb2.Reply.FromString,
)
self.OutputNames = channel.unary_unary(
'/mmdeploy.Inference/OutputNames',
request_serializer=inference__pb2.Empty.SerializeToString,
response_deserializer=inference__pb2.Names.FromString,
)
self.Inference = channel.unary_unary(
'/mmdeploy.Inference/Inference',
request_serializer=inference__pb2.TensorList.SerializeToString,
response_deserializer=inference__pb2.Reply.FromString,
)
self.Destroy = channel.unary_unary(
'/mmdeploy.Inference/Destroy',
request_serializer=inference__pb2.Empty.SerializeToString,
response_deserializer=inference__pb2.Reply.FromString,
)
class InferenceServicer(object):
"""The inference service definition.
"""
def Echo(self, request, context):
"""Missing associated documentation comment in .proto file."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def Init(self, request, context):
"""Init Model with model file
"""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def OutputNames(self, request, context):
"""Get output names
"""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def Inference(self, request, context):
"""Inference with inputs
"""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def Destroy(self, request, context):
"""Destroy handle
"""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def add_InferenceServicer_to_server(servicer, server):
rpc_method_handlers = {
'Echo': grpc.unary_unary_rpc_method_handler(
servicer.Echo,
request_deserializer=inference__pb2.Empty.FromString,
response_serializer=inference__pb2.Reply.SerializeToString,
),
'Init': grpc.unary_unary_rpc_method_handler(
servicer.Init,
request_deserializer=inference__pb2.Model.FromString,
response_serializer=inference__pb2.Reply.SerializeToString,
),
'OutputNames': grpc.unary_unary_rpc_method_handler(
servicer.OutputNames,
request_deserializer=inference__pb2.Empty.FromString,
response_serializer=inference__pb2.Names.SerializeToString,
),
'Inference': grpc.unary_unary_rpc_method_handler(
servicer.Inference,
request_deserializer=inference__pb2.TensorList.FromString,
response_serializer=inference__pb2.Reply.SerializeToString,
),
'Destroy': grpc.unary_unary_rpc_method_handler(
servicer.Destroy,
request_deserializer=inference__pb2.Empty.FromString,
response_serializer=inference__pb2.Reply.SerializeToString,
),
}
generic_handler = grpc.method_handlers_generic_handler(
'mmdeploy.Inference', rpc_method_handlers)
server.add_generic_rpc_handlers((generic_handler,))
# This class is part of an EXPERIMENTAL API.
class Inference(object):
"""The inference service definition.
"""
@staticmethod
def Echo(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/mmdeploy.Inference/Echo',
inference__pb2.Empty.SerializeToString,
inference__pb2.Reply.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
@staticmethod
def Init(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/mmdeploy.Inference/Init',
inference__pb2.Model.SerializeToString,
inference__pb2.Reply.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
@staticmethod
def OutputNames(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/mmdeploy.Inference/OutputNames',
inference__pb2.Empty.SerializeToString,
inference__pb2.Names.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
@staticmethod
def Inference(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/mmdeploy.Inference/Inference',
inference__pb2.TensorList.SerializeToString,
inference__pb2.Reply.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
@staticmethod
def Destroy(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(request, target, '/mmdeploy.Inference/Destroy',
inference__pb2.Empty.SerializeToString,
inference__pb2.Reply.FromString,
options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)

7
csrc/mmdeploy/backend_ops/snpe/service/inference.proto → csrc/mmdeploy/backend_ops/snpe/inference.proto
View File

@ -18,7 +18,7 @@ service Inference {
// Get output names
rpc OutputNames(Empty) returns (Names) {}
// Inference with inputs
// Inference with inputs
rpc Inference(TensorList) returns (Reply) {}
// Destroy handle
@ -44,12 +44,15 @@ message Empty {}
message Tensor {
// name
string name = 1;
// datatype
optional string dtype = 2;
// data
bytes data = 3;
// shape
repeated int32 shape = 4;
}
message TensorList {

2
csrc/mmdeploy/backend_ops/snpe/service/CMakeLists.txt → csrc/mmdeploy/backend_ops/snpe/server/CMakeLists.txt
View File

@ -22,7 +22,7 @@ project(SNPEServer C CXX)
include(./common.cmake)
# Proto file
get_filename_component(hw_proto "inference.proto" ABSOLUTE)
get_filename_component(hw_proto "../inference.proto" ABSOLUTE)
get_filename_component(hw_proto_path "${hw_proto}" PATH)
# Generated sources

10
csrc/mmdeploy/backend_ops/snpe/server/build_shared.sh 100755
View File

@ -0,0 +1,10 @@
export ANDROID_NDK=/home/PJLAB/konghuanjun/Downloads/android-ndk-r17c
cmake .. \
-DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake \
-DANDROID_ABI=arm64-v8a \
-DANDROID_PLATFORM=android-26 \
-DANDROID_STL=c++_shared \
-DCMAKE_BUILD_TYPE=Release \
-Dabsl_DIR=/tmp/android_grpc_install_shared/lib/cmake/absl \
-DProtobuf_DIR=/tmp/android_grpc_install_shared/lib/cmake/protobuf \
-DgRPC_DIR=/tmp/android_grpc_install_shared/lib/cmake/grpc

0
csrc/mmdeploy/backend_ops/snpe/service/common.cmake → csrc/mmdeploy/backend_ops/snpe/server/common.cmake
View File

0
csrc/mmdeploy/backend_ops/snpe/service/inference_server.cc → csrc/mmdeploy/backend_ops/snpe/server/inference_server.cc
View File

2
csrc/mmdeploy/backend_ops/snpe/service/service_impl.cpp → csrc/mmdeploy/backend_ops/snpe/server/service_impl.cpp
View File

@ -99,7 +99,7 @@ void InferenceServiceImpl::LoadFloatData(const std::string& data,
::mmdeploy::Reply* response) {
fprintf(stdout, "Stage Init: recv command\n");
// std::string filename = SaveDLC(request);
std::string filename = "alexnet.dlc";
std::string filename = "end2end.dlc";
if (snpe != nullptr) {
snpe.reset();

2
csrc/mmdeploy/backend_ops/snpe/service/service_impl.h → csrc/mmdeploy/backend_ops/snpe/server/service_impl.h
View File

@ -55,7 +55,7 @@ class InferenceServiceImpl final : public Inference::Service {
::grpc::Status Inference(::grpc::ServerContext* context,
const ::mmdeploy::TensorList* request,
::mmdeploy::Reply* response) override;
// Destroy handle
// Destory handle
::grpc::Status Destroy(::grpc::ServerContext* context,
const ::mmdeploy::Empty* request,
::mmdeploy::Reply* response) override;

4
mmdeploy/backend/snpe/__init__.py
View File

@ -4,7 +4,6 @@ import os.path as osp
from .init_plugins import get_onnx2dlc_path
from .onnx2dlc import from_onnx
def is_available():
"""Check whether ncnn and snpe-onnx-to-dlc tool are installed.
@ -23,5 +22,6 @@ if is_available():
from .wrapper import SNPEWrapper
__all__ += ['SNPEWrapper']
except Exception:
except Exception as e:
print(e)
pass

107
mmdeploy/backend/snpe/inference_pb2.py
View File

@ -1,107 +0,0 @@
# Copyright (c) OpenMMLab. All rights reserved.
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: inference.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
b'\n\x0finference.proto\x12\x08mmdeploy\"\x91\x01\n\x05Model\x12\x11\n\x04name\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x07weights\x18\x02 \x01(\x0c\x12+\n\x06\x64\x65vice\x18\x03 \x01(\x0e\x32\x16.mmdeploy.Model.DeviceH\x01\x88\x01\x01\"#\n\x06\x44\x65vice\x12\x07\n\x03\x43PU\x10\x00\x12\x07\n\x03GPU\x10\x01\x12\x07\n\x03\x44SP\x10\x02\x42\x07\n\x05_nameB\t\n\x07_device\"\x07\n\x05\x45mpty\"B\n\x06Tensor\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\x05\x64type\x18\x02 \x01(\tH\x00\x88\x01\x01\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x42\x08\n\x06_dtype\"-\n\nTensorList\x12\x1f\n\x05\x64\x61tas\x18\x01 \x03(\x0b\x32\x10.mmdeploy.Tensor\"F\n\x05Reply\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x0c\n\x04info\x18\x02 \x01(\t\x12\x1f\n\x05\x64\x61tas\x18\x03 \x03(\x0b\x32\x10.mmdeploy.Tensor\"\x16\n\x05Names\x12\r\n\x05names\x18\x01 \x03(\t2\xfb\x01\n\tInference\x12*\n\x04\x45\x63ho\x12\x0f.mmdeploy.Empty\x1a\x0f.mmdeploy.Reply\"\x00\x12*\n\x04Init\x12\x0f.mmdeploy.Model\x1a\x0f.mmdeploy.Reply\"\x00\x12\x31\n\x0bOutputNames\x12\x0f.mmdeploy.Empty\x1a\x0f.mmdeploy.Names\"\x00\x12\x34\n\tInference\x12\x14.mmdeploy.TensorList\x1a\x0f.mmdeploy.Reply\"\x00\x12-\n\x07\x44\x65stroy\x12\x0f.mmdeploy.Empty\x1a\x0f.mmdeploy.Reply\"\x00\x42%\n\rmmdeploy.snpeB\x0bSNPEWrapperP\x01\xa2\x02\x04SNPEb\x06proto3'
)
_MODEL = DESCRIPTOR.message_types_by_name['Model']
_EMPTY = DESCRIPTOR.message_types_by_name['Empty']
_TENSOR = DESCRIPTOR.message_types_by_name['Tensor']
_TENSORLIST = DESCRIPTOR.message_types_by_name['TensorList']
_REPLY = DESCRIPTOR.message_types_by_name['Reply']
_NAMES = DESCRIPTOR.message_types_by_name['Names']
_MODEL_DEVICE = _MODEL.enum_types_by_name['Device']
Model = _reflection.GeneratedProtocolMessageType(
'Model',
(_message.Message, ),
{
'DESCRIPTOR': _MODEL,
'__module__': 'inference_pb2'
# @@protoc_insertion_point(class_scope:mmdeploy.Model)
})
_sym_db.RegisterMessage(Model)
Empty = _reflection.GeneratedProtocolMessageType(
'Empty',
(_message.Message, ),
{
'DESCRIPTOR': _EMPTY,
'__module__': 'inference_pb2'
# @@protoc_insertion_point(class_scope:mmdeploy.Empty)
})
_sym_db.RegisterMessage(Empty)
Tensor = _reflection.GeneratedProtocolMessageType(
'Tensor',
(_message.Message, ),
{
'DESCRIPTOR': _TENSOR,
'__module__': 'inference_pb2'
# @@protoc_insertion_point(class_scope:mmdeploy.Tensor)
})
_sym_db.RegisterMessage(Tensor)
TensorList = _reflection.GeneratedProtocolMessageType(
'TensorList',
(_message.Message, ),
{
'DESCRIPTOR': _TENSORLIST,
'__module__': 'inference_pb2'
# @@protoc_insertion_point(class_scope:mmdeploy.TensorList)
})
_sym_db.RegisterMessage(TensorList)
Reply = _reflection.GeneratedProtocolMessageType(
'Reply',
(_message.Message, ),
{
'DESCRIPTOR': _REPLY,
'__module__': 'inference_pb2'
# @@protoc_insertion_point(class_scope:mmdeploy.Reply)
})
_sym_db.RegisterMessage(Reply)
Names = _reflection.GeneratedProtocolMessageType(
'Names',
(_message.Message, ),
{
'DESCRIPTOR': _NAMES,
'__module__': 'inference_pb2'
# @@protoc_insertion_point(class_scope:mmdeploy.Names)
})
_sym_db.RegisterMessage(Names)
_INFERENCE = DESCRIPTOR.services_by_name['Inference']
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
DESCRIPTOR._serialized_options = b'\n\rmmdeploy.snpeB\013SNPEWrapperP\001\242\002\004SNPE'
_MODEL._serialized_start = 30
_MODEL._serialized_end = 175
_MODEL_DEVICE._serialized_start = 120
_MODEL_DEVICE._serialized_end = 155
_EMPTY._serialized_start = 177
_EMPTY._serialized_end = 184
_TENSOR._serialized_start = 186
_TENSOR._serialized_end = 252
_TENSORLIST._serialized_start = 254
_TENSORLIST._serialized_end = 299
_REPLY._serialized_start = 301
_REPLY._serialized_end = 371
_NAMES._serialized_start = 373
_NAMES._serialized_end = 395
_INFERENCE._serialized_start = 398
_INFERENCE._serialized_end = 649
# @@protoc_insertion_point(module_scope)

208
mmdeploy/backend/snpe/inference_pb2_grpc.py
View File

@ -1,208 +0,0 @@
# Copyright (c) OpenMMLab. All rights reserved.
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
"""Client and server classes corresponding to protobuf-defined services."""
import grpc
import inference_pb2 as inference__pb2
class InferenceStub(object):
"""The inference service definition."""
def __init__(self, channel):
"""Constructor.
Args:
channel: A grpc.Channel.
"""
self.Echo = channel.unary_unary(
'/mmdeploy.Inference/Echo',
request_serializer=inference__pb2.Empty.SerializeToString,
response_deserializer=inference__pb2.Reply.FromString,
)
self.Init = channel.unary_unary(
'/mmdeploy.Inference/Init',
request_serializer=inference__pb2.Model.SerializeToString,
response_deserializer=inference__pb2.Reply.FromString,
)
self.OutputNames = channel.unary_unary(
'/mmdeploy.Inference/OutputNames',
request_serializer=inference__pb2.Empty.SerializeToString,
response_deserializer=inference__pb2.Names.FromString,
)
self.Inference = channel.unary_unary(
'/mmdeploy.Inference/Inference',
request_serializer=inference__pb2.TensorList.SerializeToString,
response_deserializer=inference__pb2.Reply.FromString,
)
self.Destroy = channel.unary_unary(
'/mmdeploy.Inference/Destroy',
request_serializer=inference__pb2.Empty.SerializeToString,
response_deserializer=inference__pb2.Reply.FromString,
)
class InferenceServicer(object):
"""The inference service definition."""
def Echo(self, request, context):
"""Missing associated documentation comment in .proto file."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def Init(self, request, context):
"""Init Model with model file."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def OutputNames(self, request, context):
"""Get output names."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def Inference(self, request, context):
"""Inference with inputs."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def Destroy(self, request, context):
"""Destroy handle."""
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def add_InferenceServicer_to_server(servicer, server):
rpc_method_handlers = {
'Echo':
grpc.unary_unary_rpc_method_handler(
servicer.Echo,
request_deserializer=inference__pb2.Empty.FromString,
response_serializer=inference__pb2.Reply.SerializeToString,
),
'Init':
grpc.unary_unary_rpc_method_handler(
servicer.Init,
request_deserializer=inference__pb2.Model.FromString,
response_serializer=inference__pb2.Reply.SerializeToString,
),
'OutputNames':
grpc.unary_unary_rpc_method_handler(
servicer.OutputNames,
request_deserializer=inference__pb2.Empty.FromString,
response_serializer=inference__pb2.Names.SerializeToString,
),
'Inference':
grpc.unary_unary_rpc_method_handler(
servicer.Inference,
request_deserializer=inference__pb2.TensorList.FromString,
response_serializer=inference__pb2.Reply.SerializeToString,
),
'Destroy':
grpc.unary_unary_rpc_method_handler(
servicer.Destroy,
request_deserializer=inference__pb2.Empty.FromString,
response_serializer=inference__pb2.Reply.SerializeToString,
),
}
generic_handler = grpc.method_handlers_generic_handler(
'mmdeploy.Inference', rpc_method_handlers)
server.add_generic_rpc_handlers((generic_handler, ))
# This class is part of an EXPERIMENTAL API.
class Inference(object):
"""The inference service definition."""
@staticmethod
def Echo(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(
request, target, '/mmdeploy.Inference/Echo',
inference__pb2.Empty.SerializeToString,
inference__pb2.Reply.FromString, options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout,
metadata)
@staticmethod
def Init(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(
request, target, '/mmdeploy.Inference/Init',
inference__pb2.Model.SerializeToString,
inference__pb2.Reply.FromString, options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout,
metadata)
@staticmethod
def OutputNames(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(
request, target, '/mmdeploy.Inference/OutputNames',
inference__pb2.Empty.SerializeToString,
inference__pb2.Names.FromString, options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout,
metadata)
@staticmethod
def Inference(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(
request, target, '/mmdeploy.Inference/Inference',
inference__pb2.TensorList.SerializeToString,
inference__pb2.Reply.FromString, options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout,
metadata)
@staticmethod
def Destroy(request,
target,
options=(),
channel_credentials=None,
call_credentials=None,
insecure=False,
compression=None,
wait_for_ready=None,
timeout=None,
metadata=None):
return grpc.experimental.unary_unary(
request, target, '/mmdeploy.Inference/Destroy',
inference__pb2.Empty.SerializeToString,
inference__pb2.Reply.FromString, options, channel_credentials,
insecure, call_credentials, compression, wait_for_ready, timeout,
metadata)

40
mmdeploy/backend/snpe/wrapper.py
View File

@ -1,9 +1,12 @@
# Copyright (c) OpenMMLab. All rights reserved.
import importlib
import os
import sys
from typing import Dict, Optional, Sequence
import grpc
# import mmdeploy.backend.snpe.inference_pb2
# import mmdeploy.backend.snpe.inference_pb2_grpc
import inference_pb2
import inference_pb2_grpc
import numpy as np
@ -37,7 +40,6 @@ class SNPEWrapper(BaseWrapper):
def __init__(self,
dlc_file: str,
uri: str,
output_names: Optional[Sequence[str]] = None,
**kwargs):
@ -45,6 +47,8 @@ class SNPEWrapper(BaseWrapper):
# The maximum model file size is 512MB
MAX_SIZE = 2 << 29
uri = os.environ['__MMDEPLOY_GRPC_URI']
logger.info(f'fetch uri: {uri}')
self.channel = grpc.insecure_channel(
uri,
options=(('grpc.GRPC_ARG_KEEPALIVE_TIME_MS',
@ -55,8 +59,8 @@ class SNPEWrapper(BaseWrapper):
filesize = os.stat(dlc_file).st_size
logger.info(f'reading local model file {dlc_file}')
with open(dlc_file, 'rb') as f:
weights = f.read(filesize)
# with open(dlc_file, 'rb') as f:
# weights = f.read(filesize)
stub = inference_pb2_grpc.InferenceStub(self.channel)
logger.info(f'init remote SNPE engine with RPC, please wait...')
@ -67,12 +71,15 @@ class SNPEWrapper(BaseWrapper):
logger.error(f'init SNPE model failed {resp.info}')
return
output_names = stub.OutputNames(inference_pb2.Empty())
super().__init__(output_names)
output = stub.OutputNames(inference_pb2.Empty())
output_names = output.names
def __del__(self):
stub = inference_pb2_grpc.InferenceStub(self.channel)
stub.Destroy()
super().__init__(output_names)
logger.info(f'init success, outputs {output_names}')
# def __del__(self):
# stub = inference_pb2_grpc.InferenceStub(self.channel)
# stub.Destroy()
def forward(self, inputs: Dict[str,
torch.Tensor]) -> Dict[str, torch.Tensor]:
@ -102,28 +109,29 @@ class SNPEWrapper(BaseWrapper):
snpe_inputs.append(tensor)
return self.__snpe_execute(snpe_inputs, device_type)
return self.__snpe_execute(inference_pb2.TensorList(datas=snpe_inputs), device_type)
@TimeCounter.count_time()
def __snpe_execute(self, inputs: inference_pb2.TensorList,
def __snpe_execute(self, tensorList: inference_pb2.TensorList,
device: str) -> Dict[str, torch.tensor]:
"""Run inference with snpe remote inference engine.
Args:
inputs (inference_pb2.TensorList): snpe input tensor.
tensorList (inference_pb2.TensorList): snpe input tensor.
Returns:
dict[str, torch.tensor]: Inference results of snpe model.
"""
stub = inference_pb2_grpc.InferenceStub(self.channel)
resp = stub.Inference(inputs)
resp = stub.Inference(tensorList)
result = dict()
if resp.status == 0:
for tensor in resp.datas:
ndarray = np.frombuffer(tensor.data)
result[tensor.name] = torch.from_numpy(ndarray).to(device)
ndarray = np.frombuffer(tensor.data, dtype=np.float32)
import pdb
pdb.set_trace()
result[tensor.name] = torch.from_numpy(ndarray.copy()).to(device)
else:
logger = get_root_logger()
logger.error(f'snpe inference failed {resp.info}')

2
mmdeploy/codebase/base/backend_model.py
View File

@ -108,7 +108,7 @@ class BaseBackendModel(torch.nn.Module, metaclass=ABCMeta):
output_names=output_names)
elif backend == Backend.SNPE:
from mmdeploy.backend.snpe import SNPEWrapper
return SNPEWrapper(model=backend_files[0],uri=,output_names=output_names)
return SNPEWrapper(dlc_file=backend_files[0], output_names=output_names)
else:
raise NotImplementedError(f'Unknown backend type: {backend.value}')

1
requirements/optional.txt
View File

@ -7,3 +7,4 @@ mmrazor>=0.3.0
mmsegmentation
onnxruntime>=1.8.0
openvino-dev
grpcio

24
tools/deploy.py
View File

@ -272,6 +272,7 @@ def main():
backend_files += [model_param_path, model_bin_path]
elif backend == Backend.SNPE:
os.environ['__MMDEPLOY_GRPC_URI'] = args.uri
from mmdeploy.apis.snpe import is_available as is_available
if not is_available():
@ -358,17 +359,18 @@ def main():
# for headless installation.
if not headless:
# visualize model of the backend
create_process(
f'visualize {backend.value} model',
target=visualize_model,
args=(model_cfg_path, deploy_cfg_path, backend_files,
args.test_img, args.device),
kwargs=dict(
backend=backend,
output_file=osp.join(args.work_dir,
f'output_{backend.value}.jpg'),
show_result=args.show),
ret_value=ret_value)
visualize_model(model_cfg_path, deploy_cfg_path, backend_files,args.test_img, args.device)
# create_process(
# f'visualize {backend.value} model',
# target=visualize_model,
# args=(model_cfg_path, deploy_cfg_path, backend_files,
# args.test_img, args.device),
# kwargs=dict(
# backend=backend,
# output_file=osp.join(args.work_dir,
# f'output_{backend.value}.jpg'),
# show_result=args.show),
# ret_value=ret_value)
# visualize pytorch model
create_process(