diff --git a/csrc/mmdeploy/backend_ops/snpe/client/end2end.dlc b/csrc/mmdeploy/backend_ops/snpe/client/end2end.dlc new file mode 120000 index 000000000..873fbcebd --- /dev/null +++ b/csrc/mmdeploy/backend_ops/snpe/client/end2end.dlc @@ -0,0 +1 @@ +/home/PJLAB/konghuanjun/GitProjects/mmdeploy/resnet18/end2end.dlc \ No newline at end of file diff --git a/csrc/mmdeploy/backend_ops/snpe/client/end2end.onnx b/csrc/mmdeploy/backend_ops/snpe/client/end2end.onnx new file mode 120000 index 000000000..253d07ce1 --- /dev/null +++ b/csrc/mmdeploy/backend_ops/snpe/client/end2end.onnx @@ -0,0 +1 @@ +/home/PJLAB/konghuanjun/GitProjects/mmdeploy/resnet18/end2end.onnx \ No newline at end of file diff --git a/csrc/mmdeploy/backend_ops/snpe/client/inference_client.py b/csrc/mmdeploy/backend_ops/snpe/client/inference_client.py new file mode 100644 index 000000000..5f7746f7b --- /dev/null +++ b/csrc/mmdeploy/backend_ops/snpe/client/inference_client.py @@ -0,0 +1,82 @@ +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The Python implementation of the GRPC helloworld.Greeter client.""" + +from __future__ import print_function + +import logging + +import grpc +import inference_pb2 +import inference_pb2_grpc +import os +import cv2 +import numpy as np + +def build_dummy_tensor(): + img = cv2.imread('/home/PJLAB/konghuanjun/Downloads/snpe-1.55.0.2958/models/alexnet/data/chairs.jpg') + m = cv2.resize(img, (600, 400)) + data = (m.astype(np.float32) - 127.5) / 127.5 + print(data.shape) + tensor = inference_pb2.Tensor(data=data.tobytes(), shape=list(data.shape), name='data_0', dtype='float32') + return tensor + +def run(): + # NOTE(gRPC Python Team): .close() is possible on a channel and should be + # used in circumstances in which the with statement does not fit the needs + # of the code. + filename = 'end2end.dlc' + filesize = os.stat(filename).st_size + + weights = bytes() + # with open(filename, 'rb') as f: + # weights = f.read(filesize) + # if len(weights) >= (2 << 29): + # print('model size too big') + + # https://github.com/grpc/grpc/blob/v1.46.x/include/grpc/impl/codegen/grpc_types.h + # https://grpc.io/docs/guides/performance/ + with grpc.insecure_channel('10.1.80.67:50051', + options=( + ('grpc.GRPC_ARG_KEEPALIVE_TIME_MS', 2000), + ('grpc.max_send_message_length', 2<<29), + ('grpc.keepalive_permit_without_calls', 1))) as channel: + print("channel type {}".format(type(channel))) + # with grpc.insecure_channel('[0:0:fe80::3455:bf2a]:50051') as channel: + stub = inference_pb2_grpc.InferenceStub(channel) + response = stub.Echo(inference_pb2.Empty()) + print("Response echo {}".format(response)) + + model = inference_pb2.Model(name= filename, weights=weights, device=1) + print("Sending model to init, please wait...") + response = stub.Init(model) + print("Response init {}".format(response)) + + response = stub.OutputNames(inference_pb2.Empty()) + print("Response outputnames {}".format(response)) + + tensor = build_dummy_tensor() + tensorList = inference_pb2.TensorList(datas = [tensor]) + + for x in range(1): + response = stub.Inference(tensorList) + if response.status == 0: + prob = np.frombuffer(response.datas[0].data, dtype=np.float32) + print("prob argmax: {} max: {}".format(prob.argmax(), prob.max())) + else: + print(response.info) + +if __name__ == '__main__': + logging.basicConfig() + run() diff --git a/csrc/mmdeploy/backend_ops/snpe/client/inference_pb2.py b/csrc/mmdeploy/backend_ops/snpe/client/inference_pb2.py new file mode 100644 index 000000000..f11cdf66e --- /dev/null +++ b/csrc/mmdeploy/backend_ops/snpe/client/inference_pb2.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: inference.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0finference.proto\x12\x08mmdeploy\"\x91\x01\n\x05Model\x12\x11\n\x04name\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x07weights\x18\x02 \x01(\x0c\x12+\n\x06\x64\x65vice\x18\x03 \x01(\x0e\x32\x16.mmdeploy.Model.DeviceH\x01\x88\x01\x01\"#\n\x06\x44\x65vice\x12\x07\n\x03\x43PU\x10\x00\x12\x07\n\x03GPU\x10\x01\x12\x07\n\x03\x44SP\x10\x02\x42\x07\n\x05_nameB\t\n\x07_device\"\x07\n\x05\x45mpty\"Q\n\x06Tensor\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\x05\x64type\x18\x02 \x01(\tH\x00\x88\x01\x01\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12\r\n\x05shape\x18\x04 \x03(\x05\x42\x08\n\x06_dtype\"-\n\nTensorList\x12\x1f\n\x05\x64\x61tas\x18\x01 \x03(\x0b\x32\x10.mmdeploy.Tensor\"F\n\x05Reply\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x0c\n\x04info\x18\x02 \x01(\t\x12\x1f\n\x05\x64\x61tas\x18\x03 \x03(\x0b\x32\x10.mmdeploy.Tensor\"\x16\n\x05Names\x12\r\n\x05names\x18\x01 \x03(\t2\xfb\x01\n\tInference\x12*\n\x04\x45\x63ho\x12\x0f.mmdeploy.Empty\x1a\x0f.mmdeploy.Reply\"\x00\x12*\n\x04Init\x12\x0f.mmdeploy.Model\x1a\x0f.mmdeploy.Reply\"\x00\x12\x31\n\x0bOutputNames\x12\x0f.mmdeploy.Empty\x1a\x0f.mmdeploy.Names\"\x00\x12\x34\n\tInference\x12\x14.mmdeploy.TensorList\x1a\x0f.mmdeploy.Reply\"\x00\x12-\n\x07\x44\x65stroy\x12\x0f.mmdeploy.Empty\x1a\x0f.mmdeploy.Reply\"\x00\x42%\n\rmmdeploy.snpeB\x0bSNPEWrapperP\x01\xa2\x02\x04SNPEb\x06proto3') + + + +_MODEL = DESCRIPTOR.message_types_by_name['Model'] +_EMPTY = DESCRIPTOR.message_types_by_name['Empty'] +_TENSOR = DESCRIPTOR.message_types_by_name['Tensor'] +_TENSORLIST = DESCRIPTOR.message_types_by_name['TensorList'] +_REPLY = DESCRIPTOR.message_types_by_name['Reply'] +_NAMES = DESCRIPTOR.message_types_by_name['Names'] +_MODEL_DEVICE = _MODEL.enum_types_by_name['Device'] +Model = _reflection.GeneratedProtocolMessageType('Model', (_message.Message,), { + 'DESCRIPTOR' : _MODEL, + '__module__' : 'inference_pb2' + # @@protoc_insertion_point(class_scope:mmdeploy.Model) + }) +_sym_db.RegisterMessage(Model) + +Empty = _reflection.GeneratedProtocolMessageType('Empty', (_message.Message,), { + 'DESCRIPTOR' : _EMPTY, + '__module__' : 'inference_pb2' + # @@protoc_insertion_point(class_scope:mmdeploy.Empty) + }) +_sym_db.RegisterMessage(Empty) + +Tensor = _reflection.GeneratedProtocolMessageType('Tensor', (_message.Message,), { + 'DESCRIPTOR' : _TENSOR, + '__module__' : 'inference_pb2' + # @@protoc_insertion_point(class_scope:mmdeploy.Tensor) + }) +_sym_db.RegisterMessage(Tensor) + +TensorList = _reflection.GeneratedProtocolMessageType('TensorList', (_message.Message,), { + 'DESCRIPTOR' : _TENSORLIST, + '__module__' : 'inference_pb2' + # @@protoc_insertion_point(class_scope:mmdeploy.TensorList) + }) +_sym_db.RegisterMessage(TensorList) + +Reply = _reflection.GeneratedProtocolMessageType('Reply', (_message.Message,), { + 'DESCRIPTOR' : _REPLY, + '__module__' : 'inference_pb2' + # @@protoc_insertion_point(class_scope:mmdeploy.Reply) + }) +_sym_db.RegisterMessage(Reply) + +Names = _reflection.GeneratedProtocolMessageType('Names', (_message.Message,), { + 'DESCRIPTOR' : _NAMES, + '__module__' : 'inference_pb2' + # @@protoc_insertion_point(class_scope:mmdeploy.Names) + }) +_sym_db.RegisterMessage(Names) + +_INFERENCE = DESCRIPTOR.services_by_name['Inference'] +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'\n\rmmdeploy.snpeB\013SNPEWrapperP\001\242\002\004SNPE' + _MODEL._serialized_start=30 + _MODEL._serialized_end=175 + _MODEL_DEVICE._serialized_start=120 + _MODEL_DEVICE._serialized_end=155 + _EMPTY._serialized_start=177 + _EMPTY._serialized_end=184 + _TENSOR._serialized_start=186 + _TENSOR._serialized_end=267 + _TENSORLIST._serialized_start=269 + _TENSORLIST._serialized_end=314 + _REPLY._serialized_start=316 + _REPLY._serialized_end=386 + _NAMES._serialized_start=388 + _NAMES._serialized_end=410 + _INFERENCE._serialized_start=413 + _INFERENCE._serialized_end=664 +# @@protoc_insertion_point(module_scope) diff --git a/csrc/mmdeploy/backend_ops/snpe/client/inference_pb2_grpc.py b/csrc/mmdeploy/backend_ops/snpe/client/inference_pb2_grpc.py new file mode 100644 index 000000000..aeea95479 --- /dev/null +++ b/csrc/mmdeploy/backend_ops/snpe/client/inference_pb2_grpc.py @@ -0,0 +1,205 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! +"""Client and server classes corresponding to protobuf-defined services.""" +import grpc + +import inference_pb2 as inference__pb2 + + +class InferenceStub(object): + """The inference service definition. + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.Echo = channel.unary_unary( + '/mmdeploy.Inference/Echo', + request_serializer=inference__pb2.Empty.SerializeToString, + response_deserializer=inference__pb2.Reply.FromString, + ) + self.Init = channel.unary_unary( + '/mmdeploy.Inference/Init', + request_serializer=inference__pb2.Model.SerializeToString, + response_deserializer=inference__pb2.Reply.FromString, + ) + self.OutputNames = channel.unary_unary( + '/mmdeploy.Inference/OutputNames', + request_serializer=inference__pb2.Empty.SerializeToString, + response_deserializer=inference__pb2.Names.FromString, + ) + self.Inference = channel.unary_unary( + '/mmdeploy.Inference/Inference', + request_serializer=inference__pb2.TensorList.SerializeToString, + response_deserializer=inference__pb2.Reply.FromString, + ) + self.Destroy = channel.unary_unary( + '/mmdeploy.Inference/Destroy', + request_serializer=inference__pb2.Empty.SerializeToString, + response_deserializer=inference__pb2.Reply.FromString, + ) + + +class InferenceServicer(object): + """The inference service definition. + """ + + def Echo(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Init(self, request, context): + """Init Model with model file + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def OutputNames(self, request, context): + """Get output names + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Inference(self, request, context): + """Inference with inputs + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Destroy(self, request, context): + """Destroy handle + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_InferenceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'Echo': grpc.unary_unary_rpc_method_handler( + servicer.Echo, + request_deserializer=inference__pb2.Empty.FromString, + response_serializer=inference__pb2.Reply.SerializeToString, + ), + 'Init': grpc.unary_unary_rpc_method_handler( + servicer.Init, + request_deserializer=inference__pb2.Model.FromString, + response_serializer=inference__pb2.Reply.SerializeToString, + ), + 'OutputNames': grpc.unary_unary_rpc_method_handler( + servicer.OutputNames, + request_deserializer=inference__pb2.Empty.FromString, + response_serializer=inference__pb2.Names.SerializeToString, + ), + 'Inference': grpc.unary_unary_rpc_method_handler( + servicer.Inference, + request_deserializer=inference__pb2.TensorList.FromString, + response_serializer=inference__pb2.Reply.SerializeToString, + ), + 'Destroy': grpc.unary_unary_rpc_method_handler( + servicer.Destroy, + request_deserializer=inference__pb2.Empty.FromString, + response_serializer=inference__pb2.Reply.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'mmdeploy.Inference', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + # This class is part of an EXPERIMENTAL API. +class Inference(object): + """The inference service definition. + """ + + @staticmethod + def Echo(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/mmdeploy.Inference/Echo', + inference__pb2.Empty.SerializeToString, + inference__pb2.Reply.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Init(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/mmdeploy.Inference/Init', + inference__pb2.Model.SerializeToString, + inference__pb2.Reply.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def OutputNames(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/mmdeploy.Inference/OutputNames', + inference__pb2.Empty.SerializeToString, + inference__pb2.Names.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Inference(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/mmdeploy.Inference/Inference', + inference__pb2.TensorList.SerializeToString, + inference__pb2.Reply.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def Destroy(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/mmdeploy.Inference/Destroy', + inference__pb2.Empty.SerializeToString, + inference__pb2.Reply.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/csrc/mmdeploy/backend_ops/snpe/service/inference.proto b/csrc/mmdeploy/backend_ops/snpe/inference.proto similarity index 94% rename from csrc/mmdeploy/backend_ops/snpe/service/inference.proto rename to csrc/mmdeploy/backend_ops/snpe/inference.proto index 0389f69c8..bff4d7ea4 100644 --- a/csrc/mmdeploy/backend_ops/snpe/service/inference.proto +++ b/csrc/mmdeploy/backend_ops/snpe/inference.proto @@ -18,7 +18,7 @@ service Inference { // Get output names rpc OutputNames(Empty) returns (Names) {} - // Inference with inputs + // Inference with inputs rpc Inference(TensorList) returns (Reply) {} // Destroy handle @@ -44,12 +44,15 @@ message Empty {} message Tensor { // name string name = 1; - + // datatype optional string dtype = 2; // data bytes data = 3; + + // shape + repeated int32 shape = 4; } message TensorList { diff --git a/csrc/mmdeploy/backend_ops/snpe/service/CMakeLists.txt b/csrc/mmdeploy/backend_ops/snpe/server/CMakeLists.txt similarity index 97% rename from csrc/mmdeploy/backend_ops/snpe/service/CMakeLists.txt rename to csrc/mmdeploy/backend_ops/snpe/server/CMakeLists.txt index c02f0061a..92c8c8f91 100644 --- a/csrc/mmdeploy/backend_ops/snpe/service/CMakeLists.txt +++ b/csrc/mmdeploy/backend_ops/snpe/server/CMakeLists.txt @@ -22,7 +22,7 @@ project(SNPEServer C CXX) include(./common.cmake) # Proto file -get_filename_component(hw_proto "inference.proto" ABSOLUTE) +get_filename_component(hw_proto "../inference.proto" ABSOLUTE) get_filename_component(hw_proto_path "${hw_proto}" PATH) # Generated sources diff --git a/csrc/mmdeploy/backend_ops/snpe/server/build_shared.sh b/csrc/mmdeploy/backend_ops/snpe/server/build_shared.sh new file mode 100755 index 000000000..1f1b5de5f --- /dev/null +++ b/csrc/mmdeploy/backend_ops/snpe/server/build_shared.sh @@ -0,0 +1,10 @@ +export ANDROID_NDK=/home/PJLAB/konghuanjun/Downloads/android-ndk-r17c +cmake .. \ + -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake \ + -DANDROID_ABI=arm64-v8a \ + -DANDROID_PLATFORM=android-26 \ + -DANDROID_STL=c++_shared \ + -DCMAKE_BUILD_TYPE=Release \ + -Dabsl_DIR=/tmp/android_grpc_install_shared/lib/cmake/absl \ + -DProtobuf_DIR=/tmp/android_grpc_install_shared/lib/cmake/protobuf \ + -DgRPC_DIR=/tmp/android_grpc_install_shared/lib/cmake/grpc diff --git a/csrc/mmdeploy/backend_ops/snpe/service/common.cmake b/csrc/mmdeploy/backend_ops/snpe/server/common.cmake similarity index 100% rename from csrc/mmdeploy/backend_ops/snpe/service/common.cmake rename to csrc/mmdeploy/backend_ops/snpe/server/common.cmake diff --git a/csrc/mmdeploy/backend_ops/snpe/service/inference_server.cc b/csrc/mmdeploy/backend_ops/snpe/server/inference_server.cc similarity index 100% rename from csrc/mmdeploy/backend_ops/snpe/service/inference_server.cc rename to csrc/mmdeploy/backend_ops/snpe/server/inference_server.cc diff --git a/csrc/mmdeploy/backend_ops/snpe/service/service_impl.cpp b/csrc/mmdeploy/backend_ops/snpe/server/service_impl.cpp similarity index 99% rename from csrc/mmdeploy/backend_ops/snpe/service/service_impl.cpp rename to csrc/mmdeploy/backend_ops/snpe/server/service_impl.cpp index c9806b771..3bdb29a68 100644 --- a/csrc/mmdeploy/backend_ops/snpe/service/service_impl.cpp +++ b/csrc/mmdeploy/backend_ops/snpe/server/service_impl.cpp @@ -99,7 +99,7 @@ void InferenceServiceImpl::LoadFloatData(const std::string& data, ::mmdeploy::Reply* response) { fprintf(stdout, "Stage Init: recv command\n"); // std::string filename = SaveDLC(request); - std::string filename = "alexnet.dlc"; + std::string filename = "end2end.dlc"; if (snpe != nullptr) { snpe.reset(); diff --git a/csrc/mmdeploy/backend_ops/snpe/service/service_impl.h b/csrc/mmdeploy/backend_ops/snpe/server/service_impl.h similarity index 99% rename from csrc/mmdeploy/backend_ops/snpe/service/service_impl.h rename to csrc/mmdeploy/backend_ops/snpe/server/service_impl.h index db0a73da4..d78411d09 100644 --- a/csrc/mmdeploy/backend_ops/snpe/service/service_impl.h +++ b/csrc/mmdeploy/backend_ops/snpe/server/service_impl.h @@ -55,7 +55,7 @@ class InferenceServiceImpl final : public Inference::Service { ::grpc::Status Inference(::grpc::ServerContext* context, const ::mmdeploy::TensorList* request, ::mmdeploy::Reply* response) override; - // Destroy handle + // Destory handle ::grpc::Status Destroy(::grpc::ServerContext* context, const ::mmdeploy::Empty* request, ::mmdeploy::Reply* response) override; diff --git a/mmdeploy/backend/snpe/__init__.py b/mmdeploy/backend/snpe/__init__.py index 45a05eeaa..cf6f8ea4b 100644 --- a/mmdeploy/backend/snpe/__init__.py +++ b/mmdeploy/backend/snpe/__init__.py @@ -4,7 +4,6 @@ import os.path as osp from .init_plugins import get_onnx2dlc_path from .onnx2dlc import from_onnx - def is_available(): """Check whether ncnn and snpe-onnx-to-dlc tool are installed. @@ -23,5 +22,6 @@ if is_available(): from .wrapper import SNPEWrapper __all__ += ['SNPEWrapper'] - except Exception: + except Exception as e: + print(e) pass diff --git a/mmdeploy/backend/snpe/inference_pb2.py b/mmdeploy/backend/snpe/inference_pb2.py deleted file mode 100644 index 42c31c6e6..000000000 --- a/mmdeploy/backend/snpe/inference_pb2.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: inference.proto -"""Generated protocol buffer code.""" -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database - -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x0finference.proto\x12\x08mmdeploy\"\x91\x01\n\x05Model\x12\x11\n\x04name\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x0f\n\x07weights\x18\x02 \x01(\x0c\x12+\n\x06\x64\x65vice\x18\x03 \x01(\x0e\x32\x16.mmdeploy.Model.DeviceH\x01\x88\x01\x01\"#\n\x06\x44\x65vice\x12\x07\n\x03\x43PU\x10\x00\x12\x07\n\x03GPU\x10\x01\x12\x07\n\x03\x44SP\x10\x02\x42\x07\n\x05_nameB\t\n\x07_device\"\x07\n\x05\x45mpty\"B\n\x06Tensor\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\x05\x64type\x18\x02 \x01(\tH\x00\x88\x01\x01\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x42\x08\n\x06_dtype\"-\n\nTensorList\x12\x1f\n\x05\x64\x61tas\x18\x01 \x03(\x0b\x32\x10.mmdeploy.Tensor\"F\n\x05Reply\x12\x0e\n\x06status\x18\x01 \x01(\x05\x12\x0c\n\x04info\x18\x02 \x01(\t\x12\x1f\n\x05\x64\x61tas\x18\x03 \x03(\x0b\x32\x10.mmdeploy.Tensor\"\x16\n\x05Names\x12\r\n\x05names\x18\x01 \x03(\t2\xfb\x01\n\tInference\x12*\n\x04\x45\x63ho\x12\x0f.mmdeploy.Empty\x1a\x0f.mmdeploy.Reply\"\x00\x12*\n\x04Init\x12\x0f.mmdeploy.Model\x1a\x0f.mmdeploy.Reply\"\x00\x12\x31\n\x0bOutputNames\x12\x0f.mmdeploy.Empty\x1a\x0f.mmdeploy.Names\"\x00\x12\x34\n\tInference\x12\x14.mmdeploy.TensorList\x1a\x0f.mmdeploy.Reply\"\x00\x12-\n\x07\x44\x65stroy\x12\x0f.mmdeploy.Empty\x1a\x0f.mmdeploy.Reply\"\x00\x42%\n\rmmdeploy.snpeB\x0bSNPEWrapperP\x01\xa2\x02\x04SNPEb\x06proto3' -) - -_MODEL = DESCRIPTOR.message_types_by_name['Model'] -_EMPTY = DESCRIPTOR.message_types_by_name['Empty'] -_TENSOR = DESCRIPTOR.message_types_by_name['Tensor'] -_TENSORLIST = DESCRIPTOR.message_types_by_name['TensorList'] -_REPLY = DESCRIPTOR.message_types_by_name['Reply'] -_NAMES = DESCRIPTOR.message_types_by_name['Names'] -_MODEL_DEVICE = _MODEL.enum_types_by_name['Device'] -Model = _reflection.GeneratedProtocolMessageType( - 'Model', - (_message.Message, ), - { - 'DESCRIPTOR': _MODEL, - '__module__': 'inference_pb2' - # @@protoc_insertion_point(class_scope:mmdeploy.Model) - }) -_sym_db.RegisterMessage(Model) - -Empty = _reflection.GeneratedProtocolMessageType( - 'Empty', - (_message.Message, ), - { - 'DESCRIPTOR': _EMPTY, - '__module__': 'inference_pb2' - # @@protoc_insertion_point(class_scope:mmdeploy.Empty) - }) -_sym_db.RegisterMessage(Empty) - -Tensor = _reflection.GeneratedProtocolMessageType( - 'Tensor', - (_message.Message, ), - { - 'DESCRIPTOR': _TENSOR, - '__module__': 'inference_pb2' - # @@protoc_insertion_point(class_scope:mmdeploy.Tensor) - }) -_sym_db.RegisterMessage(Tensor) - -TensorList = _reflection.GeneratedProtocolMessageType( - 'TensorList', - (_message.Message, ), - { - 'DESCRIPTOR': _TENSORLIST, - '__module__': 'inference_pb2' - # @@protoc_insertion_point(class_scope:mmdeploy.TensorList) - }) -_sym_db.RegisterMessage(TensorList) - -Reply = _reflection.GeneratedProtocolMessageType( - 'Reply', - (_message.Message, ), - { - 'DESCRIPTOR': _REPLY, - '__module__': 'inference_pb2' - # @@protoc_insertion_point(class_scope:mmdeploy.Reply) - }) -_sym_db.RegisterMessage(Reply) - -Names = _reflection.GeneratedProtocolMessageType( - 'Names', - (_message.Message, ), - { - 'DESCRIPTOR': _NAMES, - '__module__': 'inference_pb2' - # @@protoc_insertion_point(class_scope:mmdeploy.Names) - }) -_sym_db.RegisterMessage(Names) - -_INFERENCE = DESCRIPTOR.services_by_name['Inference'] -if _descriptor._USE_C_DESCRIPTORS == False: - - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'\n\rmmdeploy.snpeB\013SNPEWrapperP\001\242\002\004SNPE' - _MODEL._serialized_start = 30 - _MODEL._serialized_end = 175 - _MODEL_DEVICE._serialized_start = 120 - _MODEL_DEVICE._serialized_end = 155 - _EMPTY._serialized_start = 177 - _EMPTY._serialized_end = 184 - _TENSOR._serialized_start = 186 - _TENSOR._serialized_end = 252 - _TENSORLIST._serialized_start = 254 - _TENSORLIST._serialized_end = 299 - _REPLY._serialized_start = 301 - _REPLY._serialized_end = 371 - _NAMES._serialized_start = 373 - _NAMES._serialized_end = 395 - _INFERENCE._serialized_start = 398 - _INFERENCE._serialized_end = 649 -# @@protoc_insertion_point(module_scope) diff --git a/mmdeploy/backend/snpe/inference_pb2_grpc.py b/mmdeploy/backend/snpe/inference_pb2_grpc.py deleted file mode 100644 index e28932529..000000000 --- a/mmdeploy/backend/snpe/inference_pb2_grpc.py +++ /dev/null @@ -1,208 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! -"""Client and server classes corresponding to protobuf-defined services.""" -import grpc -import inference_pb2 as inference__pb2 - - -class InferenceStub(object): - """The inference service definition.""" - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.Echo = channel.unary_unary( - '/mmdeploy.Inference/Echo', - request_serializer=inference__pb2.Empty.SerializeToString, - response_deserializer=inference__pb2.Reply.FromString, - ) - self.Init = channel.unary_unary( - '/mmdeploy.Inference/Init', - request_serializer=inference__pb2.Model.SerializeToString, - response_deserializer=inference__pb2.Reply.FromString, - ) - self.OutputNames = channel.unary_unary( - '/mmdeploy.Inference/OutputNames', - request_serializer=inference__pb2.Empty.SerializeToString, - response_deserializer=inference__pb2.Names.FromString, - ) - self.Inference = channel.unary_unary( - '/mmdeploy.Inference/Inference', - request_serializer=inference__pb2.TensorList.SerializeToString, - response_deserializer=inference__pb2.Reply.FromString, - ) - self.Destroy = channel.unary_unary( - '/mmdeploy.Inference/Destroy', - request_serializer=inference__pb2.Empty.SerializeToString, - response_deserializer=inference__pb2.Reply.FromString, - ) - - -class InferenceServicer(object): - """The inference service definition.""" - - def Echo(self, request, context): - """Missing associated documentation comment in .proto file.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Init(self, request, context): - """Init Model with model file.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def OutputNames(self, request, context): - """Get output names.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Inference(self, request, context): - """Inference with inputs.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Destroy(self, request, context): - """Destroy handle.""" - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - -def add_InferenceServicer_to_server(servicer, server): - rpc_method_handlers = { - 'Echo': - grpc.unary_unary_rpc_method_handler( - servicer.Echo, - request_deserializer=inference__pb2.Empty.FromString, - response_serializer=inference__pb2.Reply.SerializeToString, - ), - 'Init': - grpc.unary_unary_rpc_method_handler( - servicer.Init, - request_deserializer=inference__pb2.Model.FromString, - response_serializer=inference__pb2.Reply.SerializeToString, - ), - 'OutputNames': - grpc.unary_unary_rpc_method_handler( - servicer.OutputNames, - request_deserializer=inference__pb2.Empty.FromString, - response_serializer=inference__pb2.Names.SerializeToString, - ), - 'Inference': - grpc.unary_unary_rpc_method_handler( - servicer.Inference, - request_deserializer=inference__pb2.TensorList.FromString, - response_serializer=inference__pb2.Reply.SerializeToString, - ), - 'Destroy': - grpc.unary_unary_rpc_method_handler( - servicer.Destroy, - request_deserializer=inference__pb2.Empty.FromString, - response_serializer=inference__pb2.Reply.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'mmdeploy.Inference', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler, )) - - -# This class is part of an EXPERIMENTAL API. -class Inference(object): - """The inference service definition.""" - - @staticmethod - def Echo(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary( - request, target, '/mmdeploy.Inference/Echo', - inference__pb2.Empty.SerializeToString, - inference__pb2.Reply.FromString, options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, - metadata) - - @staticmethod - def Init(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary( - request, target, '/mmdeploy.Inference/Init', - inference__pb2.Model.SerializeToString, - inference__pb2.Reply.FromString, options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, - metadata) - - @staticmethod - def OutputNames(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary( - request, target, '/mmdeploy.Inference/OutputNames', - inference__pb2.Empty.SerializeToString, - inference__pb2.Names.FromString, options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, - metadata) - - @staticmethod - def Inference(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary( - request, target, '/mmdeploy.Inference/Inference', - inference__pb2.TensorList.SerializeToString, - inference__pb2.Reply.FromString, options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, - metadata) - - @staticmethod - def Destroy(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary( - request, target, '/mmdeploy.Inference/Destroy', - inference__pb2.Empty.SerializeToString, - inference__pb2.Reply.FromString, options, channel_credentials, - insecure, call_credentials, compression, wait_for_ready, timeout, - metadata) diff --git a/mmdeploy/backend/snpe/wrapper.py b/mmdeploy/backend/snpe/wrapper.py index aa56360ba..4f1f15c1b 100644 --- a/mmdeploy/backend/snpe/wrapper.py +++ b/mmdeploy/backend/snpe/wrapper.py @@ -1,9 +1,12 @@ # Copyright (c) OpenMMLab. All rights reserved. -import importlib import os +import sys from typing import Dict, Optional, Sequence import grpc + +# import mmdeploy.backend.snpe.inference_pb2 +# import mmdeploy.backend.snpe.inference_pb2_grpc import inference_pb2 import inference_pb2_grpc import numpy as np @@ -37,7 +40,6 @@ class SNPEWrapper(BaseWrapper): def __init__(self, dlc_file: str, - uri: str, output_names: Optional[Sequence[str]] = None, **kwargs): @@ -45,6 +47,8 @@ class SNPEWrapper(BaseWrapper): # The maximum model file size is 512MB MAX_SIZE = 2 << 29 + uri = os.environ['__MMDEPLOY_GRPC_URI'] + logger.info(f'fetch uri: {uri}') self.channel = grpc.insecure_channel( uri, options=(('grpc.GRPC_ARG_KEEPALIVE_TIME_MS', @@ -55,8 +59,8 @@ class SNPEWrapper(BaseWrapper): filesize = os.stat(dlc_file).st_size logger.info(f'reading local model file {dlc_file}') - with open(dlc_file, 'rb') as f: - weights = f.read(filesize) + # with open(dlc_file, 'rb') as f: + # weights = f.read(filesize) stub = inference_pb2_grpc.InferenceStub(self.channel) logger.info(f'init remote SNPE engine with RPC, please wait...') @@ -67,12 +71,15 @@ class SNPEWrapper(BaseWrapper): logger.error(f'init SNPE model failed {resp.info}') return - output_names = stub.OutputNames(inference_pb2.Empty()) - super().__init__(output_names) + output = stub.OutputNames(inference_pb2.Empty()) + output_names = output.names - def __del__(self): - stub = inference_pb2_grpc.InferenceStub(self.channel) - stub.Destroy() + super().__init__(output_names) + logger.info(f'init success, outputs {output_names}') + + # def __del__(self): + # stub = inference_pb2_grpc.InferenceStub(self.channel) + # stub.Destroy() def forward(self, inputs: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: @@ -102,28 +109,29 @@ class SNPEWrapper(BaseWrapper): snpe_inputs.append(tensor) - return self.__snpe_execute(snpe_inputs, device_type) + return self.__snpe_execute(inference_pb2.TensorList(datas=snpe_inputs), device_type) @TimeCounter.count_time() - def __snpe_execute(self, inputs: inference_pb2.TensorList, + def __snpe_execute(self, tensorList: inference_pb2.TensorList, device: str) -> Dict[str, torch.tensor]: """Run inference with snpe remote inference engine. Args: - inputs (inference_pb2.TensorList): snpe input tensor. + tensorList (inference_pb2.TensorList): snpe input tensor. Returns: dict[str, torch.tensor]: Inference results of snpe model. """ - stub = inference_pb2_grpc.InferenceStub(self.channel) - resp = stub.Inference(inputs) + resp = stub.Inference(tensorList) result = dict() if resp.status == 0: for tensor in resp.datas: - ndarray = np.frombuffer(tensor.data) - result[tensor.name] = torch.from_numpy(ndarray).to(device) + ndarray = np.frombuffer(tensor.data, dtype=np.float32) + import pdb + pdb.set_trace() + result[tensor.name] = torch.from_numpy(ndarray.copy()).to(device) else: logger = get_root_logger() logger.error(f'snpe inference failed {resp.info}') diff --git a/mmdeploy/codebase/base/backend_model.py b/mmdeploy/codebase/base/backend_model.py index 9c9892ffd..c6d4ce4aa 100644 --- a/mmdeploy/codebase/base/backend_model.py +++ b/mmdeploy/codebase/base/backend_model.py @@ -106,6 +106,9 @@ class BaseBackendModel(torch.nn.Module, metaclass=ABCMeta): model=backend_files[0], input_names=input_names, output_names=output_names) + elif backend == Backend.SNPE: + from mmdeploy.backend.snpe import SNPEWrapper + return SNPEWrapper(dlc_file=backend_files[0], output_names=output_names) else: raise NotImplementedError(f'Unknown backend type: {backend.value}') diff --git a/requirements/optional.txt b/requirements/optional.txt index 5f5251130..f730e164c 100644 --- a/requirements/optional.txt +++ b/requirements/optional.txt @@ -7,3 +7,4 @@ mmrazor>=0.3.0 mmsegmentation onnxruntime>=1.8.0 openvino-dev +grpcio diff --git a/tools/deploy.py b/tools/deploy.py index 7495d10fe..736e3b5e7 100644 --- a/tools/deploy.py +++ b/tools/deploy.py @@ -272,6 +272,7 @@ def main(): backend_files += [model_param_path, model_bin_path] elif backend == Backend.SNPE: + os.environ['__MMDEPLOY_GRPC_URI'] = args.uri from mmdeploy.apis.snpe import is_available as is_available if not is_available(): @@ -290,7 +291,7 @@ def main(): dlc_path = get_output_model_file(onnx_path, args.work_dir) onnx_name = osp.splitext(osp.split(onnx_path)[1])[0] snpe_api.from_onnx(onnx_path, osp.join(args.work_dir, onnx_name)) - backend_files += [dlc_path] + backend_files += dlc_path elif backend == Backend.OPENVINO: from mmdeploy.apis.openvino import \ @@ -358,17 +359,18 @@ def main(): # for headless installation. if not headless: # visualize model of the backend - create_process( - f'visualize {backend.value} model', - target=visualize_model, - args=(model_cfg_path, deploy_cfg_path, backend_files, - args.test_img, args.device), - kwargs=dict( - backend=backend, - output_file=osp.join(args.work_dir, - f'output_{backend.value}.jpg'), - show_result=args.show), - ret_value=ret_value) + visualize_model(model_cfg_path, deploy_cfg_path, backend_files,args.test_img, args.device) + # create_process( + # f'visualize {backend.value} model', + # target=visualize_model, + # args=(model_cfg_path, deploy_cfg_path, backend_files, + # args.test_img, args.device), + # kwargs=dict( + # backend=backend, + # output_file=osp.join(args.work_dir, + # f'output_{backend.value}.jpg'), + # show_result=args.show), + # ret_value=ret_value) # visualize pytorch model create_process(