forked from kubeflow/pipelines
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Torchserve v2 protocol (kubeflow#1870)
* feat: v2 protocol support for torchserve Signed-off-by: Dan Sun <dsun20@bloomberg.net> Feat: Add unit test for torchserve predictor - Add e2e test Signed-off-by: Dan Sun <dsun20@bloomberg.net> Feat: Add readme doc for v2 support - fix lint error - add grpc sample yaml - add tensor input generation script - fix model archiver to support v2 protocol Signed-off-by: Dan Sun <dsun20@bloomberg.net> Feat: Add gRPC client - fix lint error - update readme for gRPC Signed-off-by: Dan Sun <dsun20@bloomberg.net> feat: Add custom handlers for v2 api Signed-off-by: Dan Sun <dsun20@bloomberg.net> Update torchserve image in test overlay Signed-off-by: Dan Sun <dsun20@bloomberg.net> Feat: Add readme doc for v2 support - fix lint error - add grpc sample yaml - add tensor input generation script - fix model archiver to support v2 protocol Signed-off-by: Dan Sun <dsun20@bloomberg.net> Update test_transformer.py Signed-off-by: Dan Sun <dsun20@bloomberg.net> Add e2e test for grpc torchserve Signed-off-by: Dan Sun <dsun20@bloomberg.net> Update test configuration Signed-off-by: Dan Sun <dsun20@bloomberg.net> Support torchserve runtime for v2 protocol Signed-off-by: Dan Sun <dsun20@bloomberg.net> Fix storage uri for v2 example Signed-off-by: Dan Sun <dsun20@bloomberg.net> Add grpc debug and retry config Signed-off-by: Dan Sun <dsun20@bloomberg.net> fix: torchserve gRPC test Signed-off-by: Jagadeesh J <jagadeeshj@ideas2it.com> * fix: skip gRPC test Signed-off-by: Jagadeesh J <jagadeeshj@ideas2it.com>
- Loading branch information
Jagadeesh J
authored
Dec 14, 2021
1 parent
f25ca1a
commit 07e4d5d
Showing
64 changed files
with
1,764 additions
and
3,806 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
2 changes: 1 addition & 1 deletion
2
...samples/v1beta1/torchserve/bert/README.md → ...ples/v1beta1/torchserve/v1/bert/README.md
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
7 changes: 5 additions & 2 deletions
7
...v1beta1/torchserve/bert/config.properties → ...eta1/torchserve/v1/bert/config.properties
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,10 +1,13 @@ | ||
| inference_address=http://0.0.0.0:8085 | ||
| management_address=http://0.0.0.0:8081 | ||
| management_address=http://0.0.0.0:8085 | ||
| metrics_address=http://0.0.0.0:8082 | ||
| grpc_inference_port=7070 | ||
| grpc_management_port=7071 | ||
| enable_metrics_api=true | ||
| metrics_format=prometheus | ||
| number_of_netty_threads=4 | ||
| job_queue_size=10 | ||
| service_envelope=kfserving | ||
| enable_envvars_config=true | ||
| install_py_dep_per_model=true | ||
| model_store=/mnt/models/model-store | ||
| model_snapshot={"name":"startup.cfg","modelCount":1,"models":{"bert":{"1.0":{"defaultVersion":true,"marName":"BERTSeqClassification.mar","minWorkers":1,"maxWorkers":5,"batchSize":1,"maxBatchDelay":5000,"responseTimeout":120}}}} |
File renamed without changes.
7 changes: 5 additions & 2 deletions
7
...ples/v1beta1/torchserve/config.properties → ...s/v1beta1/torchserve/v1/config.properties
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,10 +1,13 @@ | ||
| inference_address=http://0.0.0.0:8085 | ||
| management_address=http://0.0.0.0:8081 | ||
| management_address=http://0.0.0.0:8085 | ||
| metrics_address=http://0.0.0.0:8082 | ||
| grpc_inference_port=7070 | ||
| grpc_management_port=7071 | ||
| enable_metrics_api=true | ||
| metrics_format=prometheus | ||
| number_of_netty_threads=4 | ||
| job_queue_size=10 | ||
| service_envelope=kfserving | ||
| enable_envvars_config=true | ||
| install_py_dep_per_model=true | ||
| model_store=/mnt/models/model-store | ||
| model_snapshot={"name":"startup.cfg","modelCount":1,"models":{"mnist":{"1.0":{"defaultVersion":true,"marName":"mnist.mar","minWorkers":1,"maxWorkers":5,"batchSize":1,"maxBatchDelay":5000,"responseTimeout":120}}}} |
2 changes: 1 addition & 1 deletion
2
docs/samples/v1beta1/torchserve/gpu.yaml → docs/samples/v1beta1/torchserve/v1/gpu.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| apiVersion: serving.kserve.io/v1beta1 | ||
| kind: InferenceService | ||
| metadata: | ||
| name: "torchserve-grpc" | ||
| spec: | ||
| predictor: | ||
| pytorch: | ||
| storageUri: gs://kfserving-examples/models/torchserve/image_classifier | ||
| ports: | ||
| - containerPort: 7070 | ||
| name: h2c | ||
| protocol: TCP |
File renamed without changes
File renamed without changes
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
104 changes: 104 additions & 0 deletions
104
docs/samples/v1beta1/torchserve/v1/torchserve_grpc_client.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,104 @@ | ||
| import grpc | ||
| import inference_pb2 | ||
| import inference_pb2_grpc | ||
| import management_pb2 | ||
| import management_pb2_grpc | ||
| import sys | ||
|
|
||
|
|
||
| def get_inference_stub(): | ||
| channel = grpc.insecure_channel( | ||
| 'localhost:8080', | ||
| options=(('grpc.ssl_target_name_override', | ||
| 'torchserve-grpc.kserve-test.example.com'),)) | ||
| stub = inference_pb2_grpc.InferenceAPIsServiceStub(channel) | ||
| return stub | ||
|
|
||
|
|
||
| def get_management_stub(): | ||
| channel = grpc.insecure_channel( | ||
| 'localhost:8081', | ||
| options=(('grpc.ssl_target_name_override', | ||
| 'torchserve-grpc.kserve-test.example.com'),)) | ||
| stub = management_pb2_grpc.ManagementAPIsServiceStub(channel) | ||
| return stub | ||
|
|
||
|
|
||
| def infer(stub, model_name, model_input): | ||
| with open(model_input, 'rb') as f: | ||
| data = f.read() | ||
|
|
||
| input_data = {'data': data} | ||
| response = stub.Predictions( | ||
| inference_pb2.PredictionsRequest(model_name=model_name, | ||
| input=input_data)) | ||
|
|
||
| try: | ||
| prediction = response.prediction.decode('utf-8') | ||
| print(prediction) | ||
| except grpc.RpcError: | ||
| exit(1) | ||
|
|
||
|
|
||
| def ping(stub): | ||
| response = stub.Ping(inference_pb2.TorchServeHealthResponse()) | ||
| try: | ||
| health = response | ||
| print("Ping Response:", health) | ||
| except grpc.RpcError: | ||
| exit(1) | ||
|
|
||
|
|
||
| def register(stub, model_name, mar_set_str): | ||
| mar_set = set() | ||
| if mar_set_str: | ||
| mar_set = set(mar_set_str.split(',')) | ||
| marfile = f"{model_name}.mar" | ||
| print(f"## Check {marfile} in mar_set :", mar_set) | ||
| if marfile not in mar_set: | ||
| marfile = "https://torchserve.s3.amazonaws.com/mar_files/{}.mar".format( | ||
| model_name) | ||
|
|
||
| print(f"## Register marfile:{marfile}\n") | ||
| params = { | ||
| 'url': marfile, | ||
| 'initial_workers': 1, | ||
| 'synchronous': True, | ||
| 'model_name': model_name | ||
| } | ||
| try: | ||
| stub.RegisterModel(management_pb2.RegisterModelRequest(**params)) | ||
| print(f"Model {model_name} registered successfully") | ||
| except grpc.RpcError as e: | ||
| print(f"Failed to register model {model_name}.") | ||
| print(str(e.details())) | ||
| exit(1) | ||
|
|
||
|
|
||
| def unregister(stub, model_name): | ||
| try: | ||
| stub.UnregisterModel( | ||
| management_pb2.UnregisterModelRequest(model_name=model_name)) | ||
| print(f"Model {model_name} unregistered successfully") | ||
| except grpc.RpcError as e: | ||
| print(f"Failed to unregister model {model_name}.") | ||
| print(str(e.details())) | ||
| exit(1) | ||
|
|
||
|
|
||
| if __name__ == '__main__': | ||
| # args: | ||
| # 1-> api name [infer, register, unregister] | ||
| # 2-> model name | ||
| # 3-> model input for prediction | ||
| args = sys.argv[1:] | ||
| if args[0] == "infer": | ||
| infer(get_inference_stub(), args[1], args[2]) | ||
| elif args[0] == "ping": | ||
| ping(get_inference_stub()) | ||
| else: | ||
| api = globals()[args[0]] | ||
| if args[0] == "register": | ||
| api(get_management_stub(), args[1], args[2]) | ||
| else: | ||
| api(get_management_stub(), args[1]) |
Oops, something went wrong.