feat(cmd-api-server): support grpc web services #1189

Primary change:
--------------
The API server now contains a gRPC server in addition to what it
had before (HTTP+SocketIO servers) so that through this it can
provide the opportunity for plugins to expose their gRPC services
via the Cactus API server. It is possible for plugins to serve
their requests on multiple protocols at the same time which means
that this is a big step towards our goal of being properly multi-
protocol capable.

Secondary change(s):
-------------------
1. Custom protobuf-schema openapi generator template added because
of this issue: thesayyn/protoc-gen-ts#82
(we override the stock template to not use the public keyword)

TODO:
----

1. Implement streaming healthcheck endpiont with gRPC

2. Allow plugins to hook in their own gRPC service implementations.

Fixes #1189

Signed-off-by: Peter Somogyvari <peter.somogyvari@accenture.com>

.cspell.json

@@ -36,6 +36,7 @@
         "gopath",
         "grpc",
         "grpcs",
+        "grpcwebtext",
         "hashicorp",
         "Healthcheck",
         "HTLC",
@@ -71,12 +72,16 @@
         "NODETXPOOLACK",
         "notok",
         "Oidc",
+        "oneofs",
         "onsi",
         "OpenAPI",
         "openethereum",
         "organisation",
         "parameterizable",
         "Postgres",
+        "proto",
+        "protobuf",
+        "protoc",
         "protos",
         "RUSTC",
         "Secp",

.eslintignore

@@ -14,3 +14,5 @@
 # **/coverage/**
 
 # typings/**
+
+**/src/main/typescript/generated/proto/**

examples/cactus-example-carbon-accounting-backend/src/main/typescript/carbon-accounting-app.ts

@@ -231,6 +231,7 @@ export class CarbonAccountingApp {
       config.apiHost = addressInfoApi.address;
       config.cockpitHost = addressInfoCockpit.address;
       config.cockpitPort = addressInfoCockpit.port;
+      config.grpcPort = 0; // TODO - make this configurable as well
       config.logLevel = this.options.logLevel || "INFO";
     }
 

examples/cactus-example-carbon-accounting-backend/src/test/typescript/integration/admin-enroll-v1-endpoint.test.ts

@@ -81,6 +81,7 @@ test(testCase, async (t: Test) => {
   apiSrvOpts.apiCorsDomainCsv = "*";
   apiSrvOpts.apiPort = 0;
   apiSrvOpts.cockpitPort = 0;
+  apiSrvOpts.grpcPort = 0;
   apiSrvOpts.apiTlsEnabled = false;
   apiSrvOpts.plugins = [];
   const convictConfig = configService.newExampleConfigConvict(apiSrvOpts);

examples/cactus-example-carbon-accounting-frontend/package.json

@@ -45,7 +45,6 @@
     "@angular/compiler-cli": "12.1.1",
     "@angular/language-service": "12.1.1",
     "@ionic/angular-toolkit": "2.3.0",
-    "@types/jasminewd2": "2.0.3",
     "@types/node": "12.11.1",
     "codelyzer": "6.0.2",
     "https-browserify": "1.0.0",

examples/cactus-example-supply-chain-backend/src/main/typescript/supply-chain-app.ts

@@ -492,6 +492,7 @@ export class SupplyChainApp {
     properties.apiHost = addressInfoApi.address;
     properties.cockpitHost = addressInfoCockpit.address;
     properties.cockpitPort = addressInfoCockpit.port;
+    properties.grpcPort = 0; // TODO - make this configurable as well
     properties.logLevel = this.options.logLevel || "INFO";
 
     const apiServer = new ApiServer({

examples/cactus-example-supply-chain-frontend/package.json

@@ -46,7 +46,6 @@
     "@angular/compiler-cli": "12.1.1",
     "@angular/language-service": "12.1.1",
     "@ionic/angular-toolkit": "2.3.0",
-    "@types/jasminewd2": "2.0.3",
     "@types/node": "12.11.1",
     "codelyzer": "6.0.2",
     "constants-browserify": "1.0.0",

package.json

@@ -18,7 +18,7 @@
     "start:example-supply-chain": "cd ./examples/supply-chain-app/ && npm i --no-package-lock && npm run start",
     "start:example-carbon-accounting": "CONFIG_FILE=examples/cactus-example-carbon-accounting-backend/example-config.json node examples/cactus-example-carbon-accounting-backend/dist/lib/main/typescript/carbon-accounting-app-cli.js",
     "purge-build-cache": "del-cli .build-cache/*",
-    "clean": "npm run purge-build-cache && del-cli \"./{packages,examples,extensions}/cactus-*/{dist,.nyc_output,src/main/typescript/generated/openapi/typescript-axios/*}\"",
+    "clean": "npm run purge-build-cache && del-cli \"./{packages,examples,extensions}/cactus-*/{dist,.nyc_output,src/main/proto/generated/*,src/main/typescript/generated/proto/protoc-gen-ts/*,src/main/typescript/generated/openapi/typescript-axios/*}\"",
     "lint": "eslint '*/*/src/**/*.{js,ts}' --quiet --fix && cspell \"*/*/src/**/*.{js,ts}\"",
     "tsc": "tsc --build --verbose",
     "codegen": "lerna run codegen",
@@ -78,7 +78,6 @@
     "@commitlint/config-conventional": "8.0.0",
     "@openapitools/openapi-generator-cli": "2.3.3",
     "@types/fs-extra": "9.0.11",
-    "@types/jasminewd2": "2.0.10",
     "@types/node": "15.14.7",
     "@types/node-fetch": "2.5.4",
     "@types/tape": "4.13.0",
@@ -103,6 +102,8 @@
     "fs-extra": "10.0.0",
     "git-cz": "4.7.6",
     "globby": "10.0.2",
+    "grpc-tools": "1.11.2",
+    "grpc_tools_node_protoc_ts": "5.3.1",
     "husky": "4.2.5",
     "inquirer": "8.1.1",
     "json5": "2.2.0",
@@ -117,6 +118,7 @@
     "npm-run-all": "4.1.5",
     "npm-watch": "0.7.0",
     "prettier": "2.0.5",
+    "protoc-gen-ts": "0.4.0",
     "run-time-error": "1.4.0",
     "secp256k1": "4.0.0",
     "shebang-loader": "0.0.1",

packages/cactus-cmd-api-server/README.md

@@ -116,6 +116,7 @@ const main = async () => {
   apiServerOptions.apiCorsDomainCsv = "your.domain.example.com";
   apiServerOptions.apiPort = 3000;
   apiServerOptions.cockpitPort = 3100;
+  apiServerOptions.grpcPort = 5000;
   // Disble TLS (or provide TLS certs for secure HTTP if you are deploying to production)
   apiServerOptions.apiTlsEnabled = false;
   apiServerOptions.plugins = [

packages/cactus-cmd-api-server/package.json

@@ -14,6 +14,9 @@
   "scripts": {
     "generate-sdk": "openapi-generator-cli generate -i ./src/main/json/openapi.json -g typescript-axios -o ./src/main/typescript/generated/openapi/typescript-axios/ --reserved-words-mappings protected=protected",
     "codegen:openapi": "npm run generate-sdk",
+    "proto:openapi": "openapi-generator-cli generate -i ./src/main/json/openapi.json -g protobuf-schema --model-name-suffix=PB --additional-properties=packageName=org.hyperledger.cactus.cmd_api_server -o ./src/main/proto/generated/openapi/ -t=./src/main/openapi-generator/templates/protobuf-schema/",
+    "proto:protoc-gen-ts": "yarn run grpc_tools_node_protoc --plugin=protoc-gen-ts=../../node_modules/.bin/protoc-gen-ts --ts_out=grpc_js:./src/main/typescript/generated/proto/protoc-gen-ts/ --proto_path ./src/main/proto/generated/openapi/ ./src/main/proto/generated/openapi/services/*.proto",
+    "codegen:proto": "run-s proto:openapi proto:protoc-gen-ts",
     "codegen": "run-p 'codegen:*'",
     "watch": "npm-watch",
     "webpack": "npm-run-all webpack:dev webpack:prod",
@@ -72,6 +75,8 @@
   },
   "homepage": "https://github.com/hyperledger/cactus#readme",
   "dependencies": {
+    "@grpc/grpc-js": "1.3.6",
+    "@grpc/proto-loader": "0.6.4",
     "@hyperledger/cactus-common": "0.8.0",
     "@hyperledger/cactus-core": "0.8.0",
     "@hyperledger/cactus-core-api": "0.8.0",
@@ -109,6 +114,7 @@
     "@types/express": "4.17.8",
     "@types/express-http-proxy": "1.6.1",
     "@types/express-jwt": "6.0.1",
+    "@types/google-protobuf": "3.15.3",
     "@types/jsonwebtoken": "8.5.1",
     "@types/multer": "1.4.5",
     "@types/node-forge": "0.9.3",

packages/cactus-cmd-api-server/src/main/openapi-generator/templates/protobuf-schema/api.mustache

@@ -0,0 +1,46 @@
+{{>partial_header}}
+syntax = "proto3";
+
+package {{{packageName}}};
+
+import "google/protobuf/empty.proto";
+{{#imports}}
+{{#import}}
+import "{{{modelPackage}}}/{{{.}}}.proto";
+{{/import}}
+{{/imports}}
+
+service {{classname}} {
+{{#operations}}
+{{#operation}}
+  {{#description}}
+  // {{{.}}}
+  {{/description}}
+  rpc {{operationId}} ({{#hasParams}}{{operationId}}Request{{/hasParams}}{{^hasParams}}google.protobuf.Empty{{/hasParams}}) returns ({{#vendorExtensions.x-grpc-response}}{{.}}{{/vendorExtensions.x-grpc-response}}{{^vendorExtensions.x-grpc-response}}{{operationId}}Response{{/vendorExtensions.x-grpc-response}});
+
+{{/operation}}
+{{/operations}}
+}
+
+{{#operations}}
+{{#operation}}
+{{#hasParams}}
+message {{operationId}}Request {
+  {{#allParams}}
+  {{#description}}
+  // {{{.}}}
+  {{/description}}
+  {{#vendorExtensions.x-protobuf-type}}{{.}} {{/vendorExtensions.x-protobuf-type}}{{vendorExtensions.x-protobuf-data-type}} {{paramName}} = {{vendorExtensions.x-protobuf-index}};
+  {{/allParams}}
+
+}
+
+{{/hasParams}}
+{{^vendorExtensions.x-grpc-response}}
+message {{operationId}}Response {
+  {{{vendorExtensions.x-grpc-response-type}}} data = 1;
+}
+
+{{/vendorExtensions.x-grpc-response}}
+{{/operation}}
+{{/operations}}

packages/cactus-cmd-api-server/src/main/openapi-generator/templates/protobuf-schema/model.mustache

@@ -0,0 +1,37 @@
+{{>partial_header}}
+syntax = "proto3";
+
+package {{{packageName}}};
+
+{{#imports}}
+{{#import}}
+import "{{{modelPackage}}}/{{{import}}}.proto";
+{{/import}}
+{{/imports}}
+
+{{#models}}
+{{#model}}
+message {{classname}} {
+  {{#vars}}
+  {{#description}}
+  // {{{.}}}
+  {{/description}}
+  {{^isEnum}}
+  {{#vendorExtensions.x-protobuf-type}}{{{.}}} {{/vendorExtensions.x-protobuf-type}}{{{vendorExtensions.x-protobuf-data-type}}} {{{name}}} = {{vendorExtensions.x-protobuf-index}}{{#vendorExtensions.x-protobuf-packed}} [packed=true]{{/vendorExtensions.x-protobuf-packed}};
+  {{/isEnum}}
+  {{#isEnum}}
+  enum {{enumName}} {
+    {{#allowableValues}}
+    {{#enumVars}}
+    {{{name}}} = {{{protobuf-enum-index}}};
+    {{/enumVars}}
+    {{/allowableValues}}
+  }
+
+  {{enumName}} {{name}} = {{vendorExtensions.x-protobuf-index}};
+  {{/isEnum}}
+
+  {{/vars}}
+}
+{{/model}}
+{{/models}}

packages/cactus-cmd-api-server/src/main/openapi-generator/templates/protobuf-schema/root.mustache

@@ -0,0 +1,22 @@
+{{>partial_header}}
+syntax = "proto3";
+
+package {{{packageName}}};
+
+{{#vendorExtensions.x-grpc-options}}
+option {{{.}}};
+{{/vendorExtensions.x-grpc-options}}
+
+// Models
+{{#models}}
+{{#model}}
+import "{{modelPackage}}/{{classFilename}}.proto";
+{{/model}}
+{{/models}}
+
+// APIs
+{{#apiInfo}}
+{{#apis}}
+import "{{apiPackage}}/{{classFilename}}.proto";
+{{/apis}}
+{{/apiInfo}}

packages/cactus-cmd-api-server/src/main/proto/generated/openapi/.openapi-generator-ignore

@@ -0,0 +1,23 @@
+# OpenAPI Generator Ignore
+# Generated by openapi-generator https://github.com/openapitools/openapi-generator
+
+# Use this file to prevent files from being overwritten by the generator.
+# The patterns follow closely to .gitignore or .dockerignore.
+
+# As an example, the C# client generator defines ApiClient.cs.
+# You can make changes and tell OpenAPI Generator to ignore just this file by uncommenting the following line:
+#ApiClient.cs
+
+# You can match any string of characters against a directory, file or extension with a single asterisk (*):
+#foo/*/qux
+# The above matches foo/bar/qux and foo/baz/qux, but not foo/bar/baz/qux
+
+# You can recursively match patterns against a directory, file or extension with a double asterisk (**):
+#foo/**/qux
+# This matches foo/bar/qux, foo/baz/qux, and foo/bar/baz/qux
+
+# You can also negate patterns with an exclamation (!).
+# For example, you can ignore all files in a docs folder with the file extension .md:
+#docs/*.md
+# Then explicitly reverse the ignore rule for a single file:
+#!docs/README.md

packages/cactus-cmd-api-server/src/main/proto/generated/openapi/.openapi-generator/FILES

@@ -0,0 +1,6 @@
+.openapi-generator-ignore
+README.md
+models/health_check_response_pb.proto
+models/memory_usage_pb.proto
+models/watch_healthcheck_v1_pb.proto
+services/default_service.proto

packages/cactus-cmd-api-server/src/main/proto/generated/openapi/.openapi-generator/VERSION

@@ -0,0 +1 @@
+5.1.1

packages/cactus-cmd-api-server/src/main/proto/generated/openapi/README.md

@@ -0,0 +1,31 @@
+# gPRC for org.hyperledger.cactus.cmd_api_server
+
+Interact with a Cactus deployment through HTTP.
+
+## Overview
+These files were generated by the [OpenAPI Generator](https://openapi-generator.tech) project. 
+
+- API version: 0.0.1
+- Package version: 
+- Build package: org.openapitools.codegen.languages.ProtobufSchemaCodegen
+
+## Usage
+
+Below are some usage examples for Go and Ruby. For other languages, please refer to https://grpc.io/docs/quickstart/.
+
+### Go
+```
+# assuming `protoc-gen-go` has been installed with `go get -u github.com/golang/protobuf/protoc-gen-go`
+mkdir /var/tmp/go/
+protoc --go_out=/var/tmp/go/ services/*
+protoc --go_out=/var/tmp/go/ models/*
+```
+
+### Ruby
+```
+# assuming `grpc_tools_ruby_protoc` has been installed via `gem install grpc-tools`
+RUBY_OUTPUT_DIR="/var/tmp/ruby/org.hyperledger.cactus.cmd_api_server"
+mkdir $RUBY_OUTPUT_DIR
+grpc_tools_ruby_protoc --ruby_out=$RUBY_OUTPUT_DIR --grpc_out=$RUBY_OUTPUT_DIR/lib services/*
+grpc_tools_ruby_protoc --ruby_out=$RUBY_OUTPUT_DIR --grpc_out=$RUBY_OUTPUT_DIR/lib models/*
+```

packages/cactus-cmd-api-server/src/main/proto/generated/openapi/models/health_check_response_pb.proto

@@ -0,0 +1,24 @@
+/*
+  Hyperledger Cactus API
+ 
+  Interact with a Cactus deployment through HTTP.
+ 
+  The version of the OpenAPI document: 0.0.1
+  
+  Generated by OpenAPI Generator: https://openapi-generator.tech
+*/
+
+syntax = "proto3";
+
+package org.hyperledger.cactus.cmd_api_server;
+
+import "models/memory_usage_pb.proto";
+
+message HealthCheckResponsePB {
+  bool success = 256557056;
+
+  string createdAt = 61500732;
+
+  MemoryUsagePB memoryUsage = 335792418;
+
+}

packages/cactus-cmd-api-server/src/main/proto/generated/openapi/models/memory_usage_pb.proto

@@ -0,0 +1,27 @@
+/*
+  Hyperledger Cactus API
+ 
+  Interact with a Cactus deployment through HTTP.
+ 
+  The version of the OpenAPI document: 0.0.1
+  
+  Generated by OpenAPI Generator: https://openapi-generator.tech
+*/
+
+syntax = "proto3";
+
+package org.hyperledger.cactus.cmd_api_server;
+
+
+message MemoryUsagePB {
+  float rss = 113234;
+
+  float heapTotal = 114487480;
+
+  float heapUsed = 30910521;
+
+  float external = 210148408;
+
+  float arrayBuffers = 116952168;
+
+}

packages/cactus-cmd-api-server/src/main/proto/generated/openapi/models/watch_healthcheck_v1_pb.proto

@@ -0,0 +1,17 @@
+/*
+  Hyperledger Cactus API
+ 
+  Interact with a Cactus deployment through HTTP.
+ 
+  The version of the OpenAPI document: 0.0.1
+  
+  Generated by OpenAPI Generator: https://openapi-generator.tech
+*/
+
+syntax = "proto3";
+
+package org.hyperledger.cactus.cmd_api_server;
+
+
+message WatchHealthcheckV1PB {
+}