refactor(api-server): instantiate plugins in api-server without install them

.taprc

@@ -163,6 +163,8 @@ files:
   - ./packages/cactus-cmd-api-server/src/test/typescript/integration/jwt-unprotected-endpoint-authz.test.ts
   - ./packages/cactus-cmd-api-server/src/test/typescript/integration/jwt-endpoint-authorization.test.ts
   - ./packages/cactus-cmd-api-server/src/test/typescript/integration/jwt-unprotected-endpoint-authz-ops-confirm.test.ts
+  - ./packages/cactus-cmd-api-server/src/test/typescript/integration/plugin-import-from-github.test.ts
+  - ./packages/cactus-cmd-api-server/src/test/typescript/integration/plugin-import-without-install.test.ts
   - ./packages/cactus-cmd-api-server/src/test/typescript/unit/plugins/install-basic-plugin-keychain-memory.test.ts
   - ./packages/cactus-cmd-api-server/src/test/typescript/unit/plugins/install-basic-plugin-ledger-connector-quorum-0-7-0.test.ts
   - ./packages/cactus-cmd-api-server/src/test/typescript/unit/plugins/install-basic-plugin-ledger-connector-fabric-0-7-0.test.ts

packages/cactus-cmd-api-server/README.md

@@ -102,7 +102,7 @@ if (require.main === module) {
 ### Remote Plugin Imports at Runtime Example
 
 ```typescript
-import { PluginImportType } from "@hyperledger/cactus-core-api";
+import { PluginImportType, PluginImportAction } from "@hyperledger/cactus-core-api";
 import { ApiServer } from "@hyperledger/cactus-cmd-api-server";
 import { ConfigService } from "@hyperledger/cactus-cmd-api-server";
 import { Logger, LoggerProvider } from "@hyperledger/cactus-common";
@@ -131,6 +131,9 @@ const main = async () => {
       // being imported by the API server regardless of the language the plugin
       // was written in.
       type: PluginImportType.REMOTE,
+      // The INSTALL value means that the plugin will be installed instead of
+      // only instantiate it
+      action: PluginImportAction.INSTALL,
       // The options that will be passed in to the plugin factory
       options: {
         keychainId: "_keychainId_",
@@ -250,7 +253,7 @@ Once you've built the container, the following commands should work:
     --publish 4000:4000 \
     cas \
       ./node_modules/.bin/cactusapi \
-      --plugins='[{"packageName": "@hyperledger/cactus-plugin-ledger-connector-fabric", "type": "org.hyperledger.cactus.plugin_import_type.LOCAL",  "options": { "connectionProfile": {}, "instanceId": "some-unique-instance-id"}}]'
+      --plugins='[{"packageName": "@hyperledger/cactus-plugin-ledger-connector-fabric", "type": "org.hyperledger.cactus.plugin_import_type.LOCAL", "action": "org.hyperledger.cactus.plugin_import_action.INSTALL",  "options": { "connectionProfile": {}, "instanceId": "some-unique-instance-id"}}]'
   ```
 
 - Launch container with plugin configuration as an **environment variable**:
@@ -259,7 +262,7 @@ Once you've built the container, the following commands should work:
     --rm \
     --publish 3000:3000 \
     --publish 4000:4000 \
-    --env PLUGINS='[{"packageName": "@hyperledger/cactus-plugin-ledger-connector-besu", "type": "org.hyperledger.cactus.plugin_import_type.LOCAL",  "options": {"rpcApiHttpHost": "http://localhost:8545", "instanceId": "some-unique-besu-connector-instance-id"}}]' \
+    --env PLUGINS='[{"packageName": "@hyperledger/cactus-plugin-ledger-connector-besu", "type": "org.hyperledger.cactus.plugin_import_type.LOCAL", "action": "org.hyperledger.cactus.plugin_import_action.INSTALL",  "options": {"rpcApiHttpHost": "http://localhost:8545", "instanceId": "some-unique-besu-connector-instance-id"}}]' \
     cas
   ```
 
@@ -271,13 +274,13 @@ Once you've built the container, the following commands should work:
     --publish 4000:4000 \
     cas \
       ./node_modules/.bin/cactusapi \
-      --plugins='[{"packageName": "@hyperledger/cactus-plugin-ledger-connector-besu", "type": "org.hyperledger.cactus.plugin_import_type.LOCAL",  "options": {"rpcApiHttpHost": "http://localhost:8545", "instanceId": "some-unique-besu-connector-instance-id"}}]'
+      --plugins='[{"packageName": "@hyperledger/cactus-plugin-ledger-connector-besu", "type": "org.hyperledger.cactus.plugin_import_type.LOCAL", "action": "org.hyperledger.cactus.plugin_import_action.INSTALL",  "options": {"rpcApiHttpHost": "http://localhost:8545", "instanceId": "some-unique-besu-connector-instance-id"}}]'
   ```
 
 - Launch container with **configuration file** mounted from host machine:
   ```sh
 
-  echo '[{"packageName": "@hyperledger/cactus-plugin-ledger-connector-besu", "type": "org.hyperledger.cactus.plugin_import_type.LOCAL",  "options": {"rpcApiHttpHost": "http://localhost:8545", "instanceId": "some-unique-besu-connector-instance-id"}}]' > cactus.json
+  echo '[{"packageName": "@hyperledger/cactus-plugin-ledger-connector-besu", "type": "org.hyperledger.cactus.plugin_import_type.LOCAL", "action": "org.hyperledger.cactus.plugin_import_action.INSTALL",  "options": {"rpcApiHttpHost": "http://localhost:8545", "instanceId": "some-unique-besu-connector-instance-id"}}]' > cactus.json
 
   docker run \
     --rm \
@@ -305,7 +308,7 @@ Don't have a Besu network on hand to test with? Test or develop against our Besu
       --rm \
       --publish 3000:3000 \
       --publish 4000:4000 \
-      --env PLUGINS='[{"packageName": "@hyperledger/cactus-plugin-ledger-connector-besu", "type": "org.hyperledger.cactus.plugin_import_type.LOCAL",  "options": {"rpcApiHttpHost": "http://localhost:8545", "instanceId": "some-unique-besu-connector-instance-id"}}]' \
+      --env PLUGINS='[{"packageName": "@hyperledger/cactus-plugin-ledger-connector-besu", "type": "org.hyperledger.cactus.plugin_import_type.LOCAL", "action": "org.hyperledger.cactus.plugin_import_action.INSTALL",  "options": {"rpcApiHttpHost": "http://localhost:8545", "instanceId": "some-unique-besu-connector-instance-id"}}]' \
       cas
     ```
 

packages/cactus-cmd-api-server/src/main/typescript/api-server.ts

@@ -33,6 +33,7 @@ import {
   PluginFactoryFactory,
   PluginImport,
   Constants,
+  PluginImportAction,
 } from "@hyperledger/cactus-core-api";
 
 import { PluginRegistry } from "@hyperledger/cactus-core";
@@ -279,19 +280,26 @@ export class ApiServer {
   }
 
   public async getOrInitPluginRegistry(): Promise<PluginRegistry> {
-    if (!this.pluginRegistry) {
-      if (!this.options.pluginRegistry) {
-        this.log.info(`getOrInitPluginRegistry() initializing a new one...`);
-        this.pluginRegistry = await this.initPluginRegistry();
-      } else {
-        this.log.info(`getOrInitPluginRegistry() re-using injected one...`);
-        this.pluginRegistry = this.options.pluginRegistry;
+    try {
+      if (!this.pluginRegistry) {
+        if (!this.options.pluginRegistry) {
+          this.log.info(`getOrInitPluginRegistry() initializing a new one...`);
+          this.pluginRegistry = await this.initPluginRegistry();
+        } else {
+          this.log.info(`getOrInitPluginRegistry() re-using injected one...`);
+          this.pluginRegistry = this.options.pluginRegistry;
+        }
       }
+      await this.prometheusExporter.setTotalPluginImports(
+        await this.getPluginImportsCount(),
+      );
+      return this.pluginRegistry;
+    } catch (e) {
+      this.pluginRegistry = new PluginRegistry({ plugins: [] });
+      const errorMessage = `Failed init PluginRegistry: ${e.stack}`;
+      this.log.error(errorMessage);
+      throw new Error(errorMessage);
     }
-    await this.prometheusExporter.setTotalPluginImports(
-      await this.getPluginImportsCount(),
-    );
-    return this.pluginRegistry;
   }
 
   public async initPluginRegistry(): Promise<PluginRegistry> {
@@ -311,43 +319,44 @@ export class ApiServer {
     pluginImport: PluginImport,
     registry: PluginRegistry,
   ): Promise<ICactusPlugin> {
+    const fnTag = `${this.className}#instantiatePlugin`;
     const { logLevel } = this.options.config;
     const { packageName, options } = pluginImport;
     this.log.info(`Creating plugin from package: ${packageName}`, options);
     const pluginOptions = { ...options, logLevel, pluginRegistry: registry };
 
-    await this.installPluginPackage(pluginImport);
-
-    const packagePath = path.join(
-      this.pluginsPath,
-      options.instanceId,
-      "node_modules",
-      packageName,
-    );
-    this.log.debug("Package path: %o", packagePath);
-
-    // eslint-disable-next-line @typescript-eslint/no-var-requires
-    const pluginPackage = require(/* webpackIgnore: true */ packagePath);
-    const createPluginFactory = pluginPackage.createPluginFactory as PluginFactoryFactory;
-
-    const pluginFactoryOptions: IPluginFactoryOptions = {
-      pluginImportType: pluginImport.type,
-    };
-
-    const pluginFactory = await createPluginFactory(pluginFactoryOptions);
-
-    const plugin = await pluginFactory.create(pluginOptions);
-
-    // need to invoke the i-cactus-plugin onPluginInit functionality here before plugin registry can be used further
     try {
-      await plugin.onPluginInit();
-    } catch (error) {
-      const fnTag = `${this.className}#instantiatePlugin`;
-      const packageName = plugin.getPackageName();
-      const instanceId = plugin.getInstanceId();
+      if (pluginImport.action == PluginImportAction.Install) {
+        await this.installPluginPackage(pluginImport);
+      } else {
+        this.log.info(
+          `The installation of the plugin package ${packageName} was skipped due to the configuration flag action`,
+        );
+      }
 
-      const errorMessage = `${fnTag} failed calling onPluginInit() on the plugin '${packageName}' with the instanceId '${instanceId}'`;
+      const packagePath = path.join(
+        this.pluginsPath,
+        options.instanceId,
+        "node_modules",
+        packageName,
+      );
+      this.log.debug("Package path: %o", packagePath);
+
+      // eslint-disable-next-line @typescript-eslint/no-var-requires
+      const pluginPackage = require(/* webpackIgnore: true */ packagePath);
+      const createPluginFactory = pluginPackage.createPluginFactory as PluginFactoryFactory;
+      const pluginFactoryOptions: IPluginFactoryOptions = {
+        pluginImportType: pluginImport.type,
+      };
+      const pluginFactory = await createPluginFactory(pluginFactoryOptions);
+      const plugin = await pluginFactory.create(pluginOptions);
+
+      // need to invoke the i-cactus-plugin onPluginInit functionality here before plugin registry can be used further
+      await plugin.onPluginInit();
 
+      return plugin;
+    } catch (error) {
+      const errorMessage = `${fnTag} failed instantiating plugin '${packageName}' with the instanceId '${options.instanceId}'`;
       this.log.error(errorMessage, error);
 
       if (error instanceof Error) {
@@ -356,15 +365,15 @@ export class ApiServer {
         throw new RuntimeError(errorMessage, JSON.stringify(error));
       }
     }
-
-    return plugin;
   }
 
   private async installPluginPackage(
     pluginImport: PluginImport,
   ): Promise<void> {
     const fnTag = `ApiServer#installPluginPackage()`;
-    const { packageName: pkgName } = pluginImport;
+    const pkgName = pluginImport.options.packageSrc
+      ? pluginImport.options.packageSrc
+      : pluginImport.packageName;
 
     const instanceId = pluginImport.options.instanceId;
     const pluginPackageDir = path.join(this.pluginsPath, instanceId);
@@ -401,7 +410,14 @@ export class ApiServer {
       }
       this.log.info(`Installed ${pkgName} OK`);
     } catch (ex) {
-      throw new RuntimeError(`${fnTag} plugin install fail: ${pkgName}`, ex);
+      const errorMessage = `${fnTag} failed installing plugin '${pkgName}`;
+      this.log.error(errorMessage, ex);
+
+      if (ex instanceof Error) {
+        throw new RuntimeError(errorMessage, ex);
+      } else {
+        throw new RuntimeError(errorMessage, JSON.stringify(ex));
+      }
     }
   }
 
@@ -654,9 +670,6 @@ export class ApiServer {
       this.log.info(`Authorization request handler configured OK.`);
     }
 
-    // const openApiValidator = this.createOpenApiValidator();
-    // await openApiValidator.install(app);
-
     this.getOrCreateWebServices(app); // The API server's own endpoints
 
     this.log.info(`Starting to install web services...`);

packages/cactus-cmd-api-server/src/main/typescript/config/config-service.ts

@@ -15,6 +15,7 @@ import {
   Constants,
   PluginImport,
   PluginImportType,
+  PluginImportAction,
 } from "@hyperledger/cactus-core-api";
 
 import { FORMAT_PLUGIN_ARRAY } from "./convict-plugin-array-format";
@@ -468,10 +469,12 @@ export class ConfigService {
 
   public newExampleConfigConvict(
     cactusApiServerOptions?: ICactusApiServerOptions,
+    overrides?: boolean,
   ): Config<ICactusApiServerOptions> {
     cactusApiServerOptions = cactusApiServerOptions || this.newExampleConfig();
     const env = this.newExampleConfigEnv(cactusApiServerOptions);
-    return this.getOrCreate({ env });
+    const conf = overrides ? this.create({ env }) : this.getOrCreate({ env });
+    return conf;
   }
 
   public newExampleConfig(): ICactusApiServerOptions {
@@ -532,6 +535,7 @@ export class ConfigService {
       {
         packageName: "@hyperledger/cactus-plugin-keychain-memory",
         type: PluginImportType.Local,
+        action: PluginImportAction.Install,
         options: {
           instanceId: uuidV4(),
           keychainId: uuidV4(),
@@ -540,6 +544,7 @@ export class ConfigService {
       {
         packageName: "@hyperledger/cactus-plugin-consortium-manual",
         type: PluginImportType.Local,
+        action: PluginImportAction.Install,
         options: {
           instanceId: uuidV4(),
           keyPairPem,
@@ -610,26 +615,34 @@ export class ConfigService {
     args?: string[];
   }): Config<ICactusApiServerOptions> {
     if (!ConfigService.config) {
-      const schema: Schema<ICactusApiServerOptions> = ConfigService.getConfigSchema();
-      ConfigService.config = (convict as any)(schema, options);
-      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-      // @ts-expect-error
-      if (ConfigService.config.get("configFile")) {
-        const configFilePath = ConfigService.config.get("configFile");
-        ConfigService.config.loadFile(configFilePath);
-      }
-      ConfigService.config.validate();
-      this.validateKeyPairMatch();
-      const level = ConfigService.config.get("logLevel");
-      const logger: Logger = LoggerProvider.getOrCreate({
-        label: "config-service",
-        level,
-      });
-      logger.info("Configuration validation OK.");
+      this.create(options);
     }
     return ConfigService.config;
   }
 
+  create(options?: {
+    env?: NodeJS.ProcessEnv;
+    args?: string[];
+  }): Config<ICactusApiServerOptions> {
+    const schema: Schema<ICactusApiServerOptions> = ConfigService.getConfigSchema();
+    ConfigService.config = (convict as any)(schema, options);
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-expect-error
+    if (ConfigService.config.get("configFile")) {
+      const configFilePath = ConfigService.config.get("configFile");
+      ConfigService.config.loadFile(configFilePath);
+    }
+    ConfigService.config.validate();
+    this.validateKeyPairMatch();
+    const level = ConfigService.config.get("logLevel");
+    const logger: Logger = LoggerProvider.getOrCreate({
+      label: "config-service",
+      level,
+    });
+    logger.info("Configuration validation OK.");
+    return ConfigService.config;
+  }
+
   /**
    * Validation that prevents operators from mistakenly deploying a key pair
    * that they may not be operational for whatever reason.

packages/cactus-cmd-api-server/src/test/typescript/benchmark/artillery-api-benchmark.test.ts

@@ -13,6 +13,7 @@ import { LogLevelDesc, LoggerProvider } from "@hyperledger/cactus-common";
 import {
   PluginImportType,
   ConsortiumDatabase,
+  PluginImportAction,
 } from "@hyperledger/cactus-core-api";
 
 import {
@@ -71,6 +72,7 @@ test("Start API server, and run Artillery benchmark test.", async (t: Test) => {
     {
       packageName: "@hyperledger/cactus-plugin-keychain-memory",
       type: PluginImportType.Local,
+      action: PluginImportAction.Install,
       options: {
         instanceId: uuidv4(),
         keychainId: uuidv4(),
@@ -80,6 +82,7 @@ test("Start API server, and run Artillery benchmark test.", async (t: Test) => {
     {
       packageName: "@hyperledger/cactus-plugin-consortium-manual",
       type: PluginImportType.Local,
+      action: PluginImportAction.Install,
       options: {
         instanceId: uuidv4(),
         keyPairPem: keyPairPem,

packages/cactus-cmd-api-server/src/test/typescript/integration/jwt-endpoint-authorization.test.ts

@@ -14,6 +14,7 @@ import { LoggerProvider, LogLevelDesc } from "@hyperledger/cactus-common";
 import {
   Configuration,
   ConsortiumDatabase,
+  PluginImportAction,
   PluginImportType,
 } from "@hyperledger/cactus-core-api";
 import { AuthorizationProtocol } from "../../../main/typescript/config/authorization-protocol";
@@ -88,6 +89,7 @@ test(testCase, async (t: Test) => {
       {
         packageName: "@hyperledger/cactus-plugin-keychain-memory",
         type: PluginImportType.Local,
+        action: PluginImportAction.Install,
         options: {
           instanceId: uuidv4(),
           keychainId: uuidv4(),
@@ -97,6 +99,7 @@ test(testCase, async (t: Test) => {
       {
         packageName: "@hyperledger/cactus-plugin-consortium-manual",
         type: PluginImportType.Local,
+        action: PluginImportAction.Install,
         options: {
           instanceId: uuidv4(),
           keyPairPem: keyPairPem,