diff --git a/.cspell.json b/.cspell.json index 46f543aa46..d806c19568 100644 --- a/.cspell.json +++ b/.cspell.json @@ -8,6 +8,7 @@ "Albertirsa", "ALLFORTX", "ANYFORTX", + "AWSSM", "APIV", "approveformyorg", "Authz", diff --git a/packages/cactus-plugin-keychain-aws-sm/README.md b/packages/cactus-plugin-keychain-aws-sm/README.md index b04e984095..2671205bac 100644 --- a/packages/cactus-plugin-keychain-aws-sm/README.md +++ b/packages/cactus-plugin-keychain-aws-sm/README.md @@ -1,3 +1,167 @@ # `@hyperledger/cactus-plugin-keychain-aws-sm` -## TO-DO \ No newline at end of file +- [`@hyperledger/cactus-plugin-keychain-aws-sm`](#hyperledgercactus-plugin-keychain-aws-sm) + - [1. Usage](#1-usage) + - [1.1. Installation](#11-installation) + - [1.2. Using as a Library](#12-using-as-a-library) + - [1.3. Using via the API Client](#13-using-via-the-api-client) + - [2. Architecture](#2-architecture) + - [2.1. set-keychain-entry-endpoint](#21-set-keychain-entry-endpoint) + - [2.2. get-keychain-entry-endpoint](#22-get-keychain-entry-endpoint) + - [2.3. has-keychain-entry-endpoint](#23-has-keychain-entry-endpoint) + - [2.4. delete-keychain-entry-endpoint](#24-delete-keychain-entry-endpoint) + - [3. Monitoring](#3-monitoring) + - [3.1. Prometheus Exporter](#31-prometheus-exporter) + - [3.1.1. Usage Prometheus](#311-usage-prometheus) + - [3.1.2. Prometheus Integration](#312-prometheus-integration) + - [3.1.3. Helper code](#313-helper-code) + - [3.1.3.1. response.type.ts](#3131-responsetypets) + - [3.1.3.2. data-fetcher.ts](#3132-data-fetcherts) + - [3.1.3.3. metrics.ts](#3133-metricsts) + - [4. Contributing](#4-contributing) + - [5. License](#5-license) + - [6. Acknowledgments](#6-acknowledgments) +## 1. Usage + +This plugin provides a way to interact with the AWS Secrets Manager. +Using this one can perform: +* Set key,value pair +* Get value for a particular key +* Check if a certain key exists +* Delete a certain key,value pair + +The above functionality can either be accessed by importing hte plugin directly as a library (embedding) or by hosting it as a REST API through the [Cactus API server](https://www.npmjs.com/package/@hyperledger/cactus-cmd-api-server) + +We also publish the [Cactus API server as a container image](https://github.com/hyperledger/cactus/pkgs/container/cactus-cmd-api-server) to the Github Container Registry that you can run easily with a one liner. +The API server is also embeddable in your own NodeJS project if you choose to do so. + +### 1.1. Installation + +**npm** + +```sh +npm install @hyperledger/cactus-plugin-keychain-aws-sm +``` + +**yarn** + +```sh +yarn add @hyperledger/cactus-plugin-keychain-aws-sm +``` + +### 1.2. Using as a Library + +```typescript +import { + PluginKeychainAwsSm, + AwsCredentialType, +} from "@hyperledger/cactus-plugin-keychain-aws-sm"; + +const plugin = new PluginKeychainAwsSm({ + // See test cases for exact details on what parameters are needed +}); + +const res = await plugin.get( + // See function definition for exact details on what parameters are needed and the corresponding output +); +``` + +### 1.3. Using via the API Client + +**Prerequisites** +- An AWS account with access to AWS Secrets Manager +- You have a running Cactus API server on `$HOST:$PORT` with the AWS Secrets Manager connector plugin installed on it (and the latter configured to have access to the AWS Secrets manager from point 1) + +```typescript +import { + PluginKeychainAwsSm, + AwsCredentialType, + DefaultApi as KeychainAwsSmApi, +} from "@hyperledger/cactus-plugin-keychain-aws-sm"; + +// Step zero is to deploy the Cactus API server +const apiUrl = `https://${HOST}:${PORT}`; + +const config = new Configuration({ basePath: apiUrl }); + +const apiClient = new KeychainAwsSmApi(config); + +// Example: To set a key,value pair +const res = await apiClient.setKeychainEntryV1({ + key: key, + value: value, +}); +``` + +## 2. Architecture +The sequence diagrams for various endpoints are mentioned below + +### 2.1. set-keychain-entry-endpoint + +![set-keychain-entry-endpoint sequence diagram](docs/architecture/images/set-keychain-entry-endpoint.png) + +### 2.2. get-keychain-entry-endpoint + +![get-keychain-entry-endpoint sequence diagram](docs/architecture/images/get-keychain-entry-endpoint.png) + +### 2.3. has-keychain-entry-endpoint + +![has-keychain-entry-endpoint sequence diagram](docs/architecture/images/has-keychain-entry-endpoint.png) + +### 2.4. delete-keychain-entry-endpoint + +![delete-keychain-entry-endpoint sequence diagram](docs/architecture/images/delete-keychain-entry-endpoint.png) + +## 3. Monitoring +This section explains various monitoring tools used +### 3.1. Prometheus Exporter + +This creates a prometheus exporter, which scraps the transactions (total transaction count) for the use cases incorporating the use of AWS Secret Manager connector plugin. + + +#### 3.1.1. Usage Prometheus +The prometheus exporter object is initialized in the `PluginKeychainAwsSm` class constructor itself, so instantiating the object of the `PluginKeychainAwsSm` class, gives access to the exporter object. +You can also initialize the prometheus exporter object seperately and then pass it to the `IPluginKeychainAwsSmOptions` interface for `PluginKeychainAwsSm` constructor. + +`getPrometheusExporterMetricsEndpointV1` function returns the prometheus exporter metrics, currently displaying the total transaction count, which currently increments everytime the `set()` method of the `PluginKeychainAwsSm` class is called and decreases everytime the `delete()` method of the `PluginKeychainAwsSm` class is called. + +#### 3.1.2. Prometheus Integration +To use Prometheus with this exporter make sure to install [Prometheus main component](https://prometheus.io/download/). +Once Prometheus is setup, the corresponding scrape_config needs to be added to the prometheus.yml + +```(yaml) +- job_name: 'aws_sm_exporter' + metrics_path: 'api/v1/plugins/@hyperledger/cactus-plugin-keychain-aws-sm/get-prometheus-exporter-metrics' + scrape_interval: 5s + static_configs: + - targets: ['{host}:{port}'] +``` + +Here the `host:port` is where the prometheus exporter metrics are exposed. The test cases (For example, packages/cactus-plugin-keychain-aws-sm/src/test/typescript/integration/plugin-keychain-aws-sm.test.ts) exposes it over `0.0.0.0` and a random port(). The random port can be found in the running logs of the test case and looks like (42379 in the below mentioned URL) +`Metrics URL: http://0.0.0.0:42379/api/v1/plugins/@hyperledger/cactus-plugin-keychain-aws-sm/get-prometheus-exporter-metrics` + +Once edited, you can start the prometheus service by referencing the above edited prometheus.yml file. +On the prometheus graphical interface (defaulted to http://localhost:9090), choose **Graph** from the menu bar, then select the **Console** tab. From the **Insert metric at cursor** drop down, select **cactus_keychain_awssm_managed_key_count** and click **execute** + +#### 3.1.3. Helper code + +##### 3.1.3.1. response.type.ts +This file contains the various responses of the metrics. + +##### 3.1.3.2. data-fetcher.ts +This file contains functions encasing the logic to process the data points + +##### 3.1.3.3. metrics.ts +This file lists all the prometheus metrics and what they are used for. + +## 4. Contributing + +We welcome contributions to Hyperledger Cactus in many forms, and there’s always plenty to do! + +Please review [CONTIRBUTING.md](../../CONTRIBUTING.md) to get started. + +## 5. License + +This distribution is published under the Apache License Version 2.0 found in the [LICENSE](../../LICENSE) file. + +## 6. Acknowledgments diff --git a/packages/cactus-plugin-keychain-aws-sm/docs/architecture/delete-keychain-entry-endpoint.puml b/packages/cactus-plugin-keychain-aws-sm/docs/architecture/delete-keychain-entry-endpoint.puml new file mode 100644 index 0000000000..d68fee4100 --- /dev/null +++ b/packages/cactus-plugin-keychain-aws-sm/docs/architecture/delete-keychain-entry-endpoint.puml @@ -0,0 +1,31 @@ +@startuml Sequence Diagram - Transaction + +title Hyperledger Cactus\nSequence Diagram\nDelete Keychain Entry Endpoint + +skinparam sequenceArrowThickness 2 +skinparam roundcorner 20 +skinparam maxmessagesize 120 +skinparam sequenceParticipant underline + +box "Users" #LightBlue +actor "User A" as a +end box + +box "Hyperledger Cactus" #LightGray +entity "API Client" as apic +entity "API Server" as apis +end box + +box "AWS SM Connector" #LightGreen +database "AWS SM" as awssm +end box + +a --> apic : Tx DeleteKeychainEntryV1 +apic --> apis: Request +apis --> awssm: delete(key,value) +awssm -> awssm: awsClient = getAwsClient() +awssm -> awssm: await awsClient.deleteSecret() +awssm --> apis: Response +apis --> apic: Formatted Response +apic --> a: DetKeychainEntryResponse +@enduml \ No newline at end of file diff --git a/packages/cactus-plugin-keychain-aws-sm/docs/architecture/get-keychain-entry-endpoint.puml b/packages/cactus-plugin-keychain-aws-sm/docs/architecture/get-keychain-entry-endpoint.puml new file mode 100644 index 0000000000..195b74be5e --- /dev/null +++ b/packages/cactus-plugin-keychain-aws-sm/docs/architecture/get-keychain-entry-endpoint.puml @@ -0,0 +1,35 @@ +@startuml Sequence Diagram - Transaction + +title Hyperledger Cactus\nSequence Diagram\nGet Keychain Entry Endpoint + +skinparam sequenceArrowThickness 2 +skinparam roundcorner 20 +skinparam maxmessagesize 120 +skinparam sequenceParticipant underline + +box "Users" #LightBlue +actor "User A" as a +end box + +box "Hyperledger Cactus" #LightGray +entity "API Client" as apic +entity "API Server" as apis +end box + +box "AWS SM Connector" #LightGreen +database "AWS SM" as awssm +end box + +a --> apic : Tx GetKeychainEntryV1 +apic --> apis: Request +apis --> awssm: get(key,value) +awssm -> awssm: awsClient = getAwsClient() +group #Yellow try { await awsClient.getSecretValue() } + awssm -> apis: True +else #LightCoral catch(ex) + awssm -> apis: error= Invalid response received from AWS SecretsManager. Expected "response.SecretString" property chain to be truthy +end +awssm --> apis: Response +apis --> apic: Formatted Response +apic --> a: GetKeychainEntryResponse +@enduml \ No newline at end of file diff --git a/packages/cactus-plugin-keychain-aws-sm/docs/architecture/has-keychain-entry-endpoint.puml b/packages/cactus-plugin-keychain-aws-sm/docs/architecture/has-keychain-entry-endpoint.puml new file mode 100644 index 0000000000..12cf41debd --- /dev/null +++ b/packages/cactus-plugin-keychain-aws-sm/docs/architecture/has-keychain-entry-endpoint.puml @@ -0,0 +1,35 @@ +@startuml Sequence Diagram - Transaction + +title Hyperledger Cactus\nSequence Diagram\nHas Keychain Entry Endpoint + +skinparam sequenceArrowThickness 2 +skinparam roundcorner 20 +skinparam maxmessagesize 120 +skinparam sequenceParticipant underline + +box "Users" #LightBlue +actor "User A" as a +end box + +box "Hyperledger Cactus" #LightGray +entity "API Client" as apic +entity "API Server" as apis +end box + +box "AWS SM Connector" #LightGreen +database "AWS SM" as awssm +end box + +a --> apic : Tx HasKeychainEntryV1 +apic --> apis: Request +apis --> awssm: set(key,value) +awssm -> awssm: awsClient = getAwsClient() +group #Yellow try { await awsClient.describeSecret() } + awssm -> apis: True +else #LightCoral catch(ex) + awssm -> apis: error: Secrets Manager can't find the specified secret +end +awssm --> apis: Response +apis --> apic: Formatted Response +apic --> a: SetKeychainEntryResponse +@enduml \ No newline at end of file diff --git a/packages/cactus-plugin-keychain-aws-sm/docs/architecture/images/delete-keychain-entry-endpoint.png b/packages/cactus-plugin-keychain-aws-sm/docs/architecture/images/delete-keychain-entry-endpoint.png new file mode 100644 index 0000000000..a41ae3d7aa Binary files /dev/null and b/packages/cactus-plugin-keychain-aws-sm/docs/architecture/images/delete-keychain-entry-endpoint.png differ diff --git a/packages/cactus-plugin-keychain-aws-sm/docs/architecture/images/get-keychain-entry-endpoint.png b/packages/cactus-plugin-keychain-aws-sm/docs/architecture/images/get-keychain-entry-endpoint.png new file mode 100644 index 0000000000..6bcdf9429d Binary files /dev/null and b/packages/cactus-plugin-keychain-aws-sm/docs/architecture/images/get-keychain-entry-endpoint.png differ diff --git a/packages/cactus-plugin-keychain-aws-sm/docs/architecture/images/has-keychain-entry-endpoint.png b/packages/cactus-plugin-keychain-aws-sm/docs/architecture/images/has-keychain-entry-endpoint.png new file mode 100644 index 0000000000..134da89db1 Binary files /dev/null and b/packages/cactus-plugin-keychain-aws-sm/docs/architecture/images/has-keychain-entry-endpoint.png differ diff --git a/packages/cactus-plugin-keychain-aws-sm/docs/architecture/images/set-keychain-entry-endpoint.png b/packages/cactus-plugin-keychain-aws-sm/docs/architecture/images/set-keychain-entry-endpoint.png new file mode 100644 index 0000000000..3270978274 Binary files /dev/null and b/packages/cactus-plugin-keychain-aws-sm/docs/architecture/images/set-keychain-entry-endpoint.png differ diff --git a/packages/cactus-plugin-keychain-aws-sm/docs/architecture/set-keychain-entry-endpoint.puml b/packages/cactus-plugin-keychain-aws-sm/docs/architecture/set-keychain-entry-endpoint.puml new file mode 100644 index 0000000000..e63b1a25c2 --- /dev/null +++ b/packages/cactus-plugin-keychain-aws-sm/docs/architecture/set-keychain-entry-endpoint.puml @@ -0,0 +1,31 @@ +@startuml Sequence Diagram - Transaction + +title Hyperledger Cactus\nSequence Diagram\nSet Keychain Entry Endpoint + +skinparam sequenceArrowThickness 2 +skinparam roundcorner 20 +skinparam maxmessagesize 120 +skinparam sequenceParticipant underline + +box "Users" #LightBlue +actor "User A" as a +end box + +box "Hyperledger Cactus" #LightGray +entity "API Client" as apic +entity "API Server" as apis +end box + +box "AWS SM Connector" #LightGreen +database "AWS SM" as awssm +end box + +a --> apic : Tx SetKeychainEntryV1 +apic --> apis: Request +apis --> awssm: set(key,value) +awssm -> awssm: awsClient = getAwsClient() +awssm -> awssm: await awsClient.createSecret() +awssm --> apis: Response +apis --> apic: Formatted Response +apic --> a: SetKeychainEntryResponse +@enduml \ No newline at end of file diff --git a/packages/cactus-plugin-keychain-aws-sm/src/main/json/openapi.json b/packages/cactus-plugin-keychain-aws-sm/src/main/json/openapi.json index 626064c7a4..d991dfb3a1 100644 --- a/packages/cactus-plugin-keychain-aws-sm/src/main/json/openapi.json +++ b/packages/cactus-plugin-keychain-aws-sm/src/main/json/openapi.json @@ -11,6 +11,10 @@ }, "components": { "schemas": { + "PrometheusExporterMetricsResponse": { + "type": "string", + "nullable": false + }, "GetSecretRequest": { "type": "string", "nullable": false, @@ -145,6 +149,31 @@ } } } + }, + "/api/v1/plugins/@hyperledger/cactus-plugin-keychain-aws-sm/get-prometheus-exporter-metrics": { + "get": { + "x-hyperledger-cactus": { + "http": { + "verbLowerCase": "get", + "path": "/api/v1/plugins/@hyperledger/cactus-plugin-keychain-aws-sm/get-prometheus-exporter-metrics" + } + }, + "operationId": "getPrometheusMetricsV1", + "summary": "Get the Prometheus Metrics", + "parameters": [], + "responses": { + "200": { + "description": "OK", + "content": { + "text/plain": { + "schema": { + "$ref": "#/components/schemas/PrometheusExporterMetricsResponse" + } + } + } + } + } + } } } } diff --git a/packages/cactus-plugin-keychain-aws-sm/src/main/typescript/generated/openapi/typescript-axios/api.ts b/packages/cactus-plugin-keychain-aws-sm/src/main/typescript/generated/openapi/typescript-axios/api.ts index b52cb7f1ca..4353d01b22 100644 --- a/packages/cactus-plugin-keychain-aws-sm/src/main/typescript/generated/openapi/typescript-axios/api.ts +++ b/packages/cactus-plugin-keychain-aws-sm/src/main/typescript/generated/openapi/typescript-axios/api.ts @@ -228,6 +228,36 @@ export const DefaultApiAxiosParamCreator = function (configuration?: Configurati options: localVarRequestOptions, }; }, + /** + * + * @summary Get the Prometheus Metrics + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + getPrometheusMetricsV1: async (options: any = {}): Promise => { + const localVarPath = `/api/v1/plugins/@hyperledger/cactus-plugin-keychain-aws-sm/get-prometheus-exporter-metrics`; + // use dummy base URL string because the URL constructor only accepts absolute URLs. + const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); + let baseOptions; + if (configuration) { + baseOptions = configuration.baseOptions; + } + + const localVarRequestOptions = { method: 'GET', ...baseOptions, ...options}; + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + + + setSearchParams(localVarUrlObj, localVarQueryParameter, options.query); + let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; + localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + + return { + url: toPathString(localVarUrlObj), + options: localVarRequestOptions, + }; + }, /** * * @summary Checks that an entry exists under a key on the keychain backend @@ -332,6 +362,16 @@ export const DefaultApiFp = function(configuration?: Configuration) { const localVarAxiosArgs = await localVarAxiosParamCreator.getKeychainEntryV1(getKeychainEntryRequestV1, options); return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration); }, + /** + * + * @summary Get the Prometheus Metrics + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + async getPrometheusMetricsV1(options?: any): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.getPrometheusMetricsV1(options); + return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration); + }, /** * * @summary Checks that an entry exists under a key on the keychain backend @@ -384,6 +424,15 @@ export const DefaultApiFactory = function (configuration?: Configuration, basePa getKeychainEntryV1(getKeychainEntryRequestV1: GetKeychainEntryRequestV1, options?: any): AxiosPromise { return localVarFp.getKeychainEntryV1(getKeychainEntryRequestV1, options).then((request) => request(axios, basePath)); }, + /** + * + * @summary Get the Prometheus Metrics + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + getPrometheusMetricsV1(options?: any): AxiosPromise { + return localVarFp.getPrometheusMetricsV1(options).then((request) => request(axios, basePath)); + }, /** * * @summary Checks that an entry exists under a key on the keychain backend @@ -438,6 +487,17 @@ export class DefaultApi extends BaseAPI { return DefaultApiFp(this.configuration).getKeychainEntryV1(getKeychainEntryRequestV1, options).then((request) => request(this.axios, this.basePath)); } + /** + * + * @summary Get the Prometheus Metrics + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof DefaultApi + */ + public getPrometheusMetricsV1(options?: any) { + return DefaultApiFp(this.configuration).getPrometheusMetricsV1(options).then((request) => request(this.axios, this.basePath)); + } + /** * * @summary Checks that an entry exists under a key on the keychain backend diff --git a/packages/cactus-plugin-keychain-aws-sm/src/main/typescript/plugin-keychain-aws-sm.ts b/packages/cactus-plugin-keychain-aws-sm/src/main/typescript/plugin-keychain-aws-sm.ts index 5dcd30dd04..0c1cff2e0d 100644 --- a/packages/cactus-plugin-keychain-aws-sm/src/main/typescript/plugin-keychain-aws-sm.ts +++ b/packages/cactus-plugin-keychain-aws-sm/src/main/typescript/plugin-keychain-aws-sm.ts @@ -23,6 +23,9 @@ import { IPluginKeychain, } from "@hyperledger/cactus-core-api"; +import { PrometheusExporter } from "./prometheus-exporter/prometheus-exporter"; +import { GetPrometheusExporterMetricsEndpointV1 } from "./webservices/get-prometheus-exporter-metrics-endpoint-v1"; + import { homedir } from "os"; import { PluginRegistry } from "@hyperledger/cactus-core"; import { SetKeychainEntryV1Endpoint } from "./webservices/set-keychain-entry-endpoint-v1"; @@ -60,6 +63,11 @@ export interface IPluginKeychainAwsSmOptions extends ICactusPluginOptions { * awsCredentialType == AwsCredentialType.InMemory */ awsSecretAccessKey?: string; + /** + * Prometheus Exporter object for metrics monitoring + */ + + prometheusExporter?: PrometheusExporter; } const SECRETMANAGER_STATUS_KEY_NOT_FOUND = @@ -84,6 +92,7 @@ export class PluginKeychainAwsSm private readonly awsClient: SecretsManager; private endpoints: IWebServiceEndpoint[] | undefined; private awsCredentialType: AwsCredentialType; + public prometheusExporter: PrometheusExporter; public get className(): string { return PluginKeychainAwsSm.CLASS_NAME; @@ -143,6 +152,15 @@ export class PluginKeychainAwsSm endpoint: this.awsEndpoint, }); + this.prometheusExporter = + opts.prometheusExporter || + new PrometheusExporter({ pollingIntervalInMin: 1 }); + Checks.truthy( + this.prometheusExporter, + `${fnTag} options.prometheusExporter`, + ); + this.prometheusExporter.startMetricsCollection(); + this.log.info(`Created ${this.className}. KeychainID=${opts.keychainId}`); } @@ -150,6 +168,15 @@ export class PluginKeychainAwsSm return OAS; } + public getPrometheusExporter(): PrometheusExporter { + return this.prometheusExporter; + } + + public async getPrometheusExporterMetrics(): Promise { + const res: string = await this.prometheusExporter.getPrometheusMetrics(); + return res; + } + public getAwsClient(): SecretsManager { return this.awsClient; } @@ -181,6 +208,10 @@ export class PluginKeychainAwsSm connector: this, logLevel: this.opts.logLevel, }), + new GetPrometheusExporterMetricsEndpointV1({ + plugin: this, + logLevel: this.opts.logLevel, + }), ]; this.endpoints = endpoints; @@ -276,6 +307,7 @@ export class PluginKeychainAwsSm SecretString: value, }) .promise(); + this.prometheusExporter.setTotalKeyCounter(key, "set"); } catch (ex) { this.log.error(` ${fnTag}: Error writing secret "${key}"`); throw ex; @@ -292,6 +324,7 @@ export class PluginKeychainAwsSm ForceDeleteWithoutRecovery: true, }) .promise(); + this.prometheusExporter.setTotalKeyCounter(key, "delete"); } catch (ex) { this.log.error(`${fnTag} Error deleting secret "${key}"`); throw ex; diff --git a/packages/cactus-plugin-keychain-aws-sm/src/main/typescript/prometheus-exporter/data-fetcher.ts b/packages/cactus-plugin-keychain-aws-sm/src/main/typescript/prometheus-exporter/data-fetcher.ts new file mode 100644 index 0000000000..1759a5b7f3 --- /dev/null +++ b/packages/cactus-plugin-keychain-aws-sm/src/main/typescript/prometheus-exporter/data-fetcher.ts @@ -0,0 +1,12 @@ +import { AwsSmKeys } from "./response.type"; + +import { + totalKeyCount, + K_CACTUS_KEYCHAIN_AWSSM_MANAGED_KEY_COUNT, +} from "./metrics"; + +export async function collectMetrics(awsSmKeys: AwsSmKeys): Promise { + totalKeyCount + .labels(K_CACTUS_KEYCHAIN_AWSSM_MANAGED_KEY_COUNT) + .set(awsSmKeys.size); +} diff --git a/packages/cactus-plugin-keychain-aws-sm/src/main/typescript/prometheus-exporter/metrics.ts b/packages/cactus-plugin-keychain-aws-sm/src/main/typescript/prometheus-exporter/metrics.ts new file mode 100644 index 0000000000..759d4f5a73 --- /dev/null +++ b/packages/cactus-plugin-keychain-aws-sm/src/main/typescript/prometheus-exporter/metrics.ts @@ -0,0 +1,12 @@ +import { Gauge } from "prom-client"; + +export const K_CACTUS_KEYCHAIN_AWSSM_MANAGED_KEY_COUNT = + "cactus_keychain_awssm_managed_key_count"; + +export const totalKeyCount = new Gauge({ + registers: [], + name: K_CACTUS_KEYCHAIN_AWSSM_MANAGED_KEY_COUNT, + help: + "The number of keys that were set in the backing Aws Secret Manager deployment via this specific keychain plugin instance", + labelNames: ["type"], +}); diff --git a/packages/cactus-plugin-keychain-aws-sm/src/main/typescript/prometheus-exporter/prometheus-exporter.ts b/packages/cactus-plugin-keychain-aws-sm/src/main/typescript/prometheus-exporter/prometheus-exporter.ts new file mode 100644 index 0000000000..60c1cea39c --- /dev/null +++ b/packages/cactus-plugin-keychain-aws-sm/src/main/typescript/prometheus-exporter/prometheus-exporter.ts @@ -0,0 +1,44 @@ +import promClient, { Registry } from "prom-client"; +import { AwsSmKeys } from "./response.type"; +import { collectMetrics } from "./data-fetcher"; +import { K_CACTUS_KEYCHAIN_AWSSM_MANAGED_KEY_COUNT } from "./metrics"; +import { totalKeyCount } from "./metrics"; + +export interface IPrometheusExporterOptions { + pollingIntervalInMin?: number; +} + +export class PrometheusExporter { + public readonly metricsPollingIntervalInMin: number; + public readonly awsSmKeys: AwsSmKeys = new Map(); + public readonly registry: Registry; + + constructor( + public readonly prometheusExporterOptions: IPrometheusExporterOptions, + ) { + this.metricsPollingIntervalInMin = + prometheusExporterOptions.pollingIntervalInMin || 1; + this.registry = new Registry(); + } + + public setTotalKeyCounter(key: string, operation: string): void { + if (operation === "set") { + this.awsSmKeys.set(key, "keychain-awssm"); + } else { + this.awsSmKeys.delete(key); + } + collectMetrics(this.awsSmKeys); + } + + public async getPrometheusMetrics(): Promise { + const result = await this.registry.getSingleMetricAsString( + K_CACTUS_KEYCHAIN_AWSSM_MANAGED_KEY_COUNT, + ); + return result; + } + + public startMetricsCollection(): void { + this.registry.registerMetric(totalKeyCount); + promClient.collectDefaultMetrics({ register: this.registry }); + } +} diff --git a/packages/cactus-plugin-keychain-aws-sm/src/main/typescript/prometheus-exporter/response.type.ts b/packages/cactus-plugin-keychain-aws-sm/src/main/typescript/prometheus-exporter/response.type.ts new file mode 100644 index 0000000000..f652e8c8a6 --- /dev/null +++ b/packages/cactus-plugin-keychain-aws-sm/src/main/typescript/prometheus-exporter/response.type.ts @@ -0,0 +1 @@ +export type AwsSmKeys = Map; diff --git a/packages/cactus-plugin-keychain-aws-sm/src/main/typescript/webservices/get-prometheus-exporter-metrics-endpoint-v1.ts b/packages/cactus-plugin-keychain-aws-sm/src/main/typescript/webservices/get-prometheus-exporter-metrics-endpoint-v1.ts new file mode 100644 index 0000000000..be2d785ad9 --- /dev/null +++ b/packages/cactus-plugin-keychain-aws-sm/src/main/typescript/webservices/get-prometheus-exporter-metrics-endpoint-v1.ts @@ -0,0 +1,101 @@ +import { Express, Request, Response } from "express"; + +import { + Logger, + LoggerProvider, + LogLevelDesc, + Checks, + IAsyncProvider, +} from "@hyperledger/cactus-common"; + +import { + IWebServiceEndpoint, + IExpressRequestHandler, + IEndpointAuthzOptions, +} from "@hyperledger/cactus-core-api/"; + +import OAS from "../../json/openapi.json"; + +import { registerWebServiceEndpoint } from "@hyperledger/cactus-core"; + +import { PluginKeychainAwsSm } from "../plugin-keychain-aws-sm"; + +export interface IGetPrometheusExporterMetricsEndpointV1Options { + logLevel?: LogLevelDesc; + plugin: PluginKeychainAwsSm; +} + +export class GetPrometheusExporterMetricsEndpointV1 + implements IWebServiceEndpoint { + private readonly log: Logger; + + constructor( + public readonly opts: IGetPrometheusExporterMetricsEndpointV1Options, + ) { + const fnTag = "GetPrometheusExporterMetricsEndpointV1#constructor()"; + + Checks.truthy(opts, `${fnTag} options`); + Checks.truthy(opts.plugin, `${fnTag} options.plugin`); + + this.log = LoggerProvider.getOrCreate({ + label: "get-prometheus-exporter-metrics-v1", + level: opts.logLevel || "INFO", + }); + } + + getAuthorizationOptionsProvider(): IAsyncProvider { + // TODO: make this an injectable dependency in the constructor + return { + get: async () => ({ + isProtected: true, + requiredRoles: [], + }), + }; + } + + public getExpressRequestHandler(): IExpressRequestHandler { + return this.handleRequest.bind(this); + } + + public get oasPath(): typeof OAS.paths["/api/v1/plugins/@hyperledger/cactus-plugin-keychain-aws-sm/get-prometheus-exporter-metrics"] { + return OAS.paths[ + "/api/v1/plugins/@hyperledger/cactus-plugin-keychain-aws-sm/get-prometheus-exporter-metrics" + ]; + } + + public getPath(): string { + return this.oasPath.get["x-hyperledger-cactus"].http.path; + } + + public getVerbLowerCase(): string { + return this.oasPath.get["x-hyperledger-cactus"].http.verbLowerCase; + } + + public getOperationId(): string { + return this.oasPath.get.operationId; + } + + public async registerExpress( + expressApp: Express, + ): Promise { + await registerWebServiceEndpoint(expressApp, this); + return this; + } + + async handleRequest(req: Request, res: Response): Promise { + const fnTag = "GetPrometheusExporterMetrics#handleRequest()"; + const verbUpper = this.getVerbLowerCase().toUpperCase(); + this.log.debug(`${verbUpper} ${this.getPath()}`); + + try { + const resBody = await this.opts.plugin.getPrometheusExporterMetrics(); + res.status(200); + res.send(resBody); + } catch (ex) { + this.log.error(`${fnTag} failed to serve request`, ex); + res.status(500); + res.statusMessage = ex.message; + res.json({ error: ex.stack }); + } + } +} diff --git a/packages/cactus-plugin-keychain-aws-sm/src/test/typescript/integration/plugin-keychain-aws-sm.test.ts b/packages/cactus-plugin-keychain-aws-sm/src/test/typescript/integration/plugin-keychain-aws-sm.test.ts index 237b95ed5a..39715cbfb0 100644 --- a/packages/cactus-plugin-keychain-aws-sm/src/test/typescript/integration/plugin-keychain-aws-sm.test.ts +++ b/packages/cactus-plugin-keychain-aws-sm/src/test/typescript/integration/plugin-keychain-aws-sm.test.ts @@ -29,6 +29,8 @@ import { Configuration, } from "../../../main/typescript/generated/openapi/typescript-axios/index"; +import { K_CACTUS_KEYCHAIN_AWSSM_MANAGED_KEY_COUNT } from "../../../main/typescript/prometheus-exporter/metrics"; + import fs from "fs"; import path from "path"; import os from "os"; @@ -91,6 +93,9 @@ test("get,set,has,delete alters state as expected", async (t: Test) => { test.onFinish(async () => await Servers.shutdown(server)); const { address, port } = addressInfo; const apiHost = `http://${address}:${port}`; + t.comment( + `Metrics URL: ${apiHost}/api/v1/plugins/@hyperledger/cactus-plugin-keychain-aws-sm/get-prometheus-exporter-metrics`, + ); const config = new Configuration({ basePath: apiHost }); const apiClient = new KeychainAwsSmApi(config); @@ -130,6 +135,28 @@ test("get,set,has,delete alters state as expected", async (t: Test) => { t.ok(res3.data.checkedAt, "res3.data.checkedAt truthy OK"); t.equal(res3.data.key, key, "res3.data.key === key OK"); + { + const res = await apiClient.getPrometheusMetricsV1(); + const promMetricsOutput = + "# HELP " + + K_CACTUS_KEYCHAIN_AWSSM_MANAGED_KEY_COUNT + + " The number of keys that were set in the backing Aws Secret Manager deployment via this specific keychain plugin instance\n" + + "# TYPE " + + K_CACTUS_KEYCHAIN_AWSSM_MANAGED_KEY_COUNT + + " gauge\n" + + K_CACTUS_KEYCHAIN_AWSSM_MANAGED_KEY_COUNT + + '{type="' + + K_CACTUS_KEYCHAIN_AWSSM_MANAGED_KEY_COUNT + + '"} 1'; + t.ok(res); + t.ok(res.data); + t.equal(res.status, 200); + t.true( + res.data.includes(promMetricsOutput), + "Total Key Count 1 recorded as expected. RESULT OK", + ); + } + const res4 = await apiClient.getKeychainEntryV1({ key: key, }); @@ -151,6 +178,28 @@ test("get,set,has,delete alters state as expected", async (t: Test) => { t.ok(res6.data.checkedAt, "res6.data.checkedAt truthy OK"); t.equal(res6.data.key, key, "res6.data.key === key OK"); + { + const res = await apiClient.getPrometheusMetricsV1(); + const promMetricsOutput = + "# HELP " + + K_CACTUS_KEYCHAIN_AWSSM_MANAGED_KEY_COUNT + + " The number of keys that were set in the backing Aws Secret Manager deployment via this specific keychain plugin instance\n" + + "# TYPE " + + K_CACTUS_KEYCHAIN_AWSSM_MANAGED_KEY_COUNT + + " gauge\n" + + K_CACTUS_KEYCHAIN_AWSSM_MANAGED_KEY_COUNT + + '{type="' + + K_CACTUS_KEYCHAIN_AWSSM_MANAGED_KEY_COUNT + + '"} 0'; + t.ok(res); + t.ok(res.data); + t.equal(res.status, 200); + t.true( + res.data.includes(promMetricsOutput), + "Total Key Count 0 recorded as expected. RESULT OK", + ); + } + try { await apiClient.getKeychainEntryV1({ key }); t.fail(