From 1fd4c2dd1a2574b770ef55165e7ef31ab772f2ac Mon Sep 17 00:00:00 2001 From: seven Date: Sun, 8 Dec 2024 15:14:46 +0800 Subject: [PATCH] feat: enable support to define elasticsearch serverless (#14) feat: enable support to define Elasticsearch serverless Refs: #12 --------- Signed-off-by: seven --- package-lock.json | 37 +++++++++++--- package.json | 3 +- src/stack/iacSchema.ts | 37 ++++++++++++++ src/stack/iacStack.ts | 57 +++++++++++++++++++--- src/stack/parse.ts | 37 +++++++++++++- src/types.ts | 47 +++++++++++++++++- tests/fixtures/deployFixture.ts | 61 +++++++++++++++++++++++- tests/fixtures/serverless-insight-es.yml | 21 ++++++++ tests/fixtures/serverless-insight.yml | 19 ++++---- tests/stack/deploy.test.ts | 16 +++++++ tests/stack/parse.test.ts | 39 +++++++++++++++ 11 files changed, 347 insertions(+), 27 deletions(-) create mode 100644 tests/fixtures/serverless-insight-es.yml create mode 100644 tests/stack/parse.test.ts diff --git a/package-lock.json b/package-lock.json index 57da3bfd..22121d7f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,11 +12,12 @@ "@alicloud/openapi-client": "^0.4.12", "@alicloud/ros-cdk-apigateway": "^1.4.0", "@alicloud/ros-cdk-core": "^1.4.0", + "@alicloud/ros-cdk-elasticsearchserverless": "^1.4.0", "@alicloud/ros-cdk-fc3": "^1.4.0", "@alicloud/ros-cdk-oss": "^1.4.0", "@alicloud/ros-cdk-ossdeployment": "^1.4.0", "@alicloud/ros-cdk-ram": "^1.4.0", - "@alicloud/ros20190910": "^3.5.0", + "@alicloud/ros20190910": "^3.5.2", "ajv": "^8.17.1", "chalk": "^5.3.0", "commander": "^12.1.0", @@ -295,6 +296,19 @@ "node": ">=10" } }, + "node_modules/@alicloud/ros-cdk-elasticsearchserverless": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@alicloud/ros-cdk-elasticsearchserverless/-/ros-cdk-elasticsearchserverless-1.4.0.tgz", + "integrity": "sha512-MkXivFcy9DlPzKZbyFCVMK1KssZWO6CBIpNockOPNK5vnBkaB1ujRqvpfyLk5gfVpD94n3mP9alK2ASlk5h80A==", + "dependencies": { + "@alicloud/ros-cdk-core": "^1.0.27", + "constructs": "^3.0.4" + }, + "peerDependencies": { + "@alicloud/ros-cdk-core": "^1.0.27", + "constructs": "^3.0.4" + } + }, "node_modules/@alicloud/ros-cdk-fc": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/@alicloud/ros-cdk-fc/-/ros-cdk-fc-1.4.0.tgz", @@ -2033,9 +2047,9 @@ } }, "node_modules/@alicloud/ros20190910": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/@alicloud/ros20190910/-/ros20190910-3.5.0.tgz", - "integrity": "sha512-wXNTUzswL2xAhyqtU4c06yWdcXWlAvJ7MbTYV7kYcqhdU+B6/E5Xrr2HUoKUY0WAzM09g2qleCaLcs4Qge3MYA==", + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/@alicloud/ros20190910/-/ros20190910-3.5.2.tgz", + "integrity": "sha512-ANzxBI1+yYGTuqD4W+VxO6lROo96DOioCDTGSfRERYjY/ZjTMTzCVSiAVVpce6FixNCuQDW8BtBjpB13Pv9WsA==", "dependencies": { "@alicloud/endpoint-util": "^0.0.1", "@alicloud/openapi-client": "^0.4.12", @@ -8294,6 +8308,15 @@ } } }, + "@alicloud/ros-cdk-elasticsearchserverless": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@alicloud/ros-cdk-elasticsearchserverless/-/ros-cdk-elasticsearchserverless-1.4.0.tgz", + "integrity": "sha512-MkXivFcy9DlPzKZbyFCVMK1KssZWO6CBIpNockOPNK5vnBkaB1ujRqvpfyLk5gfVpD94n3mP9alK2ASlk5h80A==", + "requires": { + "@alicloud/ros-cdk-core": "^1.0.27", + "constructs": "^3.0.4" + } + }, "@alicloud/ros-cdk-fc": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/@alicloud/ros-cdk-fc/-/ros-cdk-fc-1.4.0.tgz", @@ -9444,9 +9467,9 @@ } }, "@alicloud/ros20190910": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/@alicloud/ros20190910/-/ros20190910-3.5.0.tgz", - "integrity": "sha512-wXNTUzswL2xAhyqtU4c06yWdcXWlAvJ7MbTYV7kYcqhdU+B6/E5Xrr2HUoKUY0WAzM09g2qleCaLcs4Qge3MYA==", + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/@alicloud/ros20190910/-/ros20190910-3.5.2.tgz", + "integrity": "sha512-ANzxBI1+yYGTuqD4W+VxO6lROo96DOioCDTGSfRERYjY/ZjTMTzCVSiAVVpce6FixNCuQDW8BtBjpB13Pv9WsA==", "requires": { "@alicloud/endpoint-util": "^0.0.1", "@alicloud/openapi-client": "^0.4.12", diff --git a/package.json b/package.json index 58ecbe71..d35528cf 100644 --- a/package.json +++ b/package.json @@ -52,11 +52,12 @@ "@alicloud/openapi-client": "^0.4.12", "@alicloud/ros-cdk-apigateway": "^1.4.0", "@alicloud/ros-cdk-core": "^1.4.0", + "@alicloud/ros-cdk-elasticsearchserverless": "^1.4.0", "@alicloud/ros-cdk-fc3": "^1.4.0", "@alicloud/ros-cdk-oss": "^1.4.0", "@alicloud/ros-cdk-ossdeployment": "^1.4.0", "@alicloud/ros-cdk-ram": "^1.4.0", - "@alicloud/ros20190910": "^3.5.0", + "@alicloud/ros20190910": "^3.5.2", "ajv": "^8.17.1", "chalk": "^5.3.0", "commander": "^12.1.0", diff --git a/src/stack/iacSchema.ts b/src/stack/iacSchema.ts index e6db27cc..7bcb8b82 100644 --- a/src/stack/iacSchema.ts +++ b/src/stack/iacSchema.ts @@ -111,6 +111,43 @@ const schema = { }, }, }, + databases: { + type: 'object', + patternProperties: { + '.*': { + type: 'object', + properties: { + name: { type: 'string' }, + type: { type: 'string', enum: ['ELASTICSEARCH_SERVERLESS'] }, + version: { type: 'string' }, + engine_mode: { type: 'string', enum: ['SEARCH', 'TIMESERIES'] }, + cu: { type: 'number' }, + storage_size: { type: 'number' }, + security: { + type: 'object', + properties: { + basic_auth: { + type: 'object', + properties: { + password: { type: 'string' }, + }, + required: ['password'], + }, + }, + required: ['basic_auth'], + }, + network: { + type: 'object', + properties: { + public: { type: 'boolean' }, + }, + }, + }, + required: ['name', 'type', 'version', 'security', 'cu', 'storage_size'], + additionalProperties: false, + }, + }, + }, }, required: ['version', 'provider', 'service'], additionalProperties: false, diff --git a/src/stack/iacStack.ts b/src/stack/iacStack.ts index ed08f8eb..c6a9bbcf 100644 --- a/src/stack/iacStack.ts +++ b/src/stack/iacStack.ts @@ -1,12 +1,19 @@ import * as ros from '@alicloud/ros-cdk-core'; import { RosParameterType } from '@alicloud/ros-cdk-core'; -import { ActionContext, EventTypes, ServerlessIac } from '../types'; +import { + ActionContext, + DatabaseEngineMode, + DatabaseEnum, + EventTypes, + ServerlessIac, +} from '../types'; import * as fc from '@alicloud/ros-cdk-fc3'; import { RosFunction } from '@alicloud/ros-cdk-fc3/lib/fc3.generated'; import * as ram from '@alicloud/ros-cdk-ram'; import * as agw from '@alicloud/ros-cdk-apigateway'; import * as oss from '@alicloud/ros-cdk-oss'; import * as ossDeployment from '@alicloud/ros-cdk-ossdeployment'; +import * as esServerless from '@alicloud/ros-cdk-elasticsearchserverless'; import { CODE_ZIP_SIZE_LIMIT, getFileSource, @@ -14,6 +21,7 @@ import { replaceReference, resolveCode, } from '../common'; +import { isEmpty } from 'lodash'; export class IacStack extends ros.Stack { private readonly service: string; @@ -47,7 +55,7 @@ export class IacStack extends ros.Stack { new ros.RosInfo(this, ros.RosInfo.description, `${this.service} stack`); const fileSources = iac.functions - .filter(({ code }) => readCodeSize(code) > CODE_ZIP_SIZE_LIMIT) + ?.filter(({ code }) => readCodeSize(code) > CODE_ZIP_SIZE_LIMIT) .map(({ code, name }) => { const fcName = replaceReference(name, context); @@ -55,7 +63,7 @@ export class IacStack extends ros.Stack { }); let destinationBucket: oss.Bucket; - if (fileSources.length > 0) { + if (!isEmpty(fileSources)) { // creat oss to store code destinationBucket = new oss.Bucket( this, @@ -70,7 +78,7 @@ export class IacStack extends ros.Stack { this, `${this.service}_artifacts_code_deployment`, { - sources: fileSources.map(({ source }) => source), + sources: fileSources!.map(({ source }) => source), destinationBucket, timeout: 300, logMonitoring: false, // 是否开启日志监控，设为false则不开启 @@ -79,14 +87,14 @@ export class IacStack extends ros.Stack { ); } - iac.functions.forEach((fnc) => { + iac.functions?.forEach((fnc) => { let code: RosFunction.CodeProperty = { zipFile: resolveCode(fnc.code), }; if (readCodeSize(fnc.code) > CODE_ZIP_SIZE_LIMIT) { code = { ossBucketName: destinationBucket.attrName, - ossObjectName: fileSources.find(({ fcName }) => fcName === fnc.name)?.objectKey, + ossObjectName: fileSources?.find(({ fcName }) => fcName === fnc.name)?.objectKey, }; } new fc.RosFunction( @@ -204,5 +212,42 @@ export class IacStack extends ros.Stack { }); }); } + iac.databases?.forEach((db) => { + if ([DatabaseEnum.ELASTICSEARCH_SERVERLESS].includes(db.type)) { + new esServerless.App( + this, + replaceReference(db.key, context), + { + appName: replaceReference(db.name, context), + appVersion: db.version, + authentication: { + basicAuth: [ + { + password: replaceReference(db.security.basicAuth.password, context), + }, + ], + }, + quotaInfo: { + cu: db.cu, + storage: db.storageSize, + appType: db.engineMode === DatabaseEngineMode.TIMESERIES ? 'TRIAL' : 'STANDARD', + }, + // network: [ + // { + // type: 'PUBLIC_KIBANA', + // enabled: true, + // whiteIpGroup: [{ groupName: 'default', ips: ['0.0.0.0/24'] }], + // }, + // { + // type: 'PUBLIC_ES', + // enabled: true, + // whiteIpGroup: [{ groupName: 'default', ips: ['0.0.0.0/24'] }], + // }, + // ], + }, + true, + ); + } + }); } } diff --git a/src/stack/parse.ts b/src/stack/parse.ts index 8b62ada8..cfa95ab3 100644 --- a/src/stack/parse.ts +++ b/src/stack/parse.ts @@ -1,7 +1,16 @@ import { parse } from 'yaml'; import { existsSync, readFileSync } from 'node:fs'; -import { Event, IacFunction, RawServerlessIac, ServerlessIac } from '../types'; +import { + DatabaseEnum, + Event, + IacDatabase, + IacFunction, + RawIacDatabase, + RawServerlessIac, + ServerlessIac, +} from '../types'; import { validateYaml } from './iacSchema'; +import { get, isEmpty } from 'lodash'; const mapToArr = (obj: Record | string | null | undefined>) => { if (!obj) { @@ -24,7 +33,30 @@ const validateExistence = (path: string) => { throw new Error(`File does not exist at path: ${path}`); } }; - +const transformDatabase = (databases?: { + [key: string]: RawIacDatabase; +}): Array | undefined => { + if (isEmpty(databases)) { + return undefined; + } + return Object.entries(databases)?.map(([key, database]) => ({ + key: key, + name: database.name, + type: database.type as DatabaseEnum, + version: database.version, + engineMode: database.engine_mode, + security: { + basicAuth: { + password: get(database, 'security.basic_auth.password'), + }, + }, + cu: database.cu, + storageSize: database.storage_size, + network: database.network && { + public: database.network?.public as boolean, + }, + })); +}; const transformYaml = (iacJson: RawServerlessIac): ServerlessIac => { return { service: iacJson.service, @@ -38,6 +70,7 @@ const transformYaml = (iacJson: RawServerlessIac): ServerlessIac => { { key: 'iac-provider', value: 'ServerlessInsight' }, ...mapToKvArr(iacJson.tags), ] as unknown as Array<{ key: string; value: string }>, + databases: transformDatabase(iacJson.databases), }; }; diff --git a/src/types.ts b/src/types.ts index 5ee483c6..a0d7b340 100644 --- a/src/types.ts +++ b/src/types.ts @@ -45,6 +45,22 @@ type Events = { type Tags = { [key: string]: string; }; +export type RawIacDatabase = { + name: string; + type: DatabaseEnum; + version: string; + engine_mode: DatabaseEngineMode; + security: { + basic_auth: { + password: string; + }; + }; + network?: { + public: boolean; + }; + cu: number; + storage_size: number; +}; export type RawServerlessIac = { version: string; @@ -55,6 +71,34 @@ export type RawServerlessIac = { tags: Tags; functions: { [key: string]: IacFunction }; events: Events; + databases: { [key: string]: RawIacDatabase }; +}; + +export enum DatabaseEnum { + ELASTICSEARCH_SERVERLESS = 'ELASTICSEARCH_SERVERLESS', +} + +export enum DatabaseEngineMode { + SEARCH = 'SEARCH', + TIMESERIES = 'TIMESERIES', +} + +export type IacDatabase = { + key: string; + name: string; + type: DatabaseEnum; + version: string; + engineMode: string; + security: { + basicAuth: { + password: string; + }; + }; + network?: { + public: boolean; + }; + cu: number; + storageSize: number; }; export type ServerlessIac = { @@ -64,8 +108,9 @@ export type ServerlessIac = { vars?: Vars; stages?: Stages; tags?: Array<{ key: string; value: string }>; - functions: Array; + functions?: Array; events?: Array; + databases?: Array; }; export type ActionContext = { diff --git a/tests/fixtures/deployFixture.ts b/tests/fixtures/deployFixture.ts index fb632650..2a6675ac 100644 --- a/tests/fixtures/deployFixture.ts +++ b/tests/fixtures/deployFixture.ts @@ -1,4 +1,4 @@ -import { ServerlessIac } from '../../src/types'; +import { DatabaseEnum, ServerlessIac } from '../../src/types'; import { cloneDeep, set } from 'lodash'; export const oneFcOneGatewayIac = { @@ -747,3 +747,62 @@ export const defaultContext = { stackName: 'my-demo-stack', stage: 'default', }; + +export const esServerlessMinimumIac: ServerlessIac = { + service: 'my-demo-es-serverless-service', + version: '0.0.1', + provider: 'aliyun', + databases: [ + { + key: 'insight_es_db_test', + name: 'insight-poc-es-test', + type: DatabaseEnum.ELASTICSEARCH_SERVERLESS, + version: '7.10', + engineMode: 'SEARCH', + security: { + basicAuth: { + password: 'test-password', + }, + }, + cu: 1, + storageSize: 20, + }, + ], +}; + +export const esServerlessMinimumRos = { + Description: 'my-demo-es-serverless-service stack', + Metadata: { 'ALIYUN::ROS::Interface': { TemplateTags: ['Create by ROS CDK'] } }, + ROSTemplateFormatVersion: '2015-09-01', + Resources: { + insight_es_db_test: { + Properties: { + AppName: 'insight-poc-es-test', + AppVersion: '7.10', + Authentication: { + BasicAuth: [ + { + Password: 'test-password', + }, + ], + }, + QuotaInfo: { + AppType: 'STANDARD', + Cu: 1, + Storage: 20, + }, + // Network: [ + // { + // Enabled: true, + // Type: 'PUBLIC_KIBANA', + // }, + // { + // Enabled: true, + // Type: 'PUBLIC_ES', + // }, + // ], + }, + Type: 'ALIYUN::ElasticSearchServerless::App', + }, + }, +}; diff --git a/tests/fixtures/serverless-insight-es.yml b/tests/fixtures/serverless-insight-es.yml new file mode 100644 index 00000000..61a70433 --- /dev/null +++ b/tests/fixtures/serverless-insight-es.yml @@ -0,0 +1,21 @@ +version: 0.0.1 +provider: aliyun + + +service: insight-es-poc + +tags: + owner: geek-fun + +databases: + insight_es_db: + name: insight-poc-es + type: ELASTICSEARCH_SERVERLESS + version: '7.10' + engine_mode: SEARCH + cu: 1 + storage_size: 20 + security: + basic_auth: + password: 'U34I6InQ8elseTgqTWT2t2oFXpoqFg' + diff --git a/tests/fixtures/serverless-insight.yml b/tests/fixtures/serverless-insight.yml index abad28e9..faa5fbb7 100644 --- a/tests/fixtures/serverless-insight.yml +++ b/tests/fixtures/serverless-insight.yml @@ -35,16 +35,17 @@ functions: TEST_VAR_EXTRA: abcds-${vars.testv}-andyou #databases: -# insight_poc_db: -# name: insight-poc-db -# type: RDS -# region: ${vars.region} -# instance_class: rds.mysql.t5.xlarge +# insight_es_db: +# name: insight-poc-es +# type: ELASTICSEARCH_SERVERLESS +# version: 7.10 +# engine_mode: SEARCH +# security: +# basic_auth: +# password: 123456 +# cu: 1 # storage_size: 20 -# username: root -# password: 123456 -# database: insight_poc_db -# + events: gateway_event: type: API_GATEWAY diff --git a/tests/stack/deploy.test.ts b/tests/stack/deploy.test.ts index 46ff962f..8b88cce9 100644 --- a/tests/stack/deploy.test.ts +++ b/tests/stack/deploy.test.ts @@ -1,6 +1,8 @@ import { deployStack } from '../../src/stack'; import { ActionContext } from '../../src/types'; import { + esServerlessMinimumIac, + esServerlessMinimumRos, largeCodeRos, minimumIac, minimumRos, @@ -124,4 +126,18 @@ describe('Unit tests for stack deployment', () => { expect(mockedResolveCode).toHaveBeenCalledTimes(1); expect(mockedRosStackDeploy).toHaveBeenCalledWith(stackName, largeCodeRos, { stackName }); }); + + describe('unit test for deploy of databases', () => { + it('should deploy elasticsearch serverless when database minimum fields provided', () => { + const stackName = 'my-demo-es-serverless-stack'; + mockedRosStackDeploy.mockResolvedValueOnce(stackName); + + deployStack(stackName, esServerlessMinimumIac, { stackName } as ActionContext); + + expect(mockedRosStackDeploy).toHaveBeenCalledTimes(1); + expect(mockedRosStackDeploy).toHaveBeenCalledWith(stackName, esServerlessMinimumRos, { + stackName, + }); + }); + }); }); diff --git a/tests/stack/parse.test.ts b/tests/stack/parse.test.ts new file mode 100644 index 00000000..a1775a98 --- /dev/null +++ b/tests/stack/parse.test.ts @@ -0,0 +1,39 @@ +import { parseYaml } from '../../src/stack'; +import path from 'node:path'; + +describe('unit test for parse', () => { + describe('domain - databases', () => { + const yamlPath = path.resolve(__dirname, '../fixtures/serverless-insight-es.yml'); + + it('should pass databases from yaml to domain instance when the yaml is valid', () => { + const databaseDomain = parseYaml(yamlPath); + expect(databaseDomain).toEqual({ + service: 'insight-es-poc', + version: '0.0.1', + provider: 'aliyun', + tags: [ + { key: 'iac-provider', value: 'ServerlessInsight' }, + { key: 'owner', value: 'geek-fun' }, + ], + events: [], + functions: [], + databases: [ + { + key: 'insight_es_db', + type: 'ELASTICSEARCH_SERVERLESS', + name: 'insight-poc-es', + engineMode: 'SEARCH', + version: '7.10', + security: { + basicAuth: { + password: 'U34I6InQ8elseTgqTWT2t2oFXpoqFg', + }, + }, + cu: 1, + storageSize: 20, + }, + ], + }); + }); + }); +});