From 75eb9330194824cdf435ae64095813191fcd6e13 Mon Sep 17 00:00:00 2001 From: Michael White Date: Sat, 18 Feb 2023 02:53:15 -0500 Subject: [PATCH] feat(logs): Add support for multiple parse and filter statements in QueryString (#24022) Currently, `QueryString` is limited to only allow a single line/statement to be provided for each query command. For some commands this makes sense (e.g. `limit`), but for `parse` and `filter` this can be limiting. Adding multiple lines for these commands is possible in the AWS console, so it makes sense for it to be supported in CDK too. In this PR, I'm adding support for `filter` and `parse` to be provided as `string` or `string[]`, and adding/modifying various utility methods to handle this ambiguity. I left the existing tests the same to verify no breaking changes, and added a new test for the newly enabled behavior. ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- packages/@aws-cdk/aws-logs/README.md | 8 ++ .../@aws-cdk/aws-logs/lib/query-definition.ts | 114 +++++++++++++----- ...efaultTestDeployAssert902BAAD5.assets.json | 2 +- ...insights-querydefinition-integ.assets.json | 6 +- ...sights-querydefinition-integ.template.json | 12 ++ .../cdk.out | 2 +- .../integ.json | 2 +- .../manifest.json | 10 +- .../tree.json | 34 +++++- ...teg.save-logs-insights-query-definition.ts | 21 ++++ .../aws-logs/test/query-definition.test.ts | 63 +++++++++- 11 files changed, 231 insertions(+), 43 deletions(-) diff --git a/packages/@aws-cdk/aws-logs/README.md b/packages/@aws-cdk/aws-logs/README.md index 784ab7e91760c..5a6a28f74547d 100644 --- a/packages/@aws-cdk/aws-logs/README.md +++ b/packages/@aws-cdk/aws-logs/README.md @@ -330,6 +330,14 @@ new logs.QueryDefinition(this, 'QueryDefinition', { queryDefinitionName: 'MyQuery', queryString: new logs.QueryString({ fields: ['@timestamp', '@message'], + parseStatements: [ + '@message "[*] *" as loggingType, loggingMessage', + '@message "<*>: *" as differentLoggingType, differentLoggingMessage', + ], + filterStatements: [ + 'loggingType = "ERROR"', + 'loggingMessage = "A very strange error occurred!"', + ], sort: '@timestamp desc', limit: 20, }), diff --git a/packages/@aws-cdk/aws-logs/lib/query-definition.ts b/packages/@aws-cdk/aws-logs/lib/query-definition.ts index 2dff1ad3d93e1..09594756d4cb2 100644 --- a/packages/@aws-cdk/aws-logs/lib/query-definition.ts +++ b/packages/@aws-cdk/aws-logs/lib/query-definition.ts @@ -16,19 +16,43 @@ export interface QueryStringProps { readonly fields?: string[]; /** - * Extracts data from a log field and creates one or more ephemeral fields that you can process further in the query. + * A single statement for parsing data from a log field and creating ephemeral fields that can + * be processed further in the query. * + * @deprecated Use `parseStatements` instead * @default - no parse in QueryString */ readonly parse?: string; /** - * Filters the results of a query that's based on one or more conditions. + * An array of one or more statements for parsing data from a log field and creating ephemeral + * fields that can be processed further in the query. Each provided statement generates a separate + * parse line in the query string. * + * Note: If provided, this property overrides any value provided for the `parse` property. + * + * @default - no parse in QueryString + */ + readonly parseStatements?: string[]; + + /** + * A single statement for filtering the results of a query based on a boolean expression. + * + * @deprecated Use `filterStatements` instead * @default - no filter in QueryString */ readonly filter?: string; + /** + * An array of one or more statements for filtering the results of a query based on a boolean + * expression. Each provided statement generates a separate filter line in the query string. + * + * Note: If provided, this property overrides any value provided for the `filter` property. + * + * @default - no filter in QueryString + */ + readonly filterStatements?: string[]; + /** * Uses log field values to calculate aggregate statistics. * @@ -58,23 +82,13 @@ export interface QueryStringProps { readonly display?: string; } -interface QueryStringMap { - readonly fields?: string, - readonly parse?: string, - readonly filter?: string, - readonly stats?: string, - readonly sort?: string, - readonly limit?: Number, - readonly display?: string, -} - /** * Define a QueryString */ export class QueryString { private readonly fields?: string[]; - private readonly parse?: string; - private readonly filter?: string; + private readonly parse: string[]; + private readonly filter: string[]; private readonly stats?: string; private readonly sort?: string; private readonly limit?: Number; @@ -82,38 +96,74 @@ export class QueryString { constructor(props: QueryStringProps = {}) { this.fields = props.fields; - this.parse = props.parse; - this.filter = props.filter; this.stats = props.stats; this.sort = props.sort; this.limit = props.limit; this.display = props.display; + + // Determine parsing by either the parseStatements or parse properties, or default to empty array + if (props.parseStatements) { + this.parse = props.parseStatements; + } else if (props.parse) { + this.parse = [props.parse]; + } else { + this.parse = []; + } + + // Determine filtering by either the filterStatements or filter properties, or default to empty array + if (props.filterStatements) { + this.filter = props.filterStatements; + } else if (props.filter) { + this.filter = [props.filter]; + } else { + this.filter = []; + } } /** * String representation of this QueryString. */ public toString(): string { - return noUndef({ - fields: this.fields !== undefined ? this.fields.join(', ') : this.fields, - parse: this.parse, - filter: this.filter, - stats: this.stats, - sort: this.sort, - limit: this.limit, - display: this.display, - }).join('\n| '); + return [ + this.buildQueryLine('fields', this.fields?.join(', ')), + ...this.buildQueryLines('parse', this.parse), + ...this.buildQueryLines('filter', this.filter), + this.buildQueryLine('stats', this.stats), + this.buildQueryLine('sort', this.sort), + this.buildQueryLine('limit', this.limit?.toString()), + this.buildQueryLine('display', this.display), + ].filter( + (queryLine) => queryLine !== undefined && queryLine.length > 0, + ).join('\n| '); } -} -function noUndef(x: QueryStringMap): string[] { - const ret: string[] = []; - for (const [key, value] of Object.entries(x)) { - if (value !== undefined) { - ret.push(`${key} ${value}`); + /** + * Build an array of query lines given a command and statement(s). + * + * @param command a query command + * @param statements one or more query statements for the specified command, or undefined + * @returns an array of the query string lines generated from the provided command and statements + */ + private buildQueryLines(command: string, statements?: string[]): string[] { + if (statements === undefined) { + return []; } + + return statements.map( + (statement: string): string => this.buildQueryLine(command, statement), + ); + } + + /** + * Build a single query line given a command and statement. + * + * @param command a query command + * @param statement a single query statement + * @returns a single query string line generated from the provided command and statement + */ + private buildQueryLine(command: string, statement?: string): string { + return statement ? `${command} ${statement}` : ''; } - return ret; } /** diff --git a/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/LogsInsightsQueryDefinitionIntegTestDefaultTestDeployAssert902BAAD5.assets.json b/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/LogsInsightsQueryDefinitionIntegTestDefaultTestDeployAssert902BAAD5.assets.json index faca19c9ec2b8..7e7b72e0995a4 100644 --- a/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/LogsInsightsQueryDefinitionIntegTestDefaultTestDeployAssert902BAAD5.assets.json +++ b/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/LogsInsightsQueryDefinitionIntegTestDefaultTestDeployAssert902BAAD5.assets.json @@ -1,5 +1,5 @@ { - "version": "21.0.0", + "version": "29.0.0", "files": { "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": { "source": { diff --git a/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/aws-cdk-logs-insights-querydefinition-integ.assets.json b/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/aws-cdk-logs-insights-querydefinition-integ.assets.json index 7388475903110..98998264c32c0 100644 --- a/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/aws-cdk-logs-insights-querydefinition-integ.assets.json +++ b/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/aws-cdk-logs-insights-querydefinition-integ.assets.json @@ -1,7 +1,7 @@ { - "version": "21.0.0", + "version": "29.0.0", "files": { - "e6a0a51961925fbc37d9b81431449c256ed453f98089eb70f83850f237b4d722": { + "3546c78647ea20567832041c960a034787f9bfc0128226ea8fbc0894366a4dd0": { "source": { "path": "aws-cdk-logs-insights-querydefinition-integ.template.json", "packaging": "file" @@ -9,7 +9,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "e6a0a51961925fbc37d9b81431449c256ed453f98089eb70f83850f237b4d722.json", + "objectKey": "3546c78647ea20567832041c960a034787f9bfc0128226ea8fbc0894366a4dd0.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/aws-cdk-logs-insights-querydefinition-integ.template.json b/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/aws-cdk-logs-insights-querydefinition-integ.template.json index d4df6bf7837ca..16808c0ca7ef0 100644 --- a/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/aws-cdk-logs-insights-querydefinition-integ.template.json +++ b/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/aws-cdk-logs-insights-querydefinition-integ.template.json @@ -19,6 +19,18 @@ } ] } + }, + "QueryDefinitionWithMultipleStatements58A3EF74": { + "Type": "AWS::Logs::QueryDefinition", + "Properties": { + "Name": "QueryDefinitionWithMultipleStatements", + "QueryString": "fields @timestamp, @message\n| parse @message \"[*] *\" as loggingType, loggingMessage\n| parse @message \"<*>: *\" as differentLoggingType, differentLoggingMessage\n| filter loggingType = \"ERROR\"\n| filter loggingMessage = \"A very strange error occurred!\"\n| sort @timestamp desc\n| limit 20\n| display loggingMessage", + "LogGroupNames": [ + { + "Ref": "LogGroupF5B46931" + } + ] + } } }, "Parameters": { diff --git a/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/cdk.out b/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/cdk.out index 8ecc185e9dbee..d8b441d447f8a 100644 --- a/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/cdk.out +++ b/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/cdk.out @@ -1 +1 @@ -{"version":"21.0.0"} \ No newline at end of file +{"version":"29.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/integ.json b/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/integ.json index 5c5e893333693..f445dff967844 100644 --- a/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/integ.json +++ b/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/integ.json @@ -1,5 +1,5 @@ { - "version": "21.0.0", + "version": "29.0.0", "testCases": { "LogsInsightsQueryDefinitionIntegTest/DefaultTest": { "stacks": [ diff --git a/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/manifest.json b/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/manifest.json index d6fb5b02dcca8..a975ff9d5fae9 100644 --- a/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/manifest.json @@ -1,5 +1,5 @@ { - "version": "21.0.0", + "version": "29.0.0", "artifacts": { "aws-cdk-logs-insights-querydefinition-integ.assets": { "type": "cdk:asset-manifest", @@ -17,7 +17,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/e6a0a51961925fbc37d9b81431449c256ed453f98089eb70f83850f237b4d722.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/3546c78647ea20567832041c960a034787f9bfc0128226ea8fbc0894366a4dd0.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -45,6 +45,12 @@ "data": "QueryDefinition4190BC36" } ], + "/aws-cdk-logs-insights-querydefinition-integ/QueryDefinitionWithMultipleStatements/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "QueryDefinitionWithMultipleStatements58A3EF74" + } + ], "/aws-cdk-logs-insights-querydefinition-integ/BootstrapVersion": [ { "type": "aws:cdk:logicalId", diff --git a/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/tree.json b/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/tree.json index 9e04ca4a9a7fa..1bbbc61ba797b 100644 --- a/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/tree.json @@ -62,6 +62,36 @@ "version": "0.0.0" } }, + "QueryDefinitionWithMultipleStatements": { + "id": "QueryDefinitionWithMultipleStatements", + "path": "aws-cdk-logs-insights-querydefinition-integ/QueryDefinitionWithMultipleStatements", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-cdk-logs-insights-querydefinition-integ/QueryDefinitionWithMultipleStatements/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Logs::QueryDefinition", + "aws:cdk:cloudformation:props": { + "name": "QueryDefinitionWithMultipleStatements", + "queryString": "fields @timestamp, @message\n| parse @message \"[*] *\" as loggingType, loggingMessage\n| parse @message \"<*>: *\" as differentLoggingType, differentLoggingMessage\n| filter loggingType = \"ERROR\"\n| filter loggingMessage = \"A very strange error occurred!\"\n| sort @timestamp desc\n| limit 20\n| display loggingMessage", + "logGroupNames": [ + { + "Ref": "LogGroupF5B46931" + } + ] + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-logs.CfnQueryDefinition", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-logs.QueryDefinition", + "version": "0.0.0" + } + }, "BootstrapVersion": { "id": "BootstrapVersion", "path": "aws-cdk-logs-insights-querydefinition-integ/BootstrapVersion", @@ -97,7 +127,7 @@ "path": "LogsInsightsQueryDefinitionIntegTest/DefaultTest/Default", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.1.161" + "version": "10.1.237" } }, "DeployAssert": { @@ -143,7 +173,7 @@ "path": "Tree", "constructInfo": { "fqn": "constructs.Construct", - "version": "10.1.161" + "version": "10.1.237" } } }, diff --git a/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.ts b/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.ts index b3f19a5940d11..ad022bcc486a7 100644 --- a/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.ts +++ b/packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.ts @@ -11,6 +11,7 @@ class LogsInsightsQueryDefinitionIntegStack extends Stack { removalPolicy: RemovalPolicy.DESTROY, }); + // Test query creation with single parse and filter statements new QueryDefinition(this, 'QueryDefinition', { queryDefinitionName: 'QueryDefinition', queryString: new QueryString({ @@ -23,6 +24,26 @@ class LogsInsightsQueryDefinitionIntegStack extends Stack { }), logGroups: [logGroup], }); + + // Test query creation with multiple parse and filter statements + new QueryDefinition(this, 'QueryDefinitionWithMultipleStatements', { + queryDefinitionName: 'QueryDefinitionWithMultipleStatements', + queryString: new QueryString({ + fields: ['@timestamp', '@message'], + parseStatements: [ + '@message "[*] *" as loggingType, loggingMessage', + '@message "<*>: *" as differentLoggingType, differentLoggingMessage', + ], + filterStatements: [ + 'loggingType = "ERROR"', + 'loggingMessage = "A very strange error occurred!"', + ], + sort: '@timestamp desc', + limit: 20, + display: 'loggingMessage', + }), + logGroups: [logGroup], + }); } } diff --git a/packages/@aws-cdk/aws-logs/test/query-definition.test.ts b/packages/@aws-cdk/aws-logs/test/query-definition.test.ts index 7b9fd5cb7d4ff..10593be3c35f6 100644 --- a/packages/@aws-cdk/aws-logs/test/query-definition.test.ts +++ b/packages/@aws-cdk/aws-logs/test/query-definition.test.ts @@ -1,4 +1,5 @@ import { Template } from '@aws-cdk/assertions'; +import { testDeprecated } from '@aws-cdk/cdk-build-tools'; import { Stack } from '@aws-cdk/core'; import { LogGroup, QueryDefinition, QueryString } from '../lib'; @@ -49,7 +50,7 @@ describe('query definition', () => { }); }); - test('create a query definition with all commands', () => { + testDeprecated('create a query definition with all commands', () => { // GIVEN const stack = new Stack(); @@ -75,4 +76,64 @@ describe('query definition', () => { QueryString: 'fields @timestamp, @message\n| parse @message "[*] *" as loggingType, loggingMessage\n| filter loggingType = "ERROR"\n| sort @timestamp desc\n| limit 20\n| display loggingMessage', }); }); + + test('create a query definition with multiple statements for supported commands', () => { + // GIVEN + const stack = new Stack(); + + // WHEN + new QueryDefinition(stack, 'QueryDefinition', { + queryDefinitionName: 'MyQuery', + queryString: new QueryString({ + fields: ['@timestamp', '@message'], + parseStatements: [ + '@message "[*] *" as loggingType, loggingMessage', + '@message "<*>: *" as differentLoggingType, differentLoggingMessage', + ], + filterStatements: [ + 'loggingType = "ERROR"', + 'loggingMessage = "A very strange error occurred!"', + ], + sort: '@timestamp desc', + limit: 20, + }), + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::Logs::QueryDefinition', { + Name: 'MyQuery', + QueryString: 'fields @timestamp, @message\n| parse @message "[*] *" as loggingType, loggingMessage\n| parse @message "<*>: *" as differentLoggingType, differentLoggingMessage\n| filter loggingType = "ERROR"\n| filter loggingMessage = "A very strange error occurred!"\n| sort @timestamp desc\n| limit 20', + }); + }); + + testDeprecated('create a query with both single and multi statement properties for filtering and parsing', () => { + // GIVEN + const stack = new Stack(); + + // WHEN + new QueryDefinition(stack, 'QueryDefinition', { + queryDefinitionName: 'MyQuery', + queryString: new QueryString({ + fields: ['@timestamp', '@message'], + parse: '@message "[*] *" as loggingType, loggingMessage', + parseStatements: [ + '@message "[*] *" as loggingType, loggingMessage', + '@message "<*>: *" as differentLoggingType, differentLoggingMessage', + ], + filter: 'loggingType = "ERROR"', + filterStatements: [ + 'loggingType = "ERROR"', + 'loggingMessage = "A very strange error occurred!"', + ], + sort: '@timestamp desc', + limit: 20, + }), + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::Logs::QueryDefinition', { + Name: 'MyQuery', + QueryString: 'fields @timestamp, @message\n| parse @message "[*] *" as loggingType, loggingMessage\n| parse @message "<*>: *" as differentLoggingType, differentLoggingMessage\n| filter loggingType = "ERROR"\n| filter loggingMessage = "A very strange error occurred!"\n| sort @timestamp desc\n| limit 20', + }); + }); });