feat(logs): Add support for multiple parse and filter statements in QueryString

packages/@aws-cdk/aws-logs/README.md

@@ -330,6 +330,14 @@ new logs.QueryDefinition(this, 'QueryDefinition', {
   queryDefinitionName: 'MyQuery',
   queryString: new logs.QueryString({
     fields: ['@timestamp', '@message'],
+    parseStatements: [
+      '@message "[*] *" as loggingType, loggingMessage',
+      '@message "<*>: *" as differentLoggingType, differentLoggingMessage',
+    ],
+    filterStatements: [
+      'loggingType = "ERROR"',
+      'loggingMessage = "A very strange error occurred!"',
+    ],
     sort: '@timestamp desc',
     limit: 20,
   }),

packages/@aws-cdk/aws-logs/lib/query-definition.ts

@@ -16,19 +16,43 @@ export interface QueryStringProps {
   readonly fields?: string[];
 
   /**
-  * Extracts data from a log field and creates one or more ephemeral fields that you can process further in the query.
+  * A single statement for parsing data from a log field and creating ephemeral fields that can
+  * be processed further in the query.
   *
+  * @deprecated Use `parseStatements` instead
   * @default - no parse in QueryString
   */
   readonly parse?: string;
 
   /**
-  * Filters the results of a query that's based on one or more conditions.
+  * An array of one or more statements for parsing data from a log field and creating ephemeral
+  * fields that can be processed further in the query. Each provided statement generates a separate
+  * parse line in the query string.
   *
+  * Note: If provided, this property overrides any value provided for the `parse` property.
+  *
+  * @default - no parse in QueryString
+  */
+  readonly parseStatements?: string[];
+
+  /**
+  * A single statement for filtering the results of a query based on a boolean expression.
+  *
+  * @deprecated Use `filterStatements` instead
   * @default - no filter in QueryString
   */
   readonly filter?: string;
 
+  /**
+  * An array of one or more statements for filtering the results of a query based on a boolean
+  * expression. Each provided statement generates a separate filter line in the query string.
+  *
+  * Note: If provided, this property overrides any value provided for the `filter` property.
+  *
+  * @default - no filter in QueryString
+  */
+  readonly filterStatements?: string[];
+
   /**
   * Uses log field values to calculate aggregate statistics.
   *
@@ -58,62 +82,88 @@ export interface QueryStringProps {
   readonly display?: string;
 }
 
-interface QueryStringMap {
-  readonly fields?: string,
-  readonly parse?: string,
-  readonly filter?: string,
-  readonly stats?: string,
-  readonly sort?: string,
-  readonly limit?: Number,
-  readonly display?: string,
-}
-
 /**
  * Define a QueryString
  */
 export class QueryString {
   private readonly fields?: string[];
-  private readonly parse?: string;
-  private readonly filter?: string;
+  private readonly parse: string[];
+  private readonly filter: string[];
   private readonly stats?: string;
   private readonly sort?: string;
   private readonly limit?: Number;
   private readonly display?: string;
 
   constructor(props: QueryStringProps = {}) {
     this.fields = props.fields;
-    this.parse = props.parse;
-    this.filter = props.filter;
     this.stats = props.stats;
     this.sort = props.sort;
     this.limit = props.limit;
     this.display = props.display;
+
+    // Determine parsing by either the parseStatements or parse properties, or default to empty array
+    if (props.parseStatements) {
+      this.parse = props.parseStatements;
+    } else if (props.parse) {
+      this.parse = [props.parse];
+    } else {
+      this.parse = [];
+    }
+
+    // Determine filtering by either the filterStatements or filter properties, or default to empty array
+    if (props.filterStatements) {
+      this.filter = props.filterStatements;
+    } else if (props.filter) {
+      this.filter = [props.filter];
+    } else {
+      this.filter = [];
+    }
   }
 
   /**
   * String representation of this QueryString.
   */
   public toString(): string {
-    return noUndef({
-      fields: this.fields !== undefined ? this.fields.join(', ') : this.fields,
-      parse: this.parse,
-      filter: this.filter,
-      stats: this.stats,
-      sort: this.sort,
-      limit: this.limit,
-      display: this.display,
-    }).join('\n| ');
+    return [
+      this.buildQueryLine('fields', this.fields?.join(', ')),
+      ...this.buildQueryLines('parse', this.parse),
+      ...this.buildQueryLines('filter', this.filter),
+      this.buildQueryLine('stats', this.stats),
+      this.buildQueryLine('sort', this.sort),
+      this.buildQueryLine('limit', this.limit?.toString()),
+      this.buildQueryLine('display', this.display),
+    ].filter(
+      (queryLine) => queryLine !== undefined && queryLine.length > 0,
+    ).join('\n| ');
   }
-}
 
-function noUndef(x: QueryStringMap): string[] {
-  const ret: string[] = [];
-  for (const [key, value] of Object.entries(x)) {
-    if (value !== undefined) {
-      ret.push(`${key} ${value}`);
+  /**
+   * Build an array of query lines given a command and statement(s).
+   *
+   * @param command a query command
+   * @param statements one or more query statements for the specified command, or undefined
+   * @returns an array of the query string lines generated from the provided command and statements
+   */
+  private buildQueryLines(command: string, statements?: string[]): string[] {
+    if (statements === undefined) {
+      return [];
     }
+
+    return statements.map(
+      (statement: string): string => this.buildQueryLine(command, statement),
+    );
+  }
+
+  /**
+   * Build a single query line given a command and statement.
+   *
+   * @param command a query command
+   * @param statement a single query statement
+   * @returns a single query string line generated from the provided command and statement
+   */
+  private buildQueryLine(command: string, statement?: string): string {
+    return statement ? `${command} ${statement}` : '';
   }
-  return ret;
 }
 
 /**

packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/LogsInsightsQueryDefinitionIntegTestDefaultTestDeployAssert902BAAD5.assets.json

@@ -1,5 +1,5 @@
 {
-  "version": "21.0.0",
+  "version": "29.0.0",
   "files": {
     "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": {
       "source": {

packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/aws-cdk-logs-insights-querydefinition-integ.assets.json

@@ -1,15 +1,15 @@
 {
-  "version": "21.0.0",
+  "version": "29.0.0",
   "files": {
-    "e6a0a51961925fbc37d9b81431449c256ed453f98089eb70f83850f237b4d722": {
+    "3546c78647ea20567832041c960a034787f9bfc0128226ea8fbc0894366a4dd0": {
       "source": {
         "path": "aws-cdk-logs-insights-querydefinition-integ.template.json",
         "packaging": "file"
       },
       "destinations": {
         "current_account-current_region": {
           "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
-          "objectKey": "e6a0a51961925fbc37d9b81431449c256ed453f98089eb70f83850f237b4d722.json",
+          "objectKey": "3546c78647ea20567832041c960a034787f9bfc0128226ea8fbc0894366a4dd0.json",
           "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
         }
       }

packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/aws-cdk-logs-insights-querydefinition-integ.template.json

@@ -19,6 +19,18 @@
      }
     ]
    }
+  },
+  "QueryDefinitionWithMultipleStatements58A3EF74": {
+   "Type": "AWS::Logs::QueryDefinition",
+   "Properties": {
+    "Name": "QueryDefinitionWithMultipleStatements",
+    "QueryString": "fields @timestamp, @message\n| parse @message \"[*] *\" as loggingType, loggingMessage\n| parse @message \"<*>: *\" as differentLoggingType, differentLoggingMessage\n| filter loggingType = \"ERROR\"\n| filter loggingMessage = \"A very strange error occurred!\"\n| sort @timestamp desc\n| limit 20\n| display loggingMessage",
+    "LogGroupNames": [
+     {
+      "Ref": "LogGroupF5B46931"
+     }
+    ]
+   }
   }
  },
  "Parameters": {

packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/cdk.out

@@ -1 +1 @@
-{"version":"21.0.0"}
+{"version":"29.0.0"}

packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/integ.json

@@ -1,5 +1,5 @@
 {
-  "version": "21.0.0",
+  "version": "29.0.0",
   "testCases": {
     "LogsInsightsQueryDefinitionIntegTest/DefaultTest": {
       "stacks": [

packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/manifest.json

@@ -1,5 +1,5 @@
 {
-  "version": "21.0.0",
+  "version": "29.0.0",
   "artifacts": {
     "aws-cdk-logs-insights-querydefinition-integ.assets": {
       "type": "cdk:asset-manifest",
@@ -17,7 +17,7 @@
         "validateOnSynth": false,
         "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}",
         "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}",
-        "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/e6a0a51961925fbc37d9b81431449c256ed453f98089eb70f83850f237b4d722.json",
+        "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/3546c78647ea20567832041c960a034787f9bfc0128226ea8fbc0894366a4dd0.json",
         "requiresBootstrapStackVersion": 6,
         "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version",
         "additionalDependencies": [
@@ -45,6 +45,12 @@
             "data": "QueryDefinition4190BC36"
           }
         ],
+        "/aws-cdk-logs-insights-querydefinition-integ/QueryDefinitionWithMultipleStatements/Resource": [
+          {
+            "type": "aws:cdk:logicalId",
+            "data": "QueryDefinitionWithMultipleStatements58A3EF74"
+          }
+        ],
         "/aws-cdk-logs-insights-querydefinition-integ/BootstrapVersion": [
           {
             "type": "aws:cdk:logicalId",

packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.js.snapshot/tree.json

@@ -62,6 +62,36 @@
               "version": "0.0.0"
             }
           },
+          "QueryDefinitionWithMultipleStatements": {
+            "id": "QueryDefinitionWithMultipleStatements",
+            "path": "aws-cdk-logs-insights-querydefinition-integ/QueryDefinitionWithMultipleStatements",
+            "children": {
+              "Resource": {
+                "id": "Resource",
+                "path": "aws-cdk-logs-insights-querydefinition-integ/QueryDefinitionWithMultipleStatements/Resource",
+                "attributes": {
+                  "aws:cdk:cloudformation:type": "AWS::Logs::QueryDefinition",
+                  "aws:cdk:cloudformation:props": {
+                    "name": "QueryDefinitionWithMultipleStatements",
+                    "queryString": "fields @timestamp, @message\n| parse @message \"[*] *\" as loggingType, loggingMessage\n| parse @message \"<*>: *\" as differentLoggingType, differentLoggingMessage\n| filter loggingType = \"ERROR\"\n| filter loggingMessage = \"A very strange error occurred!\"\n| sort @timestamp desc\n| limit 20\n| display loggingMessage",
+                    "logGroupNames": [
+                      {
+                        "Ref": "LogGroupF5B46931"
+                      }
+                    ]
+                  }
+                },
+                "constructInfo": {
+                  "fqn": "@aws-cdk/aws-logs.CfnQueryDefinition",
+                  "version": "0.0.0"
+                }
+              }
+            },
+            "constructInfo": {
+              "fqn": "@aws-cdk/aws-logs.QueryDefinition",
+              "version": "0.0.0"
+            }
+          },
           "BootstrapVersion": {
             "id": "BootstrapVersion",
             "path": "aws-cdk-logs-insights-querydefinition-integ/BootstrapVersion",
@@ -97,7 +127,7 @@
                 "path": "LogsInsightsQueryDefinitionIntegTest/DefaultTest/Default",
                 "constructInfo": {
                   "fqn": "constructs.Construct",
-                  "version": "10.1.161"
+                  "version": "10.1.237"
                 }
               },
               "DeployAssert": {
@@ -143,7 +173,7 @@
         "path": "Tree",
         "constructInfo": {
           "fqn": "constructs.Construct",
-          "version": "10.1.161"
+          "version": "10.1.237"
         }
       }
     },

packages/@aws-cdk/aws-logs/test/integ.save-logs-insights-query-definition.ts

@@ -11,6 +11,7 @@ class LogsInsightsQueryDefinitionIntegStack extends Stack {
       removalPolicy: RemovalPolicy.DESTROY,
     });
 
+    // Test query creation with single parse and filter statements
     new QueryDefinition(this, 'QueryDefinition', {
       queryDefinitionName: 'QueryDefinition',
       queryString: new QueryString({
@@ -23,6 +24,26 @@ class LogsInsightsQueryDefinitionIntegStack extends Stack {
       }),
       logGroups: [logGroup],
     });
+
+    // Test query creation with multiple parse and filter statements
+    new QueryDefinition(this, 'QueryDefinitionWithMultipleStatements', {
+      queryDefinitionName: 'QueryDefinitionWithMultipleStatements',
+      queryString: new QueryString({
+        fields: ['@timestamp', '@message'],
+        parseStatements: [
+          '@message "[*] *" as loggingType, loggingMessage',
+          '@message "<*>: *" as differentLoggingType, differentLoggingMessage',
+        ],
+        filterStatements: [
+          'loggingType = "ERROR"',
+          'loggingMessage = "A very strange error occurred!"',
+        ],
+        sort: '@timestamp desc',
+        limit: 20,
+        display: 'loggingMessage',
+      }),
+      logGroups: [logGroup],
+    });
   }
 }