update the cloudtrail s3 integration assets

integrations/observability/aws_cloudtrail/assets/tables/aws_cloudtrail_s3_table-1.0.0.sql

@@ -0,0 +1,53 @@
+CREATE TABLE IF NOT EXISTS ${table} (
+ Records ARRAY<STRUCT<
+ awsRegion: STRING,
+ eventCategory: STRING,
+ eventID: STRING,
+ eventName: STRING,
+ eventSource: STRING,
+ eventTime: STRING,
+ eventType: STRING,
+ eventVersion: STRING,
+ managementEvent: STRING,
+ readOnly: BOOLEAN,
+ recipientAccountId: STRING,
+ requestID: STRING,
+ requestParameters: STRUCT<
+ assumeRolePolicyDocument: STRING,
+ description: STRING,
+ roleName: STRING
+ >,
+ responseElements: STRUCT<
+ role: STRUCT<
+ arn: STRING,
+ assumeRolePolicyDocument: STRING,
+ createDate: STRING,
+ path: STRING,
+ roleId: STRING,
+ roleName: STRING
+ >
+ >,
+ sessionCredentialFromConsole: STRING,
+ sourceIPAddress: STRING,
+ tlsDetails: STRUCT<
+ cipherSuite: STRING,
+ clientProvidedHostHeader: STRING,
+ tlsVersion: STRING
+ >,
+ userAgent: STRING,
+ userIdentity: STRUCT<
+ accessKeyId: STRING,
+ accountId: STRING,
+ arn: STRING,
+ principalId: STRING,
+ sessionContext: STRUCT<
+ attributes: STRUCT<
+ creationDate: STRING,
+ mfaAuthenticated: STRING
+ >
+ >,
+ type: STRING,
+ userName: STRING
+ >
+ >>
+) USING json OPTIONS ('path' ${bucket});

integrations/observability/aws_cloudtrail/assets/tables/aws_elb_s3_mv_index-1.0.0.sql

@@ -0,0 +1,45 @@
+CREATE VIEW AS
+ SELECT
+ exploded_records.eventVersion AS `aws.cloudtrail.eventVersion`,
+ exploded_records.userIdentity.type AS `aws.cloudtrail.userIdentity.type`,
+ exploded_records.userIdentity.principalId AS `aws.cloudtrail.userIdentity.principalId`,
+ exploded_records.userIdentity.arn AS `aws.cloudtrail.userIdentity.arn`,
+ exploded_records.userIdentity.accountId AS `aws.cloudtrail.userIdentity.accountId`,
+ exploded_records.userIdentity.accessKeyId AS `aws.cloudtrail.userIdentity.accessKeyId`,
+ exploded_records.userIdentity.userName AS `aws.cloudtrail.userIdentity.userName`,
+ exploded_records.userIdentity.sessionContext.attributes.creationDate AS `aws.cloudtrail.userIdentity.sessionContext.attributes.creationDate`,
+ exploded_records.userIdentity.sessionContext.attributes.mfaAuthenticated AS `aws.cloudtrail.userIdentity.sessionContext.attributes.mfaAuthenticated`,
+ exploded_records.eventTime AS `aws.cloudtrail.eventTime`,
+ exploded_records.eventSource AS `aws.cloudtrail.eventSource`,
+ exploded_records.eventName AS `aws.cloudtrail.eventName`,
+ exploded_records.awsRegion AS `aws.cloudtrail.awsRegion`,
+ exploded_records.sourceIPAddress AS `aws.cloudtrail.sourceIPAddress`,
+ exploded_records.userAgent AS `aws.cloudtrail.userAgent`,
+ exploded_records.requestParameters.roleName AS `aws.cloudtrail.requestParameters.roleName`,
+ exploded_records.requestParameters.description AS `aws.cloudtrail.requestParameters.description`,
+ exploded_records.requestParameters.assumeRolePolicyDocument AS `aws.cloudtrail.requestParameters.assumeRolePolicyDocument`,
+ exploded_records.responseElements.role.assumeRolePolicyDocument AS `aws.cloudtrail.responseElements.role.assumeRolePolicyDocument`,
+ exploded_records.responseElements.role.arn AS `aws.cloudtrail.responseElements.role.arn`,
+ exploded_records.responseElements.role.roleId AS `aws.cloudtrail.responseElements.role.roleId`,
+ exploded_records.responseElements.role.createDate AS `aws.cloudtrail.responseElements.role.createDate`,
+ exploded_records.responseElements.role.roleName AS `aws.cloudtrail.responseElements.role.roleName`,
+ exploded_records.responseElements.role.path AS `aws.cloudtrail.responseElements.role.path`,
+ exploded_records.requestID AS `aws.cloudtrail.requestID`,
+ exploded_records.eventID AS `aws.cloudtrail.eventID`,
+ exploded_records.readOnly AS `aws.cloudtrail.readOnly`,
+ exploded_records.eventType AS `aws.cloudtrail.eventType`,
+ exploded_records.managementEvent AS `aws.cloudtrail.managementEvent`,
+ exploded_records.recipientAccountId AS `aws.cloudtrail.recipientAccountId`,
+ exploded_records.eventCategory AS `aws.cloudtrail.eventCategory`,
+ exploded_records.tlsDetails.tlsVersion AS `aws.cloudtrail.tlsDetails.tlsVersion`,
+ exploded_records.tlsDetails.cipherSuite AS `aws.cloudtrail.tlsDetails.cipherSuite`,
+ exploded_records.tlsDetails.clientProvidedHostHeader AS `aws.cloudtrail.tlsDetails.clientProvidedHostHeader`,
+ exploded_records.sessionCredentialFromConsole AS `aws.cloudtrail.sessionCredentialFromConsole`
+ FROM (
+ SELECT explode(Records) as exploded_records
+ FROM ${table}
+ ) t;
+WITH (
+ auto_refresh = true,
+ refresh_interval = '5 Seconds'
+)

integrations/observability/aws_cloudtrail/aws_cloudtrail-1.0.0.json

@@ -40,6 +40,16 @@
  }
  ],
  "assets": {
+ "tables": [
+ {
+ "name": "aws_cloudtrail_s3_table",
+ "version": "1.0.0"
+ },
+ {
+ "name": "aws_elb_s3_mv_index",
+ "version": "1.0.0"
+ }
+ ],
  "savedObjects": {
  "name": "aws_cloudtrail",
  "version": "1.0.0"

integrations/observability/aws_cloudtrail/data/raw-sample-2.json

@@ -0,0 +1,53 @@
+{"Records": [{
+ "eventVersion": "1.08",
+ "userIdentity": {
+ "type": "IAMUser",
+ "principalId": "AIDA6ON6E4XEGITEXAMPLE",
+ "arn": "arn:aws:iam::777777777777:user/Saanvi",
+ "accountId": "777777777777",
+ "accessKeyId": "AKIAIOSFODNN7EXAMPLE",
+ "userName": "Saanvi",
+ "sessionContext": {
+ "sessionIssuer": {},
+ "webIdFederationData": {},
+ "attributes": {
+ "creationDate": "2023-07-19T21:11:57Z",
+ "mfaAuthenticated": "false"
+ }
+ }
+ },
+ "eventTime": "2023-07-19T21:29:12Z",
+ "eventSource": "iam.amazonaws.com",
+ "eventName": "CreateRole",
+ "awsRegion": "us-east-1",
+ "sourceIPAddress": "192.0.2.0",
+ "userAgent": "aws-cli/2.13.5 Python/3.11.4 Linux/4.14.255-314-253.539.amzn2.x86_64 exec-env/CloudShell exe/x86_64.amzn.2 prompt/off command/iam.create-role",
+ "requestParameters": {
+ "roleName": "TestRole",
+ "description": "Allows EC2 instances to call AWS services on your behalf.",
+ "assumeRolePolicyDocument": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"sts:AssumeRole\"],\"Principal\":{\"Service\":[\"ec2.amazonaws.com\"]}}]}"
+ },
+ "responseElements": {
+ "role": {
+ "assumeRolePolicyDocument": "%7B%22Version%22%3A%222012-10-17%22%2C%22Statement%22%3A%5B%7B%22Effect%22%3A%22Allow%22%2C%22Action%22%3A%5B%22sts%3AAssumeRole%22%5D%2C%22Principal%22%3A%7B%22Service%22%3A%5B%22ec2.amazonaws.com%22%5D%7D%7D%5D%7D",
+ "arn": "arn:aws:iam::777777777777:role/TestRole",
+ "roleId": "AROA6ON6E4XEFFEXAMPLE",
+ "createDate": "Jul 19, 2023 9:29:12 PM",
+ "roleName": "TestRole",
+ "path": "/"
+ }
+ },
+ "requestID": "ff38f36e-ebd3-425b-9939-EXAMPLE1bbe",
+ "eventID": "9da77cd0-493f-4c89-8852-EXAMPLEa887c",
+ "readOnly": false,
+ "eventType": "AwsApiCall",
+ "managementEvent": true,
+ "recipientAccountId": "777777777777",
+ "eventCategory": "Management",
+ "tlsDetails": {
+ "tlsVersion": "TLSv1.2",
+ "cipherSuite": "ECDHE-RSA-AES128-GCM-SHA256",
+ "clientProvidedHostHeader": "iam.amazonaws.com"
+ },
+ "sessionCredentialFromConsole": "true"
+}]}

integrations/observability/aws_cloudtrail/data/raw-sample-3.json

@@ -0,0 +1,79 @@
+{"Records": [{
+ "eventVersion": "1.08",
+ "userIdentity": {
+ "type": "IAMUser",
+ "principalId": "EXAMPLE6E4XEGITWATV6R",
+ "arn": "arn:aws:iam::123456789012:user/Mateo",
+ "accountId": "123456789012",
+ "accessKeyId": "AKIAIOSFODNN7EXAMPLE",
+ "userName": "Mateo",
+ "sessionContext": {
+ "sessionIssuer": {},
+ "webIdFederationData": {},
+ "attributes": {
+ "creationDate": "2023-07-19T21:11:57Z",
+ "mfaAuthenticated": "false"
+ }
+ }
+ },
+ "eventTime": "2023-07-19T21:17:28Z",
+ "eventSource": "ec2.amazonaws.com",
+ "eventName": "StartInstances",
+ "awsRegion": "us-east-1",
+ "sourceIPAddress": "192.0.2.0",
+ "userAgent": "aws-cli/2.13.5 Python/3.11.4 Linux/4.14.255-314-253.539.amzn2.x86_64 exec-env/CloudShell exe/x86_64.amzn.2 prompt/off command/ec2.start-instances",
+ "requestParameters": {
+ "instancesSet": {
+ "items": [
+ {
+ "instanceId": "i-EXAMPLE56126103cb"
+ },
+ {
+ "instanceId": "i-EXAMPLEaff4840c22"
+ }
+ ]
+ }
+ },
+ "responseElements": {
+ "requestId": "e4336db0-149f-4a6b-844d-EXAMPLEb9d16",
+ "instancesSet": {
+ "items": [
+ {
+ "instanceId": "i-EXAMPLEaff4840c22",
+ "currentState": {
+ "code": 0,
+ "name": "pending"
+ },
+ "previousState": {
+ "code": 80,
+ "name": "stopped"
+ }
+ },
+ {
+ "instanceId": "i-EXAMPLE56126103cb",
+ "currentState": {
+ "code": 0,
+ "name": "pending"
+ },
+ "previousState": {
+ "code": 80,
+ "name": "stopped"
+ }
+ }
+ ]
+ }
+ },
+ "requestID": "e4336db0-149f-4a6b-844d-EXAMPLEb9d16",
+ "eventID": "e755e09c-42f9-4c5c-9064-EXAMPLE228c7",
+ "readOnly": false,
+ "eventType": "AwsApiCall",
+ "managementEvent": true,
+ "recipientAccountId": "123456789012",
+ "eventCategory": "Management",
+ "tlsDetails": {
+ "tlsVersion": "TLSv1.2",
+ "cipherSuite": "ECDHE-RSA-AES128-GCM-SHA256",
+ "clientProvidedHostHeader": "ec2.us-east-1.amazonaws.com"
+ },
+ "sessionCredentialFromConsole": "true"
+}]}

integrations/observability/aws_cloudtrail/data/raw-sample-4.json

@@ -0,0 +1,50 @@
+{"Records": [{
+ "eventVersion": "1.08",
+ "userIdentity": {
+ "type": "IAMUser",
+ "principalId": "AIDA6ON6E4XEGITEXAMPLE",
+ "arn": "arn:aws:iam::888888888888:user/Mary",
+ "accountId": "888888888888",
+ "accessKeyId": "AKIAIOSFODNN7EXAMPLE",
+ "userName": "Mary",
+ "sessionContext": {
+ "sessionIssuer": {},
+ "webIdFederationData": {},
+ "attributes": {
+ "creationDate": "2023-07-19T21:11:57Z",
+ "mfaAuthenticated": "false"
+ }
+ }
+ },
+ "eventTime": "2023-07-19T21:25:09Z",
+ "eventSource": "iam.amazonaws.com",
+ "eventName": "CreateUser",
+ "awsRegion": "us-east-1",
+ "sourceIPAddress": "192.0.2.0",
+ "userAgent": "aws-cli/2.13.5 Python/3.11.4 Linux/4.14.255-314-253.539.amzn2.x86_64 exec-env/CloudShell exe/x86_64.amzn.2 prompt/off command/iam.create-user",
+ "requestParameters": {
+ "userName": "Richard"
+ },
+ "responseElements": {
+ "user": {
+ "path": "/",
+ "arn": "arn:aws:iam::888888888888:user/Richard",
+ "userId": "AIDA6ON6E4XEP7EXAMPLE",
+ "createDate": "Jul 19, 2023 9:25:09 PM",
+ "userName": "Richard"
+ }
+ },
+ "requestID": "2d528c76-329e-410b-9516-EXAMPLE565dc",
+ "eventID": "ba0801a1-87ec-4d26-be87-EXAMPLE75bbb",
+ "readOnly": false,
+ "eventType": "AwsApiCall",
+ "managementEvent": true,
+ "recipientAccountId": "888888888888",
+ "eventCategory": "Management",
+ "tlsDetails": {
+ "tlsVersion": "TLSv1.2",
+ "cipherSuite": "ECDHE-RSA-AES128-GCM-SHA256",
+ "clientProvidedHostHeader": "iam.amazonaws.com"
+ },
+ "sessionCredentialFromConsole": "true"
+}]} 

integrations/observability/aws_cloudtrail/data/raw-sample-5.json

@@ -0,0 +1,54 @@
+{"Records": [
+ {
+ "eventVersion": "1.08",
+ "userIdentity": {
+ "type": "IAMUser",
+ "principalId": "AIDA6ON6E4XEGITEXAMPLE",
+ "arn": "arn:aws:iam::777777777777:user/Saanvi",
+ "accountId": "777777777777",
+ "accessKeyId": "AKIAIOSFODNN7EXAMPLE",
+ "userName": "Saanvi",
+ "sessionContext": {
+ "sessionIssuer": {},
+ "webIdFederationData": {},
+ "attributes": {
+ "creationDate": "2023-07-19T21:11:57Z",
+ "mfaAuthenticated": "false"
+ }
+ }
+ },
+ "eventTime": "2023-07-19T21:29:12Z",
+ "eventSource": "iam.amazonaws.com",
+ "eventName": "CreateRole",
+ "awsRegion": "us-east-1",
+ "sourceIPAddress": "192.0.2.0",
+ "userAgent": "aws-cli/2.13.5 Python/3.11.4 Linux/4.14.255-314-253.539.amzn2.x86_64 exec-env/CloudShell exe/x86_64.amzn.2 prompt/off command/iam.create-role",
+ "requestParameters": {
+ "roleName": "TestRole",
+ "description": "Allows EC2 instances to call AWS services on your behalf.",
+ "assumeRolePolicyDocument": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"sts:AssumeRole\"],\"Principal\":{\"Service\":[\"ec2.amazonaws.com\"]}}]}"
+ },
+ "responseElements": {
+ "role": {
+ "assumeRolePolicyDocument": "%7B%22Version%22%3A%222012-10-17%22%2C%22Statement%22%3A%5B%7B%22Effect%22%3A%22Allow%22%2C%22Action%22%3A%5B%22sts%3AAssumeRole%22%5D%2C%22Principal%22%3A%7B%22Service%22%3A%5B%22ec2.amazonaws.com%22%5D%7D%7D%5D%7D",
+ "arn": "arn:aws:iam::777777777777:role/TestRole",
+ "roleId": "AROA6ON6E4XEFFEXAMPLE",
+ "createDate": "Jul 19, 2023 9:29:12 PM",
+ "roleName": "TestRole",
+ "path": "/"
+ }
+ },
+ "requestID": "ff38f36e-ebd3-425b-9939-EXAMPLE1bbe",
+ "eventID": "9da77cd0-493f-4c89-8852-EXAMPLEa887c",
+ "readOnly": false,
+ "eventType": "AwsApiCall",
+ "managementEvent": true,
+ "recipientAccountId": "777777777777",
+ "eventCategory": "Management",
+ "tlsDetails": {
+ "tlsVersion": "TLSv1.2",
+ "cipherSuite": "ECDHE-RSA-AES128-GCM-SHA256",
+ "clientProvidedHostHeader": "iam.amazonaws.com"
+ },
+ "sessionCredentialFromConsole": "true"
+}]}