From cc0958b2dcece8af0493be37ffb42c515886d525 Mon Sep 17 00:00:00 2001 From: Calvin Kim Date: Tue, 28 Mar 2023 13:14:23 -0500 Subject: [PATCH 1/6] gcp bigquery large query detection --- packs/gcp_audit.yml | 1 + .../gcp_bigquery_large_scan.py | 73 ++++++++ .../gcp_bigquery_large_scan.yml | 164 ++++++++++++++++++ 3 files changed, 238 insertions(+) create mode 100644 rules/gcp_audit_rules/gcp_bigquery_large_scan.py create mode 100644 rules/gcp_audit_rules/gcp_bigquery_large_scan.yml diff --git a/packs/gcp_audit.yml b/packs/gcp_audit.yml index c2e63a1f6..e195bd3ee 100644 --- a/packs/gcp_audit.yml +++ b/packs/gcp_audit.yml @@ -12,6 +12,7 @@ PackDefinition: - GCP.Access.Attempts.Violating.VPC.Service.Controls - GCP.Access.Attempts.Violating.IAP.Access.Controls - GCP.VPC.Flow.Logs.Disabled + - GCP.BigQuery.Large.Scan # Data model - Standard.GCP.AuditLog # Globals used in these rules/policies diff --git a/rules/gcp_audit_rules/gcp_bigquery_large_scan.py b/rules/gcp_audit_rules/gcp_bigquery_large_scan.py new file mode 100644 index 000000000..d6d712b83 --- /dev/null +++ b/rules/gcp_audit_rules/gcp_bigquery_large_scan.py @@ -0,0 +1,73 @@ +from panther_base_helpers import deep_get + +QUERY_THRESHOLD_BYTES = 1073741824 + + +def rule(event): + return all( + [ + deep_get(event, "resource", "type", default="").startswith("bigquery"), + deep_get(event, "operation", "last") is True, + deep_get(event, "protopayload", "metadata", "jobChange", "job", "jobConfig", "type") + == "QUERY", + deep_get( + event, + "protopayload", + "metadata", + "jobChange", + "job", + "jobConfig", + "queryConfig", + "statementType", + ) + == "SELECT", + int( + deep_get( + event, + "protopayload", + "metadata", + "jobChange", + "job", + "jobStats", + "queryStats", + "totalBilledBytes", + default=0, + ) + ) + > QUERY_THRESHOLD_BYTES, + ] + ) + + +def title(event): + actor = deep_get( + event, "protopayload", "authenticationInfo", "principalEmail", default="" + ) + query_size = deep_get( + event, + "protopayload", + "metadata", + "jobChange", + "job", + "jobStats", + "queryStats", + "totalBilledBytes", + default=0, + ) + return f"GCP: [{actor}] ran a large BigQuery query of [{query_size}] bytes." + + +def alert_context(event): + return { + "query": deep_get( + event, + "protopayload", + "metadata", + "jobChange", + "job", + "jobConfig", + "queryConfig", + "query", + default="", + ) + } diff --git a/rules/gcp_audit_rules/gcp_bigquery_large_scan.yml b/rules/gcp_audit_rules/gcp_bigquery_large_scan.yml new file mode 100644 index 000000000..042b747a4 --- /dev/null +++ b/rules/gcp_audit_rules/gcp_bigquery_large_scan.yml @@ -0,0 +1,164 @@ +AnalysisType: rule +Description: Detect any BigQuery query that is doing a very large scan (> 1 GB). +DisplayName: GCP BigQuery Large Scan +Enabled: false +Filename: gcp_bigquery_large_scan.py +Severity: Info +Tests: + - ExpectedResult: false + Log: + insertid: ABCDEFGHIJKL + logname: projects/gcp-project1/logs/cloudaudit.googleapis.com%2Fdata_access + operation: + id: 0123456789012-gcp-project1:abcdefg_hijklmnop_1234567abcd + last: true + producer: bigquery.googleapis.com + p_any_emails: + - user@company.io + p_any_ip_addresses: + - 1.2.3.4 + p_event_time: "2023-03-28 17:37:02.096" + p_log_type: GCP.AuditLog + p_parse_time: "2023-03-28 17:38:14.621" + p_row_id: de00d3dcdaeee4b4d5f7fa9d17b4c203 + p_schema_version: 0 + p_source_id: 964c7894-9a0d-4ddf-864f-0193438221d6 + p_source_label: gcp-logsource + protopayload: + at_sign_type: type.googleapis.com/google.cloud.audit.AuditLog + authenticationInfo: + principalEmail: user@company.io + authorizationInfo: + - granted: true + permission: bigquery.jobs.create + resource: projects/gcp-project1 + metadata: + '@type': type.googleapis.com/google.cloud.audit.BigQueryAuditMetadata + jobChange: + after: DONE + job: + jobConfig: + queryConfig: + createDisposition: CREATE_IF_NEEDED + destinationTable: projects/gcp-project1/datasets/_c2b49f742788f188022fcec1f1622e7404b40ce5/tables/anondea8e77183dcdf1cf93a47b5003036409a525808207450ab80db17fcc8cdcf53 + priority: QUERY_INTERACTIVE + query: |- + -- This query shows a list of the daily top Google Search terms. + SELECT + * + FROM `bigquery-public-data.google_trends.top_terms` + statementType: SELECT + writeDisposition: WRITE_TRUNCATE + type: QUERY + jobName: projects/gcp-project1/jobs/abcdefg_hijklmnop_1234567abcd + jobStats: + createTime: "2023-03-28T17:36:49.087Z" + endTime: "2023-03-28T17:37:02.092Z" + queryStats: + billingTier: 1 + outputRowCount: "43683701" + referencedTables: + - projects/bigquery-public-data/datasets/google_trends/tables/top_terms + totalBilledBytes: "100000" + totalProcessedBytes: "100000" + startTime: "2023-03-28T17:36:49.224Z" + totalSlotMs: "259581" + jobStatus: + jobState: DONE + methodName: google.cloud.bigquery.v2.JobService.InsertJob + requestMetadata: + callerIP: 1.2.3.4 + callerSuppliedUserAgent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36,gzip(gfe),gzip(gfe) + resourceName: projects/gcp-project1/jobs/abcdefg_hijklmnop_1234567abcd + serviceName: bigquery.googleapis.com + status: {} + receivetimestamp: "2023-03-28 17:37:02.114" + resource: + labels: + location: US + project_id: gcp-project1 + type: bigquery_project + severity: INFO + timestamp: "2023-03-28 17:37:02.096" + Name: small query + - ExpectedResult: true + Log: + insertid: ABCDEFGHIJKL + logname: projects/gcp-project1/logs/cloudaudit.googleapis.com%2Fdata_access + operation: + id: 0123456789012-gcp-project1:abcdefg_hijklmnop_1234567abcd + last: true + producer: bigquery.googleapis.com + p_any_emails: + - user@company.io + p_any_ip_addresses: + - 1.2.3.4 + p_event_time: "2023-03-28 17:37:02.096" + p_log_type: GCP.AuditLog + p_parse_time: "2023-03-28 17:38:14.621" + p_row_id: de00d3dcdaeee4b4d5f7fa9d17b4c203 + p_schema_version: 0 + p_source_id: 964c7894-9a0d-4ddf-864f-0193438221d6 + p_source_label: gcp-logsource + protopayload: + at_sign_type: type.googleapis.com/google.cloud.audit.AuditLog + authenticationInfo: + principalEmail: user@company.io + authorizationInfo: + - granted: true + permission: bigquery.jobs.create + resource: projects/gcp-project1 + metadata: + '@type': type.googleapis.com/google.cloud.audit.BigQueryAuditMetadata + jobChange: + after: DONE + job: + jobConfig: + queryConfig: + createDisposition: CREATE_IF_NEEDED + destinationTable: projects/gcp-project1/datasets/_c2b49f742788f188022fcec1f1622e7404b40ce5/tables/anondea8e77183dcdf1cf93a47b5003036409a525808207450ab80db17fcc8cdcf53 + priority: QUERY_INTERACTIVE + query: |- + -- This query shows a list of the daily top Google Search terms. + SELECT + * + FROM `bigquery-public-data.google_trends.top_terms` + statementType: SELECT + writeDisposition: WRITE_TRUNCATE + type: QUERY + jobName: projects/gcp-project1/jobs/abcdefg_hijklmnop_1234567abcd + jobStats: + createTime: "2023-03-28T17:36:49.087Z" + endTime: "2023-03-28T17:37:02.092Z" + queryStats: + billingTier: 1 + outputRowCount: "43683701" + referencedTables: + - projects/bigquery-public-data/datasets/google_trends/tables/top_terms + totalBilledBytes: "3097493504" + totalProcessedBytes: "3096864198" + startTime: "2023-03-28T17:36:49.224Z" + totalSlotMs: "259581" + jobStatus: + jobState: DONE + methodName: google.cloud.bigquery.v2.JobService.InsertJob + requestMetadata: + callerIP: 1.2.3.4 + callerSuppliedUserAgent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36,gzip(gfe),gzip(gfe) + resourceName: projects/gcp-project1/jobs/abcdefg_hijklmnop_1234567abcd + serviceName: bigquery.googleapis.com + status: {} + receivetimestamp: "2023-03-28 17:37:02.114" + resource: + labels: + location: US + project_id: gcp-project1 + type: bigquery_project + severity: INFO + timestamp: "2023-03-28 17:37:02.096" + Name: Large Query +DedupPeriodMinutes: 60 +LogTypes: + - GCP.AuditLog +RuleID: GCP.BigQuery.Large.Scan +Threshold: 1 From c3ea9f4f61eaf715ca785e4899421f8ab6161b6a Mon Sep 17 00:00:00 2001 From: Calvin Kim Date: Tue, 28 Mar 2023 14:51:21 -0500 Subject: [PATCH 2/6] gcp destructive query detection --- packs/gcp_audit.yml | 1 + .../gcp_destructive_queries.py | 79 +++++++++++ .../gcp_destructive_queries.yml | 125 ++++++++++++++++++ 3 files changed, 205 insertions(+) create mode 100644 rules/gcp_audit_rules/gcp_destructive_queries.py create mode 100644 rules/gcp_audit_rules/gcp_destructive_queries.yml diff --git a/packs/gcp_audit.yml b/packs/gcp_audit.yml index e195bd3ee..0d7e36a35 100644 --- a/packs/gcp_audit.yml +++ b/packs/gcp_audit.yml @@ -13,6 +13,7 @@ PackDefinition: - GCP.Access.Attempts.Violating.IAP.Access.Controls - GCP.VPC.Flow.Logs.Disabled - GCP.BigQuery.Large.Scan + - GCP.Destructive.Queries # Data model - Standard.GCP.AuditLog # Globals used in these rules/policies diff --git a/rules/gcp_audit_rules/gcp_destructive_queries.py b/rules/gcp_audit_rules/gcp_destructive_queries.py new file mode 100644 index 000000000..640fa05b0 --- /dev/null +++ b/rules/gcp_audit_rules/gcp_destructive_queries.py @@ -0,0 +1,79 @@ +from panther_base_helpers import deep_get + +DESTRUCTIVE_STATEMENTS = ("UPDATE", "DELETE", "DROP_TABLE", "ALTER_TABLE", "TRUNCATE_TABLE") + + +def rule(event): + return any( + [ + all( + [ + deep_get(event, "resource", "type", default="").startswith( + "bigquery" + ), + deep_get( + event, "protopayload", "metadata", "jobChange", "job", "jobConfig", "type" + ) + == "QUERY", + deep_get( + event, + "protopayload", + "metadata", + "jobChange", + "job", + "jobConfig", + "queryConfig", + "statementType", + default="", + ) + in DESTRUCTIVE_STATEMENTS, + ] + ), + deep_get(event, "protopayload", "metadata", "tableDeletion"), + deep_get(event, "protopayload", "metadata", "datasetDeletion"), + ] + ) + + +def title(event): + actor = deep_get( + event, "protopayload", "authenticationInfo", "principalEmail", default="" + ) + statement = deep_get( + event, + "protopayload", + "metadata", + "jobChange", + "job", + "jobConfig", + "queryConfig", + "statementType", + default="", + ) + table = deep_get( + event, + "protopayload", + "metadata", + "jobChange", + "job", + "jobConfig", + "queryConfig", + "destinationTable", + ) or deep_get(event, "protopayload", "metadata", "resourceName", default="") + return f"GCP: [{actor}] performed a destructive BigQuery [{statement}] query on [{table}]." + + +def alert_context(event): + return { + "query": deep_get( + event, + "protopayload", + "metadata", + "jobChange", + "job", + "jobConfig", + "queryConfig", + "query", + default="", + ) + } diff --git a/rules/gcp_audit_rules/gcp_destructive_queries.yml b/rules/gcp_audit_rules/gcp_destructive_queries.yml new file mode 100644 index 000000000..163b6214d --- /dev/null +++ b/rules/gcp_audit_rules/gcp_destructive_queries.yml @@ -0,0 +1,125 @@ +AnalysisType: rule +Description: Detect any destructive BigQuery queries or jobs such as update, delete, drop, alter or truncate. +DisplayName: 'GCP Destructive Queries ' +Enabled: true +Filename: gcp_destructive_queries.py +Severity: Info +Tests: + - ExpectedResult: true + Log: + insertid: abcdefghijklmn + logname: projects/gcp-project1/logs/cloudaudit.googleapis.com%2Fdata_access + operation: + id: 1234567890123-gcp-project1:abcdefghijklmnopqrstuvwz + last: true + producer: bigquery.googleapis.com + p_any_emails: + - user@company.io + p_any_ip_addresses: + - 1.2.3.4 + p_event_time: "2023-03-28 18:37:06.079" + p_log_type: GCP.AuditLog + p_parse_time: "2023-03-28 18:38:14.478" + p_row_id: 06bf03d9d5dfbadba981899e1787bf05 + p_schema_version: 0 + p_source_id: 964c7894-9a0d-4ddf-864f-0193438221d6 + p_source_label: gcp-logsource + protopayload: + at_sign_type: type.googleapis.com/google.cloud.audit.AuditLog + authenticationInfo: + principalEmail: user@company.io + authorizationInfo: + - granted: true + permission: bigquery.jobs.create + resource: projects/gcp-project1 + metadata: + '@type': type.googleapis.com/google.cloud.audit.BigQueryAuditMetadata + jobChange: + after: DONE + job: + jobConfig: + queryConfig: + createDisposition: CREATE_IF_NEEDED + destinationTable: projects/gcp-project1/datasets/test1/tables/newtable + priority: QUERY_INTERACTIVE + query: DROP TABLE test1.newtable + statementType: DROP_TABLE + writeDisposition: WRITE_EMPTY + type: QUERY + jobName: projects/gcp-project1/jobs/abcdefghijklmnopqrstuvwz + jobStats: + createTime: "2023-03-28T18:37:05.842Z" + endTime: "2023-03-28T18:37:06.073Z" + queryStats: {} + startTime: "2023-03-28T18:37:05.934Z" + jobStatus: + jobState: DONE + methodName: google.cloud.bigquery.v2.JobService.InsertJob + requestMetadata: + callerIP: 1.2.3.4 + callerSuppliedUserAgent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36,gzip(gfe),gzip(gfe) + resourceName: projects/gcp-project1/jobs/abcdefghijklmnopqrstuvwz + serviceName: bigquery.googleapis.com + status: {} + receivetimestamp: "2023-03-28 18:37:06.745" + resource: + labels: + location: US + project_id: gcp-project1 + type: bigquery_project + severity: INFO + timestamp: "2023-03-28 18:37:06.079" + Name: Drop Table Event + - ExpectedResult: true + Log: + insertid: abcdefghijklmn + logname: projects/gcp-project1/logs/cloudaudit.googleapis.com%2Factivity + operation: + id: 1234567890123-gcp-project1:abcdefghijklmnopqrstuvwz + last: true + producer: bigquery.googleapis.com + p_any_emails: + - user@company.io + p_any_ip_addresses: + - 1.2.3.4 + p_event_time: "2023-03-28 18:37:06.079" + p_log_type: GCP.AuditLog + p_parse_time: "2023-03-28 18:38:14.478" + p_row_id: 06bf03d9d5dfbadba981899e1787bf05 + p_schema_version: 0 + p_source_id: 964c7894-9a0d-4ddf-864f-0193438221d6 + p_source_label: gcp-logsource + protopayload: + at_sign_type: type.googleapis.com/google.cloud.audit.AuditLog + authenticationInfo: + principalEmail: user@company.io + authorizationInfo: + - granted: true + permission: bigquery.tables.delete + resource: projects/gcp-project1/datasets/test1/tables/newtable + metadata: + '@type': type.googleapis.com/google.cloud.audit.BigQueryAuditMetadata + methodName: google.cloud.bigquery.v2.JobService.InsertJob + requestMetadata: + callerIP: 1.2.3.4 + callerSuppliedUserAgent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36,gzip(gfe),gzip(gfe) + resourceName: projects/gcp-project1/datasets/test1/tables/newtable + serviceName: bigquery.googleapis.com + status: {} + tableDeletion: + jobName: projects/gcp-project1/jobs/bquxjob_5e4a0679_18729a639d7 + reason: QUERY + receivetimestamp: "2023-03-28 18:37:06.745" + resource: + labels: + dataset_id: test1 + project_id: gcp-project1 + type: bigquery_dataset + severity: NOTICE + timestamp: "2023-03-28 18:37:06.079" + Name: TableDeletion +DedupPeriodMinutes: 60 +LogTypes: + - GCP.AuditLog +RuleID: GCP.Destructive.Queries +Threshold: 1 From a0c2ea19a4b9370dec619d0decfca456cbe90da5 Mon Sep 17 00:00:00 2001 From: Calvin Kim Date: Wed, 29 Mar 2023 09:49:25 -0500 Subject: [PATCH 3/6] updates --- .../gcp_bigquery_large_scan.py | 21 ++++- .../gcp_destructive_queries.py | 84 +++++++++++++------ 2 files changed, 77 insertions(+), 28 deletions(-) diff --git a/rules/gcp_audit_rules/gcp_bigquery_large_scan.py b/rules/gcp_audit_rules/gcp_bigquery_large_scan.py index d6d712b83..896430080 100644 --- a/rules/gcp_audit_rules/gcp_bigquery_large_scan.py +++ b/rules/gcp_audit_rules/gcp_bigquery_large_scan.py @@ -1,5 +1,6 @@ from panther_base_helpers import deep_get +# 1.07 GB QUERY_THRESHOLD_BYTES = 1073741824 @@ -69,5 +70,23 @@ def alert_context(event): "queryConfig", "query", default="", - ) + ), + "actor": deep_get( + event, + "protopayload", + "authenticationInfo", + "principalEmail", + default="", + ), + "query_size": deep_get( + event, + "protopayload", + "metadata", + "jobChange", + "job", + "jobStats", + "queryStats", + "totalBilledBytes", + default=0, + ), } diff --git a/rules/gcp_audit_rules/gcp_destructive_queries.py b/rules/gcp_audit_rules/gcp_destructive_queries.py index 640fa05b0..39f95d5c5 100644 --- a/rules/gcp_audit_rules/gcp_destructive_queries.py +++ b/rules/gcp_audit_rules/gcp_destructive_queries.py @@ -1,38 +1,39 @@ from panther_base_helpers import deep_get -DESTRUCTIVE_STATEMENTS = ("UPDATE", "DELETE", "DROP_TABLE", "ALTER_TABLE", "TRUNCATE_TABLE") +DESTRUCTIVE_STATEMENTS = ["UPDATE", "DELETE", "DROP_TABLE", "ALTER_TABLE", "TRUNCATE_TABLE"] def rule(event): - return any( + if all( [ - all( - [ - deep_get(event, "resource", "type", default="").startswith( - "bigquery" - ), - deep_get( - event, "protopayload", "metadata", "jobChange", "job", "jobConfig", "type" - ) - == "QUERY", - deep_get( - event, - "protopayload", - "metadata", - "jobChange", - "job", - "jobConfig", - "queryConfig", - "statementType", - default="", - ) - in DESTRUCTIVE_STATEMENTS, - ] + deep_get(event, "resource", "type", default="").startswith( + "bigquery" ), - deep_get(event, "protopayload", "metadata", "tableDeletion"), - deep_get(event, "protopayload", "metadata", "datasetDeletion"), + deep_get(event, "protopayload", "metadata", "jobChange", "job", "jobConfig", "type") + == "QUERY", + deep_get( + event, + "protopayload", + "metadata", + "jobChange", + "job", + "jobConfig", + "queryConfig", + "statementType", + default="", + ) + in DESTRUCTIVE_STATEMENTS, ] - ) + ): + return True + + if deep_get(event, "protopayload", "metadata", "tableDeletion"): + return True + + if deep_get(event, "protopayload", "metadata", "datasetDeletion"): + return True + + return False def title(event): @@ -75,5 +76,34 @@ def alert_context(event): "queryConfig", "query", default="", + ), + "actor": deep_get( + event, + "protopayload", + "authenticationInfo", + "principalEmail", + default="", + ), + "statement": deep_get( + event, + "protopayload", + "metadata", + "jobChange", + "job", + "jobConfig", + "queryConfig", + "statementType", + default="", + ), + "table": deep_get( + event, + "protopayload", + "metadata", + "jobChange", + "job", + "jobConfig", + "queryConfig", + "destinationTable", ) + or deep_get(event, "protopayload", "metadata", "resourceName", default=""), } From b2159dfc3965fa0de2ec8203f658752b2f51fcbe Mon Sep 17 00:00:00 2001 From: Calvin Kim Date: Tue, 4 Apr 2023 17:47:56 -0500 Subject: [PATCH 4/6] gcp logging settings modified detection --- packs/gcp_audit.yml | 1 + .../gcp_logging_settings_modified.py | 32 +++++ .../gcp_logging_settings_modified.yml | 116 ++++++++++++++++++ 3 files changed, 149 insertions(+) create mode 100644 rules/gcp_audit_rules/gcp_logging_settings_modified.py create mode 100644 rules/gcp_audit_rules/gcp_logging_settings_modified.yml diff --git a/packs/gcp_audit.yml b/packs/gcp_audit.yml index 0d7e36a35..2436a7621 100644 --- a/packs/gcp_audit.yml +++ b/packs/gcp_audit.yml @@ -14,6 +14,7 @@ PackDefinition: - GCP.VPC.Flow.Logs.Disabled - GCP.BigQuery.Large.Scan - GCP.Destructive.Queries + - GCP.Logging.Settings.Modified # Data model - Standard.GCP.AuditLog # Globals used in these rules/policies diff --git a/rules/gcp_audit_rules/gcp_logging_settings_modified.py b/rules/gcp_audit_rules/gcp_logging_settings_modified.py new file mode 100644 index 000000000..c1c55bb35 --- /dev/null +++ b/rules/gcp_audit_rules/gcp_logging_settings_modified.py @@ -0,0 +1,32 @@ +from panther_base_helpers import deep_get + + +def rule(event): + return all( + [ + deep_get(event, "protoPayload", "serviceName", default="") == "logging.googleapis.com", + "Update" in deep_get(event, "protoPayload", "methodName", default=""), + ] + ) + + +def title(event): + resource = deep_get(event, "protoPayload", "resourceName", default="") + actor = deep_get( + event, "protoPayload", "authenticationInfo", "principalEmail", default="" + ) + return f"GCP [{resource}] logging settings modified by [{actor}]." + + +def alert_context(event): + return { + "resource": deep_get(event, "protoPayload", "resourceName", default=""), + "actor": deep_get( + event, + "protoPayload", + "authenticationInfo", + "principalEmail", + default="", + ), + "method": deep_get(event, "protoPayload", "methodName", default=""), + } diff --git a/rules/gcp_audit_rules/gcp_logging_settings_modified.yml b/rules/gcp_audit_rules/gcp_logging_settings_modified.yml new file mode 100644 index 000000000..950717acd --- /dev/null +++ b/rules/gcp_audit_rules/gcp_logging_settings_modified.yml @@ -0,0 +1,116 @@ +AnalysisType: rule +Description: Detects any changes made to logging settings +DisplayName: GCP Logging Settings Modified +Enabled: true +Filename: gcp_logging_settings_modified.py +Severity: Low +Tests: + - ExpectedResult: false + Log: + insertid: abcdefghijklmn + logname: projects/gcp-project1/logs/cloudaudit.googleapis.com%2Factivity + operation: + id: 1234567890123-gcp-project1:abcdefghijklmnopqrstuvwz + last: true + producer: bigquery.googleapis.com + p_any_emails: + - user@company.io + p_any_ip_addresses: + - 1.2.3.4 + p_event_time: "2023-03-28 18:37:06.079" + p_log_type: GCP.AuditLog + p_parse_time: "2023-03-28 18:38:14.478" + p_row_id: 06bf03d9d5dfbadba981899e1787bf05 + p_schema_version: 0 + p_source_id: 964c7894-9a0d-4ddf-864f-0193438221d6 + p_source_label: gcp-logsource + protopayload: + at_sign_type: type.googleapis.com/google.cloud.audit.AuditLog + authenticationInfo: + principalEmail: user@company.io + authorizationInfo: + - granted: true + permission: bigquery.tables.delete + resource: projects/gcp-project1/datasets/test1/tables/newtable + metadata: + '@type': type.googleapis.com/google.cloud.audit.BigQueryAuditMetadata + methodName: google.cloud.bigquery.v2.JobService.InsertJob + requestMetadata: + callerIP: 1.2.3.4 + callerSuppliedUserAgent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36,gzip(gfe),gzip(gfe) + resourceName: projects/gcp-project1/datasets/test1/tables/newtable + serviceName: bigquery.googleapis.com + status: {} + tableDeletion: + jobName: projects/gcp-project1/jobs/bquxjob_5e4a0679_18729a639d7 + reason: QUERY + receivetimestamp: "2023-03-28 18:37:06.745" + resource: + labels: + dataset_id: test1 + project_id: gcp-project1 + type: bigquery_dataset + severity: NOTICE + timestamp: "2023-03-28 18:37:06.079" + Name: Other Event + - ExpectedResult: true + Log: + insertid: ezyd47c12y + logname: projects/gcp-project1/logs/cloudaudit.googleapis.com%2Factivity + p_any_ip_addresses: + - 1.2.3.4 + p_event_time: "2023-03-09 16:41:30.524" + p_log_type: GCP.AuditLog + p_parse_time: "2023-03-09 16:44:14.617" + p_row_id: "1234567909689348911" + p_source_id: 4fc88a5a-2d51-4279-9c4a-08fa7cc52566 + p_source_label: gcplogsource + protoPayload: + at_sign_type: type.googleapis.com/google.cloud.audit.AuditLog + authenticationInfo: + principalEmail: test@company.io + authorizationInfo: + - granted: true + permission: logging.sinks.update + resource: projects/gcp-project1/sinks/log-sink + resourceAttributes: + name: projects/gcp-project1/sinks/log-sink + service: logging.googleapis.com + methodName: google.logging.v2.ConfigServiceV2.UpdateSink + request: + '@type': type.googleapis.com/google.logging.v2.UpdateSinkRequest + sink: + destination: pubsub.googleapis.com/projects/gcp-project1/topics/gcp-topic1 + exclusions: + - filter: protoPayload.serviceName = 'k8s.io + name: excludek8s + name: log-sink + writerIdentity: serviceAccount:p197946410614-915152@gcp-sa-logging.iam.gserviceaccount.com + sinkName: projects/gcp-project1/sinks/log-sink + uniqueWriterIdentity: true + updateMask: exclusions + requestMetadata: + callerIP: 1.2.3.4 + callerSuppliedUserAgent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36,gzip(gfe),gzip(gfe) + destinationAttributes: {} + requestAttributes: + auth: {} + time: "2023-03-09T16:41:30.540045105Z" + resourceName: projects/gcp-project1/sinks/log-sink + serviceName: logging.googleapis.com + status: {} + receivetimestamp: "2023-03-09 16:41:32.21" + resource: + labels: + destination: "" + name: log-sink + project_id: gcp-project1 + type: logging_sink + severity: NOTICE + timestamp: "2023-03-09 16:41:30.524" + Name: Sink Update Event +DedupPeriodMinutes: 60 +LogTypes: + - GCP.AuditLog +RuleID: GCP.Logging.Settings.Modified +Threshold: 1 From bc042753bc980025318aff635dec020f8f18b222 Mon Sep 17 00:00:00 2001 From: Calvin Kim Date: Wed, 5 Apr 2023 10:25:15 -0500 Subject: [PATCH 5/6] enabled true --- rules/gcp_audit_rules/gcp_bigquery_large_scan.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rules/gcp_audit_rules/gcp_bigquery_large_scan.yml b/rules/gcp_audit_rules/gcp_bigquery_large_scan.yml index 042b747a4..49c24127e 100644 --- a/rules/gcp_audit_rules/gcp_bigquery_large_scan.yml +++ b/rules/gcp_audit_rules/gcp_bigquery_large_scan.yml @@ -1,7 +1,7 @@ AnalysisType: rule Description: Detect any BigQuery query that is doing a very large scan (> 1 GB). DisplayName: GCP BigQuery Large Scan -Enabled: false +Enabled: true Filename: gcp_bigquery_large_scan.py Severity: Info Tests: @@ -24,7 +24,7 @@ Tests: p_schema_version: 0 p_source_id: 964c7894-9a0d-4ddf-864f-0193438221d6 p_source_label: gcp-logsource - protopayload: + protoPayload: at_sign_type: type.googleapis.com/google.cloud.audit.AuditLog authenticationInfo: principalEmail: user@company.io @@ -100,7 +100,7 @@ Tests: p_schema_version: 0 p_source_id: 964c7894-9a0d-4ddf-864f-0193438221d6 p_source_label: gcp-logsource - protopayload: + protoPayload: at_sign_type: type.googleapis.com/google.cloud.audit.AuditLog authenticationInfo: principalEmail: user@company.io From 94fcfc4a436c23c4fd1f18964a55ecaba1bd4320 Mon Sep 17 00:00:00 2001 From: Calvin Kim Date: Wed, 5 Apr 2023 10:35:41 -0500 Subject: [PATCH 6/6] protoPayload --- .../gcp_bigquery_large_scan.py | 16 ++++++------ .../gcp_destructive_queries.py | 26 +++++++++---------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/rules/gcp_audit_rules/gcp_bigquery_large_scan.py b/rules/gcp_audit_rules/gcp_bigquery_large_scan.py index 896430080..ea3e0bbd7 100644 --- a/rules/gcp_audit_rules/gcp_bigquery_large_scan.py +++ b/rules/gcp_audit_rules/gcp_bigquery_large_scan.py @@ -9,11 +9,11 @@ def rule(event): [ deep_get(event, "resource", "type", default="").startswith("bigquery"), deep_get(event, "operation", "last") is True, - deep_get(event, "protopayload", "metadata", "jobChange", "job", "jobConfig", "type") + deep_get(event, "protoPayload", "metadata", "jobChange", "job", "jobConfig", "type") == "QUERY", deep_get( event, - "protopayload", + "protoPayload", "metadata", "jobChange", "job", @@ -25,7 +25,7 @@ def rule(event): int( deep_get( event, - "protopayload", + "protoPayload", "metadata", "jobChange", "job", @@ -42,11 +42,11 @@ def rule(event): def title(event): actor = deep_get( - event, "protopayload", "authenticationInfo", "principalEmail", default="" + event, "protoPayload", "authenticationInfo", "principalEmail", default="" ) query_size = deep_get( event, - "protopayload", + "protoPayload", "metadata", "jobChange", "job", @@ -62,7 +62,7 @@ def alert_context(event): return { "query": deep_get( event, - "protopayload", + "protoPayload", "metadata", "jobChange", "job", @@ -73,14 +73,14 @@ def alert_context(event): ), "actor": deep_get( event, - "protopayload", + "protoPayload", "authenticationInfo", "principalEmail", default="", ), "query_size": deep_get( event, - "protopayload", + "protoPayload", "metadata", "jobChange", "job", diff --git a/rules/gcp_audit_rules/gcp_destructive_queries.py b/rules/gcp_audit_rules/gcp_destructive_queries.py index 39f95d5c5..72ded3214 100644 --- a/rules/gcp_audit_rules/gcp_destructive_queries.py +++ b/rules/gcp_audit_rules/gcp_destructive_queries.py @@ -9,11 +9,11 @@ def rule(event): deep_get(event, "resource", "type", default="").startswith( "bigquery" ), - deep_get(event, "protopayload", "metadata", "jobChange", "job", "jobConfig", "type") + deep_get(event, "protoPayload", "metadata", "jobChange", "job", "jobConfig", "type") == "QUERY", deep_get( event, - "protopayload", + "protoPayload", "metadata", "jobChange", "job", @@ -27,10 +27,10 @@ def rule(event): ): return True - if deep_get(event, "protopayload", "metadata", "tableDeletion"): + if deep_get(event, "protoPayload", "metadata", "tableDeletion"): return True - if deep_get(event, "protopayload", "metadata", "datasetDeletion"): + if deep_get(event, "protoPayload", "metadata", "datasetDeletion"): return True return False @@ -38,11 +38,11 @@ def rule(event): def title(event): actor = deep_get( - event, "protopayload", "authenticationInfo", "principalEmail", default="" + event, "protoPayload", "authenticationInfo", "principalEmail", default="" ) statement = deep_get( event, - "protopayload", + "protoPayload", "metadata", "jobChange", "job", @@ -53,14 +53,14 @@ def title(event): ) table = deep_get( event, - "protopayload", + "protoPayload", "metadata", "jobChange", "job", "jobConfig", "queryConfig", "destinationTable", - ) or deep_get(event, "protopayload", "metadata", "resourceName", default="") + ) or deep_get(event, "protoPayload", "metadata", "resourceName", default="") return f"GCP: [{actor}] performed a destructive BigQuery [{statement}] query on [{table}]." @@ -68,7 +68,7 @@ def alert_context(event): return { "query": deep_get( event, - "protopayload", + "protoPayload", "metadata", "jobChange", "job", @@ -79,14 +79,14 @@ def alert_context(event): ), "actor": deep_get( event, - "protopayload", + "protoPayload", "authenticationInfo", "principalEmail", default="", ), "statement": deep_get( event, - "protopayload", + "protoPayload", "metadata", "jobChange", "job", @@ -97,7 +97,7 @@ def alert_context(event): ), "table": deep_get( event, - "protopayload", + "protoPayload", "metadata", "jobChange", "job", @@ -105,5 +105,5 @@ def alert_context(event): "queryConfig", "destinationTable", ) - or deep_get(event, "protopayload", "metadata", "resourceName", default=""), + or deep_get(event, "protoPayload", "metadata", "resourceName", default=""), }