Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

gcp bigquery detections #716

Merged
merged 9 commits into from
Apr 5, 2023
Merged

gcp bigquery detections #716

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
cc0958b
gcp bigquery large query detection
calkim-panther Mar 28, 2023
c3ea9f4
gcp destructive query detection
calkim-panther Mar 28, 2023
a0c2ea1
updates
calkim-panther Mar 29, 2023
8958678
Merge branch 'master' into calkim-gcp
calkim-panther Mar 29, 2023
73c6cca
Merge branch 'master' into calkim-gcp
calkim-panther Apr 3, 2023
b2159df
gcp logging settings modified detection
calkim-panther Apr 4, 2023
201db52
Merge branch 'master' into calkim-gcp
calkim-panther Apr 4, 2023
bc04275
enabled true
calkim-panther Apr 5, 2023
94fcfc4
protoPayload
calkim-panther Apr 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions packs/gcp_audit.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ PackDefinition:
- GCP.Access.Attempts.Violating.VPC.Service.Controls
- GCP.Access.Attempts.Violating.IAP.Access.Controls
- GCP.VPC.Flow.Logs.Disabled
- GCP.BigQuery.Large.Scan
- GCP.Destructive.Queries
- GCP.Logging.Settings.Modified
# Data model
- Standard.GCP.AuditLog
# Globals used in these rules/policies
Expand Down
92 changes: 92 additions & 0 deletions rules/gcp_audit_rules/gcp_bigquery_large_scan.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
from panther_base_helpers import deep_get

# 1.07 GB
QUERY_THRESHOLD_BYTES = 1073741824
calkim-panther marked this conversation as resolved.
Show resolved Hide resolved


def rule(event):
return all(
[
deep_get(event, "resource", "type", default="<type not found>").startswith("bigquery"),
deep_get(event, "operation", "last") is True,
deep_get(event, "protoPayload", "metadata", "jobChange", "job", "jobConfig", "type")
== "QUERY",
deep_get(
event,
"protoPayload",
"metadata",
"jobChange",
"job",
"jobConfig",
"queryConfig",
"statementType",
)
== "SELECT",
int(
deep_get(
event,
"protoPayload",
"metadata",
"jobChange",
"job",
"jobStats",
"queryStats",
"totalBilledBytes",
default=0,
)
)
> QUERY_THRESHOLD_BYTES,
]
)


def title(event):
actor = deep_get(
event, "protoPayload", "authenticationInfo", "principalEmail", default="<ACTOR_NOT_FOUND>"
)
query_size = deep_get(
event,
"protoPayload",
"metadata",
"jobChange",
"job",
"jobStats",
"queryStats",
"totalBilledBytes",
default=0,
)
return f"GCP: [{actor}] ran a large BigQuery query of [{query_size}] bytes."


def alert_context(event):
return {
calkim-panther marked this conversation as resolved.
Show resolved Hide resolved
"query": deep_get(
event,
"protoPayload",
"metadata",
"jobChange",
"job",
"jobConfig",
"queryConfig",
"query",
default="<QUERY_NOT_FOUND>",
),
"actor": deep_get(
event,
"protoPayload",
"authenticationInfo",
"principalEmail",
default="<ACTOR_NOT_FOUND>",
),
"query_size": deep_get(
event,
"protoPayload",
"metadata",
"jobChange",
"job",
"jobStats",
"queryStats",
"totalBilledBytes",
default=0,
),
}
164 changes: 164 additions & 0 deletions rules/gcp_audit_rules/gcp_bigquery_large_scan.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
AnalysisType: rule
Description: Detect any BigQuery query that is doing a very large scan (> 1 GB).
DisplayName: GCP BigQuery Large Scan
Enabled: true
Filename: gcp_bigquery_large_scan.py
Severity: Info
Tests:
- ExpectedResult: false
Log:
insertid: ABCDEFGHIJKL
logname: projects/gcp-project1/logs/cloudaudit.googleapis.com%2Fdata_access
operation:
id: 0123456789012-gcp-project1:abcdefg_hijklmnop_1234567abcd
last: true
producer: bigquery.googleapis.com
p_any_emails:
- user@company.io
p_any_ip_addresses:
- 1.2.3.4
p_event_time: "2023-03-28 17:37:02.096"
p_log_type: GCP.AuditLog
p_parse_time: "2023-03-28 17:38:14.621"
p_row_id: de00d3dcdaeee4b4d5f7fa9d17b4c203
p_schema_version: 0
p_source_id: 964c7894-9a0d-4ddf-864f-0193438221d6
p_source_label: gcp-logsource
protoPayload:
at_sign_type: type.googleapis.com/google.cloud.audit.AuditLog
authenticationInfo:
principalEmail: user@company.io
authorizationInfo:
- granted: true
permission: bigquery.jobs.create
resource: projects/gcp-project1
metadata:
'@type': type.googleapis.com/google.cloud.audit.BigQueryAuditMetadata
jobChange:
after: DONE
job:
jobConfig:
queryConfig:
createDisposition: CREATE_IF_NEEDED
destinationTable: projects/gcp-project1/datasets/_c2b49f742788f188022fcec1f1622e7404b40ce5/tables/anondea8e77183dcdf1cf93a47b5003036409a525808207450ab80db17fcc8cdcf53
priority: QUERY_INTERACTIVE
query: |-
-- This query shows a list of the daily top Google Search terms.
SELECT
*
FROM `bigquery-public-data.google_trends.top_terms`
statementType: SELECT
writeDisposition: WRITE_TRUNCATE
type: QUERY
jobName: projects/gcp-project1/jobs/abcdefg_hijklmnop_1234567abcd
jobStats:
createTime: "2023-03-28T17:36:49.087Z"
endTime: "2023-03-28T17:37:02.092Z"
queryStats:
billingTier: 1
outputRowCount: "43683701"
referencedTables:
- projects/bigquery-public-data/datasets/google_trends/tables/top_terms
totalBilledBytes: "100000"
totalProcessedBytes: "100000"
startTime: "2023-03-28T17:36:49.224Z"
totalSlotMs: "259581"
jobStatus:
jobState: DONE
methodName: google.cloud.bigquery.v2.JobService.InsertJob
requestMetadata:
callerIP: 1.2.3.4
callerSuppliedUserAgent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36,gzip(gfe),gzip(gfe)
resourceName: projects/gcp-project1/jobs/abcdefg_hijklmnop_1234567abcd
serviceName: bigquery.googleapis.com
status: {}
receivetimestamp: "2023-03-28 17:37:02.114"
resource:
labels:
location: US
project_id: gcp-project1
type: bigquery_project
severity: INFO
timestamp: "2023-03-28 17:37:02.096"
Name: small query
- ExpectedResult: true
Log:
insertid: ABCDEFGHIJKL
logname: projects/gcp-project1/logs/cloudaudit.googleapis.com%2Fdata_access
operation:
id: 0123456789012-gcp-project1:abcdefg_hijklmnop_1234567abcd
last: true
producer: bigquery.googleapis.com
p_any_emails:
- user@company.io
p_any_ip_addresses:
- 1.2.3.4
p_event_time: "2023-03-28 17:37:02.096"
p_log_type: GCP.AuditLog
p_parse_time: "2023-03-28 17:38:14.621"
p_row_id: de00d3dcdaeee4b4d5f7fa9d17b4c203
p_schema_version: 0
p_source_id: 964c7894-9a0d-4ddf-864f-0193438221d6
p_source_label: gcp-logsource
protoPayload:
at_sign_type: type.googleapis.com/google.cloud.audit.AuditLog
authenticationInfo:
principalEmail: user@company.io
authorizationInfo:
- granted: true
permission: bigquery.jobs.create
resource: projects/gcp-project1
metadata:
'@type': type.googleapis.com/google.cloud.audit.BigQueryAuditMetadata
jobChange:
after: DONE
job:
jobConfig:
queryConfig:
createDisposition: CREATE_IF_NEEDED
destinationTable: projects/gcp-project1/datasets/_c2b49f742788f188022fcec1f1622e7404b40ce5/tables/anondea8e77183dcdf1cf93a47b5003036409a525808207450ab80db17fcc8cdcf53
priority: QUERY_INTERACTIVE
query: |-
-- This query shows a list of the daily top Google Search terms.
SELECT
*
FROM `bigquery-public-data.google_trends.top_terms`
statementType: SELECT
writeDisposition: WRITE_TRUNCATE
type: QUERY
jobName: projects/gcp-project1/jobs/abcdefg_hijklmnop_1234567abcd
jobStats:
createTime: "2023-03-28T17:36:49.087Z"
endTime: "2023-03-28T17:37:02.092Z"
queryStats:
billingTier: 1
outputRowCount: "43683701"
referencedTables:
- projects/bigquery-public-data/datasets/google_trends/tables/top_terms
totalBilledBytes: "3097493504"
totalProcessedBytes: "3096864198"
startTime: "2023-03-28T17:36:49.224Z"
totalSlotMs: "259581"
jobStatus:
jobState: DONE
methodName: google.cloud.bigquery.v2.JobService.InsertJob
requestMetadata:
callerIP: 1.2.3.4
callerSuppliedUserAgent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36,gzip(gfe),gzip(gfe)
resourceName: projects/gcp-project1/jobs/abcdefg_hijklmnop_1234567abcd
serviceName: bigquery.googleapis.com
status: {}
receivetimestamp: "2023-03-28 17:37:02.114"
resource:
labels:
location: US
project_id: gcp-project1
type: bigquery_project
severity: INFO
timestamp: "2023-03-28 17:37:02.096"
Name: Large Query
DedupPeriodMinutes: 60
LogTypes:
- GCP.AuditLog
RuleID: GCP.BigQuery.Large.Scan
Threshold: 1
109 changes: 109 additions & 0 deletions rules/gcp_audit_rules/gcp_destructive_queries.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
from panther_base_helpers import deep_get

DESTRUCTIVE_STATEMENTS = ["UPDATE", "DELETE", "DROP_TABLE", "ALTER_TABLE", "TRUNCATE_TABLE"]


def rule(event):
if all(
[
deep_get(event, "resource", "type", default="<RESOURCE_NOT_FOUND>").startswith(
"bigquery"
),
deep_get(event, "protoPayload", "metadata", "jobChange", "job", "jobConfig", "type")
== "QUERY",
deep_get(
event,
"protoPayload",
"metadata",
"jobChange",
"job",
"jobConfig",
"queryConfig",
"statementType",
default="<STATEMENT_NOT_FOUND>",
)
in DESTRUCTIVE_STATEMENTS,
]
):
return True

if deep_get(event, "protoPayload", "metadata", "tableDeletion"):
return True

if deep_get(event, "protoPayload", "metadata", "datasetDeletion"):
return True

return False


def title(event):
actor = deep_get(
event, "protoPayload", "authenticationInfo", "principalEmail", default="<ACTOR_NOT_FOUND>"
)
statement = deep_get(
event,
"protoPayload",
"metadata",
"jobChange",
"job",
"jobConfig",
"queryConfig",
"statementType",
default="<STATEMENT_NOT_FOUND>",
)
table = deep_get(
event,
"protoPayload",
"metadata",
"jobChange",
"job",
"jobConfig",
"queryConfig",
"destinationTable",
) or deep_get(event, "protoPayload", "metadata", "resourceName", default="<TABLE_NOT_FOUND>")
return f"GCP: [{actor}] performed a destructive BigQuery [{statement}] query on [{table}]."


def alert_context(event):
return {
calkim-panther marked this conversation as resolved.
Show resolved Hide resolved
"query": deep_get(
event,
"protoPayload",
"metadata",
"jobChange",
"job",
"jobConfig",
"queryConfig",
"query",
default="<QUERY_NOT_FOUND>",
),
"actor": deep_get(
event,
"protoPayload",
"authenticationInfo",
"principalEmail",
default="<ACTOR_NOT_FOUND>",
),
"statement": deep_get(
event,
"protoPayload",
"metadata",
"jobChange",
"job",
"jobConfig",
"queryConfig",
"statementType",
default="<STATEMENT_NOT_FOUND>",
),
"table": deep_get(
event,
"protoPayload",
"metadata",
"jobChange",
"job",
"jobConfig",
"queryConfig",
"destinationTable",
)
or deep_get(event, "protoPayload", "metadata", "resourceName", default="<TABLE_NOT_FOUND>"),
}
Loading