From 7b31558edfed3a5305c0dc85b5abb6ec4c22b6d6 Mon Sep 17 00:00:00 2001 From: Oleksandr Kylymnychenko Date: Thu, 13 Oct 2022 17:25:22 +0200 Subject: [PATCH 1/4] ref(pii): consider all token as sensitive --- relay-general/src/pii/convert.rs | 31 ++++++++++ relay-general/src/pii/regexes.rs | 2 +- ...convert__tests__safe_fields_for_token.snap | 57 +++++++++++++++++++ 3 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 relay-general/src/pii/snapshots/relay_general__pii__convert__tests__safe_fields_for_token.snap diff --git a/relay-general/src/pii/convert.rs b/relay-general/src/pii/convert.rs index 12bac2391e..f5f87b47f9 100644 --- a/relay-general/src/pii/convert.rs +++ b/relay-general/src/pii/convert.rs @@ -1368,4 +1368,35 @@ THd+9FBxiHLGXNKhG/FRSyREXEt+NyYIf/0cyByc9tNksat794ddUqnLOg0vwSkv let pii_config = pii_config.unwrap(); insta::assert_json_snapshot!(pii_config); } + + #[test] + fn test_safe_fields_for_token() { + let mut data = Event::from_value( + serde_json::json!({ + "extra": { + "password": "foo", + "github_token": "bar", + "access_token": "quz", + "stripetoken": "baz", + "my-token": "secret", + "new_token": "hidden," + } + }) + .into(), + ); + let pii_config = to_pii_config(&DataScrubbingConfig { + sensitive_fields: vec![], + exclude_fields: vec![ + "GITHUB_TOKEN".to_owned(), + "access_token".to_owned(), + "stripetoken".to_owned(), + ], + ..simple_enabled_config() + }); + + let pii_config = pii_config.unwrap(); + let mut pii_processor = PiiProcessor::new(pii_config.compiled()); + process_value(&mut data, &mut pii_processor, ProcessingState::root()).unwrap(); + assert_annotated_snapshot!(data); + } } diff --git a/relay-general/src/pii/regexes.rs b/relay-general/src/pii/regexes.rs index f4fa011de9..9b331f0cc7 100644 --- a/relay-general/src/pii/regexes.rs +++ b/relay-general/src/pii/regexes.rs @@ -269,6 +269,6 @@ static US_SSN_REGEX: Lazy = Lazy::new(|| { static PASSWORD_KEY_REGEX: Lazy = Lazy::new(|| { Regex::new( - r"(?i)(password|secret|passwd|api_key|apikey|access_token|auth|credentials|mysql_pwd|stripetoken|privatekey|private_key|github_token)" + r"(?i)(password|secret|passwd|api_key|apikey|auth|credentials|mysql_pwd|privatekey|private_key|.*token)" ).unwrap() }); diff --git a/relay-general/src/pii/snapshots/relay_general__pii__convert__tests__safe_fields_for_token.snap b/relay-general/src/pii/snapshots/relay_general__pii__convert__tests__safe_fields_for_token.snap new file mode 100644 index 0000000000..1ef32cbc4d --- /dev/null +++ b/relay-general/src/pii/snapshots/relay_general__pii__convert__tests__safe_fields_for_token.snap @@ -0,0 +1,57 @@ +--- +source: relay-general/src/pii/convert.rs +expression: data +--- +{ + "extra": { + "access_token": "quz", + "github_token": "bar", + "my-token": "[Filtered]", + "new_token": "[Filtered]", + "password": "[Filtered]", + "stripetoken": "baz" + }, + "_meta": { + "extra": { + "my-token": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 6 + } + }, + "new_token": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 7 + } + }, + "password": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 3 + } + } + } + } +} From b9fabf0df94981ab1a6ae14365ccb6ad3b664d8e Mon Sep 17 00:00:00 2001 From: Oleksandr Kylymnychenko Date: Fri, 14 Oct 2022 10:19:54 +0200 Subject: [PATCH 2/4] ref(pii): filter all the tokens by default --- relay-general/src/pii/convert.rs | 3 ++- relay-general/src/pii/regexes.rs | 2 +- ...i__convert__tests__safe_fields_for_token.snap | 16 +++++++++++++++- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/relay-general/src/pii/convert.rs b/relay-general/src/pii/convert.rs index f5f87b47f9..5d83ebcee8 100644 --- a/relay-general/src/pii/convert.rs +++ b/relay-general/src/pii/convert.rs @@ -1379,7 +1379,8 @@ THd+9FBxiHLGXNKhG/FRSyREXEt+NyYIf/0cyByc9tNksat794ddUqnLOg0vwSkv "access_token": "quz", "stripetoken": "baz", "my-token": "secret", - "new_token": "hidden," + "new_token": "hidden", + "secret-token-here": "ops" } }) .into(), diff --git a/relay-general/src/pii/regexes.rs b/relay-general/src/pii/regexes.rs index 9b331f0cc7..f49b2e3ecb 100644 --- a/relay-general/src/pii/regexes.rs +++ b/relay-general/src/pii/regexes.rs @@ -269,6 +269,6 @@ static US_SSN_REGEX: Lazy = Lazy::new(|| { static PASSWORD_KEY_REGEX: Lazy = Lazy::new(|| { Regex::new( - r"(?i)(password|secret|passwd|api_key|apikey|auth|credentials|mysql_pwd|privatekey|private_key|.*token)" + r"(?i)(password|secret|passwd|api_key|apikey|auth|credentials|mysql_pwd|privatekey|private_key|.*token.*)" ).unwrap() }); diff --git a/relay-general/src/pii/snapshots/relay_general__pii__convert__tests__safe_fields_for_token.snap b/relay-general/src/pii/snapshots/relay_general__pii__convert__tests__safe_fields_for_token.snap index 1ef32cbc4d..633640f886 100644 --- a/relay-general/src/pii/snapshots/relay_general__pii__convert__tests__safe_fields_for_token.snap +++ b/relay-general/src/pii/snapshots/relay_general__pii__convert__tests__safe_fields_for_token.snap @@ -9,6 +9,7 @@ expression: data "my-token": "[Filtered]", "new_token": "[Filtered]", "password": "[Filtered]", + "secret-token-here": "[Filtered]", "stripetoken": "baz" }, "_meta": { @@ -36,7 +37,7 @@ expression: data 10 ] ], - "len": 7 + "len": 6 } }, "password": { @@ -51,6 +52,19 @@ expression: data ], "len": 3 } + }, + "secret-token-here": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 3 + } } } } From ffb06d068132453a471a53d80ee640cdd165f39a Mon Sep 17 00:00:00 2001 From: Oleksandr Kylymnychenko Date: Mon, 17 Oct 2022 11:28:37 +0200 Subject: [PATCH 3/4] Add CHANGELOG entries --- CHANGELOG.md | 1 + py/CHANGELOG.md | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a88065b40b..6449141a87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Limit the number of custom measurements per event. ([#1483](https://github.com/getsentry/relay/pull/1483))) - Add INP web vital as a measurement. ([#1487](https://github.com/getsentry/relay/pull/1487)) +- Consider all tokens as sensitive, filter out all `*token*` from the input. ([#1527](https://github.com/getsentry/relay/pull/1527)) ** Bug Fixes**: diff --git a/py/CHANGELOG.md b/py/CHANGELOG.md index 30d74617a7..14bb670227 100644 --- a/py/CHANGELOG.md +++ b/py/CHANGELOG.md @@ -5,6 +5,7 @@ - Add `transaction_info` to event payloads, including the transaction's source and internal original transaction name. ([#1330](https://github.com/getsentry/relay/pull/1330)) - Add user-agent parsing to replays processor. ([#1420](https://github.com/getsentry/relay/pull/1420)) - `convert_datascrubbing_config` will now return an error string when conversion fails on big regexes. ([#1474](https://github.com/getsentry/relay/pull/1474)) +- Consider all tokens as sensitive, filter out all `*token*` from the input. ([#1527](https://github.com/getsentry/relay/pull/1527)) ## 0.8.13 From d7f693adf10f906503b5334c33590155939aeaa5 Mon Sep 17 00:00:00 2001 From: Oleksandr <1931331+olksdr@users.noreply.github.com> Date: Mon, 17 Oct 2022 11:39:30 +0200 Subject: [PATCH 4/4] Apply suggestions from code review Co-authored-by: Joris Bayer --- CHANGELOG.md | 2 +- py/CHANGELOG.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6449141a87..94f51d2d89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ - Limit the number of custom measurements per event. ([#1483](https://github.com/getsentry/relay/pull/1483))) - Add INP web vital as a measurement. ([#1487](https://github.com/getsentry/relay/pull/1487)) -- Consider all tokens as sensitive, filter out all `*token*` from the input. ([#1527](https://github.com/getsentry/relay/pull/1527)) +- PII scrubbing now treats any key containing `token` as a password. ([#1527](https://github.com/getsentry/relay/pull/1527)) ** Bug Fixes**: diff --git a/py/CHANGELOG.md b/py/CHANGELOG.md index 14bb670227..109add7ce2 100644 --- a/py/CHANGELOG.md +++ b/py/CHANGELOG.md @@ -5,7 +5,7 @@ - Add `transaction_info` to event payloads, including the transaction's source and internal original transaction name. ([#1330](https://github.com/getsentry/relay/pull/1330)) - Add user-agent parsing to replays processor. ([#1420](https://github.com/getsentry/relay/pull/1420)) - `convert_datascrubbing_config` will now return an error string when conversion fails on big regexes. ([#1474](https://github.com/getsentry/relay/pull/1474)) -- Consider all tokens as sensitive, filter out all `*token*` from the input. ([#1527](https://github.com/getsentry/relay/pull/1527)) +- `relay_pii_strip_event` now treats any key containing `token` as a password. ([#1527](https://github.com/getsentry/relay/pull/1527)) ## 0.8.13