From ee595d3ed8cbcacb5591bdbcc811d1baa857758c Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Wed, 27 Sep 2023 08:37:43 +0200 Subject: [PATCH 01/30] wip --- relay-sampling/src/config.rs | 4 + relay-sampling/src/evaluation.rs | 65 +- relay-server/src/actors/processor.rs | 1302 +--------------------- relay-server/src/actors/project_cache.rs | 42 + relay-server/src/actors/project_redis.rs | 28 + 5 files changed, 121 insertions(+), 1320 deletions(-) diff --git a/relay-sampling/src/config.rs b/relay-sampling/src/config.rs index a07b38ca7d..642d3bd2d9 100644 --- a/relay-sampling/src/config.rs +++ b/relay-sampling/src/config.rs @@ -81,6 +81,10 @@ impl SamplingRule { self.condition.supported() && self.ty != RuleType::Unsupported } + pub fn is_reservoir(&self) -> bool { + todo!() + } + /// Returns the sample rate if the rule is active. pub fn sample_rate(&self, now: DateTime) -> Option { if !self.time_range.contains(now) { diff --git a/relay-sampling/src/evaluation.rs b/relay-sampling/src/evaluation.rs index 00117cdd57..af969da550 100644 --- a/relay-sampling/src/evaluation.rs +++ b/relay-sampling/src/evaluation.rs @@ -99,10 +99,13 @@ impl SamplingEvaluator { match sampling_value { SamplingValue::Factor { value } => self.factor *= value, + SamplingValue::SampleRate { .. } if rule.is_reservoir() => { + return Evaluation::Matched(SamplingMatch::new_reservoir(rule.id)); + } SamplingValue::SampleRate { value } => { let sample_rate = (value * self.factor).clamp(0.0, 1.0); - return Evaluation::Matched(SamplingMatch::new( + return Evaluation::Matched(SamplingMatch::new_standard( self.adjusted_sample_rate(sample_rate), seed, self.rule_ids, @@ -163,29 +166,34 @@ fn sampling_match(sample_rate: f64, seed: Uuid) -> bool { /// Represents the specification for sampling an incoming event. #[derive(Clone, Debug, PartialEq)] -pub struct SamplingMatch { - /// The sample rate to use for the incoming event. - sample_rate: f64, - /// The seed to feed to the random number generator which allows the same number to be - /// generated given the same seed. - /// - /// This is especially important for trace sampling, even though we can have inconsistent - /// traces due to multi-matching. - seed: Uuid, - /// The list of rule ids that have matched the incoming event and/or dynamic sampling context. - matched_rules: MatchedRuleIds, - /// Whether this sampling match results in the item getting sampled. - /// It's essentially a cache, as the value can be deterministically derived from - /// the sample rate and the seed. - should_keep: bool, +pub enum SamplingMatch { + Reservoir { + rule: RuleId, + }, + Standard { + /// The sample rate to use for the incoming event. + sample_rate: f64, + /// The seed to feed to the random number generator which allows the same number to be + /// generated given the same seed. + /// + /// This is especially important for trace sampling, even though we can have inconsistent + /// traces due to multi-matching. + seed: Uuid, + /// The list of rule ids that have matched the incoming event and/or dynamic sampling context. + matched_rules: MatchedRuleIds, + /// Whether this sampling match results in the item getting sampled. + /// It's essentially a cache, as the value can be deterministically derived from + /// the sample rate and the seed. + should_keep: bool, + }, } impl SamplingMatch { - fn new(sample_rate: f64, seed: Uuid, matched_rules: Vec) -> Self { + fn new_standard(sample_rate: f64, seed: Uuid, matched_rules: Vec) -> Self { let matched_rules = MatchedRuleIds(matched_rules); let should_keep = sampling_match(sample_rate, seed); - Self { + Self::Standard { sample_rate, seed, matched_rules, @@ -193,9 +201,16 @@ impl SamplingMatch { } } + fn new_reservoir(rule: RuleId) -> Self { + Self::Reservoir { rule } + } + /// Returns the sample rate. pub fn sample_rate(&self) -> f64 { - self.sample_rate + match self { + SamplingMatch::Reservoir { .. } => 1.0, + SamplingMatch::Standard { sample_rate, .. } => *sample_rate, + } } /// Returns the matched rules for the sampling match. @@ -203,12 +218,18 @@ impl SamplingMatch { /// Takes ownership, useful if you don't need the [`SamplingMatch`] anymore /// and you want to avoid allocations. pub fn into_matched_rules(self) -> MatchedRuleIds { - self.matched_rules + match self { + SamplingMatch::Reservoir { rule } => MatchedRuleIds(vec![rule]), + SamplingMatch::Standard { matched_rules, .. } => matched_rules, + } } /// Returns true if event should be kept. pub fn should_keep(&self) -> bool { - self.should_keep + match self { + SamplingMatch::Reservoir { .. } => true, + SamplingMatch::Standard { should_keep, .. } => *should_keep, + } } /// Returns true if event should be dropped. @@ -285,7 +306,7 @@ mod tests { fn matches_rule_ids(rule_ids: &[u32], rules: &[SamplingRule], instance: &impl Getter) -> bool { let matched_rule_ids = MatchedRuleIds(rule_ids.iter().map(|num| RuleId(*num)).collect()); let sampling_match = get_sampling_match(rules, instance); - matched_rule_ids == sampling_match.matched_rules + matched_rule_ids == sampling_match.into_matched_rules() } /// Helper function to create a dsc with the provided getter-values set. diff --git a/relay-server/src/actors/processor.rs b/relay-server/src/actors/processor.rs index bc99698561..80508cb902 100644 --- a/relay-server/src/actors/processor.rs +++ b/relay-server/src/actors/processor.rs @@ -42,7 +42,7 @@ use relay_protocol::{Annotated, Array, Empty, FromValue, Object, Value}; use relay_quotas::{DataCategory, ReasonCode}; use relay_redis::RedisPool; use relay_replays::recording::RecordingScrubber; -use relay_sampling::config::{RuleType, SamplingMode}; +use relay_sampling::config::{RuleId, RuleType, SamplingMode}; use relay_sampling::evaluation::{Evaluation, MatchedRuleIds, SamplingEvaluator}; use relay_sampling::{DynamicSamplingContext, SamplingConfig}; use relay_statsd::metric; @@ -424,6 +424,7 @@ pub struct ProcessEnvelope { pub envelope: ManagedEnvelope, pub project_state: Arc, pub sampling_project_state: Option>, + pub counters: BTreeMap, } /// Parses a list of metrics or metric buckets and pushes them to the project's aggregator. @@ -1318,6 +1319,7 @@ impl EnvelopeProcessorService { envelope: mut managed_envelope, project_state, sampling_project_state, + .. } = message; let envelope = managed_envelope.envelope_mut(); @@ -2345,7 +2347,7 @@ impl EnvelopeProcessorService { &state.project_state.config.transaction_metrics { if config.is_enabled() { - state.sampling_result = Self::compute_sampling_decision( + let res = Self::compute_sampling_decision( self.inner.config.processing_enabled(), state.project_state.config.dynamic_sampling.as_ref(), state.event.value(), @@ -2358,7 +2360,6 @@ impl EnvelopeProcessorService { } } } - _ => {} } } @@ -2942,1298 +2943,3 @@ impl Service for EnvelopeProcessorService { }); } } - -#[cfg(test)] -mod tests { - use std::env; - use std::str::FromStr; - - use chrono::{DateTime, TimeZone, Utc}; - use relay_base_schema::metrics::{DurationUnit, MetricUnit}; - use relay_common::glob2::LazyGlob; - use relay_event_normalization::{MeasurementsConfig, RedactionRule, TransactionNameRule}; - use relay_event_schema::protocol::{EventId, TransactionSource}; - use relay_pii::DataScrubbingConfig; - use relay_sampling::condition::RuleCondition; - use relay_sampling::config::{ - DecayingFunction, RuleId, RuleType, SamplingConfig, SamplingMode, SamplingRule, - SamplingValue, TimeRange, - }; - use relay_sampling::evaluation::SamplingMatch; - use relay_test::mock_service; - use similar_asserts::assert_eq; - use uuid::Uuid; - - use crate::actors::test_store::TestStore; - use crate::extractors::RequestMeta; - use crate::metrics_extraction::transactions::types::{ - CommonTags, TransactionMeasurementTags, TransactionMetric, - }; - use crate::metrics_extraction::IntoMetric; - - use crate::testutils::{new_envelope, state_with_rule_and_condition}; - use crate::utils::Semaphore as TestSemaphore; - - use super::*; - - struct TestProcessSessionArguments<'a> { - item: Item, - received: DateTime, - client: Option<&'a str>, - client_addr: Option, - metrics_config: SessionMetricsConfig, - clock_drift_processor: ClockDriftProcessor, - extracted_metrics: Vec, - } - - impl<'a> TestProcessSessionArguments<'a> { - fn run_session_producer(&mut self) -> bool { - let proc = create_test_processor(Default::default()); - proc.process_session( - &mut self.item, - self.received, - self.client, - self.client_addr, - self.metrics_config, - &self.clock_drift_processor, - &mut self.extracted_metrics, - ) - } - - fn default() -> Self { - let mut item = Item::new(ItemType::Event); - - let session = r#"{ - "init": false, - "started": "2021-04-26T08:00:00+0100", - "timestamp": "2021-04-26T08:00:00+0100", - "attrs": { - "release": "1.0.0" - }, - "did": "user123", - "status": "this is not a valid status!", - "duration": 123.4 - }"#; - - item.set_payload(ContentType::Json, session); - let received = DateTime::from_str("2021-04-26T08:00:00+0100").unwrap(); - - Self { - item, - received, - client: None, - client_addr: None, - metrics_config: serde_json::from_str( - " - { - \"version\": 0, - \"drop\": true - }", - ) - .unwrap(), - clock_drift_processor: ClockDriftProcessor::new(None, received), - extracted_metrics: vec![], - } - } - } - - fn mocked_event(event_type: EventType, transaction: &str, release: &str) -> Event { - Event { - id: Annotated::new(EventId::new()), - ty: Annotated::new(event_type), - transaction: Annotated::new(transaction.to_string()), - release: Annotated::new(LenientString(release.to_string())), - ..Event::default() - } - } - - /// Checks that the default test-arguments leads to the item being kept, which helps ensure the - /// other tests are valid. - #[tokio::test] - async fn test_process_session_keep_item() { - let mut args = TestProcessSessionArguments::default(); - assert!(args.run_session_producer()); - } - - #[tokio::test] - async fn test_process_session_invalid_json() { - let mut args = TestProcessSessionArguments::default(); - args.item - .set_payload(ContentType::Json, "this isnt valid json"); - assert!(!args.run_session_producer()); - } - - #[tokio::test] - async fn test_process_session_sequence_overflow() { - let mut args = TestProcessSessionArguments::default(); - args.item.set_payload( - ContentType::Json, - r#"{ - "init": false, - "started": "2021-04-26T08:00:00+0100", - "timestamp": "2021-04-26T08:00:00+0100", - "seq": 18446744073709551615, - "attrs": { - "release": "1.0.0" - }, - "did": "user123", - "status": "this is not a valid status!", - "duration": 123.4 - }"#, - ); - assert!(!args.run_session_producer()); - } - - #[tokio::test] - async fn test_process_session_invalid_timestamp() { - let mut args = TestProcessSessionArguments::default(); - args.received = DateTime::from_str("2021-05-26T08:00:00+0100").unwrap(); - assert!(!args.run_session_producer()); - } - - #[tokio::test] - async fn test_process_session_metrics_extracted() { - let mut args = TestProcessSessionArguments::default(); - args.item.set_metrics_extracted(true); - assert!(!args.run_session_producer()); - } - - fn create_breadcrumbs_item(breadcrumbs: &[(Option>, &str)]) -> Item { - let mut data = Vec::new(); - - for (date, message) in breadcrumbs { - let mut breadcrumb = BTreeMap::new(); - breadcrumb.insert("message", (*message).to_string()); - if let Some(date) = date { - breadcrumb.insert("timestamp", date.to_rfc3339()); - } - - rmp_serde::encode::write(&mut data, &breadcrumb).expect("write msgpack"); - } - - let mut item = Item::new(ItemType::Attachment); - item.set_payload(ContentType::MsgPack, data); - item - } - - fn breadcrumbs_from_event(event: &Annotated) -> &Vec> { - event - .value() - .unwrap() - .breadcrumbs - .value() - .unwrap() - .values - .value() - .unwrap() - } - - fn services() -> (Addr, Addr) { - let (outcome_aggregator, _) = mock_service("outcome_aggregator", (), |&mut (), _| {}); - let (test_store, _) = mock_service("test_store", (), |&mut (), _| {}); - (outcome_aggregator, test_store) - } - - #[tokio::test] - async fn test_dsc_respects_metrics_extracted() { - relay_test::setup(); - let (outcome_aggregator, test_store) = services(); - - let config = Config::from_json_value(serde_json::json!({ - "processing": { - "enabled": true, - "kafka_config": [], - } - })) - .unwrap(); - - let service: EnvelopeProcessorService = create_test_processor(config); - - // Gets a ProcessEnvelopeState, either with or without the metrics_exracted flag toggled. - let get_state = |version: Option| { - let event = Event { - id: Annotated::new(EventId::new()), - ty: Annotated::new(EventType::Transaction), - transaction: Annotated::new("testing".to_owned()), - ..Event::default() - }; - - let mut project_state = state_with_rule_and_condition( - Some(0.0), - RuleType::Transaction, - RuleCondition::all(), - ); - - if let Some(version) = version { - project_state.config.transaction_metrics = - ErrorBoundary::Ok(relay_dynamic_config::TransactionMetricsConfig { - version, - ..Default::default() - }) - .into(); - } - - ProcessEnvelopeState { - event: Annotated::from(event), - metrics: Default::default(), - sample_rates: None, - sampling_result: SamplingResult::Pending, - extracted_metrics: Default::default(), - project_state: Arc::new(project_state), - sampling_project_state: None, - project_id: ProjectId::new(42), - managed_envelope: ManagedEnvelope::new( - new_envelope(false, "foo"), - TestSemaphore::new(42).try_acquire().unwrap(), - outcome_aggregator.clone(), - test_store.clone(), - ), - has_profile: false, - event_metrics_extracted: false, - } - }; - - // None represents no TransactionMetricsConfig, DS will not be run - let mut state = get_state(None); - service.run_dynamic_sampling(&mut state); - assert!(state.sampling_result.should_keep()); - - // Current version is 1, so it won't run DS if it's outdated - let mut state = get_state(Some(0)); - service.run_dynamic_sampling(&mut state); - assert!(state.sampling_result.should_keep()); - - // Dynamic sampling is run, as the transactionmetrics version is up to date. - let mut state = get_state(Some(1)); - service.run_dynamic_sampling(&mut state); - assert!(state.sampling_result.should_drop()); - } - - #[test] - fn test_it_keeps_or_drops_transactions() { - let event = Event { - id: Annotated::new(EventId::new()), - ty: Annotated::new(EventType::Transaction), - transaction: Annotated::new("testing".to_owned()), - ..Event::default() - }; - - for (sample_rate, should_keep) in [(0.0, false), (1.0, true)] { - let sampling_config = SamplingConfig { - rules: vec![], - rules_v2: vec![SamplingRule { - condition: RuleCondition::all(), - sampling_value: SamplingValue::SampleRate { value: sample_rate }, - ty: RuleType::Transaction, - id: RuleId(1), - time_range: Default::default(), - decaying_fn: DecayingFunction::Constant, - }], - mode: SamplingMode::Received, - }; - - // TODO: This does not test if the sampling decision is actually applied. This should be - // refactored to send a proper Envelope in and call process_state to cover the full - // pipeline. - let res = EnvelopeProcessorService::compute_sampling_decision( - false, - Some(&sampling_config), - Some(&event), - None, - None, - ); - assert_eq!(res.should_keep(), should_keep); - } - } - - #[test] - fn test_breadcrumbs_file1() { - let item = create_breadcrumbs_item(&[(None, "item1")]); - - // NOTE: using (Some, None) here: - let result = EnvelopeProcessorService::event_from_attachments( - &Config::default(), - None, - Some(item), - None, - ); - - let event = result.unwrap().0; - let breadcrumbs = breadcrumbs_from_event(&event); - - assert_eq!(breadcrumbs.len(), 1); - let first_breadcrumb_message = breadcrumbs[0].value().unwrap().message.value().unwrap(); - assert_eq!("item1", first_breadcrumb_message); - } - - #[test] - fn test_breadcrumbs_file2() { - let item = create_breadcrumbs_item(&[(None, "item2")]); - - // NOTE: using (None, Some) here: - let result = EnvelopeProcessorService::event_from_attachments( - &Config::default(), - None, - None, - Some(item), - ); - - let event = result.unwrap().0; - let breadcrumbs = breadcrumbs_from_event(&event); - assert_eq!(breadcrumbs.len(), 1); - - let first_breadcrumb_message = breadcrumbs[0].value().unwrap().message.value().unwrap(); - assert_eq!("item2", first_breadcrumb_message); - } - - #[test] - fn test_breadcrumbs_truncation() { - let item1 = create_breadcrumbs_item(&[(None, "crumb1")]); - let item2 = create_breadcrumbs_item(&[(None, "crumb2"), (None, "crumb3")]); - - let result = EnvelopeProcessorService::event_from_attachments( - &Config::default(), - None, - Some(item1), - Some(item2), - ); - - let event = result.unwrap().0; - let breadcrumbs = breadcrumbs_from_event(&event); - assert_eq!(breadcrumbs.len(), 2); - } - - #[test] - fn test_breadcrumbs_order_with_none() { - let d1 = Utc.with_ymd_and_hms(2019, 10, 10, 12, 10, 10).unwrap(); - let d2 = Utc.with_ymd_and_hms(2019, 10, 11, 12, 10, 10).unwrap(); - - let item1 = create_breadcrumbs_item(&[(None, "none"), (Some(d1), "d1")]); - let item2 = create_breadcrumbs_item(&[(Some(d2), "d2")]); - - let result = EnvelopeProcessorService::event_from_attachments( - &Config::default(), - None, - Some(item1), - Some(item2), - ); - - let event = result.unwrap().0; - let breadcrumbs = breadcrumbs_from_event(&event); - assert_eq!(breadcrumbs.len(), 2); - - assert_eq!(Some("d1"), breadcrumbs[0].value().unwrap().message.as_str()); - assert_eq!(Some("d2"), breadcrumbs[1].value().unwrap().message.as_str()); - } - - #[test] - fn test_breadcrumbs_reversed_with_none() { - let d1 = Utc.with_ymd_and_hms(2019, 10, 10, 12, 10, 10).unwrap(); - let d2 = Utc.with_ymd_and_hms(2019, 10, 11, 12, 10, 10).unwrap(); - - let item1 = create_breadcrumbs_item(&[(Some(d2), "d2")]); - let item2 = create_breadcrumbs_item(&[(None, "none"), (Some(d1), "d1")]); - - let result = EnvelopeProcessorService::event_from_attachments( - &Config::default(), - None, - Some(item1), - Some(item2), - ); - - let event = result.unwrap().0; - let breadcrumbs = breadcrumbs_from_event(&event); - assert_eq!(breadcrumbs.len(), 2); - - assert_eq!(Some("d1"), breadcrumbs[0].value().unwrap().message.as_str()); - assert_eq!(Some("d2"), breadcrumbs[1].value().unwrap().message.as_str()); - } - - #[test] - fn test_empty_breadcrumbs_item() { - let item1 = create_breadcrumbs_item(&[]); - let item2 = create_breadcrumbs_item(&[]); - let item3 = create_breadcrumbs_item(&[]); - - let result = EnvelopeProcessorService::event_from_attachments( - &Config::default(), - Some(item1), - Some(item2), - Some(item3), - ); - - // regression test to ensure we don't fail parsing an empty file - result.expect("event_from_attachments"); - } - - fn create_test_processor(config: Config) -> EnvelopeProcessorService { - let (envelope_manager, _) = mock_service("envelope_manager", (), |&mut (), _| {}); - let (outcome_aggregator, _) = mock_service("outcome_aggregator", (), |&mut (), _| {}); - let (project_cache, _) = mock_service("project_cache", (), |&mut (), _| {}); - let (upstream_relay, _) = mock_service("upstream_relay", (), |&mut (), _| {}); - let (global_config, _) = mock_service("global_config", (), |&mut (), _| {}); - let inner = InnerProcessor { - config: Arc::new(config), - envelope_manager, - project_cache, - outcome_aggregator, - upstream_relay, - #[cfg(feature = "processing")] - rate_limiter: None, - geoip_lookup: None, - global_config, - }; - - EnvelopeProcessorService { - global_config: Arc::default(), - inner: Arc::new(inner), - } - } - - #[tokio::test] - async fn test_user_report_invalid() { - let processor = create_test_processor(Default::default()); - let (outcome_aggregator, test_store) = services(); - let event_id = EventId::new(); - - let dsn = "https://e12d836b15bb49d7bbf99e64295d995b:@sentry.io/42" - .parse() - .unwrap(); - - let request_meta = RequestMeta::new(dsn); - let mut envelope = Envelope::from_request(Some(event_id), request_meta); - - envelope.add_item({ - let mut item = Item::new(ItemType::UserReport); - item.set_payload(ContentType::Json, r#"{"foo": "bar"}"#); - item - }); - - envelope.add_item({ - let mut item = Item::new(ItemType::Event); - item.set_payload(ContentType::Json, "{}"); - item - }); - - let message = ProcessEnvelope { - envelope: ManagedEnvelope::standalone(envelope, outcome_aggregator, test_store), - project_state: Arc::new(ProjectState::allowed()), - sampling_project_state: None, - }; - - let envelope_response = processor.process(message).unwrap(); - let ctx = envelope_response.envelope.unwrap(); - let new_envelope = ctx.envelope(); - - assert_eq!(new_envelope.len(), 1); - assert_eq!(new_envelope.items().next().unwrap().ty(), &ItemType::Event); - } - - fn process_envelope_with_root_project_state( - envelope: Box, - sampling_project_state: Option>, - ) -> Envelope { - let processor = create_test_processor(Default::default()); - let (outcome_aggregator, test_store) = services(); - - let message = ProcessEnvelope { - envelope: ManagedEnvelope::standalone(envelope, outcome_aggregator, test_store), - project_state: Arc::new(ProjectState::allowed()), - sampling_project_state, - }; - - let envelope_response = processor.process(message).unwrap(); - let ctx = envelope_response.envelope.unwrap(); - ctx.envelope().clone() - } - - fn extract_first_event_from_envelope(envelope: Envelope) -> Event { - let item = envelope.items().next().unwrap(); - let annotated_event: Annotated = - Annotated::from_json_bytes(&item.payload()).unwrap(); - annotated_event.into_value().unwrap() - } - - fn mocked_error_item() -> Item { - let mut item = Item::new(ItemType::Event); - item.set_payload( - ContentType::Json, - r#"{ - "event_id": "52df9022835246eeb317dbd739ccd059", - "exception": { - "values": [ - { - "type": "mytype", - "value": "myvalue", - "module": "mymodule", - "thread_id": 42, - "other": "value" - } - ] - } - }"#, - ); - item - } - - fn project_state_with_single_rule(sample_rate: f64) -> ProjectState { - let sampling_config = SamplingConfig { - rules: vec![], - rules_v2: vec![SamplingRule { - condition: RuleCondition::all(), - sampling_value: SamplingValue::SampleRate { value: sample_rate }, - ty: RuleType::Trace, - id: RuleId(1), - time_range: Default::default(), - decaying_fn: Default::default(), - }], - mode: SamplingMode::Received, - }; - let mut sampling_project_state = ProjectState::allowed(); - sampling_project_state.config.dynamic_sampling = Some(sampling_config); - sampling_project_state - } - - #[tokio::test] - async fn test_error_is_tagged_correctly_if_trace_sampling_result_is_some() { - let event_id = EventId::new(); - let dsn = "https://e12d836b15bb49d7bbf99e64295d995b:@sentry.io/42" - .parse() - .unwrap(); - let request_meta = RequestMeta::new(dsn); - let mut envelope = Envelope::from_request(Some(event_id), request_meta); - let dsc = DynamicSamplingContext { - trace_id: Uuid::new_v4(), - public_key: ProjectKey::parse("abd0f232775f45feab79864e580d160b").unwrap(), - release: Some("1.1.1".to_string()), - user: Default::default(), - replay_id: None, - environment: None, - transaction: Some("transaction1".into()), - sample_rate: None, - sampled: Some(true), - other: BTreeMap::new(), - }; - envelope.set_dsc(dsc); - envelope.add_item(mocked_error_item()); - - // We test with sample rate equal to 100%. - let sampling_project_state = project_state_with_single_rule(1.0); - let new_envelope = process_envelope_with_root_project_state( - envelope.clone(), - Some(Arc::new(sampling_project_state)), - ); - let event = extract_first_event_from_envelope(new_envelope); - let trace_context = event.context::().unwrap(); - assert!(trace_context.sampled.value().unwrap()); - - // We test with sample rate equal to 0%. - let sampling_project_state = project_state_with_single_rule(0.0); - let new_envelope = process_envelope_with_root_project_state( - envelope, - Some(Arc::new(sampling_project_state)), - ); - let event = extract_first_event_from_envelope(new_envelope); - let trace_context = event.context::().unwrap(); - assert!(!trace_context.sampled.value().unwrap()); - } - - #[tokio::test] - async fn test_error_is_not_tagged_if_already_tagged() { - let event_id = EventId::new(); - let dsn = "https://e12d836b15bb49d7bbf99e64295d995b:@sentry.io/42" - .parse() - .unwrap(); - let request_meta = RequestMeta::new(dsn); - - // We test tagging with an incoming event that has already been tagged by downstream Relay. - let mut envelope = Envelope::from_request(Some(event_id), request_meta); - let mut item = Item::new(ItemType::Event); - item.set_payload( - ContentType::Json, - r#"{ - "event_id": "52df9022835246eeb317dbd739ccd059", - "exception": { - "values": [ - { - "type": "mytype", - "value": "myvalue", - "module": "mymodule", - "thread_id": 42, - "other": "value" - } - ] - }, - "contexts": { - "trace": { - "sampled": true - } - } - }"#, - ); - envelope.add_item(item); - let sampling_project_state = project_state_with_single_rule(0.0); - let new_envelope = process_envelope_with_root_project_state( - envelope, - Some(Arc::new(sampling_project_state)), - ); - let event = extract_first_event_from_envelope(new_envelope); - let trace_context = event.context::().unwrap(); - assert!(trace_context.sampled.value().unwrap()); - } - - #[tokio::test] - async fn test_error_is_tagged_correctly_if_trace_sampling_result_is_none() { - let event_id = EventId::new(); - let dsn = "https://e12d836b15bb49d7bbf99e64295d995b:@sentry.io/42" - .parse() - .unwrap(); - let request_meta = RequestMeta::new(dsn); - - // We test tagging when root project state and dsc are none. - let mut envelope = Envelope::from_request(Some(event_id), request_meta); - envelope.add_item(mocked_error_item()); - let new_envelope = process_envelope_with_root_project_state(envelope, None); - let event = extract_first_event_from_envelope(new_envelope); - - assert!(event.contexts.value().is_none()); - } - - #[tokio::test] - async fn test_browser_version_extraction_with_pii_like_data() { - let processor = create_test_processor(Default::default()); - let (outcome_aggregator, test_store) = services(); - let event_id = EventId::new(); - - let dsn = "https://e12d836b15bb49d7bbf99e64295d995b:@sentry.io/42" - .parse() - .unwrap(); - - let request_meta = RequestMeta::new(dsn); - let mut envelope = Envelope::from_request(Some(event_id), request_meta); - - envelope.add_item({ - let mut item = Item::new(ItemType::Event); - item.set_payload( - ContentType::Json, - r#" - { - "request": { - "headers": [ - ["User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36"] - ] - } - } - "#, - ); - item - }); - - let mut datascrubbing_settings = DataScrubbingConfig::default(); - // enable all the default scrubbing - datascrubbing_settings.scrub_data = true; - datascrubbing_settings.scrub_defaults = true; - datascrubbing_settings.scrub_ip_addresses = true; - - // Make sure to mask any IP-like looking data - let pii_config = serde_json::from_str(r#"{"applications": {"**": ["@ip:mask"]}}"#).unwrap(); - - let config = ProjectConfig { - datascrubbing_settings, - pii_config: Some(pii_config), - ..Default::default() - }; - - let mut project_state = ProjectState::allowed(); - project_state.config = config; - let message = ProcessEnvelope { - envelope: ManagedEnvelope::standalone(envelope, outcome_aggregator, test_store), - project_state: Arc::new(project_state), - sampling_project_state: None, - }; - - let envelope_response = processor.process(message).unwrap(); - let new_envelope = envelope_response.envelope.unwrap(); - let new_envelope = new_envelope.envelope(); - - let event_item = new_envelope.items().last().unwrap(); - let annotated_event: Annotated = - Annotated::from_json_bytes(&event_item.payload()).unwrap(); - let event = annotated_event.into_value().unwrap(); - let headers = event - .request - .into_value() - .unwrap() - .headers - .into_value() - .unwrap(); - - // IP-like data must be masked - assert_eq!(Some("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/********* Safari/537.36"), headers.get_header("User-Agent")); - // But we still get correct browser and version number - let contexts = event.contexts.into_value().unwrap(); - let browser = contexts.0.get("browser").unwrap(); - assert_eq!( - r#"{"name":"Chrome","version":"103.0.0","type":"browser"}"#, - browser.to_json().unwrap() - ); - } - - #[tokio::test] - async fn test_client_report_removal() { - relay_test::setup(); - let (outcome_aggregator, test_store) = services(); - - let config = Config::from_json_value(serde_json::json!({ - "outcomes": { - "emit_outcomes": true, - "emit_client_outcomes": true - } - })) - .unwrap(); - - let processor = create_test_processor(config); - - let dsn = "https://e12d836b15bb49d7bbf99e64295d995b:@sentry.io/42" - .parse() - .unwrap(); - - let request_meta = RequestMeta::new(dsn); - let mut envelope = Envelope::from_request(None, request_meta); - - envelope.add_item({ - let mut item = Item::new(ItemType::ClientReport); - item.set_payload( - ContentType::Json, - r#" - { - "discarded_events": [ - ["queue_full", "error", 42] - ] - } - "#, - ); - item - }); - - let message = ProcessEnvelope { - envelope: ManagedEnvelope::standalone(envelope, outcome_aggregator, test_store), - project_state: Arc::new(ProjectState::allowed()), - sampling_project_state: None, - }; - - let envelope_response = processor.process(message).unwrap(); - assert!(envelope_response.envelope.is_none()); - } - - #[tokio::test] - async fn test_client_report_forwarding() { - relay_test::setup(); - let (outcome_aggregator, test_store) = services(); - - let config = Config::from_json_value(serde_json::json!({ - "outcomes": { - "emit_outcomes": false, - // a relay need to emit outcomes at all to not process. - "emit_client_outcomes": true - } - })) - .unwrap(); - - let processor = create_test_processor(config); - - let dsn = "https://e12d836b15bb49d7bbf99e64295d995b:@sentry.io/42" - .parse() - .unwrap(); - - let request_meta = RequestMeta::new(dsn); - let mut envelope = Envelope::from_request(None, request_meta); - - envelope.add_item({ - let mut item = Item::new(ItemType::ClientReport); - item.set_payload( - ContentType::Json, - r#" - { - "discarded_events": [ - ["queue_full", "error", 42] - ] - } - "#, - ); - item - }); - - let message = ProcessEnvelope { - envelope: ManagedEnvelope::standalone(envelope, outcome_aggregator, test_store), - project_state: Arc::new(ProjectState::allowed()), - sampling_project_state: None, - }; - - let envelope_response = processor.process(message).unwrap(); - let ctx = envelope_response.envelope.unwrap(); - let item = ctx.envelope().items().next().unwrap(); - assert_eq!(item.ty(), &ItemType::ClientReport); - - ctx.accept(); // do not try to capture or emit outcomes - } - - #[tokio::test] - #[cfg(feature = "processing")] - async fn test_client_report_removal_in_processing() { - relay_test::setup(); - let (outcome_aggregator, test_store) = services(); - - let config = Config::from_json_value(serde_json::json!({ - "outcomes": { - "emit_outcomes": true, - "emit_client_outcomes": false, - }, - "processing": { - "enabled": true, - "kafka_config": [], - } - })) - .unwrap(); - - let processor = create_test_processor(config); - - let dsn = "https://e12d836b15bb49d7bbf99e64295d995b:@sentry.io/42" - .parse() - .unwrap(); - - let request_meta = RequestMeta::new(dsn); - let mut envelope = Envelope::from_request(None, request_meta); - - envelope.add_item({ - let mut item = Item::new(ItemType::ClientReport); - item.set_payload( - ContentType::Json, - r#" - { - "discarded_events": [ - ["queue_full", "error", 42] - ] - } - "#, - ); - item - }); - - let message = ProcessEnvelope { - envelope: ManagedEnvelope::standalone(envelope, outcome_aggregator, test_store), - project_state: Arc::new(ProjectState::allowed()), - sampling_project_state: None, - }; - - let envelope_response = processor.process(message).unwrap(); - assert!(envelope_response.envelope.is_none()); - } - - #[test] - #[cfg(feature = "processing")] - fn test_unprintable_fields() { - let event = Annotated::new(Event { - environment: Annotated::new(String::from( - "�9�~YY���)�����9�~YY���)�����9�~YY���)�����9�~YY���)�����", - )), - ..Default::default() - }); - assert!(has_unprintable_fields(&event)); - - let event = Annotated::new(Event { - release: Annotated::new( - String::from("���7��#1G����7��#1G����7��#1G����7��#1G����7��#").into(), - ), - ..Default::default() - }); - assert!(has_unprintable_fields(&event)); - - let event = Annotated::new(Event { - environment: Annotated::new(String::from("production")), - ..Default::default() - }); - assert!(!has_unprintable_fields(&event)); - - let event = Annotated::new(Event { - release: Annotated::new( - String::from("release with\t some\n normal\r\nwhitespace").into(), - ), - ..Default::default() - }); - assert!(!has_unprintable_fields(&event)); - } - - #[test] - fn test_from_outcome_type_sampled() { - assert!(outcome_from_parts(ClientReportField::FilteredSampling, "adsf").is_err()); - - assert!(outcome_from_parts(ClientReportField::FilteredSampling, "Sampled:").is_err()); - - assert!(outcome_from_parts(ClientReportField::FilteredSampling, "Sampled:foo").is_err()); - - assert!(matches!( - outcome_from_parts(ClientReportField::FilteredSampling, "Sampled:"), - Err(()) - )); - - assert!(matches!( - outcome_from_parts(ClientReportField::FilteredSampling, "Sampled:;"), - Err(()) - )); - - assert!(matches!( - outcome_from_parts(ClientReportField::FilteredSampling, "Sampled:ab;12"), - Err(()) - )); - - assert_eq!( - outcome_from_parts(ClientReportField::FilteredSampling, "Sampled:123,456"), - Ok(Outcome::FilteredSampling(MatchedRuleIds(vec![ - RuleId(123), - RuleId(456), - ]))) - ); - - assert_eq!( - outcome_from_parts(ClientReportField::FilteredSampling, "Sampled:123"), - Ok(Outcome::FilteredSampling(MatchedRuleIds(vec![RuleId(123)]))) - ); - } - - #[test] - fn test_from_outcome_type_filtered() { - assert!(matches!( - outcome_from_parts(ClientReportField::Filtered, "error-message"), - Ok(Outcome::Filtered(FilterStatKey::ErrorMessage)) - )); - assert!(outcome_from_parts(ClientReportField::Filtered, "adsf").is_err()); - } - - #[test] - fn test_from_outcome_type_client_discard() { - assert_eq!( - outcome_from_parts(ClientReportField::ClientDiscard, "foo_reason").unwrap(), - Outcome::ClientDiscard("foo_reason".into()) - ); - } - - #[test] - fn test_from_outcome_type_rate_limited() { - assert!(matches!( - outcome_from_parts(ClientReportField::RateLimited, ""), - Ok(Outcome::RateLimited(None)) - )); - assert_eq!( - outcome_from_parts(ClientReportField::RateLimited, "foo_reason").unwrap(), - Outcome::RateLimited(Some(ReasonCode::new("foo_reason"))) - ); - } - - fn capture_test_event(transaction_name: &str, source: TransactionSource) -> Vec { - let mut event = Annotated::::from_json( - r#" - { - "type": "transaction", - "transaction": "/foo/", - "timestamp": 946684810.0, - "start_timestamp": 946684800.0, - "contexts": { - "trace": { - "trace_id": "4c79f60c11214eb38604f4ae0781bfb2", - "span_id": "fa90fdead5f74053", - "op": "http.server", - "type": "trace" - } - }, - "transaction_info": { - "source": "url" - } - } - "#, - ) - .unwrap(); - let e = event.value_mut().as_mut().unwrap(); - e.transaction.set_value(Some(transaction_name.into())); - - e.transaction_info - .value_mut() - .as_mut() - .unwrap() - .source - .set_value(Some(source)); - - relay_statsd::with_capturing_test_client(|| { - utils::log_transaction_name_metrics(&mut event, |event| { - let config = LightNormalizationConfig { - transaction_name_config: TransactionNameConfig { - rules: &[TransactionNameRule { - pattern: LazyGlob::new("/foo/*/**".to_owned()), - expiry: DateTime::::MAX_UTC, - redaction: RedactionRule::Replace { - substitution: "*".to_owned(), - }, - }], - }, - ..Default::default() - }; - relay_event_normalization::light_normalize_event(event, config) - }) - .unwrap(); - }) - } - - #[test] - fn test_log_transaction_metrics_none() { - let captures = capture_test_event("/nothing", TransactionSource::Url); - insta::assert_debug_snapshot!(captures, @r#" - [ - "event.transaction_name_changes:1|c|#source_in:url,changes:none,source_out:sanitized,is_404:false", - ] - "#); - } - - #[test] - fn test_log_transaction_metrics_rule() { - let captures = capture_test_event("/foo/john/denver", TransactionSource::Url); - insta::assert_debug_snapshot!(captures, @r#" - [ - "event.transaction_name_changes:1|c|#source_in:url,changes:rule,source_out:sanitized,is_404:false", - ] - "#); - } - - #[test] - fn test_log_transaction_metrics_pattern() { - let captures = capture_test_event("/something/12345", TransactionSource::Url); - insta::assert_debug_snapshot!(captures, @r#" - [ - "event.transaction_name_changes:1|c|#source_in:url,changes:pattern,source_out:sanitized,is_404:false", - ] - "#); - } - - #[test] - fn test_log_transaction_metrics_both() { - let captures = capture_test_event("/foo/john/12345", TransactionSource::Url); - insta::assert_debug_snapshot!(captures, @r#" - [ - "event.transaction_name_changes:1|c|#source_in:url,changes:both,source_out:sanitized,is_404:false", - ] - "#); - } - - #[test] - fn test_log_transaction_metrics_no_match() { - let captures = capture_test_event("/foo/john/12345", TransactionSource::Route); - insta::assert_debug_snapshot!(captures, @r#" - [ - "event.transaction_name_changes:1|c|#source_in:route,changes:none,source_out:route,is_404:false", - ] - "#); - } - - /// This is a stand-in test to assert panicking behavior for spawn_blocking. - /// - /// [`EnvelopeProcessorService`] relies on tokio to restart the worker threads for blocking - /// tasks if there is a panic during processing. Tokio does not explicitly mention this behavior - /// in documentation, though the `spawn_blocking` contract suggests that this is intentional. - /// - /// This test should be moved if the worker pool is extracted into a utility. - #[test] - fn test_processor_panics() { - let future = async { - let semaphore = Arc::new(Semaphore::new(1)); - - // loop multiple times to prove that the runtime creates new threads - for _ in 0..3 { - // the previous permit should have been released during panic unwind - let permit = semaphore.clone().acquire_owned().await.unwrap(); - - let handle = tokio::task::spawn_blocking(move || { - let _permit = permit; // drop(permit) after panic!() would warn as "unreachable" - panic!("ignored"); - }); - - assert!(handle.await.is_err()); - } - }; - - tokio::runtime::Builder::new_current_thread() - .max_blocking_threads(1) - .build() - .unwrap() - .block_on(future); - } - - /// Confirms that the hardcoded value we use for the fixed length of the measurement MRI is - /// correct. Unit test is placed here because it has dependencies to relay-server and therefore - /// cannot be called from relay-metrics. - #[test] - fn test_mri_overhead_constant() { - let hardcoded_value = MeasurementsConfig::MEASUREMENT_MRI_OVERHEAD; - - let derived_value = { - let name = "foobar".to_string(); - let value = 5.0; // Arbitrary value. - let unit = MetricUnit::Duration(DurationUnit::default()); - let tags = TransactionMeasurementTags { - measurement_rating: None, - universal_tags: CommonTags(BTreeMap::new()), - }; - - let measurement = TransactionMetric::Measurement { - name: name.clone(), - value, - unit, - tags, - }; - - let metric: Bucket = measurement.into_metric(UnixTimestamp::now()); - metric.name.len() - unit.to_string().len() - name.len() - }; - assert_eq!( - hardcoded_value, derived_value, - "Update `MEASUREMENT_MRI_OVERHEAD` if the naming scheme changed." - ); - } - - // Helper to extract the sampling match from SamplingResult if thats the variant. - fn get_sampling_match(sampling_result: SamplingResult) -> SamplingMatch { - if let SamplingResult::Match(sampling_match) = sampling_result { - sampling_match - } else { - panic!() - } - } - - /// Happy path test for compute_sampling_decision. - #[test] - fn test_compute_sampling_decision_matching() { - let event = mocked_event(EventType::Transaction, "foo", "bar"); - let rule = SamplingRule { - condition: RuleCondition::all(), - sampling_value: SamplingValue::SampleRate { value: 1.0 }, - ty: RuleType::Transaction, - id: RuleId(0), - time_range: TimeRange::default(), - decaying_fn: Default::default(), - }; - - let sampling_config = SamplingConfig { - rules: vec![], - rules_v2: vec![rule], - mode: SamplingMode::Received, - }; - - let res = EnvelopeProcessorService::compute_sampling_decision( - false, - Some(&sampling_config), - Some(&event), - None, - None, - ); - assert!(res.is_match()); - } - - #[test] - fn test_matching_with_unsupported_rule() { - let event = mocked_event(EventType::Transaction, "foo", "bar"); - let rule = SamplingRule { - condition: RuleCondition::all(), - sampling_value: SamplingValue::SampleRate { value: 1.0 }, - ty: RuleType::Transaction, - id: RuleId(0), - time_range: TimeRange::default(), - decaying_fn: Default::default(), - }; - - let unsupported_rule = SamplingRule { - condition: RuleCondition::all(), - sampling_value: SamplingValue::SampleRate { value: 1.0 }, - ty: RuleType::Unsupported, - id: RuleId(0), - time_range: TimeRange::default(), - decaying_fn: Default::default(), - }; - - let sampling_config = SamplingConfig { - rules: vec![], - rules_v2: vec![rule, unsupported_rule], - mode: SamplingMode::Received, - }; - - // Unsupported rule should result in no match if processing is not enabled. - let res = EnvelopeProcessorService::compute_sampling_decision( - false, - Some(&sampling_config), - Some(&event), - None, - None, - ); - assert!(res.is_no_match()); - - // Match if processing is enabled. - let res = EnvelopeProcessorService::compute_sampling_decision( - true, - Some(&sampling_config), - Some(&event), - None, - None, - ); - assert!(res.is_match()); - } - - #[test] - fn test_client_sample_rate() { - let dsc = DynamicSamplingContext { - trace_id: Uuid::new_v4(), - public_key: ProjectKey::parse("abd0f232775f45feab79864e580d160b").unwrap(), - release: Some("1.1.1".to_string()), - user: Default::default(), - replay_id: None, - environment: None, - transaction: Some("transaction1".into()), - sample_rate: Some(0.5), - sampled: Some(true), - other: BTreeMap::new(), - }; - - let rule = SamplingRule { - condition: RuleCondition::all(), - sampling_value: SamplingValue::SampleRate { value: 0.2 }, - ty: RuleType::Trace, - id: RuleId(0), - time_range: TimeRange::default(), - decaying_fn: Default::default(), - }; - - let mut sampling_config = SamplingConfig { - rules: vec![], - rules_v2: vec![rule], - mode: SamplingMode::Received, - }; - - let res = EnvelopeProcessorService::compute_sampling_decision( - false, - None, - None, - Some(&sampling_config), - Some(&dsc), - ); - - assert_eq!(get_sampling_match(res).sample_rate(), 0.2); - - sampling_config.mode = SamplingMode::Total; - - let res = EnvelopeProcessorService::compute_sampling_decision( - false, - None, - None, - Some(&sampling_config), - Some(&dsc), - ); - - assert_eq!(get_sampling_match(res).sample_rate(), 0.4); - } -} diff --git a/relay-server/src/actors/project_cache.rs b/relay-server/src/actors/project_cache.rs index 08696e9b4f..b1c8b84960 100644 --- a/relay-server/src/actors/project_cache.rs +++ b/relay-server/src/actors/project_cache.rs @@ -7,6 +7,7 @@ use relay_config::{Config, RelayMode}; use relay_metrics::{self, Aggregator, FlushBuckets, MergeBuckets}; use relay_quotas::RateLimits; use relay_redis::RedisPool; +use relay_sampling::config::RuleId; use relay_statsd::metric; use relay_system::{Addr, FromMessage, Interface, Sender, Service}; use tokio::sync::mpsc; @@ -201,6 +202,7 @@ pub struct SpoolHealth; /// /// See the enumerated variants for a full list of available messages for this service. pub enum ProjectCache { + UpdateReservoir(UpdateCount), RequestUpdate(RequestUpdate), Get(GetProjectState, ProjectSender), GetCached(GetCachedProjectState, Sender>>), @@ -218,6 +220,19 @@ pub enum ProjectCache { impl Interface for ProjectCache {} +pub struct UpdateCount { + pub project_key: ProjectKey, + pub rule_id: RuleId, +} + +impl FromMessage for ProjectCache { + type Response = relay_system::NoResponse; + + fn from_message(message: UpdateCount, _: ()) -> Self { + Self::UpdateReservoir(message) + } +} + impl FromMessage for ProjectCache { type Response = relay_system::NoResponse; @@ -557,6 +572,30 @@ impl ProjectCacheBroker { Project::new(project_key, config) }) } + fn remove_outdated_biased_rules(&mut self, state: Arc) { + let project_key = state.public_keys[0].public_key; + let project = self.projects.get_mut(&project_key); + if let (Some(sampling_config), Some(project)) = (&state.config.dynamic_sampling, project) { + let rules: Vec = sampling_config + .rules_v2 + .clone() + .into_iter() + .map(|x| x.id) + .collect(); + + let counter_map = project.bias_counter(); + + let keys_to_remove: Vec<_> = counter_map + .keys() + .filter(|&k| !rules.contains(k)) + .cloned() + .collect(); + + for key in keys_to_remove { + counter_map.remove(&key); + } + } + } /// Updates the [`Project`] with received [`ProjectState`]. /// @@ -576,6 +615,8 @@ impl ProjectCacheBroker { no_cache, ); + self.remove_outdated_biased_rules(state.clone()); + if !state.invalid() { self.dequeue(project_key); } @@ -688,6 +729,7 @@ impl ProjectCacheBroker { envelope: managed_envelope, project_state: own_project_state.clone(), sampling_project_state: None, + counters: BTreeMap::default(), }; if let Some(sampling_state) = sampling_state { diff --git a/relay-server/src/actors/project_redis.rs b/relay-server/src/actors/project_redis.rs index 6f8efa1809..d59259ff90 100644 --- a/relay-server/src/actors/project_redis.rs +++ b/relay-server/src/actors/project_redis.rs @@ -3,11 +3,39 @@ use std::sync::Arc; use relay_base_schema::project::ProjectKey; use relay_config::Config; use relay_redis::{RedisError, RedisPool}; +use relay_sampling::config::RuleId; use relay_statsd::metric; use crate::actors::project::ProjectState; use crate::statsd::{RelayCounters, RelayHistograms, RelayTimers}; +pub struct BiasRedisKey(String); + +impl BiasRedisKey { + pub fn new(project_key: &ProjectKey, rule_id: RuleId) -> Self { + Self(format!("bias:{}:{}", project_key, rule_id)) + } + + pub fn as_str(&self) -> &str { + self.0.as_str() + } +} + +/// Increments the count, it will be initialized automatically if it doesn't exist. +/// +/// INCR docs: [`https://redis.io/commands/incr/`] +pub fn increment_bias_rule_count( + redis: RedisPool, + project_key: ProjectKey, + rule_id: RuleId, +) -> anyhow::Result { + let key = BiasRedisKey::new(&project_key, rule_id); + let mut command = relay_redis::redis::cmd("INCR"); + command.arg(key.as_str()); + let new_count: i64 = command.query(&mut redis.client()?.connection()?)?; + Ok(new_count) +} + #[derive(Debug, Clone)] pub struct RedisProjectSource { config: Arc, From e03b982ff3047d11069f46e0395fc1f17a9a1872 Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Wed, 27 Sep 2023 15:44:01 +0200 Subject: [PATCH 02/30] wip --- Cargo.lock | 1 + relay-sampling/Cargo.toml | 1 + relay-sampling/src/config.rs | 37 +++-- relay-sampling/src/evaluation.rs | 158 +++++++++++++++++++-- relay-server/src/actors/processor.rs | 28 ++-- relay-server/src/actors/project.rs | 7 + relay-server/src/actors/project_cache.rs | 43 ++---- relay-server/src/actors/project_redis.rs | 28 ---- relay-server/src/utils/dynamic_sampling.rs | 41 ++++-- 9 files changed, 245 insertions(+), 99 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 69a65ddbc8..41a9388a9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3831,6 +3831,7 @@ dependencies = [ "relay-event-schema", "relay-log", "relay-protocol", + "relay-redis", "serde", "serde_json", "similar-asserts", diff --git a/relay-sampling/Cargo.toml b/relay-sampling/Cargo.toml index 0ec2eb128d..dc2c1d44ce 100644 --- a/relay-sampling/Cargo.toml +++ b/relay-sampling/Cargo.toml @@ -18,6 +18,7 @@ relay-common = { path = "../relay-common" } relay-event-schema = { path = "../relay-event-schema" } relay-log = { path = "../relay-log" } relay-protocol = { path = "../relay-protocol" } +relay-redis = { path = "../relay-redis" } serde = { workspace = true } serde_json = { workspace = true } unicase = "2.6.0" diff --git a/relay-sampling/src/config.rs b/relay-sampling/src/config.rs index 642d3bd2d9..61fd036746 100644 --- a/relay-sampling/src/config.rs +++ b/relay-sampling/src/config.rs @@ -1,11 +1,13 @@ //! Dynamic sampling rule configuration. use std::fmt; +use std::sync::Arc; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use crate::condition::RuleCondition; +use crate::evaluation::ReservoirStuff; use crate::utils; /// Represents the dynamic sampling configuration available to a project. @@ -86,13 +88,25 @@ impl SamplingRule { } /// Returns the sample rate if the rule is active. - pub fn sample_rate(&self, now: DateTime) -> Option { + pub fn sample_rate( + &self, + now: DateTime, + reservoir: Arc, + ) -> Option { if !self.time_range.contains(now) { // Return None if rule is inactive. return None; } - let sampling_base_value = self.sampling_value.value(); + let sampling_base_value = match self.sampling_value { + SamplingValue::SampleRate { value } => value, + SamplingValue::Factor { value } => value, + SamplingValue::Reservoir { limit } => { + return reservoir + .evaluate_rule(self.id, limit) + .then_some(SamplingValue::Reservoir { limit }) + } + }; let value = match self.decaying_fn { DecayingFunction::Linear { decayed_value } => { @@ -118,6 +132,7 @@ impl SamplingRule { match self.sampling_value { SamplingValue::SampleRate { .. } => Some(SamplingValue::SampleRate { value }), SamplingValue::Factor { .. } => Some(SamplingValue::Factor { value }), + x => Some(x), } } } @@ -144,18 +159,12 @@ pub enum SamplingValue { /// until a sample rate rule is found. The matched rule's factor will be multiplied with the /// accumulated factors before moving onto the next possible match. Factor { - /// The fator to apply on another matched sample rate. + /// The factor to apply on another matched sample rate. value: f64, }, -} - -impl SamplingValue { - pub(crate) fn value(&self) -> f64 { - *match self { - SamplingValue::SampleRate { value } => value, - SamplingValue::Factor { value } => value, - } - } + Reservoir { + limit: i64, + }, } /// Defines what a dynamic sampling rule applies to. @@ -178,7 +187,7 @@ pub enum RuleType { /// /// This number must be unique within a Sentry organization, as it is recorded in outcomes and used /// to infer which sampling rule caused data to be dropped. -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct RuleId(pub u32); impl fmt::Display for RuleId { @@ -277,6 +286,7 @@ impl Default for SamplingMode { } } +/* #[cfg(test)] mod tests { use chrono::TimeZone; @@ -654,3 +664,4 @@ mod tests { ); } } +*/ diff --git a/relay-sampling/src/evaluation.rs b/relay-sampling/src/evaluation.rs index af969da550..a8b36d00a1 100644 --- a/relay-sampling/src/evaluation.rs +++ b/relay-sampling/src/evaluation.rs @@ -1,17 +1,22 @@ //! Evaluation of dynamic sampling rules. +use std::collections::{BTreeMap, BTreeSet}; use std::fmt; use std::num::ParseIntError; +use std::sync::{Arc, Mutex, MutexGuard}; use chrono::{DateTime, Utc}; use rand::distributions::Uniform; use rand::Rng; use rand_pcg::Pcg32; +use relay_base_schema::project::ProjectKey; use relay_protocol::Getter; +use relay_redis::{RedisError, RedisPool}; use serde::Serialize; use uuid::Uuid; use crate::config::{RuleId, SamplingRule, SamplingValue}; +use crate::SamplingConfig; /// Generates a pseudo random number by seeding the generator with the given id. /// @@ -54,6 +59,127 @@ impl Evaluation { } } +pub struct BiasRedisKey(String); + +impl BiasRedisKey { + pub fn new(project_key: &ProjectKey, rule_id: RuleId) -> Self { + Self(format!("bias:{}:{}", project_key, rule_id)) + } + + pub fn as_str(&self) -> &str { + self.0.as_str() + } +} + +/// Increments the count, it will be initialized automatically if it doesn't exist. +/// +/// INCR docs: [`https://redis.io/commands/incr/`] +#[cfg(feature = "processing")] +fn increment_bias_rule_count( + redis: &RedisPool, + project_key: ProjectKey, + rule_id: RuleId, +) -> Result { + let key = BiasRedisKey::new(&project_key, rule_id); + let mut command = relay_redis::redis::cmd("INCR"); + command.arg(key.as_str()); + let new_count: i64 = command.query(&mut redis.client()?.connection()?).unwrap(); + Ok(new_count) +} + +#[derive(Debug)] +pub struct ReservoirStuff { + #[cfg(feature = "processing")] + redis: Option, + project_key: ProjectKey, + map: Mutex>, +} + +impl ReservoirStuff { + #[cfg(feature = "processing")] + pub fn new(redis: Option, project_key: ProjectKey) -> Self { + Self { + redis, + project_key, + map: Mutex::default(), + } + } + + #[cfg(not(feature = "processing"))] + pub fn new(project_key: ProjectKey) -> Self { + Self { + project_key, + map: Mutex::default(), + } + } + + pub fn evaluate_rule(&self, rule: RuleId, limit: i64) -> bool { + let Ok(mut map_guard) = self.map.try_lock() else { + return false; + }; + + if Self::limit_exceeded(&mut map_guard, rule, limit).unwrap_or(true) { + return false; + } + + if let Some(val) = map_guard.get_mut(&rule) { + *val += 1; + } + + true + } + + fn limit_exceeded( + guard: &mut MutexGuard>, + rule: RuleId, + limit: i64, + ) -> Option { + guard.get(&rule).map(|val| *val > limit) + } + + // if a rule is no longer in the sampling config, we can assume it's deleted and no longer needed. + pub fn delete_expired_rules(&self, config: &SamplingConfig) { + let reservoir_rules: BTreeSet = config + .rules_v2 + .iter() + .filter_map(|rule| rule.is_reservoir().then_some(rule.id)) + .collect(); + + self.map + .try_lock() + .unwrap() + .retain(|key, _| reservoir_rules.contains(key)); + } + + // if limit isn't reached yet, we wanna increment. + // when incrementing, if processing is enabled, we increment redis and insert back the value + // otherwise we just increment directly + #[cfg(feature = "processing")] + pub fn increment(map_guard: &mut MutexGuard>, rule: RuleId) { + match self.redis.as_ref() { + Some(redis_pool) => { + let new_value = + increment_bias_rule_count(redis_pool, self.project_key, rule).ok()?; + + if let Some(val) = map_guard.get_mut(&rule) { + *val = new_value; + } + } + None => { + if let Some(val) = map_guard.get_mut(&rule) { + *val += 1; + } + } + }; + } + #[cfg(not(feature = "processing"))] + pub fn increment(map_guard: &mut MutexGuard>, rule: RuleId) { + if let Some(val) = map_guard.get_mut(&rule) { + *val += 1; + } + } +} + /// State machine for dynamic sampling. #[derive(Debug)] pub struct SamplingEvaluator { @@ -61,16 +187,18 @@ pub struct SamplingEvaluator { rule_ids: Vec, factor: f64, client_sample_rate: Option, + reservoir: Arc, } impl SamplingEvaluator { /// Constructor for [`SamplingEvaluator`]. - pub fn new(now: DateTime) -> Self { + pub fn new(now: DateTime, reservoir: Arc) -> Self { Self { now, rule_ids: vec![], factor: 1.0, client_sample_rate: None, + reservoir, } } @@ -91,7 +219,7 @@ impl SamplingEvaluator { continue; }; - let Some(sampling_value) = rule.sample_rate(self.now) else { + let Some(sampling_value) = rule.sample_rate(self.now, self.reservoir.clone()) else { continue; }; @@ -111,6 +239,7 @@ impl SamplingEvaluator { self.rule_ids, )); } + SamplingValue::Reservoir { limit } => todo!(), } } Evaluation::Continue(self) @@ -290,9 +419,16 @@ mod tests { use super::*; + fn dummy_reservoir() -> Arc { + let project_key = "12345678123456781234567812345678" + .parse::() + .unwrap(); + ReservoirStuff::new(project_key).into() + } + /// Helper to extract the sampling match after evaluating rules. fn get_sampling_match(rules: &[SamplingRule], instance: &impl Getter) -> SamplingMatch { - match SamplingEvaluator::new(Utc::now()).match_rules( + match SamplingEvaluator::new(Utc::now(), dummy_reservoir()).match_rules( Uuid::default(), instance, rules.iter(), @@ -344,7 +480,7 @@ mod tests { #[test] fn test_adjust_sample_rate() { // return the same as input if no client sample rate set in the sampling evaluator. - let eval = SamplingEvaluator::new(Utc::now()); + let eval = SamplingEvaluator::new(Utc::now(), dummy_reservoir()); assert_eq!(eval.adjusted_sample_rate(0.2), 0.2); let eval = eval.adjust_client_sample_rate(Some(0.5)); @@ -372,7 +508,7 @@ mod tests { let dsc = mocked_dsc_with_getter_values(vec![]); - let res = SamplingEvaluator::new(Utc::now()) + let res = SamplingEvaluator::new(Utc::now(), dummy_reservoir()) .adjust_client_sample_rate(Some(0.2)) .match_rules(Uuid::default(), &dsc, rules.iter()); @@ -438,17 +574,17 @@ mod tests { // Baseline test. let within_timerange = Utc.with_ymd_and_hms(1970, 10, 11, 0, 0, 0).unwrap(); - assert!(SamplingEvaluator::new(within_timerange) + assert!(SamplingEvaluator::new(within_timerange, dummy_reservoir()) .match_rules(Uuid::default(), &dsc, [rule.clone()].iter()) .is_match()); let before_timerange = Utc.with_ymd_and_hms(1969, 1, 1, 0, 0, 0).unwrap(); - assert!(SamplingEvaluator::new(before_timerange) + assert!(SamplingEvaluator::new(before_timerange, dummy_reservoir()) .match_rules(Uuid::default(), &dsc, [rule.clone()].iter()) .is_no_match()); let after_timerange = Utc.with_ymd_and_hms(1971, 1, 1, 0, 0, 0).unwrap(); - assert!(SamplingEvaluator::new(after_timerange) + assert!(SamplingEvaluator::new(after_timerange, dummy_reservoir()) .match_rules(Uuid::default(), &dsc, [rule].iter()) .is_no_match()); } @@ -575,7 +711,11 @@ mod tests { fn test_get_sampling_match_result_with_no_match() { let dsc = mocked_dsc_with_getter_values(vec![]); - let res = SamplingEvaluator::new(Utc::now()).match_rules(Uuid::default(), &dsc, [].iter()); + let res = SamplingEvaluator::new(Utc::now(), dummy_reservoir()).match_rules( + Uuid::default(), + &dsc, + [].iter(), + ); assert!(res.is_no_match()); } diff --git a/relay-server/src/actors/processor.rs b/relay-server/src/actors/processor.rs index 80508cb902..2163e2f3e4 100644 --- a/relay-server/src/actors/processor.rs +++ b/relay-server/src/actors/processor.rs @@ -42,8 +42,8 @@ use relay_protocol::{Annotated, Array, Empty, FromValue, Object, Value}; use relay_quotas::{DataCategory, ReasonCode}; use relay_redis::RedisPool; use relay_replays::recording::RecordingScrubber; -use relay_sampling::config::{RuleId, RuleType, SamplingMode}; -use relay_sampling::evaluation::{Evaluation, MatchedRuleIds, SamplingEvaluator}; +use relay_sampling::config::{RuleType, SamplingMode}; +use relay_sampling::evaluation::{Evaluation, MatchedRuleIds, ReservoirStuff, SamplingEvaluator}; use relay_sampling::{DynamicSamplingContext, SamplingConfig}; use relay_statsd::metric; use relay_system::{Addr, FromMessage, NoResponse, Service}; @@ -306,6 +306,9 @@ struct ProcessEnvelopeState { /// Whether there is a profiling item in the envelope. has_profile: bool, + + /// Reservoir stuff + reservoir: Arc, } impl ProcessEnvelopeState { @@ -424,7 +427,7 @@ pub struct ProcessEnvelope { pub envelope: ManagedEnvelope, pub project_state: Arc, pub sampling_project_state: Option>, - pub counters: BTreeMap, + pub reservoir: Arc, } /// Parses a list of metrics or metric buckets and pushes them to the project's aggregator. @@ -1319,7 +1322,7 @@ impl EnvelopeProcessorService { envelope: mut managed_envelope, project_state, sampling_project_state, - .. + reservoir, } = message; let envelope = managed_envelope.envelope_mut(); @@ -1365,6 +1368,7 @@ impl EnvelopeProcessorService { project_id, managed_envelope, has_profile: false, + reservoir, }) } @@ -2347,8 +2351,9 @@ impl EnvelopeProcessorService { &state.project_state.config.transaction_metrics { if config.is_enabled() { - let res = Self::compute_sampling_decision( + state.sampling_result = Self::compute_sampling_decision( self.inner.config.processing_enabled(), + state.reservoir.clone(), state.project_state.config.dynamic_sampling.as_ref(), state.event.value(), state @@ -2367,6 +2372,7 @@ impl EnvelopeProcessorService { /// Computes the sampling decision on the incoming transaction. fn compute_sampling_decision( processing_enabled: bool, + reservoir: Arc, sampling_config: Option<&SamplingConfig>, event: Option<&Event>, root_sampling_config: Option<&SamplingConfig>, @@ -2406,8 +2412,8 @@ impl EnvelopeProcessorService { } }; - let mut evaluator = - SamplingEvaluator::new(Utc::now()).adjust_client_sample_rate(adjustment_rate); + let mut evaluator = SamplingEvaluator::new(Utc::now(), reservoir) + .adjust_client_sample_rate(adjustment_rate); if let (Some(event), Some(sampling_state)) = (event, sampling_config) { if let Some(seed) = event.id.value().map(|id| id.0) { @@ -2449,8 +2455,12 @@ impl EnvelopeProcessorService { return; }; - let sampled = - utils::is_trace_fully_sampled(self.inner.config.processing_enabled(), config, dsc); + let sampled = utils::is_trace_fully_sampled( + self.inner.config.processing_enabled(), + state.reservoir.clone(), + config, + dsc, + ); let (Some(event), Some(sampled)) = (state.event.value_mut(), sampled) else { return; diff --git a/relay-server/src/actors/project.rs b/relay-server/src/actors/project.rs index 23b3f2dc81..4195a211b8 100644 --- a/relay-server/src/actors/project.rs +++ b/relay-server/src/actors/project.rs @@ -8,6 +8,7 @@ use relay_dynamic_config::{Feature, LimitedProjectConfig, ProjectConfig}; use relay_filter::matches_any_origin; use relay_metrics::{Aggregator, Bucket, MergeBuckets, MetricNamespace, MetricResourceIdentifier}; use relay_quotas::{Quota, RateLimits, Scoping}; +use relay_sampling::evaluation::ReservoirStuff; use relay_statsd::metric; use relay_system::{Addr, BroadcastChannel}; use serde::{Deserialize, Serialize}; @@ -393,6 +394,7 @@ pub struct Project { state_channel: Option, rate_limits: RateLimits, last_no_cache: Instant, + reservoir: Arc, } impl Project { @@ -408,6 +410,7 @@ impl Project { state_channel: None, rate_limits: RateLimits::new(), last_no_cache: Instant::now(), + reservoir: Arc::new(ReservoirStuff::new(key)), } } @@ -421,6 +424,10 @@ impl Project { } } + pub fn reservoir(&self) -> Arc { + self.reservoir.clone() + } + pub fn merge_rate_limits(&mut self, rate_limits: RateLimits) { self.rate_limits.merge(rate_limits); } diff --git a/relay-server/src/actors/project_cache.rs b/relay-server/src/actors/project_cache.rs index b1c8b84960..51b3d6cefb 100644 --- a/relay-server/src/actors/project_cache.rs +++ b/relay-server/src/actors/project_cache.rs @@ -7,7 +7,6 @@ use relay_config::{Config, RelayMode}; use relay_metrics::{self, Aggregator, FlushBuckets, MergeBuckets}; use relay_quotas::RateLimits; use relay_redis::RedisPool; -use relay_sampling::config::RuleId; use relay_statsd::metric; use relay_system::{Addr, FromMessage, Interface, Sender, Service}; use tokio::sync::mpsc; @@ -202,7 +201,7 @@ pub struct SpoolHealth; /// /// See the enumerated variants for a full list of available messages for this service. pub enum ProjectCache { - UpdateReservoir(UpdateCount), + //UpdateReservoir(UpdateCount), RequestUpdate(RequestUpdate), Get(GetProjectState, ProjectSender), GetCached(GetCachedProjectState, Sender>>), @@ -220,6 +219,7 @@ pub enum ProjectCache { impl Interface for ProjectCache {} +/* pub struct UpdateCount { pub project_key: ProjectKey, pub rule_id: RuleId, @@ -232,6 +232,7 @@ impl FromMessage for ProjectCache { Self::UpdateReservoir(message) } } +*/ impl FromMessage for ProjectCache { type Response = relay_system::NoResponse; @@ -572,30 +573,6 @@ impl ProjectCacheBroker { Project::new(project_key, config) }) } - fn remove_outdated_biased_rules(&mut self, state: Arc) { - let project_key = state.public_keys[0].public_key; - let project = self.projects.get_mut(&project_key); - if let (Some(sampling_config), Some(project)) = (&state.config.dynamic_sampling, project) { - let rules: Vec = sampling_config - .rules_v2 - .clone() - .into_iter() - .map(|x| x.id) - .collect(); - - let counter_map = project.bias_counter(); - - let keys_to_remove: Vec<_> = counter_map - .keys() - .filter(|&k| !rules.contains(k)) - .cloned() - .collect(); - - for key in keys_to_remove { - counter_map.remove(&key); - } - } - } /// Updates the [`Project`] with received [`ProjectState`]. /// @@ -615,7 +592,15 @@ impl ProjectCacheBroker { no_cache, ); - self.remove_outdated_biased_rules(state.clone()); + if let Some(dynamic_sampling_config) = state.config.dynamic_sampling.as_ref() { + if let Some(reservoir) = self + .projects + .get(&project_key) + .map(|project| project.reservoir()) + { + reservoir.delete_expired_rules(dynamic_sampling_config); + } + }; if !state.invalid() { self.dequeue(project_key); @@ -721,6 +706,8 @@ impl ProjectCacheBroker { .. }) = project.check_envelope(managed_envelope, self.services.outcome_aggregator.clone()) { + let reservoir = project.reservoir(); + let sampling_state = utils::get_sampling_key(managed_envelope.envelope()) .and_then(|key| self.projects.get(&key)) .and_then(|p| p.valid_state()); @@ -729,7 +716,7 @@ impl ProjectCacheBroker { envelope: managed_envelope, project_state: own_project_state.clone(), sampling_project_state: None, - counters: BTreeMap::default(), + reservoir, }; if let Some(sampling_state) = sampling_state { diff --git a/relay-server/src/actors/project_redis.rs b/relay-server/src/actors/project_redis.rs index d59259ff90..6f8efa1809 100644 --- a/relay-server/src/actors/project_redis.rs +++ b/relay-server/src/actors/project_redis.rs @@ -3,39 +3,11 @@ use std::sync::Arc; use relay_base_schema::project::ProjectKey; use relay_config::Config; use relay_redis::{RedisError, RedisPool}; -use relay_sampling::config::RuleId; use relay_statsd::metric; use crate::actors::project::ProjectState; use crate::statsd::{RelayCounters, RelayHistograms, RelayTimers}; -pub struct BiasRedisKey(String); - -impl BiasRedisKey { - pub fn new(project_key: &ProjectKey, rule_id: RuleId) -> Self { - Self(format!("bias:{}:{}", project_key, rule_id)) - } - - pub fn as_str(&self) -> &str { - self.0.as_str() - } -} - -/// Increments the count, it will be initialized automatically if it doesn't exist. -/// -/// INCR docs: [`https://redis.io/commands/incr/`] -pub fn increment_bias_rule_count( - redis: RedisPool, - project_key: ProjectKey, - rule_id: RuleId, -) -> anyhow::Result { - let key = BiasRedisKey::new(&project_key, rule_id); - let mut command = relay_redis::redis::cmd("INCR"); - command.arg(key.as_str()); - let new_count: i64 = command.query(&mut redis.client()?.connection()?)?; - Ok(new_count) -} - #[derive(Debug, Clone)] pub struct RedisProjectSource { config: Arc, diff --git a/relay-server/src/utils/dynamic_sampling.rs b/relay-server/src/utils/dynamic_sampling.rs index d9f5cc71b3..daa0d2b29f 100644 --- a/relay-server/src/utils/dynamic_sampling.rs +++ b/relay-server/src/utils/dynamic_sampling.rs @@ -1,8 +1,10 @@ //! Functionality for calculating if a trace should be processed or dropped. +use std::sync::Arc; + use chrono::Utc; use relay_base_schema::project::ProjectKey; use relay_sampling::config::{RuleType, SamplingMode}; -use relay_sampling::evaluation::{Evaluation, SamplingEvaluator, SamplingMatch}; +use relay_sampling::evaluation::{Evaluation, ReservoirStuff, SamplingEvaluator, SamplingMatch}; use relay_sampling::{DynamicSamplingContext, SamplingConfig}; use crate::envelope::{Envelope, ItemType}; @@ -63,6 +65,7 @@ impl From for SamplingResult { /// sampling. pub fn is_trace_fully_sampled( processing_enabled: bool, + reservoir: Arc, root_project_config: &SamplingConfig, dsc: &DynamicSamplingContext, ) -> Option { @@ -88,7 +91,8 @@ pub fn is_trace_fully_sampled( }; // TODO(tor): pass correct now timestamp - let evaluator = SamplingEvaluator::new(Utc::now()).adjust_client_sample_rate(adjustment_rate); + let evaluator = + SamplingEvaluator::new(Utc::now(), reservoir).adjust_client_sample_rate(adjustment_rate); let rules = root_project_config.filter_rules(RuleType::Trace); @@ -121,6 +125,13 @@ mod tests { }; use uuid::Uuid; + fn dummy_reservoir() -> Arc { + let project_key = "12345678123456781234567812345678" + .parse::() + .unwrap(); + ReservoirStuff::new(project_key).into() + } + fn mocked_event(event_type: EventType, transaction: &str, release: &str) -> Event { Event { id: Annotated::new(EventId::new()), @@ -180,7 +191,7 @@ mod tests { let rules = vec![mocked_sampling_rule(1, RuleType::Transaction, 1.0)]; let seed = Uuid::default(); - let result: SamplingResult = SamplingEvaluator::new(Utc::now()) + let result: SamplingResult = SamplingEvaluator::new(Utc::now(), dummy_reservoir()) .match_rules(seed, &event, rules.iter()) .into(); @@ -194,7 +205,7 @@ mod tests { let rules = vec![mocked_sampling_rule(1, RuleType::Transaction, 0.0)]; let seed = Uuid::default(); - let result: SamplingResult = SamplingEvaluator::new(Utc::now()) + let result: SamplingResult = SamplingEvaluator::new(Utc::now(), dummy_reservoir()) .match_rules(seed, &event, rules.iter()) .into(); @@ -217,7 +228,7 @@ mod tests { let event = mocked_event(EventType::Transaction, "bar", "2.0"); let seed = Uuid::default(); - let result: SamplingResult = SamplingEvaluator::new(Utc::now()) + let result: SamplingResult = SamplingEvaluator::new(Utc::now(), dummy_reservoir()) .match_rules(seed, &event, rules.iter()) .into(); @@ -231,7 +242,7 @@ mod tests { let rules = vec![mocked_sampling_rule(1, RuleType::Trace, 1.0)]; let dsc = mocked_simple_dynamic_sampling_context(Some(1.0), Some("3.0"), None, None, None); - let result: SamplingResult = SamplingEvaluator::new(Utc::now()) + let result: SamplingResult = SamplingEvaluator::new(Utc::now(), dummy_reservoir()) .match_rules(Uuid::default(), &dsc, rules.iter()) .into(); @@ -253,10 +264,16 @@ mod tests { let dsc = mocked_simple_dynamic_sampling_context(None, None, None, None, Some(true)); // Return true if any unsupported rules. - assert_eq!(is_trace_fully_sampled(false, &config, &dsc), Some(true)); + assert_eq!( + is_trace_fully_sampled(false, dummy_reservoir(), &config, &dsc), + Some(true) + ); // If processing is enabled, we simply log an error and otherwise proceed as usual. - assert_eq!(is_trace_fully_sampled(true, &config, &dsc), Some(false)); + assert_eq!( + is_trace_fully_sampled(true, dummy_reservoir(), &config, &dsc), + Some(false) + ); } #[test] @@ -273,7 +290,7 @@ mod tests { let dsc = mocked_simple_dynamic_sampling_context(Some(1.0), Some("3.0"), None, None, Some(true)); - let result = is_trace_fully_sampled(false, &config, &dsc).unwrap(); + let result = is_trace_fully_sampled(false, dummy_reservoir(), &config, &dsc).unwrap(); assert!(result); // We test with `sampled = true` and 0% rule. @@ -286,7 +303,7 @@ mod tests { let dsc = mocked_simple_dynamic_sampling_context(Some(1.0), Some("3.0"), None, None, Some(true)); - let result = is_trace_fully_sampled(false, &config, &dsc).unwrap(); + let result = is_trace_fully_sampled(false, dummy_reservoir(), &config, &dsc).unwrap(); assert!(!result); // We test with `sampled = false` and 100% rule. @@ -299,7 +316,7 @@ mod tests { let dsc = mocked_simple_dynamic_sampling_context(Some(1.0), Some("3.0"), None, None, Some(false)); - let result = is_trace_fully_sampled(false, &config, &dsc).unwrap(); + let result = is_trace_fully_sampled(false, dummy_reservoir(), &config, &dsc).unwrap(); assert!(!result); } @@ -314,7 +331,7 @@ mod tests { }; let dsc = mocked_simple_dynamic_sampling_context(Some(1.0), Some("3.0"), None, None, None); - let result = is_trace_fully_sampled(false, &config, &dsc); + let result = is_trace_fully_sampled(false, dummy_reservoir(), &config, &dsc); assert!(result.is_none()); } } From 92ec68f98fa4b051c0f90da767d869a5249d0b97 Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Wed, 27 Sep 2023 16:26:47 +0200 Subject: [PATCH 03/30] wip --- relay-sampling/Cargo.toml | 6 +- relay-sampling/src/evaluation.rs | 122 ++++++++++++++----------------- relay-server/Cargo.toml | 1 + 3 files changed, 60 insertions(+), 69 deletions(-) diff --git a/relay-sampling/Cargo.toml b/relay-sampling/Cargo.toml index dc2c1d44ce..03cd4818c3 100644 --- a/relay-sampling/Cargo.toml +++ b/relay-sampling/Cargo.toml @@ -9,6 +9,10 @@ edition = "2021" license-file = "../LICENSE" publish = false +[features] +default = [] +redis = ["dep:relay-redis"] + [dependencies] chrono = { workspace = true } rand = { workspace = true } @@ -18,7 +22,7 @@ relay-common = { path = "../relay-common" } relay-event-schema = { path = "../relay-event-schema" } relay-log = { path = "../relay-log" } relay-protocol = { path = "../relay-protocol" } -relay-redis = { path = "../relay-redis" } +relay-redis = { path = "../relay-redis", optional = true } serde = { workspace = true } serde_json = { workspace = true } unicase = "2.6.0" diff --git a/relay-sampling/src/evaluation.rs b/relay-sampling/src/evaluation.rs index a8b36d00a1..49c7b3112d 100644 --- a/relay-sampling/src/evaluation.rs +++ b/relay-sampling/src/evaluation.rs @@ -74,7 +74,7 @@ impl BiasRedisKey { /// Increments the count, it will be initialized automatically if it doesn't exist. /// /// INCR docs: [`https://redis.io/commands/incr/`] -#[cfg(feature = "processing")] +#[cfg(feature = "redis")] fn increment_bias_rule_count( redis: &RedisPool, project_key: ProjectKey, @@ -89,14 +89,14 @@ fn increment_bias_rule_count( #[derive(Debug)] pub struct ReservoirStuff { - #[cfg(feature = "processing")] + #[cfg(feature = "redis")] redis: Option, project_key: ProjectKey, map: Mutex>, } impl ReservoirStuff { - #[cfg(feature = "processing")] + #[cfg(feature = "redis")] pub fn new(redis: Option, project_key: ProjectKey) -> Self { Self { redis, @@ -105,7 +105,7 @@ impl ReservoirStuff { } } - #[cfg(not(feature = "processing"))] + #[cfg(not(feature = "redis"))] pub fn new(project_key: ProjectKey) -> Self { Self { project_key, @@ -122,9 +122,7 @@ impl ReservoirStuff { return false; } - if let Some(val) = map_guard.get_mut(&rule) { - *val += 1; - } + Self::increment(&mut map_guard, rule); true } @@ -137,25 +135,11 @@ impl ReservoirStuff { guard.get(&rule).map(|val| *val > limit) } - // if a rule is no longer in the sampling config, we can assume it's deleted and no longer needed. - pub fn delete_expired_rules(&self, config: &SamplingConfig) { - let reservoir_rules: BTreeSet = config - .rules_v2 - .iter() - .filter_map(|rule| rule.is_reservoir().then_some(rule.id)) - .collect(); - - self.map - .try_lock() - .unwrap() - .retain(|key, _| reservoir_rules.contains(key)); - } - // if limit isn't reached yet, we wanna increment. // when incrementing, if processing is enabled, we increment redis and insert back the value // otherwise we just increment directly - #[cfg(feature = "processing")] - pub fn increment(map_guard: &mut MutexGuard>, rule: RuleId) { + #[cfg(feature = "redis")] + fn increment(map_guard: &mut MutexGuard>, rule: RuleId) { match self.redis.as_ref() { Some(redis_pool) => { let new_value = @@ -172,12 +156,26 @@ impl ReservoirStuff { } }; } - #[cfg(not(feature = "processing"))] - pub fn increment(map_guard: &mut MutexGuard>, rule: RuleId) { + #[cfg(not(feature = "redis"))] + fn increment(map_guard: &mut MutexGuard>, rule: RuleId) { if let Some(val) = map_guard.get_mut(&rule) { *val += 1; } } + + // if a rule is no longer in the sampling config, we can assume it's deleted and no longer needed. + pub fn delete_expired_rules(&self, config: &SamplingConfig) { + let reservoir_rules: BTreeSet = config + .rules_v2 + .iter() + .filter_map(|rule| rule.is_reservoir().then_some(rule.id)) + .collect(); + + self.map + .try_lock() + .unwrap() + .retain(|key, _| reservoir_rules.contains(key)); + } } /// State machine for dynamic sampling. @@ -227,19 +225,18 @@ impl SamplingEvaluator { match sampling_value { SamplingValue::Factor { value } => self.factor *= value, - SamplingValue::SampleRate { .. } if rule.is_reservoir() => { - return Evaluation::Matched(SamplingMatch::new_reservoir(rule.id)); - } SamplingValue::SampleRate { value } => { let sample_rate = (value * self.factor).clamp(0.0, 1.0); - return Evaluation::Matched(SamplingMatch::new_standard( + return Evaluation::Matched(SamplingMatch::new( self.adjusted_sample_rate(sample_rate), seed, self.rule_ids, )); } - SamplingValue::Reservoir { limit } => todo!(), + SamplingValue::Reservoir { .. } => { + return Evaluation::Matched(SamplingMatch::new(1.0, seed, vec![rule.id])); + } } } Evaluation::Continue(self) @@ -277,6 +274,12 @@ impl SamplingEvaluator { } fn sampling_match(sample_rate: f64, seed: Uuid) -> bool { + if sample_rate == 0.0 { + return false; + } else if sample_rate == 1.0 { + return true; + } + let random_number = pseudo_random_from_uuid(seed); relay_log::trace!( sample_rate, @@ -295,34 +298,30 @@ fn sampling_match(sample_rate: f64, seed: Uuid) -> bool { /// Represents the specification for sampling an incoming event. #[derive(Clone, Debug, PartialEq)] -pub enum SamplingMatch { - Reservoir { - rule: RuleId, - }, - Standard { - /// The sample rate to use for the incoming event. - sample_rate: f64, - /// The seed to feed to the random number generator which allows the same number to be - /// generated given the same seed. - /// - /// This is especially important for trace sampling, even though we can have inconsistent - /// traces due to multi-matching. - seed: Uuid, - /// The list of rule ids that have matched the incoming event and/or dynamic sampling context. - matched_rules: MatchedRuleIds, - /// Whether this sampling match results in the item getting sampled. - /// It's essentially a cache, as the value can be deterministically derived from - /// the sample rate and the seed. - should_keep: bool, - }, +pub struct SamplingMatch { + /// The sample rate to use for the incoming event. + sample_rate: f64, + /// The seed to feed to the random number generator which allows the same number to be + /// generated given the same seed. + /// + /// This is especially important for trace sampling, even though we can have inconsistent + /// traces due to multi-matching. + seed: Uuid, + /// The list of rule ids that have matched the incoming event and/or dynamic sampling context. + matched_rules: MatchedRuleIds, + /// Whether this sampling match results in the item getting sampled. + /// It's essentially a cache, as the value can be deterministically derived from + /// the sample rate and the seed. + should_keep: bool, } impl SamplingMatch { - fn new_standard(sample_rate: f64, seed: Uuid, matched_rules: Vec) -> Self { + fn new(sample_rate: f64, seed: Uuid, matched_rules: Vec) -> Self { let matched_rules = MatchedRuleIds(matched_rules); + let should_keep = sampling_match(sample_rate, seed); - Self::Standard { + Self { sample_rate, seed, matched_rules, @@ -330,16 +329,9 @@ impl SamplingMatch { } } - fn new_reservoir(rule: RuleId) -> Self { - Self::Reservoir { rule } - } - /// Returns the sample rate. pub fn sample_rate(&self) -> f64 { - match self { - SamplingMatch::Reservoir { .. } => 1.0, - SamplingMatch::Standard { sample_rate, .. } => *sample_rate, - } + self.sample_rate } /// Returns the matched rules for the sampling match. @@ -347,18 +339,12 @@ impl SamplingMatch { /// Takes ownership, useful if you don't need the [`SamplingMatch`] anymore /// and you want to avoid allocations. pub fn into_matched_rules(self) -> MatchedRuleIds { - match self { - SamplingMatch::Reservoir { rule } => MatchedRuleIds(vec![rule]), - SamplingMatch::Standard { matched_rules, .. } => matched_rules, - } + self.matched_rules } /// Returns true if event should be kept. pub fn should_keep(&self) -> bool { - match self { - SamplingMatch::Reservoir { .. } => true, - SamplingMatch::Standard { should_keep, .. } => *should_keep, - } + self.should_keep } /// Returns true if event should be dropped. diff --git a/relay-server/Cargo.toml b/relay-server/Cargo.toml index 66c353043a..5fadb1e2f8 100644 --- a/relay-server/Cargo.toml +++ b/relay-server/Cargo.toml @@ -28,6 +28,7 @@ processing = [ "relay-kafka/producer", "relay-quotas/redis", "relay-redis/impl", + "relay-sampling/redis", ] [dependencies] From 48eacfc0a8d992836efacd80e3f0a1d1dcbcaf3f Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Wed, 27 Sep 2023 18:22:21 +0200 Subject: [PATCH 04/30] wip --- relay-sampling/src/config.rs | 2 +- relay-sampling/src/evaluation.rs | 76 ++++++++++++++++++++++---------- 2 files changed, 53 insertions(+), 25 deletions(-) diff --git a/relay-sampling/src/config.rs b/relay-sampling/src/config.rs index 61fd036746..87fceee824 100644 --- a/relay-sampling/src/config.rs +++ b/relay-sampling/src/config.rs @@ -103,7 +103,7 @@ impl SamplingRule { SamplingValue::Factor { value } => value, SamplingValue::Reservoir { limit } => { return reservoir - .evaluate_rule(self.id, limit) + .evaluate_rule(None, self.id, limit) .then_some(SamplingValue::Reservoir { limit }) } }; diff --git a/relay-sampling/src/evaluation.rs b/relay-sampling/src/evaluation.rs index 49c7b3112d..35de4f2735 100644 --- a/relay-sampling/src/evaluation.rs +++ b/relay-sampling/src/evaluation.rs @@ -62,7 +62,7 @@ impl Evaluation { pub struct BiasRedisKey(String); impl BiasRedisKey { - pub fn new(project_key: &ProjectKey, rule_id: RuleId) -> Self { + pub fn new(project_key: u64, rule_id: RuleId) -> Self { Self(format!("bias:{}:{}", project_key, rule_id)) } @@ -77,10 +77,10 @@ impl BiasRedisKey { #[cfg(feature = "redis")] fn increment_bias_rule_count( redis: &RedisPool, - project_key: ProjectKey, + project_key: u64, rule_id: RuleId, ) -> Result { - let key = BiasRedisKey::new(&project_key, rule_id); + let key = BiasRedisKey::new(project_key, rule_id); let mut command = relay_redis::redis::cmd("INCR"); command.arg(key.as_str()); let new_count: i64 = command.query(&mut redis.client()?.connection()?).unwrap(); @@ -89,41 +89,45 @@ fn increment_bias_rule_count( #[derive(Debug)] pub struct ReservoirStuff { - #[cfg(feature = "redis")] - redis: Option, - project_key: ProjectKey, + org_id: u64, map: Mutex>, } impl ReservoirStuff { - #[cfg(feature = "redis")] - pub fn new(redis: Option, project_key: ProjectKey) -> Self { + pub fn new(org_id: u64) -> Self { Self { - redis, - project_key, + org_id, map: Mutex::default(), } } #[cfg(not(feature = "redis"))] - pub fn new(project_key: ProjectKey) -> Self { - Self { - project_key, - map: Mutex::default(), + pub fn evaluate_rule(&self, redis_pool: Option<()>, rule: RuleId, limit: i64) -> bool { + let Ok(mut map_guard) = self.map.try_lock() else { + return false; + }; + + self.increment(&mut map_guard, rule, redis_pool); + + if Self::limit_exceeded(&mut map_guard, rule, limit).unwrap_or(true) { + return false; } + + true } - pub fn evaluate_rule(&self, rule: RuleId, limit: i64) -> bool { + #[cfg(feature = "redis")] + pub fn evaluate_rule(&self, redis_pool: Option<&RedisPool>, rule: RuleId, limit: i64) -> bool { let Ok(mut map_guard) = self.map.try_lock() else { return false; }; + self.increment(&mut map_guard, rule, redis_pool); + if Self::limit_exceeded(&mut map_guard, rule, limit).unwrap_or(true) { return false; } - Self::increment(&mut map_guard, rule); - true } @@ -139,11 +143,17 @@ impl ReservoirStuff { // when incrementing, if processing is enabled, we increment redis and insert back the value // otherwise we just increment directly #[cfg(feature = "redis")] - fn increment(map_guard: &mut MutexGuard>, rule: RuleId) { - match self.redis.as_ref() { + fn increment( + &self, + map_guard: &mut MutexGuard>, + rule: RuleId, + redis_pool: Option<&RedisPool>, + ) { + match redis_pool { Some(redis_pool) => { - let new_value = - increment_bias_rule_count(redis_pool, self.project_key, rule).ok()?; + let new_value = increment_bias_rule_count(redis_pool, self.org_id, rule) + .ok() + .unwrap(); if let Some(val) = map_guard.get_mut(&rule) { *val = new_value; @@ -156,8 +166,14 @@ impl ReservoirStuff { } }; } + #[cfg(not(feature = "redis"))] - fn increment(map_guard: &mut MutexGuard>, rule: RuleId) { + fn increment( + &self, + map_guard: &mut MutexGuard>, + rule: RuleId, + _: Option<()>, + ) { if let Some(val) = map_guard.get_mut(&rule) { *val += 1; } @@ -186,6 +202,8 @@ pub struct SamplingEvaluator { factor: f64, client_sample_rate: Option, reservoir: Arc, + #[cfg(feature = "redis")] + redis_pool: Option>, } impl SamplingEvaluator { @@ -197,9 +215,17 @@ impl SamplingEvaluator { factor: 1.0, client_sample_rate: None, reservoir, + #[cfg(feature = "redis")] + redis_pool: None, } } + #[cfg(feature = "redis")] + pub fn set_redis_pool(mut self, redis: Option>) -> Self { + self.redis_pool = redis; + self + } + /// Sets a new client sample rate value. pub fn adjust_client_sample_rate(mut self, client_sample_rate: Option) -> Self { self.client_sample_rate = client_sample_rate; @@ -207,10 +233,10 @@ impl SamplingEvaluator { } /// Attemps to find a match for sampling rules. - pub fn match_rules<'a, I, G>(mut self, seed: Uuid, instance: &G, rules: I) -> Evaluation + pub fn match_rules<'b, I, G>(mut self, seed: Uuid, instance: &'b G, rules: I) -> Evaluation where G: Getter, - I: Iterator, + I: Iterator, { for rule in rules { if !rule.condition.matches(instance) { @@ -388,6 +414,7 @@ impl fmt::Display for MatchedRuleIds { Ok(()) } } +/* #[cfg(test)] mod tests { use std::str::FromStr; @@ -706,3 +733,4 @@ mod tests { assert!(res.is_no_match()); } } +*/ From aef9de81c5f0fd137eccf2321e6a7eab46eed503 Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Wed, 27 Sep 2023 19:49:19 +0200 Subject: [PATCH 05/30] wip --- relay-kafka/src/lib.rs | 2 +- relay-sampling/src/config.rs | 12 +- relay-sampling/src/evaluation.rs | 157 ++++++--------------- relay-server/src/actors/processor.rs | 19 ++- relay-server/src/actors/project.rs | 34 ++++- relay-server/src/actors/project_cache.rs | 14 +- relay-server/src/utils/dynamic_sampling.rs | 6 +- 7 files changed, 103 insertions(+), 141 deletions(-) diff --git a/relay-kafka/src/lib.rs b/relay-kafka/src/lib.rs index efc9f2d039..20b87fbc93 100644 --- a/relay-kafka/src/lib.rs +++ b/relay-kafka/src/lib.rs @@ -20,7 +20,7 @@ //! //! // build the client //! let kafka_client = builder.build(); -//! +//! //! // send the message //! kafka_client.send_message(KafkaTopic::Events, 1u64, &kafka_message).unwrap(); //! ``` diff --git a/relay-sampling/src/config.rs b/relay-sampling/src/config.rs index 87fceee824..c980502ef1 100644 --- a/relay-sampling/src/config.rs +++ b/relay-sampling/src/config.rs @@ -4,6 +4,7 @@ use std::fmt; use std::sync::Arc; use chrono::{DateTime, Utc}; + use serde::{Deserialize, Serialize}; use crate::condition::RuleCondition; @@ -83,8 +84,9 @@ impl SamplingRule { self.condition.supported() && self.ty != RuleType::Unsupported } + /// Returns `true` if rule is a reservoir rule. pub fn is_reservoir(&self) -> bool { - todo!() + matches!(&self.sampling_value, &SamplingValue::Reservoir { .. }) } /// Returns the sample rate if the rule is active. @@ -103,8 +105,8 @@ impl SamplingRule { SamplingValue::Factor { value } => value, SamplingValue::Reservoir { limit } => { return reservoir - .evaluate_rule(None, self.id, limit) - .then_some(SamplingValue::Reservoir { limit }) + .evaluate_rule(self.id, limit) + .then_some(SamplingValue::Reservoir { limit }); } }; @@ -162,7 +164,11 @@ pub enum SamplingValue { /// The factor to apply on another matched sample rate. value: f64, }, + /// A reservoir limit. + /// + /// Rule will match if less than `limit` rules have been sampled. Reservoir { + /// The limit of how many transactions with this rule will be sampled. limit: i64, }, } diff --git a/relay-sampling/src/evaluation.rs b/relay-sampling/src/evaluation.rs index 2008e356bf..952ceb5cea 100644 --- a/relay-sampling/src/evaluation.rs +++ b/relay-sampling/src/evaluation.rs @@ -1,6 +1,6 @@ //! Evaluation of dynamic sampling rules. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use std::fmt; use std::num::ParseIntError; use std::ops::ControlFlow; @@ -16,7 +16,6 @@ use serde::Serialize; use uuid::Uuid; use crate::config::{RuleId, SamplingRule, SamplingValue}; -use crate::SamplingConfig; /// Generates a pseudo random number by seeding the generator with the given id. /// @@ -28,138 +27,81 @@ fn pseudo_random_from_uuid(id: Uuid) -> f64 { generator.sample(dist) } -pub struct BiasRedisKey(String); - -impl BiasRedisKey { - pub fn new(project_key: u64, rule_id: RuleId) -> Self { - Self(format!("bias:{}:{}", project_key, rule_id)) - } - - pub fn as_str(&self) -> &str { - self.0.as_str() - } -} - /// Increments the count, it will be initialized automatically if it doesn't exist. /// /// INCR docs: [`https://redis.io/commands/incr/`] #[cfg(feature = "redis")] fn increment_bias_rule_count( - redis: &RedisPool, - project_key: u64, + redis: Arc, + org_id: u64, rule_id: RuleId, ) -> Result { - let key = BiasRedisKey::new(project_key, rule_id); + let key = format!("bias:{}:{}", org_id, rule_id); let mut command = relay_redis::redis::cmd("INCR"); command.arg(key.as_str()); let new_count: i64 = command.query(&mut redis.client()?.connection()?).unwrap(); Ok(new_count) } +/// The amount of transactions sampled of a given rule id. +pub type ReservoirCounters = Arc>>; + +/// Reservoir utility. #[derive(Debug)] pub struct ReservoirStuff { + #[cfg(feature = "redis")] + redis_pool: Option>, org_id: u64, - map: Mutex>, + map: ReservoirCounters, } impl ReservoirStuff { - pub fn new(org_id: u64) -> Self { + /// Creates a new whatever ill call this struct. + pub fn new( + org_id: u64, + map: ReservoirCounters, + #[cfg(feature = "redis")] redis_pool: Option>, + ) -> Self { Self { org_id, - map: Mutex::default(), - } - } - - #[cfg(not(feature = "redis"))] - pub fn evaluate_rule(&self, redis_pool: Option<()>, rule: RuleId, limit: i64) -> bool { - let Ok(mut map_guard) = self.map.try_lock() else { - return false; - }; - - self.increment(&mut map_guard, rule, redis_pool); - - if Self::limit_exceeded(&mut map_guard, rule, limit).unwrap_or(true) { - return false; + map, + #[cfg(feature = "redis")] + redis_pool, } - - true } - #[cfg(feature = "redis")] - pub fn evaluate_rule(&self, redis_pool: Option<&RedisPool>, rule: RuleId, limit: i64) -> bool { + /// Evaluates a reservoir rule, returning true if it should be sampled. + pub fn evaluate_rule(&self, rule: RuleId, limit: i64) -> bool { let Ok(mut map_guard) = self.map.try_lock() else { return false; }; - self.increment(&mut map_guard, rule, redis_pool); - - if Self::limit_exceeded(&mut map_guard, rule, limit).unwrap_or(true) { - return false; - } + let incremented_value = self.increment(&mut map_guard, rule); - true - } - - fn limit_exceeded( - guard: &mut MutexGuard>, - rule: RuleId, - limit: i64, - ) -> Option { - guard.get(&rule).map(|val| *val > limit) + incremented_value < limit } // if limit isn't reached yet, we wanna increment. // when incrementing, if processing is enabled, we increment redis and insert back the value // otherwise we just increment directly - #[cfg(feature = "redis")] - fn increment( - &self, - map_guard: &mut MutexGuard>, - rule: RuleId, - redis_pool: Option<&RedisPool>, - ) { - match redis_pool { - Some(redis_pool) => { - let new_value = increment_bias_rule_count(redis_pool, self.org_id, rule) - .ok() - .unwrap(); - - if let Some(val) = map_guard.get_mut(&rule) { - *val = new_value; + fn increment(&self, map_guard: &mut MutexGuard>, rule: RuleId) -> i64 { + let mut increment_value: i64 = 1; + + #[cfg(feature = "redis")] + { + if let Some(pool) = self.redis_pool.as_ref() { + if let Ok(new_val_from_redis) = + increment_bias_rule_count(pool.clone(), self.org_id, rule) + { + increment_value = new_val_from_redis; } } - None => { - if let Some(val) = map_guard.get_mut(&rule) { - *val += 1; - } - } - }; - } - - #[cfg(not(feature = "redis"))] - fn increment( - &self, - map_guard: &mut MutexGuard>, - rule: RuleId, - _: Option<()>, - ) { - if let Some(val) = map_guard.get_mut(&rule) { - *val += 1; } - } - // if a rule is no longer in the sampling config, we can assume it's deleted and no longer needed. - pub fn delete_expired_rules(&self, config: &SamplingConfig) { - let reservoir_rules: BTreeSet = config - .rules_v2 - .iter() - .filter_map(|rule| rule.is_reservoir().then_some(rule.id)) - .collect(); - - self.map - .try_lock() - .unwrap() - .retain(|key, _| reservoir_rules.contains(key)); + let val = map_guard.entry(rule).or_insert(0); + *val += increment_value; + + *val } } @@ -171,8 +113,6 @@ pub struct SamplingEvaluator { factor: f64, client_sample_rate: Option, reservoir: Arc, - #[cfg(feature = "redis")] - redis_pool: Option>, } impl SamplingEvaluator { @@ -184,17 +124,9 @@ impl SamplingEvaluator { factor: 1.0, client_sample_rate: None, reservoir, - #[cfg(feature = "redis")] - redis_pool: None, } } - #[cfg(feature = "redis")] - pub fn set_redis_pool(mut self, redis: Option>) -> Self { - self.redis_pool = redis; - self - } - /// Sets a new client sample rate value. pub fn adjust_client_sample_rate(mut self, client_sample_rate: Option) -> Self { self.client_sample_rate = client_sample_rate; @@ -398,7 +330,6 @@ impl fmt::Display for MatchedRuleIds { Ok(()) } } -/* #[cfg(test)] mod tests { use std::str::FromStr; @@ -416,10 +347,7 @@ mod tests { use super::*; fn dummy_reservoir() -> Arc { - let project_key = "12345678123456781234567812345678" - .parse::() - .unwrap(); - ReservoirStuff::new(project_key).into() + ReservoirStuff::new(0, ReservoirCounters::default(), None).into() } /// Helper to extract the sampling match after evaluating rules. @@ -574,7 +502,7 @@ mod tests { // Baseline test. let within_timerange = Utc.with_ymd_and_hms(1970, 10, 11, 0, 0, 0).unwrap(); - let res = SamplingEvaluator::new(within_timerange).match_rules( + let res = SamplingEvaluator::new(within_timerange, dummy_reservoir()).match_rules( Uuid::default(), &dsc, [rule.clone()].iter(), @@ -583,7 +511,7 @@ mod tests { assert!(evaluation_is_match(res)); let before_timerange = Utc.with_ymd_and_hms(1969, 1, 1, 0, 0, 0).unwrap(); - let res = SamplingEvaluator::new(before_timerange).match_rules( + let res = SamplingEvaluator::new(before_timerange, dummy_reservoir()).match_rules( Uuid::default(), &dsc, [rule.clone()].iter(), @@ -591,7 +519,7 @@ mod tests { assert!(!evaluation_is_match(res)); let after_timerange = Utc.with_ymd_and_hms(1971, 1, 1, 0, 0, 0).unwrap(); - let res = SamplingEvaluator::new(after_timerange).match_rules( + let res = SamplingEvaluator::new(after_timerange, dummy_reservoir()).match_rules( Uuid::default(), &dsc, [rule].iter(), @@ -726,4 +654,3 @@ mod tests { assert!(!evaluation_is_match(res)); } } -*/ diff --git a/relay-server/src/actors/processor.rs b/relay-server/src/actors/processor.rs index d677f98f0a..7e86659373 100644 --- a/relay-server/src/actors/processor.rs +++ b/relay-server/src/actors/processor.rs @@ -44,7 +44,9 @@ use relay_quotas::{DataCategory, ReasonCode}; use relay_redis::RedisPool; use relay_replays::recording::RecordingScrubber; use relay_sampling::config::{RuleType, SamplingMode}; -use relay_sampling::evaluation::{MatchedRuleIds, ReservoirStuff, SamplingEvaluator}; +use relay_sampling::evaluation::{ + MatchedRuleIds, ReservoirCounters, ReservoirStuff, SamplingEvaluator, +}; use relay_sampling::{DynamicSamplingContext, SamplingConfig}; use relay_statsd::metric; use relay_system::{Addr, FromMessage, NoResponse, Service}; @@ -428,7 +430,7 @@ pub struct ProcessEnvelope { pub envelope: ManagedEnvelope, pub project_state: Arc, pub sampling_project_state: Option>, - pub reservoir: Arc, + pub reservoir_counters: ReservoirCounters, } /// Parses a list of metrics or metric buckets and pushes them to the project's aggregator. @@ -537,6 +539,7 @@ pub struct EnvelopeProcessorService { struct InnerProcessor { config: Arc, + redis_pool: Option>, envelope_manager: Addr, project_cache: Addr, global_config: Addr, @@ -569,6 +572,8 @@ impl EnvelopeProcessorService { }); let inner = InnerProcessor { + #[cfg(feature = "processing")] + redis_pool: _redis.clone().map(Arc::new), #[cfg(feature = "processing")] rate_limiter: _redis .map(|pool| RedisRateLimiter::new(pool).max_limit(config.max_rate_limit())), @@ -1323,7 +1328,7 @@ impl EnvelopeProcessorService { envelope: mut managed_envelope, project_state, sampling_project_state, - reservoir, + reservoir_counters, } = message; let envelope = managed_envelope.envelope_mut(); @@ -1357,6 +1362,14 @@ impl EnvelopeProcessorService { // 2. The DSN was moved and the envelope sent to the old project ID. envelope.meta_mut().set_project_id(project_id); + let reservoir = ReservoirStuff::new( + managed_envelope.scoping().organization_id, + reservoir_counters, + #[cfg(feature = "processing")] + self.inner.redis_pool.clone(), + ) + .into(); + Ok(ProcessEnvelopeState { event: Annotated::empty(), event_metrics_extracted: false, diff --git a/relay-server/src/actors/project.rs b/relay-server/src/actors/project.rs index 2fc1da4684..b7d0e24a66 100644 --- a/relay-server/src/actors/project.rs +++ b/relay-server/src/actors/project.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeSet; use std::sync::Arc; use std::time::Duration; @@ -8,7 +9,8 @@ use relay_dynamic_config::{Feature, LimitedProjectConfig, ProjectConfig}; use relay_filter::matches_any_origin; use relay_metrics::{Aggregator, Bucket, MergeBuckets, MetricNamespace, MetricResourceIdentifier}; use relay_quotas::{Quota, RateLimits, Scoping}; -use relay_sampling::evaluation::ReservoirStuff; +use relay_sampling::config::RuleId; +use relay_sampling::evaluation::ReservoirCounters; use relay_statsd::metric; use relay_system::{Addr, BroadcastChannel}; use serde::{Deserialize, Serialize}; @@ -394,7 +396,7 @@ pub struct Project { state_channel: Option, rate_limits: RateLimits, last_no_cache: Instant, - reservoir: Arc, + reservoir_counters: ReservoirCounters, } impl Project { @@ -410,7 +412,7 @@ impl Project { state_channel: None, rate_limits: RateLimits::new(), last_no_cache: Instant::now(), - reservoir: Arc::new(ReservoirStuff::new(0)), + reservoir_counters: Arc::default(), } } @@ -424,8 +426,30 @@ impl Project { } } - pub fn reservoir(&self) -> Arc { - self.reservoir.clone() + pub fn reservoir_counters(&self) -> ReservoirCounters { + self.reservoir_counters.clone() + } + + /// if a rule is no longer in the sampling config, we can assume it's deleted and no longer needed. + pub fn delete_expired_rules(&self) { + let Some(config) = self + .state + .as_ref() + .and_then(|state| state.config.dynamic_sampling.as_ref()) + else { + return; + }; + + let reservoir_rules: BTreeSet = config + .rules_v2 + .iter() + .filter_map(|rule| rule.is_reservoir().then_some(rule.id)) + .collect(); + + self.reservoir_counters + .try_lock() + .unwrap() + .retain(|key, _| reservoir_rules.contains(key)); } pub fn merge_rate_limits(&mut self, rate_limits: RateLimits) { diff --git a/relay-server/src/actors/project_cache.rs b/relay-server/src/actors/project_cache.rs index 51b3d6cefb..50a9b79fc7 100644 --- a/relay-server/src/actors/project_cache.rs +++ b/relay-server/src/actors/project_cache.rs @@ -592,14 +592,8 @@ impl ProjectCacheBroker { no_cache, ); - if let Some(dynamic_sampling_config) = state.config.dynamic_sampling.as_ref() { - if let Some(reservoir) = self - .projects - .get(&project_key) - .map(|project| project.reservoir()) - { - reservoir.delete_expired_rules(dynamic_sampling_config); - } + if let Some(project) = self.projects.get(&project_key) { + project.delete_expired_rules(); }; if !state.invalid() { @@ -706,7 +700,7 @@ impl ProjectCacheBroker { .. }) = project.check_envelope(managed_envelope, self.services.outcome_aggregator.clone()) { - let reservoir = project.reservoir(); + let reservoir_counters = project.reservoir_counters(); let sampling_state = utils::get_sampling_key(managed_envelope.envelope()) .and_then(|key| self.projects.get(&key)) @@ -716,7 +710,7 @@ impl ProjectCacheBroker { envelope: managed_envelope, project_state: own_project_state.clone(), sampling_project_state: None, - reservoir, + reservoir_counters, }; if let Some(sampling_state) = sampling_state { diff --git a/relay-server/src/utils/dynamic_sampling.rs b/relay-server/src/utils/dynamic_sampling.rs index aab3948c02..8ad4c4ca6f 100644 --- a/relay-server/src/utils/dynamic_sampling.rs +++ b/relay-server/src/utils/dynamic_sampling.rs @@ -167,13 +167,11 @@ mod tests { use relay_sampling::config::{ RuleId, RuleType, SamplingConfig, SamplingMode, SamplingRule, SamplingValue, }; + use relay_sampling::evaluation::ReservoirCounters; use uuid::Uuid; fn dummy_reservoir() -> Arc { - let project_key = "12345678123456781234567812345678" - .parse::() - .unwrap(); - ReservoirStuff::new(0).into() + ReservoirStuff::new(0, ReservoirCounters::default(), None).into() } fn mocked_event(event_type: EventType, transaction: &str, release: &str) -> Event { From f7e01da0ab1018cdfb5138fc2cdf65593a3b30c9 Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Wed, 27 Sep 2023 20:00:20 +0200 Subject: [PATCH 06/30] wip --- relay-sampling/src/config.rs | 82 +- relay-sampling/src/evaluation.rs | 2 +- relay-server/src/actors/processor.rs | 1314 ++++++++++++++++++++++++++ 3 files changed, 1373 insertions(+), 25 deletions(-) diff --git a/relay-sampling/src/config.rs b/relay-sampling/src/config.rs index c980502ef1..3a2380f0c3 100644 --- a/relay-sampling/src/config.rs +++ b/relay-sampling/src/config.rs @@ -4,7 +4,6 @@ use std::fmt; use std::sync::Arc; use chrono::{DateTime, Utc}; - use serde::{Deserialize, Serialize}; use crate::condition::RuleCondition; @@ -134,7 +133,7 @@ impl SamplingRule { match self.sampling_value { SamplingValue::SampleRate { .. } => Some(SamplingValue::SampleRate { value }), SamplingValue::Factor { .. } => Some(SamplingValue::Factor { value }), - x => Some(x), + _ => unreachable!(), } } } @@ -292,13 +291,18 @@ impl Default for SamplingMode { } } -/* #[cfg(test)] mod tests { use chrono::TimeZone; + use crate::evaluation::ReservoirCounters; + use super::*; + fn dummy_reservoir() -> Arc { + ReservoirStuff::new(0, ReservoirCounters::default(), None).into() + } + #[test] fn config_deserialize() { let json = include_str!("../tests/fixtures/sampling_config.json"); @@ -535,19 +539,19 @@ mod tests { // At the start of the time range, sample rate is equal to the rule's initial sampling value. assert_eq!( - rule.sample_rate(start).unwrap(), + rule.sample_rate(start, dummy_reservoir()).unwrap(), SamplingValue::SampleRate { value: 1.0 } ); // Halfway in the time range, the value is exactly between 1.0 and 0.5. assert_eq!( - rule.sample_rate(halfway).unwrap(), + rule.sample_rate(halfway, dummy_reservoir()).unwrap(), SamplingValue::SampleRate { value: 0.75 } ); // Approaches 0.5 at the end. assert_eq!( - rule.sample_rate(end).unwrap(), + rule.sample_rate(end, dummy_reservoir()).unwrap(), SamplingValue::SampleRate { // It won't go to exactly 0.5 because the time range is end-exclusive. value: 0.5000028935185186 @@ -561,7 +565,9 @@ mod tests { rule }; - assert!(rule_without_start.sample_rate(halfway).is_none()); + assert!(rule_without_start + .sample_rate(halfway, dummy_reservoir()) + .is_none()); let rule_without_end = { let mut rule = rule.clone(); @@ -569,7 +575,9 @@ mod tests { rule }; - assert!(rule_without_end.sample_rate(halfway).is_none()); + assert!(rule_without_end + .sample_rate(halfway, dummy_reservoir()) + .is_none()); } /// If the decayingfunction is set to `Constant` then it shouldn't adjust the sample rate. @@ -591,7 +599,10 @@ mod tests { let halfway = Utc.with_ymd_and_hms(1970, 10, 11, 0, 0, 0).unwrap(); - assert_eq!(rule.sample_rate(halfway), Some(sampling_value)); + assert_eq!( + rule.sample_rate(halfway, dummy_reservoir()), + Some(sampling_value) + ); } /// Validates the `sample_rate` method for different time range configurations. @@ -617,30 +628,54 @@ mod tests { time_range, decaying_fn: DecayingFunction::Constant, }; - assert!(rule.sample_rate(before_time_range).is_none()); - assert!(rule.sample_rate(during_time_range).is_some()); - assert!(rule.sample_rate(after_time_range).is_none()); + assert!(rule + .sample_rate(before_time_range, dummy_reservoir()) + .is_none()); + assert!(rule + .sample_rate(during_time_range, dummy_reservoir()) + .is_some()); + assert!(rule + .sample_rate(after_time_range, dummy_reservoir()) + .is_none()); // [start..] let mut rule_without_end = rule.clone(); rule_without_end.time_range.end = None; - assert!(rule_without_end.sample_rate(before_time_range).is_none()); - assert!(rule_without_end.sample_rate(during_time_range).is_some()); - assert!(rule_without_end.sample_rate(after_time_range).is_some()); + assert!(rule_without_end + .sample_rate(before_time_range, dummy_reservoir()) + .is_none()); + assert!(rule_without_end + .sample_rate(during_time_range, dummy_reservoir()) + .is_some()); + assert!(rule_without_end + .sample_rate(after_time_range, dummy_reservoir()) + .is_some()); // [..end] let mut rule_without_start = rule.clone(); rule_without_start.time_range.start = None; - assert!(rule_without_start.sample_rate(before_time_range).is_some()); - assert!(rule_without_start.sample_rate(during_time_range).is_some()); - assert!(rule_without_start.sample_rate(after_time_range).is_none()); + assert!(rule_without_start + .sample_rate(before_time_range, dummy_reservoir()) + .is_some()); + assert!(rule_without_start + .sample_rate(during_time_range, dummy_reservoir()) + .is_some()); + assert!(rule_without_start + .sample_rate(after_time_range, dummy_reservoir()) + .is_none()); // [..] let mut rule_without_range = rule.clone(); rule_without_range.time_range = TimeRange::default(); - assert!(rule_without_range.sample_rate(before_time_range).is_some()); - assert!(rule_without_range.sample_rate(during_time_range).is_some()); - assert!(rule_without_range.sample_rate(after_time_range).is_some()); + assert!(rule_without_range + .sample_rate(before_time_range, dummy_reservoir()) + .is_some()); + assert!(rule_without_range + .sample_rate(during_time_range, dummy_reservoir()) + .is_some()); + assert!(rule_without_range + .sample_rate(after_time_range, dummy_reservoir()) + .is_some()); } /// You can pass in a SamplingValue of either variant, and it should return the same one if @@ -659,15 +694,14 @@ mod tests { }; matches!( - rule.sample_rate(Utc::now()).unwrap(), + rule.sample_rate(Utc::now(), dummy_reservoir()).unwrap(), SamplingValue::SampleRate { .. } ); rule.sampling_value = SamplingValue::Factor { value: 0.42 }; matches!( - rule.sample_rate(Utc::now()).unwrap(), + rule.sample_rate(Utc::now(), dummy_reservoir()).unwrap(), SamplingValue::Factor { .. } ); } } -*/ diff --git a/relay-sampling/src/evaluation.rs b/relay-sampling/src/evaluation.rs index 952ceb5cea..195e1aed97 100644 --- a/relay-sampling/src/evaluation.rs +++ b/relay-sampling/src/evaluation.rs @@ -260,7 +260,6 @@ pub struct SamplingMatch { impl SamplingMatch { fn new(sample_rate: f64, seed: Uuid, matched_rules: Vec) -> Self { let matched_rules = MatchedRuleIds(matched_rules); - let should_keep = sampling_match(sample_rate, seed); Self { @@ -330,6 +329,7 @@ impl fmt::Display for MatchedRuleIds { Ok(()) } } + #[cfg(test)] mod tests { use std::str::FromStr; diff --git a/relay-server/src/actors/processor.rs b/relay-server/src/actors/processor.rs index 7e86659373..e21d7866b1 100644 --- a/relay-server/src/actors/processor.rs +++ b/relay-server/src/actors/processor.rs @@ -2963,3 +2963,1317 @@ impl Service for EnvelopeProcessorService { }); } } + +#[cfg(test)] +mod tests { + use std::env; + use std::str::FromStr; + + use chrono::{DateTime, TimeZone, Utc}; + use relay_base_schema::metrics::{DurationUnit, MetricUnit}; + use relay_common::glob2::LazyGlob; + use relay_event_normalization::{MeasurementsConfig, RedactionRule, TransactionNameRule}; + use relay_event_schema::protocol::{EventId, TransactionSource}; + use relay_pii::DataScrubbingConfig; + use relay_sampling::condition::RuleCondition; + use relay_sampling::config::{ + DecayingFunction, RuleId, RuleType, SamplingConfig, SamplingMode, SamplingRule, + SamplingValue, TimeRange, + }; + use relay_sampling::evaluation::SamplingMatch; + use relay_test::mock_service; + use similar_asserts::assert_eq; + use uuid::Uuid; + + use crate::actors::test_store::TestStore; + use crate::extractors::RequestMeta; + use crate::metrics_extraction::transactions::types::{ + CommonTags, TransactionMeasurementTags, TransactionMetric, + }; + use crate::metrics_extraction::IntoMetric; + + use crate::testutils::{new_envelope, state_with_rule_and_condition}; + use crate::utils::Semaphore as TestSemaphore; + + use super::*; + + struct TestProcessSessionArguments<'a> { + item: Item, + received: DateTime, + client: Option<&'a str>, + client_addr: Option, + metrics_config: SessionMetricsConfig, + clock_drift_processor: ClockDriftProcessor, + extracted_metrics: Vec, + } + + impl<'a> TestProcessSessionArguments<'a> { + fn run_session_producer(&mut self) -> bool { + let proc = create_test_processor(Default::default()); + proc.process_session( + &mut self.item, + self.received, + self.client, + self.client_addr, + self.metrics_config, + &self.clock_drift_processor, + &mut self.extracted_metrics, + ) + } + + fn default() -> Self { + let mut item = Item::new(ItemType::Event); + + let session = r#"{ + "init": false, + "started": "2021-04-26T08:00:00+0100", + "timestamp": "2021-04-26T08:00:00+0100", + "attrs": { + "release": "1.0.0" + }, + "did": "user123", + "status": "this is not a valid status!", + "duration": 123.4 + }"#; + + item.set_payload(ContentType::Json, session); + let received = DateTime::from_str("2021-04-26T08:00:00+0100").unwrap(); + + Self { + item, + received, + client: None, + client_addr: None, + metrics_config: serde_json::from_str( + " + { + \"version\": 0, + \"drop\": true + }", + ) + .unwrap(), + clock_drift_processor: ClockDriftProcessor::new(None, received), + extracted_metrics: vec![], + } + } + } + + fn dummy_reservoir() -> Arc { + ReservoirStuff::new(0, ReservoirCounters::default(), None).into() + } + + fn mocked_event(event_type: EventType, transaction: &str, release: &str) -> Event { + Event { + id: Annotated::new(EventId::new()), + ty: Annotated::new(event_type), + transaction: Annotated::new(transaction.to_string()), + release: Annotated::new(LenientString(release.to_string())), + ..Event::default() + } + } + + /// Checks that the default test-arguments leads to the item being kept, which helps ensure the + /// other tests are valid. + #[tokio::test] + async fn test_process_session_keep_item() { + let mut args = TestProcessSessionArguments::default(); + assert!(args.run_session_producer()); + } + + #[tokio::test] + async fn test_process_session_invalid_json() { + let mut args = TestProcessSessionArguments::default(); + args.item + .set_payload(ContentType::Json, "this isnt valid json"); + assert!(!args.run_session_producer()); + } + + #[tokio::test] + async fn test_process_session_sequence_overflow() { + let mut args = TestProcessSessionArguments::default(); + args.item.set_payload( + ContentType::Json, + r#"{ + "init": false, + "started": "2021-04-26T08:00:00+0100", + "timestamp": "2021-04-26T08:00:00+0100", + "seq": 18446744073709551615, + "attrs": { + "release": "1.0.0" + }, + "did": "user123", + "status": "this is not a valid status!", + "duration": 123.4 + }"#, + ); + assert!(!args.run_session_producer()); + } + + #[tokio::test] + async fn test_process_session_invalid_timestamp() { + let mut args = TestProcessSessionArguments::default(); + args.received = DateTime::from_str("2021-05-26T08:00:00+0100").unwrap(); + assert!(!args.run_session_producer()); + } + + #[tokio::test] + async fn test_process_session_metrics_extracted() { + let mut args = TestProcessSessionArguments::default(); + args.item.set_metrics_extracted(true); + assert!(!args.run_session_producer()); + } + + fn create_breadcrumbs_item(breadcrumbs: &[(Option>, &str)]) -> Item { + let mut data = Vec::new(); + + for (date, message) in breadcrumbs { + let mut breadcrumb = BTreeMap::new(); + breadcrumb.insert("message", (*message).to_string()); + if let Some(date) = date { + breadcrumb.insert("timestamp", date.to_rfc3339()); + } + + rmp_serde::encode::write(&mut data, &breadcrumb).expect("write msgpack"); + } + + let mut item = Item::new(ItemType::Attachment); + item.set_payload(ContentType::MsgPack, data); + item + } + + fn breadcrumbs_from_event(event: &Annotated) -> &Vec> { + event + .value() + .unwrap() + .breadcrumbs + .value() + .unwrap() + .values + .value() + .unwrap() + } + + fn services() -> (Addr, Addr) { + let (outcome_aggregator, _) = mock_service("outcome_aggregator", (), |&mut (), _| {}); + let (test_store, _) = mock_service("test_store", (), |&mut (), _| {}); + (outcome_aggregator, test_store) + } + + #[tokio::test] + async fn test_dsc_respects_metrics_extracted() { + relay_test::setup(); + let (outcome_aggregator, test_store) = services(); + + let config = Config::from_json_value(serde_json::json!({ + "processing": { + "enabled": true, + "kafka_config": [], + } + })) + .unwrap(); + + let service: EnvelopeProcessorService = create_test_processor(config); + + // Gets a ProcessEnvelopeState, either with or without the metrics_exracted flag toggled. + let get_state = |version: Option| { + let event = Event { + id: Annotated::new(EventId::new()), + ty: Annotated::new(EventType::Transaction), + transaction: Annotated::new("testing".to_owned()), + ..Event::default() + }; + + let mut project_state = state_with_rule_and_condition( + Some(0.0), + RuleType::Transaction, + RuleCondition::all(), + ); + + if let Some(version) = version { + project_state.config.transaction_metrics = + ErrorBoundary::Ok(relay_dynamic_config::TransactionMetricsConfig { + version, + ..Default::default() + }) + .into(); + } + + ProcessEnvelopeState { + event: Annotated::from(event), + metrics: Default::default(), + sample_rates: None, + sampling_result: SamplingResult::Pending, + extracted_metrics: Default::default(), + project_state: Arc::new(project_state), + sampling_project_state: None, + project_id: ProjectId::new(42), + managed_envelope: ManagedEnvelope::new( + new_envelope(false, "foo"), + TestSemaphore::new(42).try_acquire().unwrap(), + outcome_aggregator.clone(), + test_store.clone(), + ), + has_profile: false, + event_metrics_extracted: false, + reservoir: dummy_reservoir(), + } + }; + + // None represents no TransactionMetricsConfig, DS will not be run + let mut state = get_state(None); + service.run_dynamic_sampling(&mut state); + assert!(state.sampling_result.should_keep()); + + // Current version is 1, so it won't run DS if it's outdated + let mut state = get_state(Some(0)); + service.run_dynamic_sampling(&mut state); + assert!(state.sampling_result.should_keep()); + + // Dynamic sampling is run, as the transactionmetrics version is up to date. + let mut state = get_state(Some(1)); + service.run_dynamic_sampling(&mut state); + assert!(state.sampling_result.should_drop()); + } + + #[test] + fn test_it_keeps_or_drops_transactions() { + let event = Event { + id: Annotated::new(EventId::new()), + ty: Annotated::new(EventType::Transaction), + transaction: Annotated::new("testing".to_owned()), + ..Event::default() + }; + + for (sample_rate, should_keep) in [(0.0, false), (1.0, true)] { + let sampling_config = SamplingConfig { + rules: vec![], + rules_v2: vec![SamplingRule { + condition: RuleCondition::all(), + sampling_value: SamplingValue::SampleRate { value: sample_rate }, + ty: RuleType::Transaction, + id: RuleId(1), + time_range: Default::default(), + decaying_fn: DecayingFunction::Constant, + }], + mode: SamplingMode::Received, + }; + + // TODO: This does not test if the sampling decision is actually applied. This should be + // refactored to send a proper Envelope in and call process_state to cover the full + // pipeline. + let res = EnvelopeProcessorService::compute_sampling_decision( + false, + dummy_reservoir(), + Some(&sampling_config), + Some(&event), + None, + None, + ); + assert_eq!(res.should_keep(), should_keep); + } + } + + #[test] + fn test_breadcrumbs_file1() { + let item = create_breadcrumbs_item(&[(None, "item1")]); + + // NOTE: using (Some, None) here: + let result = EnvelopeProcessorService::event_from_attachments( + &Config::default(), + None, + Some(item), + None, + ); + + let event = result.unwrap().0; + let breadcrumbs = breadcrumbs_from_event(&event); + + assert_eq!(breadcrumbs.len(), 1); + let first_breadcrumb_message = breadcrumbs[0].value().unwrap().message.value().unwrap(); + assert_eq!("item1", first_breadcrumb_message); + } + + #[test] + fn test_breadcrumbs_file2() { + let item = create_breadcrumbs_item(&[(None, "item2")]); + + // NOTE: using (None, Some) here: + let result = EnvelopeProcessorService::event_from_attachments( + &Config::default(), + None, + None, + Some(item), + ); + + let event = result.unwrap().0; + let breadcrumbs = breadcrumbs_from_event(&event); + assert_eq!(breadcrumbs.len(), 1); + + let first_breadcrumb_message = breadcrumbs[0].value().unwrap().message.value().unwrap(); + assert_eq!("item2", first_breadcrumb_message); + } + + #[test] + fn test_breadcrumbs_truncation() { + let item1 = create_breadcrumbs_item(&[(None, "crumb1")]); + let item2 = create_breadcrumbs_item(&[(None, "crumb2"), (None, "crumb3")]); + + let result = EnvelopeProcessorService::event_from_attachments( + &Config::default(), + None, + Some(item1), + Some(item2), + ); + + let event = result.unwrap().0; + let breadcrumbs = breadcrumbs_from_event(&event); + assert_eq!(breadcrumbs.len(), 2); + } + + #[test] + fn test_breadcrumbs_order_with_none() { + let d1 = Utc.with_ymd_and_hms(2019, 10, 10, 12, 10, 10).unwrap(); + let d2 = Utc.with_ymd_and_hms(2019, 10, 11, 12, 10, 10).unwrap(); + + let item1 = create_breadcrumbs_item(&[(None, "none"), (Some(d1), "d1")]); + let item2 = create_breadcrumbs_item(&[(Some(d2), "d2")]); + + let result = EnvelopeProcessorService::event_from_attachments( + &Config::default(), + None, + Some(item1), + Some(item2), + ); + + let event = result.unwrap().0; + let breadcrumbs = breadcrumbs_from_event(&event); + assert_eq!(breadcrumbs.len(), 2); + + assert_eq!(Some("d1"), breadcrumbs[0].value().unwrap().message.as_str()); + assert_eq!(Some("d2"), breadcrumbs[1].value().unwrap().message.as_str()); + } + + #[test] + fn test_breadcrumbs_reversed_with_none() { + let d1 = Utc.with_ymd_and_hms(2019, 10, 10, 12, 10, 10).unwrap(); + let d2 = Utc.with_ymd_and_hms(2019, 10, 11, 12, 10, 10).unwrap(); + + let item1 = create_breadcrumbs_item(&[(Some(d2), "d2")]); + let item2 = create_breadcrumbs_item(&[(None, "none"), (Some(d1), "d1")]); + + let result = EnvelopeProcessorService::event_from_attachments( + &Config::default(), + None, + Some(item1), + Some(item2), + ); + + let event = result.unwrap().0; + let breadcrumbs = breadcrumbs_from_event(&event); + assert_eq!(breadcrumbs.len(), 2); + + assert_eq!(Some("d1"), breadcrumbs[0].value().unwrap().message.as_str()); + assert_eq!(Some("d2"), breadcrumbs[1].value().unwrap().message.as_str()); + } + + #[test] + fn test_empty_breadcrumbs_item() { + let item1 = create_breadcrumbs_item(&[]); + let item2 = create_breadcrumbs_item(&[]); + let item3 = create_breadcrumbs_item(&[]); + + let result = EnvelopeProcessorService::event_from_attachments( + &Config::default(), + Some(item1), + Some(item2), + Some(item3), + ); + + // regression test to ensure we don't fail parsing an empty file + result.expect("event_from_attachments"); + } + + fn create_test_processor(config: Config) -> EnvelopeProcessorService { + let (envelope_manager, _) = mock_service("envelope_manager", (), |&mut (), _| {}); + let (outcome_aggregator, _) = mock_service("outcome_aggregator", (), |&mut (), _| {}); + let (project_cache, _) = mock_service("project_cache", (), |&mut (), _| {}); + let (upstream_relay, _) = mock_service("upstream_relay", (), |&mut (), _| {}); + let (global_config, _) = mock_service("global_config", (), |&mut (), _| {}); + let inner = InnerProcessor { + config: Arc::new(config), + envelope_manager, + project_cache, + outcome_aggregator, + upstream_relay, + #[cfg(feature = "processing")] + rate_limiter: None, + #[cfg(feature = "processing")] + redis_pool: None, + geoip_lookup: None, + global_config, + }; + + EnvelopeProcessorService { + global_config: Arc::default(), + inner: Arc::new(inner), + } + } + + #[tokio::test] + async fn test_user_report_invalid() { + let processor = create_test_processor(Default::default()); + let (outcome_aggregator, test_store) = services(); + let event_id = EventId::new(); + + let dsn = "https://e12d836b15bb49d7bbf99e64295d995b:@sentry.io/42" + .parse() + .unwrap(); + + let request_meta = RequestMeta::new(dsn); + let mut envelope = Envelope::from_request(Some(event_id), request_meta); + + envelope.add_item({ + let mut item = Item::new(ItemType::UserReport); + item.set_payload(ContentType::Json, r#"{"foo": "bar"}"#); + item + }); + + envelope.add_item({ + let mut item = Item::new(ItemType::Event); + item.set_payload(ContentType::Json, "{}"); + item + }); + + let message = ProcessEnvelope { + envelope: ManagedEnvelope::standalone(envelope, outcome_aggregator, test_store), + project_state: Arc::new(ProjectState::allowed()), + sampling_project_state: None, + reservoir_counters: ReservoirCounters::default(), + }; + + let envelope_response = processor.process(message).unwrap(); + let ctx = envelope_response.envelope.unwrap(); + let new_envelope = ctx.envelope(); + + assert_eq!(new_envelope.len(), 1); + assert_eq!(new_envelope.items().next().unwrap().ty(), &ItemType::Event); + } + + fn process_envelope_with_root_project_state( + envelope: Box, + sampling_project_state: Option>, + ) -> Envelope { + let processor = create_test_processor(Default::default()); + let (outcome_aggregator, test_store) = services(); + + let message = ProcessEnvelope { + envelope: ManagedEnvelope::standalone(envelope, outcome_aggregator, test_store), + project_state: Arc::new(ProjectState::allowed()), + sampling_project_state, + reservoir_counters: ReservoirCounters::default(), + }; + + let envelope_response = processor.process(message).unwrap(); + let ctx = envelope_response.envelope.unwrap(); + ctx.envelope().clone() + } + + fn extract_first_event_from_envelope(envelope: Envelope) -> Event { + let item = envelope.items().next().unwrap(); + let annotated_event: Annotated = + Annotated::from_json_bytes(&item.payload()).unwrap(); + annotated_event.into_value().unwrap() + } + + fn mocked_error_item() -> Item { + let mut item = Item::new(ItemType::Event); + item.set_payload( + ContentType::Json, + r#"{ + "event_id": "52df9022835246eeb317dbd739ccd059", + "exception": { + "values": [ + { + "type": "mytype", + "value": "myvalue", + "module": "mymodule", + "thread_id": 42, + "other": "value" + } + ] + } + }"#, + ); + item + } + + fn project_state_with_single_rule(sample_rate: f64) -> ProjectState { + let sampling_config = SamplingConfig { + rules: vec![], + rules_v2: vec![SamplingRule { + condition: RuleCondition::all(), + sampling_value: SamplingValue::SampleRate { value: sample_rate }, + ty: RuleType::Trace, + id: RuleId(1), + time_range: Default::default(), + decaying_fn: Default::default(), + }], + mode: SamplingMode::Received, + }; + let mut sampling_project_state = ProjectState::allowed(); + sampling_project_state.config.dynamic_sampling = Some(sampling_config); + sampling_project_state + } + + #[tokio::test] + async fn test_error_is_tagged_correctly_if_trace_sampling_result_is_some() { + let event_id = EventId::new(); + let dsn = "https://e12d836b15bb49d7bbf99e64295d995b:@sentry.io/42" + .parse() + .unwrap(); + let request_meta = RequestMeta::new(dsn); + let mut envelope = Envelope::from_request(Some(event_id), request_meta); + let dsc = DynamicSamplingContext { + trace_id: Uuid::new_v4(), + public_key: ProjectKey::parse("abd0f232775f45feab79864e580d160b").unwrap(), + release: Some("1.1.1".to_string()), + user: Default::default(), + replay_id: None, + environment: None, + transaction: Some("transaction1".into()), + sample_rate: None, + sampled: Some(true), + other: BTreeMap::new(), + }; + envelope.set_dsc(dsc); + envelope.add_item(mocked_error_item()); + + // We test with sample rate equal to 100%. + let sampling_project_state = project_state_with_single_rule(1.0); + let new_envelope = process_envelope_with_root_project_state( + envelope.clone(), + Some(Arc::new(sampling_project_state)), + ); + let event = extract_first_event_from_envelope(new_envelope); + let trace_context = event.context::().unwrap(); + assert!(trace_context.sampled.value().unwrap()); + + // We test with sample rate equal to 0%. + let sampling_project_state = project_state_with_single_rule(0.0); + let new_envelope = process_envelope_with_root_project_state( + envelope, + Some(Arc::new(sampling_project_state)), + ); + let event = extract_first_event_from_envelope(new_envelope); + let trace_context = event.context::().unwrap(); + assert!(!trace_context.sampled.value().unwrap()); + } + + #[tokio::test] + async fn test_error_is_not_tagged_if_already_tagged() { + let event_id = EventId::new(); + let dsn = "https://e12d836b15bb49d7bbf99e64295d995b:@sentry.io/42" + .parse() + .unwrap(); + let request_meta = RequestMeta::new(dsn); + + // We test tagging with an incoming event that has already been tagged by downstream Relay. + let mut envelope = Envelope::from_request(Some(event_id), request_meta); + let mut item = Item::new(ItemType::Event); + item.set_payload( + ContentType::Json, + r#"{ + "event_id": "52df9022835246eeb317dbd739ccd059", + "exception": { + "values": [ + { + "type": "mytype", + "value": "myvalue", + "module": "mymodule", + "thread_id": 42, + "other": "value" + } + ] + }, + "contexts": { + "trace": { + "sampled": true + } + } + }"#, + ); + envelope.add_item(item); + let sampling_project_state = project_state_with_single_rule(0.0); + let new_envelope = process_envelope_with_root_project_state( + envelope, + Some(Arc::new(sampling_project_state)), + ); + let event = extract_first_event_from_envelope(new_envelope); + let trace_context = event.context::().unwrap(); + assert!(trace_context.sampled.value().unwrap()); + } + + #[tokio::test] + async fn test_error_is_tagged_correctly_if_trace_sampling_result_is_none() { + let event_id = EventId::new(); + let dsn = "https://e12d836b15bb49d7bbf99e64295d995b:@sentry.io/42" + .parse() + .unwrap(); + let request_meta = RequestMeta::new(dsn); + + // We test tagging when root project state and dsc are none. + let mut envelope = Envelope::from_request(Some(event_id), request_meta); + envelope.add_item(mocked_error_item()); + let new_envelope = process_envelope_with_root_project_state(envelope, None); + let event = extract_first_event_from_envelope(new_envelope); + + assert!(event.contexts.value().is_none()); + } + + #[tokio::test] + async fn test_browser_version_extraction_with_pii_like_data() { + let processor = create_test_processor(Default::default()); + let (outcome_aggregator, test_store) = services(); + let event_id = EventId::new(); + + let dsn = "https://e12d836b15bb49d7bbf99e64295d995b:@sentry.io/42" + .parse() + .unwrap(); + + let request_meta = RequestMeta::new(dsn); + let mut envelope = Envelope::from_request(Some(event_id), request_meta); + + envelope.add_item({ + let mut item = Item::new(ItemType::Event); + item.set_payload( + ContentType::Json, + r#" + { + "request": { + "headers": [ + ["User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36"] + ] + } + } + "#, + ); + item + }); + + let mut datascrubbing_settings = DataScrubbingConfig::default(); + // enable all the default scrubbing + datascrubbing_settings.scrub_data = true; + datascrubbing_settings.scrub_defaults = true; + datascrubbing_settings.scrub_ip_addresses = true; + + // Make sure to mask any IP-like looking data + let pii_config = serde_json::from_str(r#"{"applications": {"**": ["@ip:mask"]}}"#).unwrap(); + + let config = ProjectConfig { + datascrubbing_settings, + pii_config: Some(pii_config), + ..Default::default() + }; + + let mut project_state = ProjectState::allowed(); + project_state.config = config; + let message = ProcessEnvelope { + envelope: ManagedEnvelope::standalone(envelope, outcome_aggregator, test_store), + project_state: Arc::new(project_state), + sampling_project_state: None, + reservoir_counters: ReservoirCounters::default(), + }; + + let envelope_response = processor.process(message).unwrap(); + let new_envelope = envelope_response.envelope.unwrap(); + let new_envelope = new_envelope.envelope(); + + let event_item = new_envelope.items().last().unwrap(); + let annotated_event: Annotated = + Annotated::from_json_bytes(&event_item.payload()).unwrap(); + let event = annotated_event.into_value().unwrap(); + let headers = event + .request + .into_value() + .unwrap() + .headers + .into_value() + .unwrap(); + + // IP-like data must be masked + assert_eq!(Some("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/********* Safari/537.36"), headers.get_header("User-Agent")); + // But we still get correct browser and version number + let contexts = event.contexts.into_value().unwrap(); + let browser = contexts.0.get("browser").unwrap(); + assert_eq!( + r#"{"name":"Chrome","version":"103.0.0","type":"browser"}"#, + browser.to_json().unwrap() + ); + } + + #[tokio::test] + async fn test_client_report_removal() { + relay_test::setup(); + let (outcome_aggregator, test_store) = services(); + + let config = Config::from_json_value(serde_json::json!({ + "outcomes": { + "emit_outcomes": true, + "emit_client_outcomes": true + } + })) + .unwrap(); + + let processor = create_test_processor(config); + + let dsn = "https://e12d836b15bb49d7bbf99e64295d995b:@sentry.io/42" + .parse() + .unwrap(); + + let request_meta = RequestMeta::new(dsn); + let mut envelope = Envelope::from_request(None, request_meta); + + envelope.add_item({ + let mut item = Item::new(ItemType::ClientReport); + item.set_payload( + ContentType::Json, + r#" + { + "discarded_events": [ + ["queue_full", "error", 42] + ] + } + "#, + ); + item + }); + + let message = ProcessEnvelope { + envelope: ManagedEnvelope::standalone(envelope, outcome_aggregator, test_store), + project_state: Arc::new(ProjectState::allowed()), + sampling_project_state: None, + reservoir_counters: ReservoirCounters::default(), + }; + + let envelope_response = processor.process(message).unwrap(); + assert!(envelope_response.envelope.is_none()); + } + + #[tokio::test] + async fn test_client_report_forwarding() { + relay_test::setup(); + let (outcome_aggregator, test_store) = services(); + + let config = Config::from_json_value(serde_json::json!({ + "outcomes": { + "emit_outcomes": false, + // a relay need to emit outcomes at all to not process. + "emit_client_outcomes": true + } + })) + .unwrap(); + + let processor = create_test_processor(config); + + let dsn = "https://e12d836b15bb49d7bbf99e64295d995b:@sentry.io/42" + .parse() + .unwrap(); + + let request_meta = RequestMeta::new(dsn); + let mut envelope = Envelope::from_request(None, request_meta); + + envelope.add_item({ + let mut item = Item::new(ItemType::ClientReport); + item.set_payload( + ContentType::Json, + r#" + { + "discarded_events": [ + ["queue_full", "error", 42] + ] + } + "#, + ); + item + }); + + let message = ProcessEnvelope { + envelope: ManagedEnvelope::standalone(envelope, outcome_aggregator, test_store), + project_state: Arc::new(ProjectState::allowed()), + sampling_project_state: None, + reservoir_counters: ReservoirCounters::default(), + }; + + let envelope_response = processor.process(message).unwrap(); + let ctx = envelope_response.envelope.unwrap(); + let item = ctx.envelope().items().next().unwrap(); + assert_eq!(item.ty(), &ItemType::ClientReport); + + ctx.accept(); // do not try to capture or emit outcomes + } + + #[tokio::test] + #[cfg(feature = "processing")] + async fn test_client_report_removal_in_processing() { + relay_test::setup(); + let (outcome_aggregator, test_store) = services(); + + let config = Config::from_json_value(serde_json::json!({ + "outcomes": { + "emit_outcomes": true, + "emit_client_outcomes": false, + }, + "processing": { + "enabled": true, + "kafka_config": [], + } + })) + .unwrap(); + + let processor = create_test_processor(config); + + let dsn = "https://e12d836b15bb49d7bbf99e64295d995b:@sentry.io/42" + .parse() + .unwrap(); + + let request_meta = RequestMeta::new(dsn); + let mut envelope = Envelope::from_request(None, request_meta); + + envelope.add_item({ + let mut item = Item::new(ItemType::ClientReport); + item.set_payload( + ContentType::Json, + r#" + { + "discarded_events": [ + ["queue_full", "error", 42] + ] + } + "#, + ); + item + }); + + let message = ProcessEnvelope { + envelope: ManagedEnvelope::standalone(envelope, outcome_aggregator, test_store), + project_state: Arc::new(ProjectState::allowed()), + sampling_project_state: None, + reservoir_counters: ReservoirCounters::default(), + }; + + let envelope_response = processor.process(message).unwrap(); + assert!(envelope_response.envelope.is_none()); + } + + #[test] + #[cfg(feature = "processing")] + fn test_unprintable_fields() { + let event = Annotated::new(Event { + environment: Annotated::new(String::from( + "�9�~YY���)�����9�~YY���)�����9�~YY���)�����9�~YY���)�����", + )), + ..Default::default() + }); + assert!(has_unprintable_fields(&event)); + + let event = Annotated::new(Event { + release: Annotated::new( + String::from("���7��#1G����7��#1G����7��#1G����7��#1G����7��#").into(), + ), + ..Default::default() + }); + assert!(has_unprintable_fields(&event)); + + let event = Annotated::new(Event { + environment: Annotated::new(String::from("production")), + ..Default::default() + }); + assert!(!has_unprintable_fields(&event)); + + let event = Annotated::new(Event { + release: Annotated::new( + String::from("release with\t some\n normal\r\nwhitespace").into(), + ), + ..Default::default() + }); + assert!(!has_unprintable_fields(&event)); + } + + #[test] + fn test_from_outcome_type_sampled() { + assert!(outcome_from_parts(ClientReportField::FilteredSampling, "adsf").is_err()); + + assert!(outcome_from_parts(ClientReportField::FilteredSampling, "Sampled:").is_err()); + + assert!(outcome_from_parts(ClientReportField::FilteredSampling, "Sampled:foo").is_err()); + + assert!(matches!( + outcome_from_parts(ClientReportField::FilteredSampling, "Sampled:"), + Err(()) + )); + + assert!(matches!( + outcome_from_parts(ClientReportField::FilteredSampling, "Sampled:;"), + Err(()) + )); + + assert!(matches!( + outcome_from_parts(ClientReportField::FilteredSampling, "Sampled:ab;12"), + Err(()) + )); + + assert_eq!( + outcome_from_parts(ClientReportField::FilteredSampling, "Sampled:123,456"), + Ok(Outcome::FilteredSampling(MatchedRuleIds(vec![ + RuleId(123), + RuleId(456), + ]))) + ); + + assert_eq!( + outcome_from_parts(ClientReportField::FilteredSampling, "Sampled:123"), + Ok(Outcome::FilteredSampling(MatchedRuleIds(vec![RuleId(123)]))) + ); + } + + #[test] + fn test_from_outcome_type_filtered() { + assert!(matches!( + outcome_from_parts(ClientReportField::Filtered, "error-message"), + Ok(Outcome::Filtered(FilterStatKey::ErrorMessage)) + )); + assert!(outcome_from_parts(ClientReportField::Filtered, "adsf").is_err()); + } + + #[test] + fn test_from_outcome_type_client_discard() { + assert_eq!( + outcome_from_parts(ClientReportField::ClientDiscard, "foo_reason").unwrap(), + Outcome::ClientDiscard("foo_reason".into()) + ); + } + + #[test] + fn test_from_outcome_type_rate_limited() { + assert!(matches!( + outcome_from_parts(ClientReportField::RateLimited, ""), + Ok(Outcome::RateLimited(None)) + )); + assert_eq!( + outcome_from_parts(ClientReportField::RateLimited, "foo_reason").unwrap(), + Outcome::RateLimited(Some(ReasonCode::new("foo_reason"))) + ); + } + + fn capture_test_event(transaction_name: &str, source: TransactionSource) -> Vec { + let mut event = Annotated::::from_json( + r#" + { + "type": "transaction", + "transaction": "/foo/", + "timestamp": 946684810.0, + "start_timestamp": 946684800.0, + "contexts": { + "trace": { + "trace_id": "4c79f60c11214eb38604f4ae0781bfb2", + "span_id": "fa90fdead5f74053", + "op": "http.server", + "type": "trace" + } + }, + "transaction_info": { + "source": "url" + } + } + "#, + ) + .unwrap(); + let e = event.value_mut().as_mut().unwrap(); + e.transaction.set_value(Some(transaction_name.into())); + + e.transaction_info + .value_mut() + .as_mut() + .unwrap() + .source + .set_value(Some(source)); + + relay_statsd::with_capturing_test_client(|| { + utils::log_transaction_name_metrics(&mut event, |event| { + let config = LightNormalizationConfig { + transaction_name_config: TransactionNameConfig { + rules: &[TransactionNameRule { + pattern: LazyGlob::new("/foo/*/**".to_owned()), + expiry: DateTime::::MAX_UTC, + redaction: RedactionRule::Replace { + substitution: "*".to_owned(), + }, + }], + }, + ..Default::default() + }; + relay_event_normalization::light_normalize_event(event, config) + }) + .unwrap(); + }) + } + + #[test] + fn test_log_transaction_metrics_none() { + let captures = capture_test_event("/nothing", TransactionSource::Url); + insta::assert_debug_snapshot!(captures, @r#" + [ + "event.transaction_name_changes:1|c|#source_in:url,changes:none,source_out:sanitized,is_404:false", + ] + "#); + } + + #[test] + fn test_log_transaction_metrics_rule() { + let captures = capture_test_event("/foo/john/denver", TransactionSource::Url); + insta::assert_debug_snapshot!(captures, @r#" + [ + "event.transaction_name_changes:1|c|#source_in:url,changes:rule,source_out:sanitized,is_404:false", + ] + "#); + } + + #[test] + fn test_log_transaction_metrics_pattern() { + let captures = capture_test_event("/something/12345", TransactionSource::Url); + insta::assert_debug_snapshot!(captures, @r#" + [ + "event.transaction_name_changes:1|c|#source_in:url,changes:pattern,source_out:sanitized,is_404:false", + ] + "#); + } + + #[test] + fn test_log_transaction_metrics_both() { + let captures = capture_test_event("/foo/john/12345", TransactionSource::Url); + insta::assert_debug_snapshot!(captures, @r#" + [ + "event.transaction_name_changes:1|c|#source_in:url,changes:both,source_out:sanitized,is_404:false", + ] + "#); + } + + #[test] + fn test_log_transaction_metrics_no_match() { + let captures = capture_test_event("/foo/john/12345", TransactionSource::Route); + insta::assert_debug_snapshot!(captures, @r#" + [ + "event.transaction_name_changes:1|c|#source_in:route,changes:none,source_out:route,is_404:false", + ] + "#); + } + + /// This is a stand-in test to assert panicking behavior for spawn_blocking. + /// + /// [`EnvelopeProcessorService`] relies on tokio to restart the worker threads for blocking + /// tasks if there is a panic during processing. Tokio does not explicitly mention this behavior + /// in documentation, though the `spawn_blocking` contract suggests that this is intentional. + /// + /// This test should be moved if the worker pool is extracted into a utility. + #[test] + fn test_processor_panics() { + let future = async { + let semaphore = Arc::new(Semaphore::new(1)); + + // loop multiple times to prove that the runtime creates new threads + for _ in 0..3 { + // the previous permit should have been released during panic unwind + let permit = semaphore.clone().acquire_owned().await.unwrap(); + + let handle = tokio::task::spawn_blocking(move || { + let _permit = permit; // drop(permit) after panic!() would warn as "unreachable" + panic!("ignored"); + }); + + assert!(handle.await.is_err()); + } + }; + + tokio::runtime::Builder::new_current_thread() + .max_blocking_threads(1) + .build() + .unwrap() + .block_on(future); + } + + /// Confirms that the hardcoded value we use for the fixed length of the measurement MRI is + /// correct. Unit test is placed here because it has dependencies to relay-server and therefore + /// cannot be called from relay-metrics. + #[test] + fn test_mri_overhead_constant() { + let hardcoded_value = MeasurementsConfig::MEASUREMENT_MRI_OVERHEAD; + + let derived_value = { + let name = "foobar".to_string(); + let value = 5.0; // Arbitrary value. + let unit = MetricUnit::Duration(DurationUnit::default()); + let tags = TransactionMeasurementTags { + measurement_rating: None, + universal_tags: CommonTags(BTreeMap::new()), + }; + + let measurement = TransactionMetric::Measurement { + name: name.clone(), + value, + unit, + tags, + }; + + let metric: Bucket = measurement.into_metric(UnixTimestamp::now()); + metric.name.len() - unit.to_string().len() - name.len() + }; + assert_eq!( + hardcoded_value, derived_value, + "Update `MEASUREMENT_MRI_OVERHEAD` if the naming scheme changed." + ); + } + + // Helper to extract the sampling match from SamplingResult if thats the variant. + fn get_sampling_match(sampling_result: SamplingResult) -> SamplingMatch { + if let SamplingResult::Match(sampling_match) = sampling_result { + sampling_match + } else { + panic!() + } + } + + /// Happy path test for compute_sampling_decision. + #[test] + fn test_compute_sampling_decision_matching() { + let event = mocked_event(EventType::Transaction, "foo", "bar"); + let rule = SamplingRule { + condition: RuleCondition::all(), + sampling_value: SamplingValue::SampleRate { value: 1.0 }, + ty: RuleType::Transaction, + id: RuleId(0), + time_range: TimeRange::default(), + decaying_fn: Default::default(), + }; + + let sampling_config = SamplingConfig { + rules: vec![], + rules_v2: vec![rule], + mode: SamplingMode::Received, + }; + + let res = EnvelopeProcessorService::compute_sampling_decision( + false, + dummy_reservoir(), + Some(&sampling_config), + Some(&event), + None, + None, + ); + assert!(res.is_match()); + } + + #[test] + fn test_matching_with_unsupported_rule() { + let event = mocked_event(EventType::Transaction, "foo", "bar"); + let rule = SamplingRule { + condition: RuleCondition::all(), + sampling_value: SamplingValue::SampleRate { value: 1.0 }, + ty: RuleType::Transaction, + id: RuleId(0), + time_range: TimeRange::default(), + decaying_fn: Default::default(), + }; + + let unsupported_rule = SamplingRule { + condition: RuleCondition::all(), + sampling_value: SamplingValue::SampleRate { value: 1.0 }, + ty: RuleType::Unsupported, + id: RuleId(0), + time_range: TimeRange::default(), + decaying_fn: Default::default(), + }; + + let sampling_config = SamplingConfig { + rules: vec![], + rules_v2: vec![rule, unsupported_rule], + mode: SamplingMode::Received, + }; + + // Unsupported rule should result in no match if processing is not enabled. + let res = EnvelopeProcessorService::compute_sampling_decision( + false, + dummy_reservoir(), + Some(&sampling_config), + Some(&event), + None, + None, + ); + assert!(res.is_no_match()); + + // Match if processing is enabled. + let res = EnvelopeProcessorService::compute_sampling_decision( + true, + dummy_reservoir(), + Some(&sampling_config), + Some(&event), + None, + None, + ); + assert!(res.is_match()); + } + + #[test] + fn test_client_sample_rate() { + let dsc = DynamicSamplingContext { + trace_id: Uuid::new_v4(), + public_key: ProjectKey::parse("abd0f232775f45feab79864e580d160b").unwrap(), + release: Some("1.1.1".to_string()), + user: Default::default(), + replay_id: None, + environment: None, + transaction: Some("transaction1".into()), + sample_rate: Some(0.5), + sampled: Some(true), + other: BTreeMap::new(), + }; + + let rule = SamplingRule { + condition: RuleCondition::all(), + sampling_value: SamplingValue::SampleRate { value: 0.2 }, + ty: RuleType::Trace, + id: RuleId(0), + time_range: TimeRange::default(), + decaying_fn: Default::default(), + }; + + let mut sampling_config = SamplingConfig { + rules: vec![], + rules_v2: vec![rule], + mode: SamplingMode::Received, + }; + + let res = EnvelopeProcessorService::compute_sampling_decision( + false, + dummy_reservoir(), + None, + None, + Some(&sampling_config), + Some(&dsc), + ); + + assert_eq!(get_sampling_match(res).sample_rate(), 0.2); + + sampling_config.mode = SamplingMode::Total; + + let res = EnvelopeProcessorService::compute_sampling_decision( + false, + dummy_reservoir(), + None, + None, + Some(&sampling_config), + Some(&dsc), + ); + + assert_eq!(get_sampling_match(res).sample_rate(), 0.4); + } +} From 9c2700a4ebd022aeff1213f85c6533ab36eecb26 Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Wed, 27 Sep 2023 20:50:39 +0200 Subject: [PATCH 07/30] wip --- relay-sampling/src/evaluation.rs | 1 + relay-server/src/actors/project.rs | 13 +++---------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/relay-sampling/src/evaluation.rs b/relay-sampling/src/evaluation.rs index 195e1aed97..487700a60a 100644 --- a/relay-sampling/src/evaluation.rs +++ b/relay-sampling/src/evaluation.rs @@ -11,6 +11,7 @@ use rand::distributions::Uniform; use rand::Rng; use rand_pcg::Pcg32; use relay_protocol::Getter; +#[cfg(feature = "redis")] use relay_redis::{RedisError, RedisPool}; use serde::Serialize; use uuid::Uuid; diff --git a/relay-server/src/actors/project.rs b/relay-server/src/actors/project.rs index b7d0e24a66..3791aa6826 100644 --- a/relay-server/src/actors/project.rs +++ b/relay-server/src/actors/project.rs @@ -440,16 +440,9 @@ impl Project { return; }; - let reservoir_rules: BTreeSet = config - .rules_v2 - .iter() - .filter_map(|rule| rule.is_reservoir().then_some(rule.id)) - .collect(); - - self.reservoir_counters - .try_lock() - .unwrap() - .retain(|key, _| reservoir_rules.contains(key)); + if let Ok(mut guard) = self.reservoir_counters.try_lock() { + guard.retain(|key, _| config.rules_v2.iter().any(|rule| rule.id == *key)); + } } pub fn merge_rate_limits(&mut self, rate_limits: RateLimits) { From c86932892bcb091192022f3377f940e3b6e3762f Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Wed, 27 Sep 2023 21:16:27 +0200 Subject: [PATCH 08/30] wip --- Cargo.lock | 1 + relay-sampling/Cargo.toml | 1 + relay-sampling/src/config.rs | 4 +++- relay-sampling/src/evaluation.rs | 23 +++++++++++------------ relay-server/src/actors/processor.rs | 1 + relay-server/src/actors/project.rs | 2 -- relay-server/src/actors/project_cache.rs | 16 ---------------- 7 files changed, 17 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ed485ba96e..4a2611b74d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3822,6 +3822,7 @@ dependencies = [ name = "relay-sampling" version = "23.9.1" dependencies = [ + "anyhow", "chrono", "insta", "rand", diff --git a/relay-sampling/Cargo.toml b/relay-sampling/Cargo.toml index fe1c0523ed..2ac41296c3 100644 --- a/relay-sampling/Cargo.toml +++ b/relay-sampling/Cargo.toml @@ -14,6 +14,7 @@ default = [] redis = ["dep:relay-redis"] [dependencies] +anyhow = { workspace = true } chrono = { workspace = true } rand = { workspace = true } rand_pcg = "0.3.1" diff --git a/relay-sampling/src/config.rs b/relay-sampling/src/config.rs index 3a2380f0c3..d132cb9723 100644 --- a/relay-sampling/src/config.rs +++ b/relay-sampling/src/config.rs @@ -133,7 +133,9 @@ impl SamplingRule { match self.sampling_value { SamplingValue::SampleRate { .. } => Some(SamplingValue::SampleRate { value }), SamplingValue::Factor { .. } => Some(SamplingValue::Factor { value }), - _ => unreachable!(), + // This should be impossible. + // Todo(tor): refactor so we don't run into this invalid state. + _ => None, } } } diff --git a/relay-sampling/src/evaluation.rs b/relay-sampling/src/evaluation.rs index 487700a60a..8388831b9a 100644 --- a/relay-sampling/src/evaluation.rs +++ b/relay-sampling/src/evaluation.rs @@ -12,7 +12,7 @@ use rand::Rng; use rand_pcg::Pcg32; use relay_protocol::Getter; #[cfg(feature = "redis")] -use relay_redis::{RedisError, RedisPool}; +use relay_redis::RedisPool; use serde::Serialize; use uuid::Uuid; @@ -36,12 +36,11 @@ fn increment_bias_rule_count( redis: Arc, org_id: u64, rule_id: RuleId, -) -> Result { +) -> anyhow::Result { let key = format!("bias:{}:{}", org_id, rule_id); let mut command = relay_redis::redis::cmd("INCR"); command.arg(key.as_str()); - let new_count: i64 = command.query(&mut redis.client()?.connection()?).unwrap(); - Ok(new_count) + Ok(command.query(&mut redis.client()?.connection()?)?) } /// The amount of transactions sampled of a given rule id. @@ -79,6 +78,7 @@ impl ReservoirStuff { let incremented_value = self.increment(&mut map_guard, rule); + // If the incremented value is less than the limit, we still want to sample. incremented_value < limit } @@ -86,22 +86,21 @@ impl ReservoirStuff { // when incrementing, if processing is enabled, we increment redis and insert back the value // otherwise we just increment directly fn increment(&self, map_guard: &mut MutexGuard>, rule: RuleId) -> i64 { - let mut increment_value: i64 = 1; + let val = map_guard.entry(rule).or_insert(0); + + let mut new_val: i64 = *val + 1; #[cfg(feature = "redis")] { if let Some(pool) = self.redis_pool.as_ref() { - if let Ok(new_val_from_redis) = - increment_bias_rule_count(pool.clone(), self.org_id, rule) - { - increment_value = new_val_from_redis; + match increment_bias_rule_count(pool.clone(), self.org_id, rule) { + Ok(redis_val) => new_val = redis_val, + Err(e) => relay_log::error!("failed to increment redis reservoir count: {}", e), } } } - let val = map_guard.entry(rule).or_insert(0); - *val += increment_value; - + *val = new_val; *val } } diff --git a/relay-server/src/actors/processor.rs b/relay-server/src/actors/processor.rs index e21d7866b1..3e10cc4cdb 100644 --- a/relay-server/src/actors/processor.rs +++ b/relay-server/src/actors/processor.rs @@ -539,6 +539,7 @@ pub struct EnvelopeProcessorService { struct InnerProcessor { config: Arc, + #[cfg(feature = "processing")] redis_pool: Option>, envelope_manager: Addr, project_cache: Addr, diff --git a/relay-server/src/actors/project.rs b/relay-server/src/actors/project.rs index 3791aa6826..1a5f231363 100644 --- a/relay-server/src/actors/project.rs +++ b/relay-server/src/actors/project.rs @@ -1,4 +1,3 @@ -use std::collections::BTreeSet; use std::sync::Arc; use std::time::Duration; @@ -9,7 +8,6 @@ use relay_dynamic_config::{Feature, LimitedProjectConfig, ProjectConfig}; use relay_filter::matches_any_origin; use relay_metrics::{Aggregator, Bucket, MergeBuckets, MetricNamespace, MetricResourceIdentifier}; use relay_quotas::{Quota, RateLimits, Scoping}; -use relay_sampling::config::RuleId; use relay_sampling::evaluation::ReservoirCounters; use relay_statsd::metric; use relay_system::{Addr, BroadcastChannel}; diff --git a/relay-server/src/actors/project_cache.rs b/relay-server/src/actors/project_cache.rs index 50a9b79fc7..c2cba4cd0c 100644 --- a/relay-server/src/actors/project_cache.rs +++ b/relay-server/src/actors/project_cache.rs @@ -201,7 +201,6 @@ pub struct SpoolHealth; /// /// See the enumerated variants for a full list of available messages for this service. pub enum ProjectCache { - //UpdateReservoir(UpdateCount), RequestUpdate(RequestUpdate), Get(GetProjectState, ProjectSender), GetCached(GetCachedProjectState, Sender>>), @@ -219,21 +218,6 @@ pub enum ProjectCache { impl Interface for ProjectCache {} -/* -pub struct UpdateCount { - pub project_key: ProjectKey, - pub rule_id: RuleId, -} - -impl FromMessage for ProjectCache { - type Response = relay_system::NoResponse; - - fn from_message(message: UpdateCount, _: ()) -> Self { - Self::UpdateReservoir(message) - } -} -*/ - impl FromMessage for ProjectCache { type Response = relay_system::NoResponse; From 1b2e136798e6f6ab08c84d776e89df70211fc76b Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Wed, 27 Sep 2023 21:25:43 +0200 Subject: [PATCH 09/30] wip --- relay-sampling/src/config.rs | 8 +++++++- relay-sampling/src/evaluation.rs | 16 +++++++++++----- relay-server/src/actors/processor.rs | 8 +++++++- relay-server/src/utils/dynamic_sampling.rs | 8 +++++++- 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/relay-sampling/src/config.rs b/relay-sampling/src/config.rs index d132cb9723..7ce7dfc479 100644 --- a/relay-sampling/src/config.rs +++ b/relay-sampling/src/config.rs @@ -302,7 +302,13 @@ mod tests { use super::*; fn dummy_reservoir() -> Arc { - ReservoirStuff::new(0, ReservoirCounters::default(), None).into() + ReservoirStuff::new( + 0, + ReservoirCounters::default(), + #[cfg(feature = "redis")] + None, + ) + .into() } #[test] diff --git a/relay-sampling/src/evaluation.rs b/relay-sampling/src/evaluation.rs index 8388831b9a..416c9c2843 100644 --- a/relay-sampling/src/evaluation.rs +++ b/relay-sampling/src/evaluation.rs @@ -51,7 +51,7 @@ pub type ReservoirCounters = Arc>>; pub struct ReservoirStuff { #[cfg(feature = "redis")] redis_pool: Option>, - org_id: u64, + _org_id: u64, map: ReservoirCounters, } @@ -63,7 +63,7 @@ impl ReservoirStuff { #[cfg(feature = "redis")] redis_pool: Option>, ) -> Self { Self { - org_id, + _org_id: org_id, map, #[cfg(feature = "redis")] redis_pool, @@ -78,7 +78,6 @@ impl ReservoirStuff { let incremented_value = self.increment(&mut map_guard, rule); - // If the incremented value is less than the limit, we still want to sample. incremented_value < limit } @@ -88,12 +87,13 @@ impl ReservoirStuff { fn increment(&self, map_guard: &mut MutexGuard>, rule: RuleId) -> i64 { let val = map_guard.entry(rule).or_insert(0); + #[cfg_attr(not(feature = "redis"), allow(unused_mut))] let mut new_val: i64 = *val + 1; #[cfg(feature = "redis")] { if let Some(pool) = self.redis_pool.as_ref() { - match increment_bias_rule_count(pool.clone(), self.org_id, rule) { + match increment_bias_rule_count(pool.clone(), self._org_id, rule) { Ok(redis_val) => new_val = redis_val, Err(e) => relay_log::error!("failed to increment redis reservoir count: {}", e), } @@ -347,7 +347,13 @@ mod tests { use super::*; fn dummy_reservoir() -> Arc { - ReservoirStuff::new(0, ReservoirCounters::default(), None).into() + ReservoirStuff::new( + 0, + ReservoirCounters::default(), + #[cfg(feature = "redis")] + None, + ) + .into() } /// Helper to extract the sampling match after evaluating rules. diff --git a/relay-server/src/actors/processor.rs b/relay-server/src/actors/processor.rs index 3e10cc4cdb..f3ca99e6e4 100644 --- a/relay-server/src/actors/processor.rs +++ b/relay-server/src/actors/processor.rs @@ -3060,7 +3060,13 @@ mod tests { } fn dummy_reservoir() -> Arc { - ReservoirStuff::new(0, ReservoirCounters::default(), None).into() + ReservoirStuff::new( + 0, + ReservoirCounters::default(), + #[cfg(feature = "processing")] + None, + ) + .into() } fn mocked_event(event_type: EventType, transaction: &str, release: &str) -> Event { diff --git a/relay-server/src/utils/dynamic_sampling.rs b/relay-server/src/utils/dynamic_sampling.rs index 8ad4c4ca6f..a1a64a54a7 100644 --- a/relay-server/src/utils/dynamic_sampling.rs +++ b/relay-server/src/utils/dynamic_sampling.rs @@ -171,7 +171,13 @@ mod tests { use uuid::Uuid; fn dummy_reservoir() -> Arc { - ReservoirStuff::new(0, ReservoirCounters::default(), None).into() + ReservoirStuff::new( + 0, + ReservoirCounters::default(), + #[cfg(feature = "processing")] + None, + ) + .into() } fn mocked_event(event_type: EventType, transaction: &str, release: &str) -> Event { From c59481030e7cf6363f06d92f2706949d52591e0a Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Wed, 27 Sep 2023 22:37:25 +0200 Subject: [PATCH 10/30] wip --- relay-sampling/src/config.rs | 75 ++++++++-------------- relay-sampling/src/evaluation.rs | 40 +++++------- relay-server/src/actors/processor.rs | 14 ++-- relay-server/src/utils/dynamic_sampling.rs | 44 ++++--------- 4 files changed, 60 insertions(+), 113 deletions(-) diff --git a/relay-sampling/src/config.rs b/relay-sampling/src/config.rs index 7ce7dfc479..da09be902d 100644 --- a/relay-sampling/src/config.rs +++ b/relay-sampling/src/config.rs @@ -92,7 +92,7 @@ impl SamplingRule { pub fn sample_rate( &self, now: DateTime, - reservoir: Arc, + reservoir: Option>, ) -> Option { if !self.time_range.contains(now) { // Return None if rule is inactive. @@ -103,9 +103,11 @@ impl SamplingRule { SamplingValue::SampleRate { value } => value, SamplingValue::Factor { value } => value, SamplingValue::Reservoir { limit } => { - return reservoir - .evaluate_rule(self.id, limit) - .then_some(SamplingValue::Reservoir { limit }); + return reservoir.and_then(|reservoir| { + reservoir + .evaluate_rule(self.id, limit) + .then_some(SamplingValue::Reservoir { limit }) + }); } }; @@ -297,20 +299,8 @@ impl Default for SamplingMode { mod tests { use chrono::TimeZone; - use crate::evaluation::ReservoirCounters; - use super::*; - fn dummy_reservoir() -> Arc { - ReservoirStuff::new( - 0, - ReservoirCounters::default(), - #[cfg(feature = "redis")] - None, - ) - .into() - } - #[test] fn config_deserialize() { let json = include_str!("../tests/fixtures/sampling_config.json"); @@ -547,19 +537,19 @@ mod tests { // At the start of the time range, sample rate is equal to the rule's initial sampling value. assert_eq!( - rule.sample_rate(start, dummy_reservoir()).unwrap(), + rule.sample_rate(start, None).unwrap(), SamplingValue::SampleRate { value: 1.0 } ); // Halfway in the time range, the value is exactly between 1.0 and 0.5. assert_eq!( - rule.sample_rate(halfway, dummy_reservoir()).unwrap(), + rule.sample_rate(halfway, None).unwrap(), SamplingValue::SampleRate { value: 0.75 } ); // Approaches 0.5 at the end. assert_eq!( - rule.sample_rate(end, dummy_reservoir()).unwrap(), + rule.sample_rate(end, None).unwrap(), SamplingValue::SampleRate { // It won't go to exactly 0.5 because the time range is end-exclusive. value: 0.5000028935185186 @@ -573,9 +563,7 @@ mod tests { rule }; - assert!(rule_without_start - .sample_rate(halfway, dummy_reservoir()) - .is_none()); + assert!(rule_without_start.sample_rate(halfway, None).is_none()); let rule_without_end = { let mut rule = rule.clone(); @@ -583,9 +571,7 @@ mod tests { rule }; - assert!(rule_without_end - .sample_rate(halfway, dummy_reservoir()) - .is_none()); + assert!(rule_without_end.sample_rate(halfway, None).is_none()); } /// If the decayingfunction is set to `Constant` then it shouldn't adjust the sample rate. @@ -607,10 +593,7 @@ mod tests { let halfway = Utc.with_ymd_and_hms(1970, 10, 11, 0, 0, 0).unwrap(); - assert_eq!( - rule.sample_rate(halfway, dummy_reservoir()), - Some(sampling_value) - ); + assert_eq!(rule.sample_rate(halfway, None), Some(sampling_value)); } /// Validates the `sample_rate` method for different time range configurations. @@ -636,53 +619,47 @@ mod tests { time_range, decaying_fn: DecayingFunction::Constant, }; - assert!(rule - .sample_rate(before_time_range, dummy_reservoir()) - .is_none()); - assert!(rule - .sample_rate(during_time_range, dummy_reservoir()) - .is_some()); - assert!(rule - .sample_rate(after_time_range, dummy_reservoir()) - .is_none()); + assert!(rule.sample_rate(before_time_range, None).is_none()); + assert!(rule.sample_rate(during_time_range, None).is_some()); + assert!(rule.sample_rate(after_time_range, None).is_none()); // [start..] let mut rule_without_end = rule.clone(); rule_without_end.time_range.end = None; assert!(rule_without_end - .sample_rate(before_time_range, dummy_reservoir()) + .sample_rate(before_time_range, None) .is_none()); assert!(rule_without_end - .sample_rate(during_time_range, dummy_reservoir()) + .sample_rate(during_time_range, None) .is_some()); assert!(rule_without_end - .sample_rate(after_time_range, dummy_reservoir()) + .sample_rate(after_time_range, None) .is_some()); // [..end] let mut rule_without_start = rule.clone(); rule_without_start.time_range.start = None; assert!(rule_without_start - .sample_rate(before_time_range, dummy_reservoir()) + .sample_rate(before_time_range, None) .is_some()); assert!(rule_without_start - .sample_rate(during_time_range, dummy_reservoir()) + .sample_rate(during_time_range, None) .is_some()); assert!(rule_without_start - .sample_rate(after_time_range, dummy_reservoir()) + .sample_rate(after_time_range, None) .is_none()); // [..] let mut rule_without_range = rule.clone(); rule_without_range.time_range = TimeRange::default(); assert!(rule_without_range - .sample_rate(before_time_range, dummy_reservoir()) + .sample_rate(before_time_range, None) .is_some()); assert!(rule_without_range - .sample_rate(during_time_range, dummy_reservoir()) + .sample_rate(during_time_range, None) .is_some()); assert!(rule_without_range - .sample_rate(after_time_range, dummy_reservoir()) + .sample_rate(after_time_range, None) .is_some()); } @@ -702,13 +679,13 @@ mod tests { }; matches!( - rule.sample_rate(Utc::now(), dummy_reservoir()).unwrap(), + rule.sample_rate(Utc::now(), None).unwrap(), SamplingValue::SampleRate { .. } ); rule.sampling_value = SamplingValue::Factor { value: 0.42 }; matches!( - rule.sample_rate(Utc::now(), dummy_reservoir()).unwrap(), + rule.sample_rate(Utc::now(), None).unwrap(), SamplingValue::Factor { .. } ); } diff --git a/relay-sampling/src/evaluation.rs b/relay-sampling/src/evaluation.rs index 416c9c2843..c215f7b6d6 100644 --- a/relay-sampling/src/evaluation.rs +++ b/relay-sampling/src/evaluation.rs @@ -112,21 +112,27 @@ pub struct SamplingEvaluator { rule_ids: Vec, factor: f64, client_sample_rate: Option, - reservoir: Arc, + reservoir: Option>, } impl SamplingEvaluator { /// Constructor for [`SamplingEvaluator`]. - pub fn new(now: DateTime, reservoir: Arc) -> Self { + pub fn new(now: DateTime) -> Self { Self { now, rule_ids: vec![], factor: 1.0, client_sample_rate: None, - reservoir, + reservoir: None, } } + /// Sets a new client sample rate value. + pub fn set_reservoir(mut self, reservoir: Option>) -> Self { + self.reservoir = reservoir; + self + } + /// Sets a new client sample rate value. pub fn adjust_client_sample_rate(mut self, client_sample_rate: Option) -> Self { self.client_sample_rate = client_sample_rate; @@ -346,19 +352,9 @@ mod tests { use super::*; - fn dummy_reservoir() -> Arc { - ReservoirStuff::new( - 0, - ReservoirCounters::default(), - #[cfg(feature = "redis")] - None, - ) - .into() - } - /// Helper to extract the sampling match after evaluating rules. fn get_sampling_match(rules: &[SamplingRule], instance: &impl Getter) -> SamplingMatch { - match SamplingEvaluator::new(Utc::now(), dummy_reservoir()).match_rules( + match SamplingEvaluator::new(Utc::now()).match_rules( Uuid::default(), instance, rules.iter(), @@ -414,7 +410,7 @@ mod tests { #[test] fn test_adjust_sample_rate() { // return the same as input if no client sample rate set in the sampling evaluator. - let eval = SamplingEvaluator::new(Utc::now(), dummy_reservoir()); + let eval = SamplingEvaluator::new(Utc::now()); assert_eq!(eval.adjusted_sample_rate(0.2), 0.2); let eval = eval.adjust_client_sample_rate(Some(0.5)); @@ -442,7 +438,7 @@ mod tests { let dsc = mocked_dsc_with_getter_values(vec![]); - let res = SamplingEvaluator::new(Utc::now(), dummy_reservoir()) + let res = SamplingEvaluator::new(Utc::now()) .adjust_client_sample_rate(Some(0.2)) .match_rules(Uuid::default(), &dsc, rules.iter()); @@ -508,7 +504,7 @@ mod tests { // Baseline test. let within_timerange = Utc.with_ymd_and_hms(1970, 10, 11, 0, 0, 0).unwrap(); - let res = SamplingEvaluator::new(within_timerange, dummy_reservoir()).match_rules( + let res = SamplingEvaluator::new(within_timerange).match_rules( Uuid::default(), &dsc, [rule.clone()].iter(), @@ -517,7 +513,7 @@ mod tests { assert!(evaluation_is_match(res)); let before_timerange = Utc.with_ymd_and_hms(1969, 1, 1, 0, 0, 0).unwrap(); - let res = SamplingEvaluator::new(before_timerange, dummy_reservoir()).match_rules( + let res = SamplingEvaluator::new(before_timerange).match_rules( Uuid::default(), &dsc, [rule.clone()].iter(), @@ -525,7 +521,7 @@ mod tests { assert!(!evaluation_is_match(res)); let after_timerange = Utc.with_ymd_and_hms(1971, 1, 1, 0, 0, 0).unwrap(); - let res = SamplingEvaluator::new(after_timerange, dummy_reservoir()).match_rules( + let res = SamplingEvaluator::new(after_timerange).match_rules( Uuid::default(), &dsc, [rule].iter(), @@ -651,11 +647,7 @@ mod tests { fn test_get_sampling_match_result_with_no_match() { let dsc = mocked_dsc_with_getter_values(vec![]); - let res = SamplingEvaluator::new(Utc::now(), dummy_reservoir()).match_rules( - Uuid::default(), - &dsc, - [].iter(), - ); + let res = SamplingEvaluator::new(Utc::now()).match_rules(Uuid::default(), &dsc, [].iter()); assert!(!evaluation_is_match(res)); } diff --git a/relay-server/src/actors/processor.rs b/relay-server/src/actors/processor.rs index f3ca99e6e4..ccbbd347e3 100644 --- a/relay-server/src/actors/processor.rs +++ b/relay-server/src/actors/processor.rs @@ -2376,6 +2376,7 @@ impl EnvelopeProcessorService { } } } + _ => {} } } @@ -2423,8 +2424,9 @@ impl EnvelopeProcessorService { } }; - let mut evaluator = SamplingEvaluator::new(Utc::now(), reservoir) - .adjust_client_sample_rate(adjustment_rate); + let mut evaluator = SamplingEvaluator::new(Utc::now()) + .adjust_client_sample_rate(adjustment_rate) + .set_reservoir(Some(reservoir)); if let (Some(event), Some(sampling_state)) = (event, sampling_config) { if let Some(seed) = event.id.value().map(|id| id.0) { @@ -2466,12 +2468,8 @@ impl EnvelopeProcessorService { return; }; - let sampled = utils::is_trace_fully_sampled( - self.inner.config.processing_enabled(), - state.reservoir.clone(), - config, - dsc, - ); + let sampled = + utils::is_trace_fully_sampled(self.inner.config.processing_enabled(), config, dsc); let (Some(event), Some(sampled)) = (state.event.value_mut(), sampled) else { return; diff --git a/relay-server/src/utils/dynamic_sampling.rs b/relay-server/src/utils/dynamic_sampling.rs index a1a64a54a7..e127893ab6 100644 --- a/relay-server/src/utils/dynamic_sampling.rs +++ b/relay-server/src/utils/dynamic_sampling.rs @@ -1,6 +1,5 @@ //! Functionality for calculating if a trace should be processed or dropped. use std::ops::ControlFlow; -use std::sync::Arc; use chrono::Utc; use relay_base_schema::events::EventType; @@ -8,7 +7,7 @@ use relay_base_schema::project::ProjectKey; use relay_event_schema::protocol::{Event, TraceContext}; use relay_sampling::config::{RuleType, SamplingConfig, SamplingMode}; use relay_sampling::dsc::{DynamicSamplingContext, TraceUserContext}; -use relay_sampling::evaluation::{ReservoirStuff, SamplingEvaluator, SamplingMatch}; +use relay_sampling::evaluation::{SamplingEvaluator, SamplingMatch}; use crate::envelope::{Envelope, ItemType}; @@ -68,7 +67,6 @@ impl From> for SamplingResult { /// sampling. pub fn is_trace_fully_sampled( processing_enabled: bool, - reservoir: Arc, root_project_config: &SamplingConfig, dsc: &DynamicSamplingContext, ) -> Option { @@ -94,8 +92,7 @@ pub fn is_trace_fully_sampled( }; // TODO(tor): pass correct now timestamp - let evaluator = - SamplingEvaluator::new(Utc::now(), reservoir).adjust_client_sample_rate(adjustment_rate); + let evaluator = SamplingEvaluator::new(Utc::now()).adjust_client_sample_rate(adjustment_rate); let rules = root_project_config.filter_rules(RuleType::Trace); @@ -167,19 +164,8 @@ mod tests { use relay_sampling::config::{ RuleId, RuleType, SamplingConfig, SamplingMode, SamplingRule, SamplingValue, }; - use relay_sampling::evaluation::ReservoirCounters; use uuid::Uuid; - fn dummy_reservoir() -> Arc { - ReservoirStuff::new( - 0, - ReservoirCounters::default(), - #[cfg(feature = "processing")] - None, - ) - .into() - } - fn mocked_event(event_type: EventType, transaction: &str, release: &str) -> Event { Event { id: Annotated::new(EventId::new()), @@ -231,7 +217,7 @@ mod tests { let rules = vec![mocked_sampling_rule(1, RuleType::Transaction, 1.0)]; let seed = Uuid::default(); - let result: SamplingResult = SamplingEvaluator::new(Utc::now(), dummy_reservoir()) + let result: SamplingResult = SamplingEvaluator::new(Utc::now()) .match_rules(seed, &event, rules.iter()) .into(); @@ -245,7 +231,7 @@ mod tests { let rules = vec![mocked_sampling_rule(1, RuleType::Transaction, 0.0)]; let seed = Uuid::default(); - let result: SamplingResult = SamplingEvaluator::new(Utc::now(), dummy_reservoir()) + let result: SamplingResult = SamplingEvaluator::new(Utc::now()) .match_rules(seed, &event, rules.iter()) .into(); @@ -268,7 +254,7 @@ mod tests { let event = mocked_event(EventType::Transaction, "bar", "2.0"); let seed = Uuid::default(); - let result: SamplingResult = SamplingEvaluator::new(Utc::now(), dummy_reservoir()) + let result: SamplingResult = SamplingEvaluator::new(Utc::now()) .match_rules(seed, &event, rules.iter()) .into(); @@ -282,7 +268,7 @@ mod tests { let rules = vec![mocked_sampling_rule(1, RuleType::Trace, 1.0)]; let dsc = mocked_simple_dynamic_sampling_context(Some(1.0), Some("3.0"), None, None, None); - let result: SamplingResult = SamplingEvaluator::new(Utc::now(), dummy_reservoir()) + let result: SamplingResult = SamplingEvaluator::new(Utc::now()) .match_rules(Uuid::default(), &dsc, rules.iter()) .into(); @@ -304,16 +290,10 @@ mod tests { let dsc = mocked_simple_dynamic_sampling_context(None, None, None, None, Some(true)); // Return true if any unsupported rules. - assert_eq!( - is_trace_fully_sampled(false, dummy_reservoir(), &config, &dsc), - Some(true) - ); + assert_eq!(is_trace_fully_sampled(false, &config, &dsc), Some(true)); // If processing is enabled, we simply log an error and otherwise proceed as usual. - assert_eq!( - is_trace_fully_sampled(true, dummy_reservoir(), &config, &dsc), - Some(false) - ); + assert_eq!(is_trace_fully_sampled(true, &config, &dsc), Some(false)); } #[test] @@ -330,7 +310,7 @@ mod tests { let dsc = mocked_simple_dynamic_sampling_context(Some(1.0), Some("3.0"), None, None, Some(true)); - let result = is_trace_fully_sampled(false, dummy_reservoir(), &config, &dsc).unwrap(); + let result = is_trace_fully_sampled(false, &config, &dsc).unwrap(); assert!(result); // We test with `sampled = true` and 0% rule. @@ -343,7 +323,7 @@ mod tests { let dsc = mocked_simple_dynamic_sampling_context(Some(1.0), Some("3.0"), None, None, Some(true)); - let result = is_trace_fully_sampled(false, dummy_reservoir(), &config, &dsc).unwrap(); + let result = is_trace_fully_sampled(false, &config, &dsc).unwrap(); assert!(!result); // We test with `sampled = false` and 100% rule. @@ -356,7 +336,7 @@ mod tests { let dsc = mocked_simple_dynamic_sampling_context(Some(1.0), Some("3.0"), None, None, Some(false)); - let result = is_trace_fully_sampled(false, dummy_reservoir(), &config, &dsc).unwrap(); + let result = is_trace_fully_sampled(false, &config, &dsc).unwrap(); assert!(!result); } @@ -371,7 +351,7 @@ mod tests { }; let dsc = mocked_simple_dynamic_sampling_context(Some(1.0), Some("3.0"), None, None, None); - let result = is_trace_fully_sampled(false, dummy_reservoir(), &config, &dsc); + let result = is_trace_fully_sampled(false, &config, &dsc); assert!(result.is_none()); } } From 2828452c19f40ccbd5ed121a048596615ff27be3 Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Thu, 28 Sep 2023 07:46:19 +0200 Subject: [PATCH 11/30] wip --- relay-sampling/src/config.rs | 4 +-- relay-sampling/src/evaluation.rs | 41 ++++++++++++++++------------ relay-server/src/actors/processor.rs | 30 +++++++++----------- 3 files changed, 39 insertions(+), 36 deletions(-) diff --git a/relay-sampling/src/config.rs b/relay-sampling/src/config.rs index da09be902d..95be0194da 100644 --- a/relay-sampling/src/config.rs +++ b/relay-sampling/src/config.rs @@ -7,7 +7,7 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use crate::condition::RuleCondition; -use crate::evaluation::ReservoirStuff; +use crate::evaluation::ReservoirEvaluator; use crate::utils; /// Represents the dynamic sampling configuration available to a project. @@ -92,7 +92,7 @@ impl SamplingRule { pub fn sample_rate( &self, now: DateTime, - reservoir: Option>, + reservoir: Option<&Arc>, ) -> Option { if !self.time_range.contains(now) { // Return None if rule is inactive. diff --git a/relay-sampling/src/evaluation.rs b/relay-sampling/src/evaluation.rs index c215f7b6d6..8890617180 100644 --- a/relay-sampling/src/evaluation.rs +++ b/relay-sampling/src/evaluation.rs @@ -46,30 +46,37 @@ fn increment_bias_rule_count( /// The amount of transactions sampled of a given rule id. pub type ReservoirCounters = Arc>>; -/// Reservoir utility. +/// Utility for evaluating reservoir rules. +/// +/// #[derive(Debug)] -pub struct ReservoirStuff { +pub struct ReservoirEvaluator { #[cfg(feature = "redis")] redis_pool: Option>, - _org_id: u64, + #[cfg(feature = "redis")] + org_id: Option, map: ReservoirCounters, } -impl ReservoirStuff { - /// Creates a new whatever ill call this struct. - pub fn new( - org_id: u64, - map: ReservoirCounters, - #[cfg(feature = "redis")] redis_pool: Option>, - ) -> Self { +impl ReservoirEvaluator { + /// Constructor for [`ReservoirEvaluator`]. + pub fn new(map: ReservoirCounters) -> Self { Self { - _org_id: org_id, map, #[cfg(feature = "redis")] - redis_pool, + org_id: None, + #[cfg(feature = "redis")] + redis_pool: None, } } + #[cfg(feature = "redis")] + pub fn set_redis(mut self, org_id: Option, redis_pool: Option>) -> Self { + self.org_id = org_id; + self.redis_pool = redis_pool; + self + } + /// Evaluates a reservoir rule, returning true if it should be sampled. pub fn evaluate_rule(&self, rule: RuleId, limit: i64) -> bool { let Ok(mut map_guard) = self.map.try_lock() else { @@ -92,8 +99,8 @@ impl ReservoirStuff { #[cfg(feature = "redis")] { - if let Some(pool) = self.redis_pool.as_ref() { - match increment_bias_rule_count(pool.clone(), self._org_id, rule) { + if let (Some(pool), Some(org_id)) = (self.redis_pool.as_ref(), self.org_id) { + match increment_bias_rule_count(pool.clone(), org_id, rule) { Ok(redis_val) => new_val = redis_val, Err(e) => relay_log::error!("failed to increment redis reservoir count: {}", e), } @@ -112,7 +119,7 @@ pub struct SamplingEvaluator { rule_ids: Vec, factor: f64, client_sample_rate: Option, - reservoir: Option>, + reservoir: Option>, } impl SamplingEvaluator { @@ -128,7 +135,7 @@ impl SamplingEvaluator { } /// Sets a new client sample rate value. - pub fn set_reservoir(mut self, reservoir: Option>) -> Self { + pub fn set_reservoir(mut self, reservoir: Option>) -> Self { self.reservoir = reservoir; self } @@ -165,7 +172,7 @@ impl SamplingEvaluator { continue; }; - let Some(sampling_value) = rule.sample_rate(self.now, self.reservoir.clone()) else { + let Some(sampling_value) = rule.sample_rate(self.now, self.reservoir.as_ref()) else { continue; }; diff --git a/relay-server/src/actors/processor.rs b/relay-server/src/actors/processor.rs index ccbbd347e3..2bbbda8d7e 100644 --- a/relay-server/src/actors/processor.rs +++ b/relay-server/src/actors/processor.rs @@ -45,7 +45,7 @@ use relay_redis::RedisPool; use relay_replays::recording::RecordingScrubber; use relay_sampling::config::{RuleType, SamplingMode}; use relay_sampling::evaluation::{ - MatchedRuleIds, ReservoirCounters, ReservoirStuff, SamplingEvaluator, + MatchedRuleIds, ReservoirCounters, ReservoirEvaluator, SamplingEvaluator, }; use relay_sampling::{DynamicSamplingContext, SamplingConfig}; use relay_statsd::metric; @@ -311,7 +311,7 @@ struct ProcessEnvelopeState { has_profile: bool, /// Reservoir stuff - reservoir: Arc, + reservoir: Arc, } impl ProcessEnvelopeState { @@ -1363,12 +1363,14 @@ impl EnvelopeProcessorService { // 2. The DSN was moved and the envelope sent to the old project ID. envelope.meta_mut().set_project_id(project_id); - let reservoir = ReservoirStuff::new( - managed_envelope.scoping().organization_id, - reservoir_counters, - #[cfg(feature = "processing")] - self.inner.redis_pool.clone(), - ) + let reservoir: Arc = if cfg!(feature = "processing") { + let redis = self.inner.redis_pool.clone(); + let org_id = managed_envelope.scoping().organization_id; + + ReservoirEvaluator::new(reservoir_counters).set_redis(Some(org_id), redis) + } else { + ReservoirEvaluator::new(reservoir_counters) + } .into(); Ok(ProcessEnvelopeState { @@ -2384,7 +2386,7 @@ impl EnvelopeProcessorService { /// Computes the sampling decision on the incoming transaction. fn compute_sampling_decision( processing_enabled: bool, - reservoir: Arc, + reservoir: Arc, sampling_config: Option<&SamplingConfig>, event: Option<&Event>, root_sampling_config: Option<&SamplingConfig>, @@ -3057,14 +3059,8 @@ mod tests { } } - fn dummy_reservoir() -> Arc { - ReservoirStuff::new( - 0, - ReservoirCounters::default(), - #[cfg(feature = "processing")] - None, - ) - .into() + fn dummy_reservoir() -> Arc { + ReservoirEvaluator::new(ReservoirCounters::default()).into() } fn mocked_event(event_type: EventType, transaction: &str, release: &str) -> Event { From 3e10fe9a67cb7578e3308085d7d9fa5ea4968415 Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Thu, 28 Sep 2023 09:06:40 +0200 Subject: [PATCH 12/30] wip --- relay-sampling/src/config.rs | 58 ++++++++++----------- relay-sampling/src/evaluation.rs | 77 +++++++++++++++++----------- relay-server/src/actors/processor.rs | 21 ++++---- 3 files changed, 86 insertions(+), 70 deletions(-) diff --git a/relay-sampling/src/config.rs b/relay-sampling/src/config.rs index 95be0194da..b862840913 100644 --- a/relay-sampling/src/config.rs +++ b/relay-sampling/src/config.rs @@ -88,8 +88,8 @@ impl SamplingRule { matches!(&self.sampling_value, &SamplingValue::Reservoir { .. }) } - /// Returns the sample rate if the rule is active. - pub fn sample_rate( + /// Returns the updated sampling value if it's valid. + pub fn evaluate( &self, now: DateTime, reservoir: Option<&Arc>, @@ -105,7 +105,7 @@ impl SamplingRule { SamplingValue::Reservoir { limit } => { return reservoir.and_then(|reservoir| { reservoir - .evaluate_rule(self.id, limit) + .evaluate(self.id, limit) .then_some(SamplingValue::Reservoir { limit }) }); } @@ -167,11 +167,13 @@ pub enum SamplingValue { /// The factor to apply on another matched sample rate. value: f64, }, + /// A reservoir limit. /// - /// Rule will match if less than `limit` rules have been sampled. + /// A rule with a reservoir limit will be sampled if the rule have been matched fewer times + /// than the limit. Reservoir { - /// The limit of how many transactions with this rule will be sampled. + /// The limit of how many times this rule will be sampled before this rule is invalid. limit: i64, }, } @@ -537,19 +539,19 @@ mod tests { // At the start of the time range, sample rate is equal to the rule's initial sampling value. assert_eq!( - rule.sample_rate(start, None).unwrap(), + rule.evaluate(start, None).unwrap(), SamplingValue::SampleRate { value: 1.0 } ); // Halfway in the time range, the value is exactly between 1.0 and 0.5. assert_eq!( - rule.sample_rate(halfway, None).unwrap(), + rule.evaluate(halfway, None).unwrap(), SamplingValue::SampleRate { value: 0.75 } ); // Approaches 0.5 at the end. assert_eq!( - rule.sample_rate(end, None).unwrap(), + rule.evaluate(end, None).unwrap(), SamplingValue::SampleRate { // It won't go to exactly 0.5 because the time range is end-exclusive. value: 0.5000028935185186 @@ -563,7 +565,7 @@ mod tests { rule }; - assert!(rule_without_start.sample_rate(halfway, None).is_none()); + assert!(rule_without_start.evaluate(halfway, None).is_none()); let rule_without_end = { let mut rule = rule.clone(); @@ -571,7 +573,7 @@ mod tests { rule }; - assert!(rule_without_end.sample_rate(halfway, None).is_none()); + assert!(rule_without_end.evaluate(halfway, None).is_none()); } /// If the decayingfunction is set to `Constant` then it shouldn't adjust the sample rate. @@ -593,7 +595,7 @@ mod tests { let halfway = Utc.with_ymd_and_hms(1970, 10, 11, 0, 0, 0).unwrap(); - assert_eq!(rule.sample_rate(halfway, None), Some(sampling_value)); + assert_eq!(rule.evaluate(halfway, None), Some(sampling_value)); } /// Validates the `sample_rate` method for different time range configurations. @@ -619,47 +621,41 @@ mod tests { time_range, decaying_fn: DecayingFunction::Constant, }; - assert!(rule.sample_rate(before_time_range, None).is_none()); - assert!(rule.sample_rate(during_time_range, None).is_some()); - assert!(rule.sample_rate(after_time_range, None).is_none()); + assert!(rule.evaluate(before_time_range, None).is_none()); + assert!(rule.evaluate(during_time_range, None).is_some()); + assert!(rule.evaluate(after_time_range, None).is_none()); // [start..] let mut rule_without_end = rule.clone(); rule_without_end.time_range.end = None; - assert!(rule_without_end - .sample_rate(before_time_range, None) - .is_none()); - assert!(rule_without_end - .sample_rate(during_time_range, None) - .is_some()); - assert!(rule_without_end - .sample_rate(after_time_range, None) - .is_some()); + assert!(rule_without_end.evaluate(before_time_range, None).is_none()); + assert!(rule_without_end.evaluate(during_time_range, None).is_some()); + assert!(rule_without_end.evaluate(after_time_range, None).is_some()); // [..end] let mut rule_without_start = rule.clone(); rule_without_start.time_range.start = None; assert!(rule_without_start - .sample_rate(before_time_range, None) + .evaluate(before_time_range, None) .is_some()); assert!(rule_without_start - .sample_rate(during_time_range, None) + .evaluate(during_time_range, None) .is_some()); assert!(rule_without_start - .sample_rate(after_time_range, None) + .evaluate(after_time_range, None) .is_none()); // [..] let mut rule_without_range = rule.clone(); rule_without_range.time_range = TimeRange::default(); assert!(rule_without_range - .sample_rate(before_time_range, None) + .evaluate(before_time_range, None) .is_some()); assert!(rule_without_range - .sample_rate(during_time_range, None) + .evaluate(during_time_range, None) .is_some()); assert!(rule_without_range - .sample_rate(after_time_range, None) + .evaluate(after_time_range, None) .is_some()); } @@ -679,13 +675,13 @@ mod tests { }; matches!( - rule.sample_rate(Utc::now(), None).unwrap(), + rule.evaluate(Utc::now(), None).unwrap(), SamplingValue::SampleRate { .. } ); rule.sampling_value = SamplingValue::Factor { value: 0.42 }; matches!( - rule.sample_rate(Utc::now(), None).unwrap(), + rule.evaluate(Utc::now(), None).unwrap(), SamplingValue::Factor { .. } ); } diff --git a/relay-sampling/src/evaluation.rs b/relay-sampling/src/evaluation.rs index 8890617180..05fd06c2cb 100644 --- a/relay-sampling/src/evaluation.rs +++ b/relay-sampling/src/evaluation.rs @@ -28,41 +28,53 @@ fn pseudo_random_from_uuid(id: Uuid) -> f64 { generator.sample(dist) } -/// Increments the count, it will be initialized automatically if it doesn't exist. +/// Increments the reservoir count for a given rule in redis. /// -/// INCR docs: [`https://redis.io/commands/incr/`] +/// - INCR docs: [`https://redis.io/commands/incr/`] +/// - If the counter doesn't exist in redis, a new one will be inserted. #[cfg(feature = "redis")] -fn increment_bias_rule_count( - redis: Arc, +fn increment_redis_reservoir_count( + redis: &Arc, org_id: u64, rule_id: RuleId, ) -> anyhow::Result { - let key = format!("bias:{}:{}", org_id, rule_id); let mut command = relay_redis::redis::cmd("INCR"); - command.arg(key.as_str()); + + let key = format!("bias:{}:{}", org_id, rule_id); + command.arg(key); + Ok(command.query(&mut redis.client()?.connection()?)?) } /// The amount of transactions sampled of a given rule id. pub type ReservoirCounters = Arc>>; -/// Utility for evaluating reservoir rules. +/// Utility for evaluating reservoir-based sampling rules. /// +/// A "reservoir limit" rule samples every match until its limit is reached, after which +/// the rule is disabled. /// +/// This utility uses a dual-counter system for enforcing this limit: +/// +/// - Local Counter: Each relay instance maintains a local counter to track sampled events. +/// +/// - Redis Counter: For processing relays, a Redis-based counter provides synchronization +/// across multiple relay-instances. When incremented, the Redis counter returns the current global +/// count for the given rule, which is then used to update the local counter. #[derive(Debug)] pub struct ReservoirEvaluator { + counters: ReservoirCounters, #[cfg(feature = "redis")] redis_pool: Option>, #[cfg(feature = "redis")] org_id: Option, - map: ReservoirCounters, } impl ReservoirEvaluator { /// Constructor for [`ReservoirEvaluator`]. pub fn new(map: ReservoirCounters) -> Self { Self { - map, + counters: map, #[cfg(feature = "redis")] org_id: None, #[cfg(feature = "redis")] @@ -70,45 +82,50 @@ impl ReservoirEvaluator { } } + /// Sets the Redis pool and organiation ID for the [`ReservoirEvaluator`]. + /// + /// These values are needed to synchronize with Redis. #[cfg(feature = "redis")] - pub fn set_redis(mut self, org_id: Option, redis_pool: Option>) -> Self { - self.org_id = org_id; - self.redis_pool = redis_pool; + pub fn set_redis(mut self, org_id: u64, redis_pool: Arc) -> Self { + self.org_id = Some(org_id); + self.redis_pool = Some(redis_pool); self } - /// Evaluates a reservoir rule, returning true if it should be sampled. - pub fn evaluate_rule(&self, rule: RuleId, limit: i64) -> bool { - let Ok(mut map_guard) = self.map.try_lock() else { + /// Evaluates a reservoir rule, returning `true` if it should be sampled. + pub fn evaluate(&self, rule: RuleId, limit: i64) -> bool { + // If the mutex is already locked, we abort the match, for performance reasons. + let Ok(mut map_guard) = self.counters.try_lock() else { return false; }; - let incremented_value = self.increment(&mut map_guard, rule); + let counter_value = map_guard.entry(rule).or_insert(0); - incremented_value < limit - } - - // if limit isn't reached yet, we wanna increment. - // when incrementing, if processing is enabled, we increment redis and insert back the value - // otherwise we just increment directly - fn increment(&self, map_guard: &mut MutexGuard>, rule: RuleId) -> i64 { - let val = map_guard.entry(rule).or_insert(0); + // Avoid Redis call if limit has already been reached. + if *counter_value >= limit { + return false; + } + // The new value is either the current value + 1, or the value from redis + // if we have access to redis. #[cfg_attr(not(feature = "redis"), allow(unused_mut))] - let mut new_val: i64 = *val + 1; + let mut new_val: i64 = *counter_value + 1; #[cfg(feature = "redis")] { if let (Some(pool), Some(org_id)) = (self.redis_pool.as_ref(), self.org_id) { - match increment_bias_rule_count(pool.clone(), org_id, rule) { + match increment_redis_reservoir_count(pool, org_id, rule) { Ok(redis_val) => new_val = redis_val, Err(e) => relay_log::error!("failed to increment redis reservoir count: {}", e), } } } - *val = new_val; - *val + // Update the counter. + *counter_value = new_val; + + // We sample the rule if the limit has not been reached. + *counter_value < limit } } @@ -134,7 +151,7 @@ impl SamplingEvaluator { } } - /// Sets a new client sample rate value. + /// Sets a [`ReservoirEvaluator`]. pub fn set_reservoir(mut self, reservoir: Option>) -> Self { self.reservoir = reservoir; self @@ -172,7 +189,7 @@ impl SamplingEvaluator { continue; }; - let Some(sampling_value) = rule.sample_rate(self.now, self.reservoir.as_ref()) else { + let Some(sampling_value) = rule.evaluate(self.now, self.reservoir.as_ref()) else { continue; }; diff --git a/relay-server/src/actors/processor.rs b/relay-server/src/actors/processor.rs index 2bbbda8d7e..5e279c1fc6 100644 --- a/relay-server/src/actors/processor.rs +++ b/relay-server/src/actors/processor.rs @@ -310,7 +310,7 @@ struct ProcessEnvelopeState { /// Whether there is a profiling item in the envelope. has_profile: bool, - /// Reservoir stuff + /// Reservoir evaluator that we use for dynamic sampling. reservoir: Arc, } @@ -1363,15 +1363,18 @@ impl EnvelopeProcessorService { // 2. The DSN was moved and the envelope sent to the old project ID. envelope.meta_mut().set_project_id(project_id); - let reservoir: Arc = if cfg!(feature = "processing") { - let redis = self.inner.redis_pool.clone(); - let org_id = managed_envelope.scoping().organization_id; + let reservoir: Arc = { + #[allow(unused_mut)] + let mut reservoir = ReservoirEvaluator::new(reservoir_counters); - ReservoirEvaluator::new(reservoir_counters).set_redis(Some(org_id), redis) - } else { - ReservoirEvaluator::new(reservoir_counters) - } - .into(); + #[cfg(feature = "processing")] + if let Some(redis_pool) = self.inner.redis_pool.as_ref() { + let org_id = managed_envelope.scoping().organization_id; + reservoir = reservoir.set_redis(org_id, redis_pool.clone()); + } + + Arc::new(reservoir) + }; Ok(ProcessEnvelopeState { event: Annotated::empty(), From c8bb0f4af12dd698a1f533585f4e775ca17ab8d0 Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Thu, 28 Sep 2023 09:52:42 +0200 Subject: [PATCH 13/30] wip --- relay-sampling/src/config.rs | 2 +- relay-sampling/src/evaluation.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/relay-sampling/src/config.rs b/relay-sampling/src/config.rs index b862840913..a68515e4b4 100644 --- a/relay-sampling/src/config.rs +++ b/relay-sampling/src/config.rs @@ -88,7 +88,7 @@ impl SamplingRule { matches!(&self.sampling_value, &SamplingValue::Reservoir { .. }) } - /// Returns the updated sampling value if it's valid. + /// Returns the updated [`SamplingValue`] if it's valid. pub fn evaluate( &self, now: DateTime, diff --git a/relay-sampling/src/evaluation.rs b/relay-sampling/src/evaluation.rs index 05fd06c2cb..008637e4e9 100644 --- a/relay-sampling/src/evaluation.rs +++ b/relay-sampling/src/evaluation.rs @@ -4,7 +4,7 @@ use std::collections::BTreeMap; use std::fmt; use std::num::ParseIntError; use std::ops::ControlFlow; -use std::sync::{Arc, Mutex, MutexGuard}; +use std::sync::{Arc, Mutex}; use chrono::{DateTime, Utc}; use rand::distributions::Uniform; @@ -95,6 +95,7 @@ impl ReservoirEvaluator { /// Evaluates a reservoir rule, returning `true` if it should be sampled. pub fn evaluate(&self, rule: RuleId, limit: i64) -> bool { // If the mutex is already locked, we abort the match, for performance reasons. + let Ok(mut map_guard) = self.counters.try_lock() else { return false; }; From 195bc3123190876674652e46561b8bc79c4cff3f Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Thu, 28 Sep 2023 10:10:36 +0200 Subject: [PATCH 14/30] wip --- relay-sampling/src/evaluation.rs | 6 +++--- relay-server/src/actors/project.rs | 4 ++-- relay-server/src/actors/project_cache.rs | 12 +++--------- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/relay-sampling/src/evaluation.rs b/relay-sampling/src/evaluation.rs index 008637e4e9..999c540d2f 100644 --- a/relay-sampling/src/evaluation.rs +++ b/relay-sampling/src/evaluation.rs @@ -46,7 +46,7 @@ fn increment_redis_reservoir_count( Ok(command.query(&mut redis.client()?.connection()?)?) } -/// The amount of transactions sampled of a given rule id. +/// The amount of matches for each reservoir rule in a given project. pub type ReservoirCounters = Arc>>; /// Utility for evaluating reservoir-based sampling rules. @@ -107,8 +107,8 @@ impl ReservoirEvaluator { return false; } - // The new value is either the current value + 1, or the value from redis - // if we have access to redis. + // The new value for the counter will be incremented by one, or updated with the + // global redis counter if it's available. #[cfg_attr(not(feature = "redis"), allow(unused_mut))] let mut new_val: i64 = *counter_value + 1; diff --git a/relay-server/src/actors/project.rs b/relay-server/src/actors/project.rs index 1a5f231363..45453ea9be 100644 --- a/relay-server/src/actors/project.rs +++ b/relay-server/src/actors/project.rs @@ -428,8 +428,8 @@ impl Project { self.reservoir_counters.clone() } - /// if a rule is no longer in the sampling config, we can assume it's deleted and no longer needed. - pub fn delete_expired_rules(&self) { + /// if a reservoir rule is no longer in the sampling config, we can assume it's no longer needed. + pub fn remove_expired_reservoir_rules(&self) { let Some(config) = self .state .as_ref() diff --git a/relay-server/src/actors/project_cache.rs b/relay-server/src/actors/project_cache.rs index c2cba4cd0c..e69d7945cd 100644 --- a/relay-server/src/actors/project_cache.rs +++ b/relay-server/src/actors/project_cache.rs @@ -570,15 +570,9 @@ impl ProjectCacheBroker { } = message; let project_cache = self.services.project_cache.clone(); - self.get_or_create_project(project_key).update_state( - project_cache, - state.clone(), - no_cache, - ); - - if let Some(project) = self.projects.get(&project_key) { - project.delete_expired_rules(); - }; + let project = self.get_or_create_project(project_key); + project.update_state(project_cache, state.clone(), no_cache); + project.remove_expired_reservoir_rules(); if !state.invalid() { self.dequeue(project_key); From cd1ce03b82b013cb728cfa0054c7a0c51e5cbe5e Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Thu, 28 Sep 2023 10:15:05 +0200 Subject: [PATCH 15/30] wip --- relay-sampling/src/evaluation.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/relay-sampling/src/evaluation.rs b/relay-sampling/src/evaluation.rs index 999c540d2f..3b5ba41297 100644 --- a/relay-sampling/src/evaluation.rs +++ b/relay-sampling/src/evaluation.rs @@ -72,9 +72,9 @@ pub struct ReservoirEvaluator { impl ReservoirEvaluator { /// Constructor for [`ReservoirEvaluator`]. - pub fn new(map: ReservoirCounters) -> Self { + pub fn new(counters: ReservoirCounters) -> Self { Self { - counters: map, + counters, #[cfg(feature = "redis")] org_id: None, #[cfg(feature = "redis")] From cca10cece22f9308a0492c216cb624fa1bfaf447 Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Thu, 28 Sep 2023 10:31:21 +0200 Subject: [PATCH 16/30] wip --- relay-sampling/src/evaluation.rs | 3 +-- relay-server/src/actors/processor.rs | 2 +- relay-server/src/actors/project.rs | 3 ++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/relay-sampling/src/evaluation.rs b/relay-sampling/src/evaluation.rs index 3b5ba41297..50dfae9ec1 100644 --- a/relay-sampling/src/evaluation.rs +++ b/relay-sampling/src/evaluation.rs @@ -95,7 +95,6 @@ impl ReservoirEvaluator { /// Evaluates a reservoir rule, returning `true` if it should be sampled. pub fn evaluate(&self, rule: RuleId, limit: i64) -> bool { // If the mutex is already locked, we abort the match, for performance reasons. - let Ok(mut map_guard) = self.counters.try_lock() else { return false; }; @@ -397,7 +396,7 @@ mod tests { fn matches_rule_ids(rule_ids: &[u32], rules: &[SamplingRule], instance: &impl Getter) -> bool { let matched_rule_ids = MatchedRuleIds(rule_ids.iter().map(|num| RuleId(*num)).collect()); let sampling_match = get_sampling_match(rules, instance); - matched_rule_ids == sampling_match.into_matched_rules() + matched_rule_ids == sampling_match.matched_rules } /// Helper function to create a dsc with the provided getter-values set. diff --git a/relay-server/src/actors/processor.rs b/relay-server/src/actors/processor.rs index 5e279c1fc6..dcdfcd1a3f 100644 --- a/relay-server/src/actors/processor.rs +++ b/relay-server/src/actors/processor.rs @@ -1363,7 +1363,7 @@ impl EnvelopeProcessorService { // 2. The DSN was moved and the envelope sent to the old project ID. envelope.meta_mut().set_project_id(project_id); - let reservoir: Arc = { + let reservoir = { #[allow(unused_mut)] let mut reservoir = ReservoirEvaluator::new(reservoir_counters); diff --git a/relay-server/src/actors/project.rs b/relay-server/src/actors/project.rs index 45453ea9be..a18f4f9754 100644 --- a/relay-server/src/actors/project.rs +++ b/relay-server/src/actors/project.rs @@ -424,11 +424,12 @@ impl Project { } } + /// Returns the [`ReservoirCounters`] for the project. pub fn reservoir_counters(&self) -> ReservoirCounters { self.reservoir_counters.clone() } - /// if a reservoir rule is no longer in the sampling config, we can assume it's no longer needed. + /// If a reservoir rule is no longer in the sampling config, we can assume it's no longer needed. pub fn remove_expired_reservoir_rules(&self) { let Some(config) = self .state From 6dbb2198f4715208fcc69996cb7c9164ecdf7d69 Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Thu, 28 Sep 2023 10:51:03 +0200 Subject: [PATCH 17/30] wip --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20a350a81d..50af6f3df6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,11 @@ - Exclude more spans fron metrics extraction. ([#2522](https://github.com/getsentry/relay/pull/2522), [#2525](https://github.com/getsentry/relay/pull/2525), [#2545](https://github.com/getsentry/relay/pull/2545)) - Fix hot-loop burning CPU when upstream service is unavailable. ([#2518](https://github.com/getsentry/relay/pull/2518)) +**Features**: + +- Introduce reservoir sampling rule. ([#2550](https://github.com/getsentry/relay/pull/2550)) + + ## 23.9.1 - No documented changes. From 70fc0c89bc1e834d8b04ed974d60d9e9b9f65d64 Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Thu, 28 Sep 2023 10:54:28 +0200 Subject: [PATCH 18/30] wip --- CHANGELOG.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 50af6f3df6..f8ffde64d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +**Features**: + +- Introduce reservoir sampling rule. ([#2550](https://github.com/getsentry/relay/pull/2550)) + **Bug Fixes**: - Remove profile_id from context when no profile is in the envelope. ([#2523](https://github.com/getsentry/relay/pull/2523)) @@ -16,11 +20,6 @@ - Exclude more spans fron metrics extraction. ([#2522](https://github.com/getsentry/relay/pull/2522), [#2525](https://github.com/getsentry/relay/pull/2525), [#2545](https://github.com/getsentry/relay/pull/2545)) - Fix hot-loop burning CPU when upstream service is unavailable. ([#2518](https://github.com/getsentry/relay/pull/2518)) -**Features**: - -- Introduce reservoir sampling rule. ([#2550](https://github.com/getsentry/relay/pull/2550)) - - ## 23.9.1 - No documented changes. From 5faa6c4f10caf2ff9f61a8c4ce73bbe8533320a9 Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Thu, 28 Sep 2023 14:49:24 +0200 Subject: [PATCH 19/30] wip --- CHANGELOG.md | 5 +- relay-sampling/Cargo.toml | 5 +- relay-sampling/src/config.rs | 10 +- relay-sampling/src/evaluation.rs | 117 ++++++++++++--------- relay-sampling/src/lib.rs | 2 + relay-sampling/src/redis_sampling.rs | 47 +++++++++ relay-server/src/actors/processor.rs | 41 ++++---- relay-server/src/actors/project.rs | 8 +- relay-server/src/actors/project_cache.rs | 1 - relay-server/src/utils/dynamic_sampling.rs | 2 +- 10 files changed, 148 insertions(+), 90 deletions(-) create mode 100644 relay-sampling/src/redis_sampling.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index f8ffde64d0..adb631c884 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,6 @@ ## Unreleased -**Features**: - -- Introduce reservoir sampling rule. ([#2550](https://github.com/getsentry/relay/pull/2550)) - **Bug Fixes**: - Remove profile_id from context when no profile is in the envelope. ([#2523](https://github.com/getsentry/relay/pull/2523)) @@ -19,6 +15,7 @@ - Remove filtering for Android events with missing close events. ([#2524](https://github.com/getsentry/relay/pull/2524)) - Exclude more spans fron metrics extraction. ([#2522](https://github.com/getsentry/relay/pull/2522), [#2525](https://github.com/getsentry/relay/pull/2525), [#2545](https://github.com/getsentry/relay/pull/2545)) - Fix hot-loop burning CPU when upstream service is unavailable. ([#2518](https://github.com/getsentry/relay/pull/2518)) +- Introduce reservoir sampling rule. ([#2550](https://github.com/getsentry/relay/pull/2550)) ## 23.9.1 diff --git a/relay-sampling/Cargo.toml b/relay-sampling/Cargo.toml index 2ac41296c3..fcc04aef3d 100644 --- a/relay-sampling/Cargo.toml +++ b/relay-sampling/Cargo.toml @@ -11,10 +11,11 @@ publish = false [features] default = [] -redis = ["dep:relay-redis"] +redis = ["dep:anyhow", "dep:relay-redis"] + [dependencies] -anyhow = { workspace = true } +anyhow = { workspace = true, optional = true } chrono = { workspace = true } rand = { workspace = true } rand_pcg = "0.3.1" diff --git a/relay-sampling/src/config.rs b/relay-sampling/src/config.rs index a68515e4b4..9addfceb54 100644 --- a/relay-sampling/src/config.rs +++ b/relay-sampling/src/config.rs @@ -1,7 +1,6 @@ //! Dynamic sampling rule configuration. use std::fmt; -use std::sync::Arc; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; @@ -83,16 +82,11 @@ impl SamplingRule { self.condition.supported() && self.ty != RuleType::Unsupported } - /// Returns `true` if rule is a reservoir rule. - pub fn is_reservoir(&self) -> bool { - matches!(&self.sampling_value, &SamplingValue::Reservoir { .. }) - } - /// Returns the updated [`SamplingValue`] if it's valid. pub fn evaluate( &self, now: DateTime, - reservoir: Option<&Arc>, + reservoir: Option<&ReservoirEvaluator>, ) -> Option { if !self.time_range.contains(now) { // Return None if rule is inactive. @@ -105,7 +99,7 @@ impl SamplingRule { SamplingValue::Reservoir { limit } => { return reservoir.and_then(|reservoir| { reservoir - .evaluate(self.id, limit) + .evaluate(self.id, limit, self.time_range.end.as_ref()) .then_some(SamplingValue::Reservoir { limit }) }); } diff --git a/relay-sampling/src/evaluation.rs b/relay-sampling/src/evaluation.rs index 50dfae9ec1..bf3a9a9d8b 100644 --- a/relay-sampling/src/evaluation.rs +++ b/relay-sampling/src/evaluation.rs @@ -17,6 +17,7 @@ use serde::Serialize; use uuid::Uuid; use crate::config::{RuleId, SamplingRule, SamplingValue}; +use crate::redis_sampling::{increment_redis_reservoir_count, set_redis_expiry, ReservoirRuleKey}; /// Generates a pseudo random number by seeding the generator with the given id. /// @@ -28,24 +29,6 @@ fn pseudo_random_from_uuid(id: Uuid) -> f64 { generator.sample(dist) } -/// Increments the reservoir count for a given rule in redis. -/// -/// - INCR docs: [`https://redis.io/commands/incr/`] -/// - If the counter doesn't exist in redis, a new one will be inserted. -#[cfg(feature = "redis")] -fn increment_redis_reservoir_count( - redis: &Arc, - org_id: u64, - rule_id: RuleId, -) -> anyhow::Result { - let mut command = relay_redis::redis::cmd("INCR"); - - let key = format!("bias:{}:{}", org_id, rule_id); - command.arg(key); - - Ok(command.query(&mut redis.client()?.connection()?)?) -} - /// The amount of matches for each reservoir rule in a given project. pub type ReservoirCounters = Arc>>; @@ -92,54 +75,90 @@ impl ReservoirEvaluator { self } - /// Evaluates a reservoir rule, returning `true` if it should be sampled. - pub fn evaluate(&self, rule: RuleId, limit: i64) -> bool { - // If the mutex is already locked, we abort the match, for performance reasons. - let Ok(mut map_guard) = self.counters.try_lock() else { - return false; + fn redis_count(&self, rule: RuleId, _rule_expiry: Option<&DateTime>) -> Option { + if cfg!(feature = "redis") { + if let (Some(pool), Some(org_id)) = (self.redis_pool.as_ref(), self.org_id) { + let key = ReservoirRuleKey::new(org_id, rule); + + let mut redis_client = pool.client().ok()?; + let mut redis_connection = redis_client.connection().ok()?; + + if set_redis_expiry(&mut redis_connection, &key, _rule_expiry).is_err() { + relay_log::error!("failed to set redis reservoir rule expiry"); + } + + match increment_redis_reservoir_count(&mut redis_connection, &key) { + Ok(redis_val) => return Some(redis_val), + Err(e) => relay_log::error!("failed to increment redis reservoir count: {}", e), + } + } + } + + None + } + + /// Gets the local count of a reservoir rule, and increments it. + fn local_count(&self, rule: RuleId) -> anyhow::Result { + let Ok(mut map_guard) = self.counters.lock() else { + return Err(anyhow::Error::msg("failed to lock reservoir counter mutex")); }; let counter_value = map_guard.entry(rule).or_insert(0); + *counter_value += 1; - // Avoid Redis call if limit has already been reached. - if *counter_value >= limit { - return false; - } + Ok(*counter_value) + } - // The new value for the counter will be incremented by one, or updated with the - // global redis counter if it's available. - #[cfg_attr(not(feature = "redis"), allow(unused_mut))] - let mut new_val: i64 = *counter_value + 1; + fn update_counter(&self, rule: RuleId, new_value: i64) { + let Ok(mut map_guard) = self.counters.lock() else { + relay_log::error!("failed to lock reservoir counter mutex"); + return; + }; - #[cfg(feature = "redis")] - { - if let (Some(pool), Some(org_id)) = (self.redis_pool.as_ref(), self.org_id) { - match increment_redis_reservoir_count(pool, org_id, rule) { - Ok(redis_val) => new_val = redis_val, - Err(e) => relay_log::error!("failed to increment redis reservoir count: {}", e), - } + match map_guard.get_mut(&rule) { + Some(value) => *value = new_value, + // Logging an error because at this point the value should definitively be here. + None => relay_log::error!("failed to retrieve counter entry"), + } + } + + /// Evaluates a reservoir rule, returning `true` if it should be sampled. + pub fn evaluate(&self, rule: RuleId, limit: i64, rule_expiry: Option<&DateTime>) -> bool { + let incremented_local_count = match self.local_count(rule) { + Ok(local_count) => local_count, + Err(e) => { + relay_log::error!("failed to read local reservoir count: {}", e); + return false; } + }; + + if incremented_local_count >= limit { + return false; } - // Update the counter. - *counter_value = new_val; + let redis_count = self.redis_count(rule, rule_expiry); - // We sample the rule if the limit has not been reached. - *counter_value < limit + match redis_count { + Some(redis_count) if redis_count > incremented_local_count => { + self.update_counter(rule, redis_count); + redis_count < limit + } + _ => incremented_local_count < limit, + } } } /// State machine for dynamic sampling. #[derive(Debug)] -pub struct SamplingEvaluator { +pub struct SamplingEvaluator<'a> { now: DateTime, rule_ids: Vec, factor: f64, client_sample_rate: Option, - reservoir: Option>, + reservoir: Option<&'a ReservoirEvaluator>, } -impl SamplingEvaluator { +impl<'a> SamplingEvaluator<'a> { /// Constructor for [`SamplingEvaluator`]. pub fn new(now: DateTime) -> Self { Self { @@ -152,7 +171,7 @@ impl SamplingEvaluator { } /// Sets a [`ReservoirEvaluator`]. - pub fn set_reservoir(mut self, reservoir: Option>) -> Self { + pub fn set_reservoir(mut self, reservoir: Option<&'a ReservoirEvaluator>) -> Self { self.reservoir = reservoir; self } @@ -174,7 +193,7 @@ impl SamplingEvaluator { /// - If this value is returned and there are no more rules to evaluate, it should be interpreted as "no match." /// /// - `ControlFlow::Break`: Indicates that one or more rules have successfully matched. - pub fn match_rules<'a, I, G>( + pub fn match_rules<'b, I, G>( mut self, seed: Uuid, instance: &G, @@ -182,14 +201,14 @@ impl SamplingEvaluator { ) -> ControlFlow where G: Getter, - I: Iterator, + I: Iterator, { for rule in rules { if !rule.condition.matches(instance) { continue; }; - let Some(sampling_value) = rule.evaluate(self.now, self.reservoir.as_ref()) else { + let Some(sampling_value) = rule.evaluate(self.now, self.reservoir) else { continue; }; diff --git a/relay-sampling/src/lib.rs b/relay-sampling/src/lib.rs index ce777bc4e1..b79edefbd8 100644 --- a/relay-sampling/src/lib.rs +++ b/relay-sampling/src/lib.rs @@ -77,6 +77,8 @@ pub mod condition; pub mod config; pub mod dsc; pub mod evaluation; +#[cfg(feature = "redis")] +mod redis_sampling; mod utils; pub use config::SamplingConfig; diff --git a/relay-sampling/src/redis_sampling.rs b/relay-sampling/src/redis_sampling.rs new file mode 100644 index 0000000000..a0da5421e0 --- /dev/null +++ b/relay-sampling/src/redis_sampling.rs @@ -0,0 +1,47 @@ +use chrono::{DateTime, Utc}; + +use crate::config::RuleId; + +pub struct ReservoirRuleKey(String); + +impl ReservoirRuleKey { + pub fn new(org_id: u64, rule_id: RuleId) -> Self { + Self(format!("reservoir:{}:{}", org_id, rule_id)) + } + + fn as_str(&self) -> &str { + self.0.as_str() + } +} + +/// Increments the reservoir count for a given rule in redis. +/// +/// - INCR docs: [`https://redis.io/commands/incr/`] +/// - If the counter doesn't exist in redis, a new one will be inserted. +pub fn increment_redis_reservoir_count( + redis_connection: &mut relay_redis::Connection, + key: &ReservoirRuleKey, +) -> anyhow::Result { + let mut command = relay_redis::redis::cmd("INCR"); + command.arg(key.as_str()); + let val = command.query(redis_connection)?; + + Ok(val) +} + +pub fn set_redis_expiry( + redis_connection: &mut relay_redis::Connection, + key: &ReservoirRuleKey, + rule_expiry: Option<&DateTime>, +) -> anyhow::Result<()> { + let now = Utc::now().timestamp(); + let expiry_time = rule_expiry + .map(|rule_expiry| rule_expiry.timestamp()) + .unwrap_or_else(|| now + 86400); + + let ttl = expiry_time - now; + let mut expire_command = relay_redis::redis::cmd("EXPIRE"); + expire_command.arg(key.as_str()).arg(ttl); + expire_command.query(redis_connection)?; + Ok(()) +} diff --git a/relay-server/src/actors/processor.rs b/relay-server/src/actors/processor.rs index dcdfcd1a3f..6a3b0cfb74 100644 --- a/relay-server/src/actors/processor.rs +++ b/relay-server/src/actors/processor.rs @@ -311,7 +311,7 @@ struct ProcessEnvelopeState { has_profile: bool, /// Reservoir evaluator that we use for dynamic sampling. - reservoir: Arc, + reservoir: ReservoirEvaluator, } impl ProcessEnvelopeState { @@ -1363,18 +1363,13 @@ impl EnvelopeProcessorService { // 2. The DSN was moved and the envelope sent to the old project ID. envelope.meta_mut().set_project_id(project_id); - let reservoir = { - #[allow(unused_mut)] - let mut reservoir = ReservoirEvaluator::new(reservoir_counters); - - #[cfg(feature = "processing")] - if let Some(redis_pool) = self.inner.redis_pool.as_ref() { - let org_id = managed_envelope.scoping().organization_id; - reservoir = reservoir.set_redis(org_id, redis_pool.clone()); - } - - Arc::new(reservoir) - }; + #[allow(unused_mut)] + let mut reservoir = ReservoirEvaluator::new(reservoir_counters); + #[cfg(feature = "processing")] + if let Some(redis_pool) = self.inner.redis_pool.as_ref() { + let org_id = managed_envelope.scoping().organization_id; + reservoir = reservoir.set_redis(org_id, redis_pool.clone()); + } Ok(ProcessEnvelopeState { event: Annotated::empty(), @@ -2369,7 +2364,7 @@ impl EnvelopeProcessorService { if config.is_enabled() { state.sampling_result = Self::compute_sampling_decision( self.inner.config.processing_enabled(), - state.reservoir.clone(), + &state.reservoir, state.project_state.config.dynamic_sampling.as_ref(), state.event.value(), state @@ -2389,7 +2384,7 @@ impl EnvelopeProcessorService { /// Computes the sampling decision on the incoming transaction. fn compute_sampling_decision( processing_enabled: bool, - reservoir: Arc, + reservoir: &ReservoirEvaluator, sampling_config: Option<&SamplingConfig>, event: Option<&Event>, root_sampling_config: Option<&SamplingConfig>, @@ -3062,8 +3057,8 @@ mod tests { } } - fn dummy_reservoir() -> Arc { - ReservoirEvaluator::new(ReservoirCounters::default()).into() + fn dummy_reservoir() -> ReservoirEvaluator { + ReservoirEvaluator::new(ReservoirCounters::default()) } fn mocked_event(event_type: EventType, transaction: &str, release: &str) -> Event { @@ -3267,7 +3262,7 @@ mod tests { // pipeline. let res = EnvelopeProcessorService::compute_sampling_decision( false, - dummy_reservoir(), + &dummy_reservoir(), Some(&sampling_config), Some(&event), None, @@ -4167,7 +4162,7 @@ mod tests { let res = EnvelopeProcessorService::compute_sampling_decision( false, - dummy_reservoir(), + &dummy_reservoir(), Some(&sampling_config), Some(&event), None, @@ -4206,7 +4201,7 @@ mod tests { // Unsupported rule should result in no match if processing is not enabled. let res = EnvelopeProcessorService::compute_sampling_decision( false, - dummy_reservoir(), + &dummy_reservoir(), Some(&sampling_config), Some(&event), None, @@ -4217,7 +4212,7 @@ mod tests { // Match if processing is enabled. let res = EnvelopeProcessorService::compute_sampling_decision( true, - dummy_reservoir(), + &dummy_reservoir(), Some(&sampling_config), Some(&event), None, @@ -4258,7 +4253,7 @@ mod tests { let res = EnvelopeProcessorService::compute_sampling_decision( false, - dummy_reservoir(), + &dummy_reservoir(), None, None, Some(&sampling_config), @@ -4271,7 +4266,7 @@ mod tests { let res = EnvelopeProcessorService::compute_sampling_decision( false, - dummy_reservoir(), + &dummy_reservoir(), None, None, Some(&sampling_config), diff --git a/relay-server/src/actors/project.rs b/relay-server/src/actors/project.rs index a18f4f9754..ade1417d4a 100644 --- a/relay-server/src/actors/project.rs +++ b/relay-server/src/actors/project.rs @@ -429,8 +429,8 @@ impl Project { self.reservoir_counters.clone() } - /// If a reservoir rule is no longer in the sampling config, we can assume it's no longer needed. - pub fn remove_expired_reservoir_rules(&self) { + /// If a reservoir rule is no longer in the sampling config, we will remove those counters. + fn remove_expired_reservoir_rules(&self) { let Some(config) = self .state .as_ref() @@ -439,6 +439,7 @@ impl Project { return; }; + // Using try_lock to not slow down the project cache service. if let Ok(mut guard) = self.reservoir_counters.try_lock() { guard.retain(|key, _| config.rules_v2.iter().any(|rule| rule.id == *key)); } @@ -765,6 +766,9 @@ impl Project { // Flush all waiting recipients. relay_log::debug!("project state {} updated", self.project_key); channel.inner.send(state); + + // Check if the new sampling config got rid of any reservoir rules we have counters for. + self.remove_expired_reservoir_rules(); } /// Creates `Scoping` for this project if the state is loaded. diff --git a/relay-server/src/actors/project_cache.rs b/relay-server/src/actors/project_cache.rs index e69d7945cd..b126ef1bd3 100644 --- a/relay-server/src/actors/project_cache.rs +++ b/relay-server/src/actors/project_cache.rs @@ -572,7 +572,6 @@ impl ProjectCacheBroker { let project_cache = self.services.project_cache.clone(); let project = self.get_or_create_project(project_key); project.update_state(project_cache, state.clone(), no_cache); - project.remove_expired_reservoir_rules(); if !state.invalid() { self.dequeue(project_key); diff --git a/relay-server/src/utils/dynamic_sampling.rs b/relay-server/src/utils/dynamic_sampling.rs index e127893ab6..f0b4ffaa44 100644 --- a/relay-server/src/utils/dynamic_sampling.rs +++ b/relay-server/src/utils/dynamic_sampling.rs @@ -53,7 +53,7 @@ impl SamplingResult { } } -impl From> for SamplingResult { +impl From>> for SamplingResult { fn from(value: ControlFlow) -> Self { match value { ControlFlow::Break(sampling_match) => Self::Match(sampling_match), From f431058e96adaaf64ff5dc7915e589d0f22a168b Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Thu, 28 Sep 2023 15:07:21 +0200 Subject: [PATCH 20/30] wip --- relay-sampling/src/evaluation.rs | 55 ++++++++++++++++---------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/relay-sampling/src/evaluation.rs b/relay-sampling/src/evaluation.rs index bf3a9a9d8b..dadd168060 100644 --- a/relay-sampling/src/evaluation.rs +++ b/relay-sampling/src/evaluation.rs @@ -17,7 +17,8 @@ use serde::Serialize; use uuid::Uuid; use crate::config::{RuleId, SamplingRule, SamplingValue}; -use crate::redis_sampling::{increment_redis_reservoir_count, set_redis_expiry, ReservoirRuleKey}; +#[cfg(feature = "redis")] +use crate::redis_sampling::{self, ReservoirRuleKey}; /// Generates a pseudo random number by seeding the generator with the given id. /// @@ -75,38 +76,37 @@ impl ReservoirEvaluator { self } - fn redis_count(&self, rule: RuleId, _rule_expiry: Option<&DateTime>) -> Option { - if cfg!(feature = "redis") { - if let (Some(pool), Some(org_id)) = (self.redis_pool.as_ref(), self.org_id) { - let key = ReservoirRuleKey::new(org_id, rule); + fn redis_count(&self, _rule: RuleId, _rule_expiry: Option<&DateTime>) -> Option { + #[cfg(feature = "redis")] + if let (Some(pool), Some(org_id)) = (self.redis_pool.as_ref(), self.org_id) { + let key = ReservoirRuleKey::new(org_id, _rule); - let mut redis_client = pool.client().ok()?; - let mut redis_connection = redis_client.connection().ok()?; + let mut redis_client = pool.client().ok()?; + let mut redis_connection = redis_client.connection().ok()?; - if set_redis_expiry(&mut redis_connection, &key, _rule_expiry).is_err() { - relay_log::error!("failed to set redis reservoir rule expiry"); - } + if crate::redis_sampling::set_redis_expiry(&mut redis_connection, &key, _rule_expiry) + .is_err() + { + relay_log::error!("failed to set redis reservoir rule expiry"); + } - match increment_redis_reservoir_count(&mut redis_connection, &key) { - Ok(redis_val) => return Some(redis_val), - Err(e) => relay_log::error!("failed to increment redis reservoir count: {}", e), - } + match redis_sampling::increment_redis_reservoir_count(&mut redis_connection, &key) { + Ok(redis_val) => return Some(redis_val), + Err(e) => relay_log::error!("failed to increment redis reservoir count: {}", e), } } None } - /// Gets the local count of a reservoir rule, and increments it. - fn local_count(&self, rule: RuleId) -> anyhow::Result { - let Ok(mut map_guard) = self.counters.lock() else { - return Err(anyhow::Error::msg("failed to lock reservoir counter mutex")); - }; + /// Gets the local count of a reservoir rule. Increments the count before returning it. + fn local_count(&self, rule: RuleId) -> Option { + let mut map_guard = self.counters.lock().ok()?; let counter_value = map_guard.entry(rule).or_insert(0); *counter_value += 1; - Ok(*counter_value) + Some(*counter_value) } fn update_counter(&self, rule: RuleId, new_value: i64) { @@ -124,19 +124,20 @@ impl ReservoirEvaluator { /// Evaluates a reservoir rule, returning `true` if it should be sampled. pub fn evaluate(&self, rule: RuleId, limit: i64, rule_expiry: Option<&DateTime>) -> bool { - let incremented_local_count = match self.local_count(rule) { - Ok(local_count) => local_count, - Err(e) => { - relay_log::error!("failed to read local reservoir count: {}", e); - return false; - } + let Some(incremented_local_count) = self.local_count(rule) else { + relay_log::error!("failed to read local reservoir count"); + return false; }; if incremented_local_count >= limit { return false; } - let redis_count = self.redis_count(rule, rule_expiry); + let redis_count = if cfg!(feature = "redis") { + self.redis_count(rule, rule_expiry) + } else { + None + }; match redis_count { Some(redis_count) if redis_count > incremented_local_count => { From bdb03741af0813343d1126797982f358d883607f Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Thu, 28 Sep 2023 15:14:11 +0200 Subject: [PATCH 21/30] wip --- relay-sampling/src/evaluation.rs | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/relay-sampling/src/evaluation.rs b/relay-sampling/src/evaluation.rs index dadd168060..c946331d25 100644 --- a/relay-sampling/src/evaluation.rs +++ b/relay-sampling/src/evaluation.rs @@ -76,17 +76,16 @@ impl ReservoirEvaluator { self } - fn redis_count(&self, _rule: RuleId, _rule_expiry: Option<&DateTime>) -> Option { + #[allow(unused_variables)] + fn redis_count(&self, rule: RuleId, rule_expiry: Option<&DateTime>) -> Option { #[cfg(feature = "redis")] if let (Some(pool), Some(org_id)) = (self.redis_pool.as_ref(), self.org_id) { - let key = ReservoirRuleKey::new(org_id, _rule); + let key = ReservoirRuleKey::new(org_id, rule); let mut redis_client = pool.client().ok()?; let mut redis_connection = redis_client.connection().ok()?; - if crate::redis_sampling::set_redis_expiry(&mut redis_connection, &key, _rule_expiry) - .is_err() - { + if redis_sampling::set_redis_expiry(&mut redis_connection, &key, rule_expiry).is_err() { relay_log::error!("failed to set redis reservoir rule expiry"); } @@ -133,13 +132,7 @@ impl ReservoirEvaluator { return false; } - let redis_count = if cfg!(feature = "redis") { - self.redis_count(rule, rule_expiry) - } else { - None - }; - - match redis_count { + match self.redis_count(rule, rule_expiry) { Some(redis_count) if redis_count > incremented_local_count => { self.update_counter(rule, redis_count); redis_count < limit From 7a480a84b75f82b4200de39d49db7cad4ef8b5fd Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Thu, 28 Sep 2023 15:29:01 +0200 Subject: [PATCH 22/30] wip --- relay-sampling/src/evaluation.rs | 14 ++++++++------ relay-server/src/actors/processor.rs | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/relay-sampling/src/evaluation.rs b/relay-sampling/src/evaluation.rs index c946331d25..29b0128adc 100644 --- a/relay-sampling/src/evaluation.rs +++ b/relay-sampling/src/evaluation.rs @@ -98,12 +98,14 @@ impl ReservoirEvaluator { None } - /// Gets the local count of a reservoir rule. Increments the count before returning it. - fn local_count(&self, rule: RuleId) -> Option { + /// Gets the local count of a reservoir rule. Increments the count if limit isnt reached. + fn local_count(&self, rule: RuleId, limit: i64) -> Option { let mut map_guard = self.counters.lock().ok()?; let counter_value = map_guard.entry(rule).or_insert(0); - *counter_value += 1; + if *counter_value < limit { + *counter_value += 1; + } Some(*counter_value) } @@ -123,7 +125,7 @@ impl ReservoirEvaluator { /// Evaluates a reservoir rule, returning `true` if it should be sampled. pub fn evaluate(&self, rule: RuleId, limit: i64, rule_expiry: Option<&DateTime>) -> bool { - let Some(incremented_local_count) = self.local_count(rule) else { + let Some(incremented_local_count) = self.local_count(rule, limit) else { relay_log::error!("failed to read local reservoir count"); return false; }; @@ -165,8 +167,8 @@ impl<'a> SamplingEvaluator<'a> { } /// Sets a [`ReservoirEvaluator`]. - pub fn set_reservoir(mut self, reservoir: Option<&'a ReservoirEvaluator>) -> Self { - self.reservoir = reservoir; + pub fn set_reservoir(mut self, reservoir: &'a ReservoirEvaluator) -> Self { + self.reservoir = Some(reservoir); self } diff --git a/relay-server/src/actors/processor.rs b/relay-server/src/actors/processor.rs index 6a3b0cfb74..74b0eeec35 100644 --- a/relay-server/src/actors/processor.rs +++ b/relay-server/src/actors/processor.rs @@ -2426,7 +2426,7 @@ impl EnvelopeProcessorService { let mut evaluator = SamplingEvaluator::new(Utc::now()) .adjust_client_sample_rate(adjustment_rate) - .set_reservoir(Some(reservoir)); + .set_reservoir(reservoir); if let (Some(event), Some(sampling_state)) = (event, sampling_config) { if let Some(seed) = event.id.value().map(|id| id.0) { From 8a5cb82c38a546680ed14bbdb315ad39d3d79915 Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Thu, 28 Sep 2023 15:45:11 +0200 Subject: [PATCH 23/30] wip --- relay-server/src/actors/project_cache.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/relay-server/src/actors/project_cache.rs b/relay-server/src/actors/project_cache.rs index b126ef1bd3..616d7400e6 100644 --- a/relay-server/src/actors/project_cache.rs +++ b/relay-server/src/actors/project_cache.rs @@ -570,8 +570,11 @@ impl ProjectCacheBroker { } = message; let project_cache = self.services.project_cache.clone(); - let project = self.get_or_create_project(project_key); - project.update_state(project_cache, state.clone(), no_cache); + self.get_or_create_project(project_key).update_state( + project_cache, + state.clone(), + no_cache, + ); if !state.invalid() { self.dequeue(project_key); From 9ac7012154297f0892a3eafb03d42191f29f94fe Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Thu, 28 Sep 2023 19:00:32 +0200 Subject: [PATCH 24/30] wip --- relay-sampling/src/config.rs | 1 + relay-sampling/src/evaluation.rs | 108 ++++++++++++++++----------- relay-server/src/actors/processor.rs | 14 ++-- 3 files changed, 71 insertions(+), 52 deletions(-) diff --git a/relay-sampling/src/config.rs b/relay-sampling/src/config.rs index 9addfceb54..ffc81aadc5 100644 --- a/relay-sampling/src/config.rs +++ b/relay-sampling/src/config.rs @@ -83,6 +83,7 @@ impl SamplingRule { } /// Returns the updated [`SamplingValue`] if it's valid. + /// This function is scheduled for demolition. pub fn evaluate( &self, now: DateTime, diff --git a/relay-sampling/src/evaluation.rs b/relay-sampling/src/evaluation.rs index 29b0128adc..257eec7ee9 100644 --- a/relay-sampling/src/evaluation.rs +++ b/relay-sampling/src/evaluation.rs @@ -1,5 +1,6 @@ //! Evaluation of dynamic sampling rules. +use std::cmp::Ordering; use std::collections::BTreeMap; use std::fmt; use std::num::ParseIntError; @@ -46,15 +47,16 @@ pub type ReservoirCounters = Arc>>; /// across multiple relay-instances. When incremented, the Redis counter returns the current global /// count for the given rule, which is then used to update the local counter. #[derive(Debug)] -pub struct ReservoirEvaluator { +pub struct ReservoirEvaluator<'a> { counters: ReservoirCounters, #[cfg(feature = "redis")] - redis_pool: Option>, + redis_pool: Option<&'a RedisPool>, #[cfg(feature = "redis")] org_id: Option, + _phantom: std::marker::PhantomData<&'a ()>, // Using PhantomData to associate the unused lifetime } -impl ReservoirEvaluator { +impl<'a> ReservoirEvaluator<'a> { /// Constructor for [`ReservoirEvaluator`]. pub fn new(counters: ReservoirCounters) -> Self { Self { @@ -63,6 +65,7 @@ impl ReservoirEvaluator { org_id: None, #[cfg(feature = "redis")] redis_pool: None, + _phantom: std::marker::PhantomData, } } @@ -70,49 +73,41 @@ impl ReservoirEvaluator { /// /// These values are needed to synchronize with Redis. #[cfg(feature = "redis")] - pub fn set_redis(mut self, org_id: u64, redis_pool: Arc) -> Self { + pub fn set_redis(&mut self, org_id: u64, redis_pool: &'a RedisPool) { self.org_id = Some(org_id); self.redis_pool = Some(redis_pool); - self } - #[allow(unused_variables)] - fn redis_count(&self, rule: RuleId, rule_expiry: Option<&DateTime>) -> Option { - #[cfg(feature = "redis")] - if let (Some(pool), Some(org_id)) = (self.redis_pool.as_ref(), self.org_id) { - let key = ReservoirRuleKey::new(org_id, rule); - - let mut redis_client = pool.client().ok()?; - let mut redis_connection = redis_client.connection().ok()?; - - if redis_sampling::set_redis_expiry(&mut redis_connection, &key, rule_expiry).is_err() { - relay_log::error!("failed to set redis reservoir rule expiry"); - } - - match redis_sampling::increment_redis_reservoir_count(&mut redis_connection, &key) { - Ok(redis_val) => return Some(redis_val), - Err(e) => relay_log::error!("failed to increment redis reservoir count: {}", e), + #[cfg(feature = "redis")] + fn redis_count( + &self, + key: &ReservoirRuleKey, + redis_pool: &RedisPool, + rule_expiry: Option<&DateTime>, + ) -> anyhow::Result { + let mut redis_client = redis_pool.client()?; + let mut redis_connection = redis_client.connection()?; + + let val = match redis_sampling::increment_redis_reservoir_count(&mut redis_connection, key) + { + Ok(val) => val, + Err(e) => { + relay_log::error!("failed to increment redis value: {:?}", e); + return Err(e); } - } - - None - } - - /// Gets the local count of a reservoir rule. Increments the count if limit isnt reached. - fn local_count(&self, rule: RuleId, limit: i64) -> Option { - let mut map_guard = self.counters.lock().ok()?; + }; - let counter_value = map_guard.entry(rule).or_insert(0); - if *counter_value < limit { - *counter_value += 1; + if let Err(e) = redis_sampling::set_redis_expiry(&mut redis_connection, key, rule_expiry) { + relay_log::error!("failed to set redis reservoir rule expiry"); + return Err(e); } - Some(*counter_value) + Ok(val) } + #[cfg(feature = "redis")] fn update_counter(&self, rule: RuleId, new_value: i64) { let Ok(mut map_guard) = self.counters.lock() else { - relay_log::error!("failed to lock reservoir counter mutex"); return; }; @@ -123,24 +118,47 @@ impl ReservoirEvaluator { } } + /// Gets the local count of a reserovir rule and increments it, if the limit has yet to be reached + fn local_count(&self, rule: RuleId, limit: i64) -> Option { + let Ok(mut map_guard) = self.counters.lock() else { + relay_log::error!("failed to lock reservoir counter mutex"); + return None; + }; + + let counter_value = map_guard.entry(rule).or_insert(0); + + match (*counter_value).cmp(&limit) { + Ordering::Less => *counter_value += 1, + // Limit has already been reached. + Ordering::Equal | Ordering::Greater => return None, + } + + Some(*counter_value) + } + /// Evaluates a reservoir rule, returning `true` if it should be sampled. - pub fn evaluate(&self, rule: RuleId, limit: i64, rule_expiry: Option<&DateTime>) -> bool { + pub fn evaluate(&self, rule: RuleId, limit: i64, _rule_expiry: Option<&DateTime>) -> bool { let Some(incremented_local_count) = self.local_count(rule, limit) else { - relay_log::error!("failed to read local reservoir count"); return false; }; - if incremented_local_count >= limit { - return false; - } + #[cfg(feature = "redis")] + if let (Some(org_id), Some(redis_pool)) = (self.org_id, self.redis_pool.as_ref()) { + let key = ReservoirRuleKey::new(org_id, rule); - match self.redis_count(rule, rule_expiry) { - Some(redis_count) if redis_count > incremented_local_count => { + let Ok(redis_count) = self.redis_count(&key, redis_pool, _rule_expiry) else { + return false; + }; + + if redis_count > incremented_local_count { self.update_counter(rule, redis_count); - redis_count < limit - } - _ => incremented_local_count < limit, + return redis_count <= limit; + }; } + + // We also return `true` if it's equal to the limit, since the incremented rule count + // represents the sampling that will be done after this function is run. + incremented_local_count <= limit } } @@ -151,7 +169,7 @@ pub struct SamplingEvaluator<'a> { rule_ids: Vec, factor: f64, client_sample_rate: Option, - reservoir: Option<&'a ReservoirEvaluator>, + reservoir: Option<&'a ReservoirEvaluator<'a>>, } impl<'a> SamplingEvaluator<'a> { diff --git a/relay-server/src/actors/processor.rs b/relay-server/src/actors/processor.rs index 74b0eeec35..0f47f99344 100644 --- a/relay-server/src/actors/processor.rs +++ b/relay-server/src/actors/processor.rs @@ -256,7 +256,7 @@ impl ExtractedMetrics { /// A state container for envelope processing. #[derive(Debug)] -struct ProcessEnvelopeState { +struct ProcessEnvelopeState<'a> { /// The extracted event payload. /// /// For Envelopes without event payloads, this contains `Annotated::empty`. If a single item has @@ -311,10 +311,10 @@ struct ProcessEnvelopeState { has_profile: bool, /// Reservoir evaluator that we use for dynamic sampling. - reservoir: ReservoirEvaluator, + reservoir: ReservoirEvaluator<'a>, } -impl ProcessEnvelopeState { +impl<'a> ProcessEnvelopeState<'a> { /// Returns a reference to the contained [`Envelope`]. fn envelope(&self) -> &Envelope { self.managed_envelope.envelope() @@ -540,7 +540,7 @@ pub struct EnvelopeProcessorService { struct InnerProcessor { config: Arc, #[cfg(feature = "processing")] - redis_pool: Option>, + redis_pool: Option, envelope_manager: Addr, project_cache: Addr, global_config: Addr, @@ -574,7 +574,7 @@ impl EnvelopeProcessorService { let inner = InnerProcessor { #[cfg(feature = "processing")] - redis_pool: _redis.clone().map(Arc::new), + redis_pool: _redis.clone(), #[cfg(feature = "processing")] rate_limiter: _redis .map(|pool| RedisRateLimiter::new(pool).max_limit(config.max_rate_limit())), @@ -1368,7 +1368,7 @@ impl EnvelopeProcessorService { #[cfg(feature = "processing")] if let Some(redis_pool) = self.inner.redis_pool.as_ref() { let org_id = managed_envelope.scoping().organization_id; - reservoir = reservoir.set_redis(org_id, redis_pool.clone()); + reservoir.set_redis(org_id, redis_pool); } Ok(ProcessEnvelopeState { @@ -3057,7 +3057,7 @@ mod tests { } } - fn dummy_reservoir() -> ReservoirEvaluator { + fn dummy_reservoir() -> ReservoirEvaluator<'static> { ReservoirEvaluator::new(ReservoirCounters::default()) } From ecb65da03f76aa1f27147051b09e0ed18ca35784 Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Thu, 28 Sep 2023 19:32:34 +0200 Subject: [PATCH 25/30] wip --- relay-sampling/src/config.rs | 2 +- relay-sampling/src/evaluation.rs | 20 +++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/relay-sampling/src/config.rs b/relay-sampling/src/config.rs index ffc81aadc5..352467fe94 100644 --- a/relay-sampling/src/config.rs +++ b/relay-sampling/src/config.rs @@ -83,7 +83,7 @@ impl SamplingRule { } /// Returns the updated [`SamplingValue`] if it's valid. - /// This function is scheduled for demolition. + /// todo(tor): Refactor this function. pub fn evaluate( &self, now: DateTime, diff --git a/relay-sampling/src/evaluation.rs b/relay-sampling/src/evaluation.rs index 257eec7ee9..f5970c9187 100644 --- a/relay-sampling/src/evaluation.rs +++ b/relay-sampling/src/evaluation.rs @@ -53,7 +53,8 @@ pub struct ReservoirEvaluator<'a> { redis_pool: Option<&'a RedisPool>, #[cfg(feature = "redis")] org_id: Option, - _phantom: std::marker::PhantomData<&'a ()>, // Using PhantomData to associate the unused lifetime + // Using PhantomData because the lifetimes are behind a processing flag. + _phantom: std::marker::PhantomData<&'a ()>, } impl<'a> ReservoirEvaluator<'a> { @@ -128,12 +129,15 @@ impl<'a> ReservoirEvaluator<'a> { let counter_value = map_guard.entry(rule).or_insert(0); match (*counter_value).cmp(&limit) { - Ordering::Less => *counter_value += 1, + // Limit not yet reached. Eagerly incrementing to avoid an additional lock + // in the case where it doesn't get overrwritten by the redis count. + Ordering::Less => { + *counter_value += 1; + Some(*counter_value) + } // Limit has already been reached. - Ordering::Equal | Ordering::Greater => return None, + Ordering::Equal | Ordering::Greater => None, } - - Some(*counter_value) } /// Evaluates a reservoir rule, returning `true` if it should be sampled. @@ -147,6 +151,10 @@ impl<'a> ReservoirEvaluator<'a> { let key = ReservoirRuleKey::new(org_id, rule); let Ok(redis_count) = self.redis_count(&key, redis_pool, _rule_expiry) else { + // We don't sample at all if we lost access to redis. + // Therefore we revert the previous increment. + // Seems inefficient, but this should be a rare occurence. + self.update_counter(rule, incremented_local_count - 1); return false; }; @@ -156,8 +164,6 @@ impl<'a> ReservoirEvaluator<'a> { }; } - // We also return `true` if it's equal to the limit, since the incremented rule count - // represents the sampling that will be done after this function is run. incremented_local_count <= limit } } From 9fb1c3f032c98e67a9a1224215257e479e4502c3 Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Thu, 28 Sep 2023 20:02:08 +0200 Subject: [PATCH 26/30] add test --- relay-sampling/src/evaluation.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/relay-sampling/src/evaluation.rs b/relay-sampling/src/evaluation.rs index f5970c9187..df7cfe982b 100644 --- a/relay-sampling/src/evaluation.rs +++ b/relay-sampling/src/evaluation.rs @@ -415,6 +415,18 @@ mod tests { use super::*; + fn mock_reservoir_evaluator(vals: Vec<(u32, i64)>) -> ReservoirEvaluator<'static> { + let mut map = BTreeMap::default(); + + for (rule_id, count) in vals { + map.insert(RuleId(rule_id), count); + } + + let map = Arc::new(Mutex::new(map)); + + ReservoirEvaluator::new(map) + } + /// Helper to extract the sampling match after evaluating rules. fn get_sampling_match(rules: &[SamplingRule], instance: &impl Getter) -> SamplingMatch { match SamplingEvaluator::new(Utc::now()).match_rules( @@ -470,6 +482,21 @@ mod tests { dsc } + #[test] + fn test_reservoir_evaluator_limit() { + let evaluator = mock_reservoir_evaluator(vec![(1, 0)]); + + let rule = RuleId(1); + let limit = 3; + + assert!(evaluator.evaluate(rule, limit, None)); + assert!(evaluator.evaluate(rule, limit, None)); + assert!(evaluator.evaluate(rule, limit, None)); + // After 3 samples we have reached the limit, and the following rules are not sampled. + assert!(!evaluator.evaluate(rule, limit, None)); + assert!(!evaluator.evaluate(rule, limit, None)); + } + #[test] fn test_adjust_sample_rate() { // return the same as input if no client sample rate set in the sampling evaluator. From 9b0d8fd8877863a916da1876d5e6704f160d7da4 Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Thu, 28 Sep 2023 21:44:21 +0200 Subject: [PATCH 27/30] add test --- bat | 1838 ++++++++++++++++++++++++++++++ relay-sampling/src/evaluation.rs | 53 + 2 files changed, 1891 insertions(+) create mode 100644 bat diff --git a/bat b/bat new file mode 100644 index 0000000000..e7a8bfde71 --- /dev/null +++ b/bat @@ -0,0 +1,1838 @@ +cargo test --workspace --all-features + +running 1 test +test tests::test_parse_metrics ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.04s + + +running 7 tests +test tests::test_find_rs_files ... ok +test tests::test_single_type ... ok +test tests::test_pii_false ... ok +test tests::test_scoped_paths ... ok +test tests::test_pii_true ... ok +test tests::test_pii_all ... ok +test tests::test_pii_retain_additional_properties_truth_table ... ok + +test result: ok. 7 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.04s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 12 tests +test tests::test_deserialize_old_response ... ok +test tests::test_relay_version_any_supported ... ok +test tests::test_relay_version_current ... ok +test tests::test_relay_version_from_str ... ok +test tests::test_relay_version_oldest_supported ... ok +test tests::test_relay_version_oldest ... ok +test tests::test_relay_version_parse ... ok +test tests::test_keys ... ok +test tests::test_serializing ... ok +test tests::test_signatures ... ok +test tests::test_generate_strings_for_test_auth_py ... ok +test tests::test_registration ... ok + +test result: ok. 12 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 1 test +test metrics::tests::test_custom_unit_parse ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 3 tests +test processing::pii_config_validation_invalid_regex ... ok +test processing::pii_config_validation_valid_regex ... ok +test codeowners::tests::test_translate_codeowners_pattern ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 25 tests +test glob3::tests::test_match_negative ... ok +test glob3::tests::test_match_neg_unsupported ... ok +test glob3::tests::test_match_literal ... ok +test glob3::tests::test_match_newline ... ok +test glob3::tests::test_match_newline_inner ... ok +test glob3::tests::test_match_inner ... ok +test glob3::tests::test_match_prefix ... ok +test glob3::tests::test_match_newline_pattern ... ok +test glob3::tests::test_match_utf8 ... ok +test glob3::tests::test_match_range ... ok +test time::tests::test_parse_datetime_bogus ... ok +test glob3::tests::test_match_suffix ... ok +test glob3::tests::test_match_range_neg ... ok +test time::tests::test_parse_timestamp_float ... ok +test time::tests::test_parse_timestamp_int ... ok +test time::tests::test_parse_timestamp_large_float ... ok +test glob2::tests::test_do_not_replace ... ok +test time::tests::test_parse_timestamp_neg_float ... ok +test time::tests::test_parse_timestamp_neg_int ... ok +test time::tests::test_parse_timestamp_other ... ok +test time::tests::test_parse_timestamp_str ... ok +test glob2::tests::test_glob_matcher ... ok +test glob2::tests::test_glob_replace ... ok +test glob2::tests::test_glob ... ok +test glob::tests::test_globs ... ok + +test result: ok. 25 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.04s + + +running 10 tests +test byte_size::tests::test_as_bytes ... ok +test byte_size::tests::test_infer ... ok +test config::tests::test_emit_outcomes_invalid ... ok +test byte_size::tests::test_parse ... ok +test byte_size::tests::test_serde_number ... ok +test config::tests::test_emit_outcomes ... ok +test byte_size::tests::test_serde_string ... ok +test upstream::test::test_from_dsn ... ok +test upstream::test::test_basic_parsing ... ok +test config::tests::test_event_buffer_size ... ok + +test result: ok. 10 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 39 tests +test native::bindgen_test_layout___darwin_arm_cpmu_state64 ... ok +test native::bindgen_test_layout___arm_legacy_debug_state ... ok +test native::bindgen_test_layout___darwin_arm_debug_state32 ... ok +test native::bindgen_test_layout___darwin_arm_debug_state64 ... ok +test native::bindgen_test_layout___arm_pagein_state ... ok +test native::bindgen_test_layout___darwin_arm_exception_state64 ... ok +test native::bindgen_test_layout___darwin_arm_exception_state ... ok +test native::bindgen_test_layout___darwin_arm_neon_state ... ok +test native::bindgen_test_layout___darwin_arm_neon_state64 ... ok +test native::bindgen_test_layout___darwin_arm_thread_state ... ok +test native::bindgen_test_layout___darwin_arm_thread_state64 ... ok +test native::bindgen_test_layout___darwin_arm_vfp_state ... ok +test native::bindgen_test_layout___darwin_mcontext32 ... ok +test native::bindgen_test_layout___darwin_mcontext64 ... ok +test native::bindgen_test_layout___darwin_pthread_handler_rec ... ok +test native::bindgen_test_layout___darwin_sigaltstack ... ok +test native::bindgen_test_layout___darwin_ucontext ... ok +test native::bindgen_test_layout___mbstate_t ... ok +test native::bindgen_test_layout___sigaction ... ok +test native::bindgen_test_layout___sigaction_u ... ok +test native::bindgen_test_layout___siginfo ... ok +test native::bindgen_test_layout__opaque_pthread_attr_t ... ok +test native::bindgen_test_layout__opaque_pthread_cond_t ... ok +test native::bindgen_test_layout__opaque_pthread_condattr_t ... ok +test native::bindgen_test_layout__opaque_pthread_mutex_t ... ok +test native::bindgen_test_layout__opaque_pthread_mutexattr_t ... ok +test native::bindgen_test_layout__opaque_pthread_once_t ... ok +test native::bindgen_test_layout__opaque_pthread_rwlock_t ... ok +test native::bindgen_test_layout__opaque_pthread_t ... ok +test native::bindgen_test_layout__opaque_pthread_rwlockattr_t ... ok +test native::bindgen_test_layout_sentry_ucontext_s ... ok +test native::bindgen_test_layout_imaxdiv_t ... ok +test native::bindgen_test_layout_sentry_uuid_s ... ok +test native::bindgen_test_layout_sentry_value_u ... ok +test native::bindgen_test_layout_sigaction ... ok +test native::bindgen_test_layout_sigevent ... ok +test native::bindgen_test_layout_sigvec ... ok +test native::bindgen_test_layout_sigval ... ok +test native::bindgen_test_layout_sigstack ... ok + +test result: ok. 39 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 1 test +test stats::tests::parses_metric ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 12 tests +test error_boundary::tests::test_deserialize_ok ... ok +test error_boundary::tests::test_deserialize_err ... ok +test error_boundary::tests::test_deserialize_syntax_err ... ok +test error_boundary::tests::test_serialize_err ... ok +test error_boundary::tests::test_serialize_ok ... ok +test feature::tests::roundtrip ... ok +test metrics::tests::parse_tag_spec_field ... ok +test metrics::tests::parse_tag_spec_unsupported ... ok +test global::tests::test_global_config_roundtrip ... ok +test metrics::tests::parse_tag_spec_value ... ok +test utils::tests::test_validate_json ... ok +test metrics::tests::parse_tag_mapping ... ok + +test result: ok. 12 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 423 tests +test clock_drift::tests::test_clock_drift_unix ... ok +test clock_drift::tests::test_process_datetime ... ok +test clock_drift::tests::test_clock_drift_lower_bound ... ok +test clock_drift::tests::test_no_sent_at ... ok +test event_error::tests::test_no_errors ... ok +test event_error::tests::test_nested_errors ... ok +test event_error::tests::test_original_value ... ok +test event_error::tests::test_errors_in_other ... ok +test event_error::tests::test_multiple_errors ... ok +test clock_drift::tests::test_no_clock_drift ... ok +test clock_drift::tests::test_clock_drift_from_past ... ok +test clock_drift::tests::test_clock_drift_from_future ... ok +test event_error::tests::test_top_level_errors ... ok +test normalize::breakdowns::tests::test_noop_breakdowns_with_empty_config ... ok +test normalize::breakdowns::tests::test_skip_with_empty_breakdowns_config ... ok +test normalize::breakdowns::tests::test_emit_ops_breakdown ... ok +test normalize::contexts::tests::test_broken_json_with_fallback ... ok +test normalize::contexts::tests::test_broken_json_without_fallback ... ok +test normalize::contexts::tests::test_infer_json ... ok +test normalize::contexts::tests::test_dotnet_core ... ok +test normalize::contexts::tests::test_dotnet_framework_48_without_build_id ... ok +test normalize::contexts::tests::test_dotnet_framework_472 ... ok +test normalize::contexts::tests::test_dotnet_framework_future_version ... ok +test normalize::contexts::tests::test_dotnet_native ... ok +test normalize::contexts::tests::test_macos_with_build ... ok +test normalize::contexts::tests::test_macos_without_build ... ok +test normalize::contexts::tests::test_name_not_overwritten ... ok +test normalize::contexts::tests::test_no_name ... ok +test normalize::contexts::tests::test_unity_android_api_version ... ok +test normalize::contexts::tests::test_unity_mac_os ... ok +test normalize::contexts::tests::test_linux_5_11 ... ok +test normalize::contexts::tests::test_unity_windows_os ... ok +test normalize::contexts::tests::test_version_not_overwritten ... ok +test normalize::contexts::tests::test_windows_10 ... ok +test normalize::contexts::tests::test_windows_11 ... ok +test normalize::contexts::tests::test_windows_11_future1 ... ok +test normalize::contexts::tests::test_windows_11_future2 ... ok +test normalize::contexts::tests::test_windows_7_or_server_2008 ... ok +test normalize::contexts::tests::test_windows_8_or_server_2012_or_later ... ok +test normalize::logentry::tests::test_empty_logentry ... ok +test normalize::logentry::tests::test_empty_missing_message ... ok +test normalize::contexts::tests::test_wsl_ubuntu ... ok +test normalize::contexts::tests::test_centos_runtime_info ... ok +test normalize::contexts::tests::test_unity_android_os ... ok +test normalize::contexts::tests::test_android_4_4_2 ... ok +test normalize::contexts::tests::test_centos_os_version ... ok +test normalize::contexts::tests::test_macos_runtime ... ok +test normalize::contexts::tests::test_ios_15_0 ... ok +test normalize::contexts::tests::test_macos_os_version ... ok +test normalize::logentry::tests::test_format_no_params ... ok +test normalize::logentry::tests::test_message_formatted_equal ... ok +test normalize::logentry::tests::test_only_message ... ok +test normalize::mechanism::tests::test_normalize_errno ... ok +test normalize::mechanism::tests::test_normalize_errno_fail ... ok +test normalize::mechanism::tests::test_normalize_errno_override ... ok +test normalize::mechanism::tests::test_normalize_mach_fail ... ok +test normalize::mechanism::tests::test_normalize_mach ... ok +test normalize::mechanism::tests::test_normalize_missing ... ok +test normalize::mechanism::tests::test_normalize_mach_override ... ok +test normalize::mechanism::tests::test_normalize_partial_signal ... ok +test normalize::mechanism::tests::test_normalize_signal ... ok +test normalize::mechanism::tests::test_normalize_signal_fail ... ok +test normalize::mechanism::tests::test_normalize_signal_override ... ok +test normalize::request::tests::test_broken_json_with_fallback ... ok +test normalize::request::tests::test_broken_json_without_fallback ... ok +test normalize::request::tests::test_cookies_in_header ... ok +test normalize::request::tests::test_cookies_in_header_dont_override_cookies ... ok +test normalize::request::tests::test_infer_binary ... ok +test normalize::request::tests::test_infer_json ... ok +test normalize::request::tests::test_infer_url_encoded_base64 ... ok +test normalize::request::tests::test_infer_url_encoded ... ok +test normalize::request::tests::test_infer_xml ... ok +test normalize::request::tests::test_infer_url_false_positive ... ok +test normalize::request::tests::test_query_string_empty_value ... ok +test normalize::request::tests::test_url_only_path ... ok +test normalize::request::tests::test_url_precedence ... ok +test normalize::request::tests::test_method_invalid ... ok +test normalize::request::tests::test_url_punycoded ... ok +test normalize::request::tests::test_method_valid ... ok +test normalize::request::tests::test_url_truncation ... ok +test normalize::request::tests::test_url_truncation_reversed ... ok +test normalize::request::tests::test_url_with_ellipsis ... ok +test normalize::request::tests::test_url_with_qs_and_fragment ... ok +test normalize::span::attributes::tests::test_child_spans_consumes_all_of_parent ... ok +test normalize::span::attributes::tests::test_child_spans_dont_intersect_parent ... ok +test normalize::span::attributes::tests::test_child_spans_extend_beyond_parent ... ok +test normalize::span::attributes::tests::test_childless_spans ... ok +test normalize::span::attributes::tests::test_nested_spans ... ok +test normalize::logentry::tests::test_format_python ... ok +test normalize::span::attributes::tests::test_only_immediate_child_spans_affect_calculation ... ok +test normalize::span::attributes::tests::test_overlapping_child_spans ... ok +test normalize::span::attributes::tests::test_skip_exclusive_time ... ok +test normalize::span::description::sql::tests::activerecord ... ok +test normalize::logentry::tests::test_format_python_named ... ok +test normalize::span::description::sql::parser::tests::parse_deep_expression ... ok +test normalize::span::description::sql::tests::boolean_not_in_mid_tablename_true ... ok +test normalize::span::description::sql::tests::already_scrubbed ... ok +test normalize::span::description::sql::tests::boolean_not_in_tablename_true ... ok +test normalize::span::description::sql::tests::boolean_not_in_tablename_false ... ok +test normalize::span::description::sql::tests::boolean_not_in_mid_tablename_false ... ok +test normalize::logentry::tests::test_format_java ... ok +test normalize::logentry::tests::test_format_dotnet ... ok +test normalize::span::description::sql::tests::bytesa ... ok +test normalize::span::description::sql::tests::boolean_where_false ... ok +test normalize::span::description::sql::tests::case_when ... ok +test normalize::span::description::sql::tests::boolean_where_true ... ok +test normalize::span::description::sql::tests::case_when_nested ... ok +test normalize::span::description::sql::tests::boolean_where_bool_insensitive ... ok +test normalize::span::description::sql::tests::close_cursor ... ok +test normalize::span::description::sql::tests::collapse_columns_distinct ... ok +test normalize::span::description::sql::tests::collapse_columns_nested ... ok +test normalize::span::description::sql::tests::collapse_columns ... ok +test normalize::span::description::sql::tests::collapse_columns_with_as ... ok +test normalize::span::description::sql::tests::collapse_partial_column_lists ... ok +test normalize::span::description::sql::tests::collapse_columns_with_as_without_quotes ... ok +test normalize::span::description::sql::tests::declare_cursor ... ok +test normalize::span::description::sql::tests::declare_cursor_advanced ... ok +test normalize::span::description::sql::tests::collapse_partial_column_lists_2 ... ok +test normalize::span::description::sql::tests::do_not_collapse_single_column ... ok +test normalize::span::description::sql::tests::digits_in_compound_table_name ... ok +test normalize::span::description::sql::tests::dont_scrub_double_quoted_strings_format_mysql ... ok +test normalize::span::description::sql::tests::dont_scrub_double_quoted_strings_format_postgres ... ok +test normalize::span::description::sql::tests::digits_in_table_name ... ok +test normalize::span::description::sql::tests::fetch_cursor ... ok +test normalize::span::description::sql::tests::dont_scrub_nulls ... ok +test normalize::span::description::sql::tests::jsonb ... ok +test normalize::span::description::sql::tests::not_a_comment ... ok +test normalize::span::description::sql::tests::multiple_statements ... ok +test normalize::span::description::sql::tests::mixed ... ok +test normalize::span::description::sql::tests::num_e_where ... ok +test normalize::span::description::sql::tests::num_negative_where ... ok +test normalize::span::description::sql::tests::num_limit ... ok +test normalize::span::description::sql::tests::parameters_values ... ok +test normalize::span::description::sql::tests::num_where ... ok +test normalize::span::description::sql::tests::parameters_in ... ok +test normalize::span::description::sql::tests::php_placeholders ... ok +test normalize::span::description::sql::tests::parameters_values_with_quotes ... ok +test normalize::span::description::sql::tests::quotes_in_cast ... ok +test normalize::span::description::sql::tests::qualified_wildcard ... ok +test normalize::span::description::sql::tests::quotes_in_function ... ok +test normalize::span::description::sql::tests::savepoint_lowercase ... ok +test normalize::span::description::sql::tests::quotes_in_join ... ok +test normalize::span::description::sql::tests::savepoint_quoted ... ok +test normalize::span::description::sql::tests::savepoint_uppercase ... ok +test normalize::span::description::sql::tests::savepoint_uppercase_semicolon ... ok +test normalize::span::description::sql::tests::single_digit_in_table_name ... ok +test normalize::span::description::sql::tests::single_quoted_string ... ok +test normalize::span::description::sql::tests::mysql_comment_generic ... ok +test normalize::span::description::sql::tests::activerecord_truncated ... ok +test normalize::span::description::sql::tests::mysql_comment ... ok +test normalize::span::description::sql::tests::savepoint_quoted_backtick ... ok +test normalize::span::description::sql::tests::named_placeholders ... ok +test normalize::span::description::sql::tests::strip_prefixes ... ok +test normalize::span::description::sql::tests::strip_prefixes_mysql ... ok +test normalize::span::description::sql::tests::single_quoted_string_finished ... ok +test normalize::span::description::sql::tests::strip_prefixes_ansi ... ok +test normalize::span::description::sql::tests::single_quoted_string_unfinished ... ok +test normalize::span::description::sql::tests::clickhouse ... ok +test normalize::span::description::sql::tests::unique_alias ... ok +test normalize::span::description::sql::tests::unparameterized_ins_nvarchar ... ok +test normalize::span::description::sql::tests::type_casts ... ok +test normalize::span::description::sql::tests::update_multiple ... ok +test normalize::span::description::sql::tests::strip_prefixes_truncated ... ok +test normalize::span::description::sql::tests::uuid_in_table_name ... ok +test normalize::span::description::sql::tests::unparameterized_ins_uppercase ... ok +test normalize::span::description::sql::tests::values_multi ... ok +test normalize::span::description::sql::tests::various_parameterized_ins_lowercase ... ok +test normalize::span::description::sql::tests::various_parameterized_questionmarks ... ok +test normalize::contexts::tests::test_get_product_name ... ok +test normalize::span::description::sql::tests::various_parameterized_ins_percentage ... ok +test normalize::span::description::sql::tests::various_parameterized_ins_dollar ... ok +test normalize::span::description::tests::active_record ... ok +test normalize::span::description::sql::tests::whitespace_and_comments ... ok +test normalize::span::description::tests::active_record_with_db_system ... ok +test normalize::span::description::sql::tests::various_parameterized_strings ... ok +test normalize::span::description::tests::informed_sql_parser ... ok +test normalize::span::description::tests::span_description_scrub_empty ... ok +test normalize::span::description::tests::span_description_scrub_hex ... ok +test normalize::span::description::tests::span_description_scrub_nothing_cache ... ok +test normalize::span::description::tests::span_description_scrub_cache ... ok +test normalize::span::description::tests::span_description_scrub_only_urllike_on_http_ops ... ok +test normalize::span::description::sql::tests::strip_prefixes_mysql_generic ... ok +test normalize::span::description::tests::span_description_scrub_only_dblike_on_db_ops ... ok +test normalize::span::description::tests::span_description_scrub_only_domain ... ok +test normalize::span::description::tests::span_description_scrub_path_ids_end ... ok +test normalize::span::description::tests::span_description_scrub_path_md5_hashes ... ok +test normalize::span::description::tests::span_description_scrub_path_multiple_ids ... ok +test normalize::span::description::tests::span_description_scrub_path_uuids ... ok +test normalize::span::description::tests::span_description_scrub_path_ids_middle ... ok +test normalize::span::description::tests::span_description_scrub_redis_invalid ... ok +test normalize::span::description::tests::span_description_scrub_nothing_in_resource ... ok +test normalize::span::description::tests::span_description_scrub_redis_long_command ... ok +test normalize::span::description::tests::span_description_scrub_redis_no_args ... ok +test normalize::span::description::tests::span_description_scrub_redis_set ... ok +test normalize::span::description::tests::span_description_scrub_path_sha_hashes ... ok +test normalize::span::description::tests::span_description_scrub_redis_set_quoted ... ok +test normalize::span::description::tests::span_description_scrub_redis_whitespace ... ok +test normalize::span::description::sql::tests::various_parameterized_cutoff ... ok +test normalize::span::description::tests::span_description_scrub_resource_css ... ok +test normalize::span::description::tests::span_description_scrub_resource_script_numeric_filename ... ok +test normalize::span::description::tests::span_description_scrub_ui_load ... ok +test normalize::span::description::sql::tests::update_single ... ok +test normalize::span::description::tests::span_description_scrub_resource_script ... ok +test normalize::span::tag_extraction::tests::extract_table_insert ... ok +test normalize::span::tag_extraction::tests::extract_table_delete ... ok +test normalize::span::tag_extraction::tests::extract_table_multiple_mysql ... ok +test normalize::span::tag_extraction::tests::extract_table_multiple ... ok +test normalize::span::tag_extraction::tests::extract_table_multiple_advanced ... ok +test normalize::span::description::sql::tests::unparameterized_ins_odbc_escape_sequence ... ok +test normalize::span::tag_extraction::tests::extract_table_select ... ok +test normalize::span::tag_extraction::tests::extract_table_select_nested ... ok +test normalize::span::tag_extraction::tests::extract_table_update ... ok +test normalize::span::tag_extraction::tests::test_truncate_string_no_panic ... ok +test normalize::stacktrace::tests::test_coerce_empty_filename ... ok +test normalize::stacktrace::tests::test_does_not_overwrite_filename ... ok +test normalize::stacktrace::tests::test_coerces_url_filenames ... ok +test normalize::stacktrace::tests::test_ignores_results_with_empty_path ... ok +test normalize::stacktrace::tests::test_is_url ... ok +test normalize::span::tag_extraction::tests::test_http_method_context ... ok +test normalize::span::tag_extraction::tests::test_http_method_request_prioritized ... ok +test normalize::span::tag_extraction::tests::test_http_method_txname ... ok +test normalize::stacktrace::tests::test_ignores_results_with_slash_path ... ok +test normalize::tests::test_context_line_default ... ok +test normalize::tests::test_context_line_retain ... ok +test normalize::span::tag_extraction::tests::extract_sql_action ... ok +test normalize::tests::test_drops_measurements_with_invalid_characters ... ok +test normalize::tests::test_discards_received ... ok +test normalize::tests::test_drops_too_long_measurement_names ... ok +test normalize::tests::test_empty_environment_is_removed ... ok +test normalize::tests::test_empty_environment_is_removed_and_overwritten_with_tag ... ok +test normalize::tests::test_empty_tags_removed ... ok +test normalize::tests::test_environment_tag_is_moved ... ok +test normalize::tests::test_event_level_defaulted ... ok +test normalize::tests::test_exception_invalid ... ok +test normalize::tests::test_frame_null_context_lines ... ok +test normalize::mechanism::tests::test_normalize_http_url ... ok +test normalize::tests::test_filter_custom_measurements ... ok +test normalize::tests::test_future_timestamp ... ok +test normalize::tests::test_android_medium_device_class ... ok +test normalize::tests::test_apple_high_device_class ... ok +test normalize::tests::test_apple_low_device_class ... ok +test normalize::tests::test_computed_measurements ... ok +test normalize::tests::test_android_high_device_class ... ok +test normalize::tests::test_apple_medium_device_class ... ok +test normalize::tests::test_geo_from_ip_address ... ok +test normalize::tests::test_android_low_device_class ... ok +test normalize::tests::test_internal_tags_removed ... ok +test normalize::tests::test_handles_type_in_value ... ok +test normalize::tests::test_invalid_release_removed ... ok +test normalize::tests::test_keeps_valid_measurement ... ok +test normalize::tests::test_json_value ... ok +test normalize::tests::test_grouping_config ... ok +test normalize::tests::test_light_normalization_respects_is_renormalize ... ok +test normalize::tests::test_max_custom_measurement ... ok +test normalize::tests::test_light_normalize_validates_spans ... ok +test normalize::tests::test_merge_builtin_measurement_keys ... ok +test normalize::tests::test_no_device_class ... ok +test normalize::tests::test_none_environment_errors ... ok +test normalize::tests::test_light_normalization_is_idempotent ... ok +test normalize::tests::test_logentry_error ... ok +test normalize::tests::test_normalize_app_start_cold_spans_for_react_native ... ok +test normalize::tests::test_normalize_app_start_spans_only_for_react_native_3_to_4_4 ... ok +test normalize::tests::test_normalize_app_start_measurements_does_not_add_measurements ... ok +test normalize::tests::test_normalize_dist_empty ... ok +test normalize::tests::test_normalize_dist_none ... ok +test normalize::tests::test_normalize_dist_trim ... ok +test normalize::tests::test_normalize_dist_whitespace ... ok +test normalize::tests::test_normalize_app_start_warm_spans_for_react_native ... ok +test normalize::tests::test_normalize_app_start_warm_measurements ... ok +test normalize::tests::test_normalize_app_start_cold_measurements ... ok +test normalize::tests::test_normalize_security_report ... ok +test normalize::tests::test_normalize_logger_exact_length ... ok +test normalize::tests::test_normalize_logger_empty ... ok +test normalize::tests::test_normalize_units ... ok +test normalize::tests::test_parses_sdk_info_from_header ... ok +test normalize::tests::test_normalize_logger_trimmed ... ok +test normalize::tests::test_normalize_logger_word_leading_dots ... ok +test normalize::tests::test_normalize_logger_short_no_trimming ... ok +test normalize::tests::test_normalize_logger_too_long_single_word ... ok +test normalize::tests::test_regression_backfills_abs_path_even_when_moving_stacktrace ... ok +test normalize::tests::test_rejects_empty_exception_fields ... ok +test normalize::tests::test_normalize_logger_word_trimmed_at_max ... ok +test normalize::tests::test_normalize_logger_word_trimmed_before_max ... ok +test normalize::tests::test_past_timestamp ... ok +test normalize::tests::test_replay_id_added_from_dsc ... ok +test normalize::tests::test_tags_deduplicated ... ok +test normalize::tests::test_too_long_distribution ... ok +test normalize::tests::test_top_level_keys_moved_into_tags ... ok +test normalize::tests::test_too_long_tags ... ok +test normalize::tests::test_transaction_level_untouched ... ok +test normalize::tests::test_transaction_status_defaulted_to_unknown ... ok +test normalize::tests::test_user_data_moved ... ok +test normalize::tests::test_unknown_debug_image ... ok +test normalize::tests::test_user_ip_from_client_ip_without_auto ... ok +test normalize::tests::test_user_ip_from_client_ip_without_appropriate_platform ... ok +test normalize::tests::test_user_ip_from_invalid_remote_addr ... ok +test normalize::tests::test_user_ip_from_remote_addr ... ok +test normalize::tests::test_user_ip_from_client_ip_with_auto ... ok +test normalize::user_agent::tests::test_choose_client_hints_for_os_context ... ok +test normalize::user_agent::tests::test_default_empty ... ok +test normalize::user_agent::tests::test_client_hint_parser ... ok +test normalize::user_agent::tests::test_client_hints_detected ... ok +test normalize::user_agent::tests::test_client_hints_with_unknown_browser ... ok +test normalize::user_agent::tests::test_ignore_empty_device ... ok +test normalize::user_agent::tests::test_ignore_empty_os ... ok +test normalize::user_agent::tests::test_ignore_empty_browser ... ok +test normalize::user_agent::tests::test_keep_empty_os_version ... ok +test normalize::tests::test_geo_in_light_normalize ... ok +test normalize::user_agent::tests::test_indicate_frozen_os_windows ... ok +test normalize::user_agent::tests::test_indicate_frozen_os_mac ... ok +test normalize::user_agent::tests::test_fallback_on_ua_string_for_os ... ok +test normalize::user_agent::tests::test_skip_no_user_agent ... ok +test normalize::user_agent::tests::test_strip_quotes ... ok +test normalize::user_agent::tests::test_strip_whitespace_and_quotes ... ok +test normalize::user_agent::tests::test_use_client_hints_for_device ... ok +test normalize::user_agent::tests::test_user_agent_does_not_override_prefilled ... ok +test normalize::user_agent::tests::test_verison_missing_minor ... ok +test normalize::user_agent::tests::test_version_major ... ok +test normalize::user_agent::tests::test_version_major_minor ... ok +test normalize::user_agent::tests::test_version_major_minor_patch ... ok +test normalize::user_agent::tests::test_version_none ... ok +test remove_other::tests::test_breadcrumb_errors ... ok +test remove_other::tests::test_remove_legacy_attributes ... ok +test remove_other::tests::test_remove_nested_other ... ok +test remove_other::tests::test_remove_unknown_attributes ... ok +test remove_other::tests::test_retain_context_other ... ok +test replay::tests::test_capped_values ... ok +test replay::tests::test_event_roundtrip ... ok +test replay::tests::test_lenient_release ... ok +test normalize::user_agent::tests::test_skip_unrecognizable_user_agent ... ok +test normalize::user_agent::tests::fallback_on_ua_string_when_missing_browser_field ... ok +test normalize::user_agent::tests::test_all_contexts ... ok +test normalize::user_agent::tests::test_device_context ... ok +test normalize::user_agent::tests::test_fallback_to_ua_if_no_client_hints ... ok +test replay::tests::test_truncated_list_less_than_limit ... ok +test replay::tests::test_validate_u16_segment_id ... ok +test schema::tests::test_client_sdk_missing_attribute ... ok +test schema::tests::test_invalid_email ... ok +test schema::tests::test_mechanism_missing_attributes ... ok +test schema::tests::test_newlines_release ... ok +test schema::tests::test_stacktrace_missing_attribute ... ok +test transactions::processor::tests::test_allows_transaction_event_with_empty_span_list ... ok +test transactions::processor::tests::test_allows_transaction_event_with_null_span_list ... ok +test transactions::processor::tests::test_allows_transaction_event_without_span_list ... ok +test transactions::processor::tests::test_default_transaction_source_unknown ... ok +test transactions::processor::tests::test_allows_valid_transaction_event_with_spans ... ok +test transactions::processor::tests::test_defaults_missing_op_in_context ... ok +test transactions::processor::tests::test_defaults_transaction_event_with_span_with_missing_op ... ok +test transactions::processor::tests::test_defaults_transaction_name_when_empty ... ok +test transactions::processor::tests::test_discards_on_missing_context ... ok +test transactions::processor::tests::test_discards_on_missing_contexts_map ... ok +test transactions::processor::tests::test_discards_on_missing_span_id_in_context ... ok +test transactions::processor::tests::test_discards_on_missing_trace_id_in_context ... ok +test transactions::processor::tests::test_defaults_transaction_name_when_missing ... ok +test transactions::processor::tests::test_discards_on_null_context ... ok +test transactions::processor::tests::test_discards_transaction_event_with_nulled_out_span ... ok +test transactions::processor::tests::test_discards_transaction_event_with_span_with_missing_span_id ... ok +test transactions::processor::tests::test_discards_transaction_event_with_span_with_missing_start_timestamp ... ok +test transactions::processor::tests::test_discards_transaction_event_with_span_with_missing_trace_id ... ok +test transactions::processor::tests::test_discards_when_missing_start_timestamp ... ok +test transactions::processor::tests::test_discards_when_missing_timestamp ... ok +test transactions::processor::tests::test_discards_when_timestamp_out_of_range ... ok +test transactions::processor::tests::test_is_high_cardinality_sdk_ruby_error ... ok +test transactions::processor::tests::test_is_high_cardinality_sdk_ruby_ok ... ok +test transactions::processor::tests::test_normalize_legacy_javascript ... ok +test transactions::processor::tests::test_normalize_legacy_python ... ok +test transactions::processor::tests::test_no_sanitized_if_no_rules ... ok +test transactions::processor::tests::test_normalize_twice ... ok +test transactions::processor::tests::test_replace_missing_timestamp ... ok +test normalize::user_agent::tests::test_os_context_short_version ... ok +test normalize::user_agent::tests::test_os_context ... ok +test transactions::processor::tests::test_skips_non_transaction_events ... ok +test transactions::processor::tests::test_normalize_transaction_names ... ok +test transactions::processor::tests::test_scrub_identifiers_and_apply_rules ... ok +test transactions::processor::tests::test_scrub_identifiers_before_rules ... ok +test transactions::processor::tests::test_transaction_name_normalize ... ok +test transactions::processor::tests::test_transaction_name_normalize_hex ... ok +test transactions::processor::tests::test_transaction_name_normalize_in_segments_1 ... ok +test transactions::processor::tests::test_transaction_name_normalize_in_segments_3 ... ok +test replay::tests::test_set_ip_address_missing_user_ip_address ... ok +test transactions::processor::tests::test_transaction_name_normalize_id ... ok +test transactions::processor::tests::test_transaction_name_normalize_in_segments_2 ... ok +test transactions::processor::tests::test_transaction_name_normalize_mark_as_sanitized ... ok +test transactions::processor::tests::test_transaction_name_normalize_in_segments_4 ... ok +test transactions::processor::tests::test_transaction_name_normalize_in_segments_5 ... ok +test transactions::processor::tests::test_transaction_name_normalize_mark_as_sanitized_when_ready ... ok +test transactions::processor::tests::test_transaction_name_normalize_url_encode_1 ... ok +test transactions::processor::tests::test_transaction_name_normalize_url_encode_5 ... ok +test transactions::processor::tests::test_transaction_name_normalize_url_encode_4 ... ok +test transactions::processor::tests::test_transaction_name_normalize_sha ... ok +test transactions::processor::tests::test_transaction_name_normalize_url_encode_6 ... ok +test transactions::processor::tests::test_transaction_name_normalize_url_encode_7 ... ok +test transactions::processor::tests::test_transaction_name_normalize_url_encode_3 ... ok +test transactions::processor::tests::test_transaction_name_normalize_url_encode_2 ... ok +test transactions::processor::tests::test_transaction_name_normalize_windows_path ... ok +test transactions::processor::tests::test_transaction_name_skip_original_value ... ok +test transactions::processor::tests::test_transaction_name_rename_end_slash ... ok +test transactions::processor::tests::test_transaction_name_unsupported_source ... ok +test transactions::rules::tests::test_rule_format ... ok +test normalize::user_agent::tests::test_os_context_full_version ... ok +test transactions::processor::tests::test_transaction_name_skip_replace_all ... ok +test transactions::rules::tests::test_rule_format_defaults ... ok +test transactions::rules::tests::test_rule_format_roundtrip ... ok +test transactions::rules::tests::test_rule_format_unsupported_reduction ... ok +test trimming::tests::test_basic_trimming ... ok +test transactions::processor::tests::test_transaction_name_skip_replace_all2 ... ok +test trimming::tests::test_custom_context_trimming ... ok +test transactions::processor::tests::test_transaction_name_normalize_uuid ... ok +test trimming::tests::test_databag_stripping ... ok +test trimming::tests::test_frame_hard_limit ... ok +test trimming::tests::test_frameqty_equals_limit ... ok +test trimming::tests::test_slim_frame_data_over_max ... ok +test transactions::processor::tests::test_transaction_name_rename_with_rules ... ok +test trimming::tests::test_slim_frame_data_under_max ... ok +test trimming::tests::test_string_trimming ... ok +test trimming::tests::test_tags_stripping ... ok +test trimming::tests::test_databag_state_leak ... ok +test replay::tests::test_set_user_agent_meta ... ok +test replay::tests::test_loose_type_requirements ... ok +test trimming::tests::test_extra_trimming_long_arrays ... ok +test trimming::tests::test_databag_array_stripping ... ok +test normalize::user_agent::tests::test_browser_context ... ok +test replay::tests::test_missing_user ... ok + +test result: ok. 423 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 2.68s + + +running 170 tests +test processor::chunks::tests::test_chunk_split ... ok +test protocol::breadcrumb::tests::test_python_ty_regression ... ok +test protocol::breadcrumb::tests::test_breadcrumb_default_values ... ok +test protocol::clientsdk::tests::test_client_sdk_default_values ... ok +test protocol::client_report::tests::test_client_report_roundtrip ... ok +test protocol::breadcrumb::tests::test_breadcrumb_roundtrip ... ok +test protocol::clientsdk::tests::test_client_sdk_roundtrip ... ok +test protocol::contexts::browser::tests::test_browser_context_roundtrip ... ok +test protocol::contexts::cloud_resource::tests::test_cloud_resource_context_roundtrip ... ok +test protocol::contexts::app::tests::test_app_context_roundtrip ... ok +test protocol::contexts::os::tests::test_os_context_roundtrip ... ok +test protocol::contexts::profile::tests::test_trace_context_normalization ... ok +test protocol::contexts::profile::tests::test_trace_context_roundtrip ... ok +test protocol::contexts::otel::tests::test_otel_context_roundtrip ... ok +test protocol::contexts::replay::tests::test_replay_context_normalization ... ok +test protocol::contexts::replay::tests::test_trace_context_roundtrip ... ok +test protocol::contexts::device::tests::test_device_context_roundtrip ... ok +test protocol::contexts::reprocessing::tests::test_reprocessing_context_roundtrip ... ok +test protocol::contexts::runtime::tests::test_runtime_context_roundtrip ... ok +test protocol::contexts::response::tests::test_response_context_roundtrip ... ok +test protocol::contexts::tests::test_other_context_roundtrip ... ok +test protocol::contexts::tests::test_multiple_contexts_roundtrip ... ok +test protocol::contexts::tests::test_untagged_context_deserialize ... ok +test protocol::contexts::tests::test_context_processing ... ok +test protocol::contexts::trace::tests::test_trace_context_normalization ... ok +test protocol::contexts::trace::tests::test_trace_context_roundtrip ... ok +test protocol::debugmeta::tests::test_debug_image_apple_default_values ... ok +test protocol::debugmeta::tests::test_debug_image_apple_roundtrip ... ok +test protocol::debugmeta::tests::test_debug_image_elf_roundtrip ... ok +test protocol::debugmeta::tests::test_debug_image_jvm_based_roundtrip ... ok +test protocol::debugmeta::tests::test_debug_image_macho_roundtrip ... ok +test protocol::debugmeta::tests::test_debug_image_other_roundtrip ... ok +test protocol::debugmeta::tests::test_debug_image_pe_dotnet_roundtrip ... ok +test protocol::debugmeta::tests::test_debug_image_proguard_roundtrip ... ok +test protocol::debugmeta::tests::test_debug_image_pe_roundtrip ... ok +test protocol::debugmeta::tests::test_debug_image_symbolic_default_values ... ok +test protocol::debugmeta::tests::test_debug_image_symbolic_legacy ... ok +test protocol::debugmeta::tests::test_debug_image_untagged_roundtrip ... ok +test protocol::debugmeta::tests::test_debug_image_symbolic_roundtrip ... ok +test protocol::debugmeta::tests::test_debug_meta_default_values ... ok +test protocol::debugmeta::tests::test_debug_meta_roundtrip ... ok +test protocol::debugmeta::tests::test_source_map_image_roundtrip ... ok +test protocol::event::tests::test_empty_threads ... ok +test protocol::event::tests::test_event_default_values ... ok +test protocol::event::tests::test_event_type ... ok +test protocol::event::tests::test_field_value_provider_event_empty ... ok +test protocol::event::tests::test_extra_at ... ok +test protocol::event::tests::test_event_default_values_with_meta ... ok +test protocol::event::tests::test_field_value_provider_event_filled ... ok +test protocol::event::tests::test_fingerprint_empty_string ... ok +test protocol::event::tests::test_event_roundtrip ... ok +test protocol::event::tests::test_fingerprint_null_values ... ok +test protocol::event::tests::test_lenient_release ... ok +test protocol::exception::tests::test_coerces_object_value_to_string ... ok +test protocol::exception::tests::test_exception_default_values ... ok +test protocol::exception::tests::test_exception_empty_fields ... ok +test protocol::exception::tests::test_exception_roundtrip ... ok +test protocol::exception::tests::test_explicit_none ... ok +test protocol::fingerprint::tests::test_fingerprint_bool ... ok +test protocol::fingerprint::tests::test_fingerprint_empty ... ok +test protocol::fingerprint::tests::test_fingerprint_float ... ok +test protocol::fingerprint::tests::test_fingerprint_float_bounds ... ok +test protocol::fingerprint::tests::test_fingerprint_float_strip ... ok +test protocol::fingerprint::tests::test_fingerprint_float_trunc ... ok +test protocol::fingerprint::tests::test_fingerprint_invalid_fallback ... ok +test protocol::fingerprint::tests::test_fingerprint_number ... ok +test protocol::fingerprint::tests::test_fingerprint_string ... ok +test protocol::logentry::tests::test_logentry_empty_params ... ok +test protocol::logentry::tests::test_logentry_from_message ... ok +test protocol::logentry::tests::test_logentry_invalid_params ... ok +test protocol::logentry::tests::test_logentry_named_params ... ok +test protocol::logentry::tests::test_logentry_roundtrip ... ok +test protocol::mechanism::tests::test_mechanism_default_values ... ok +test protocol::mechanism::tests::test_mechanism_empty ... ok +test protocol::mechanism::tests::test_mechanism_legacy_conversion ... ok +test protocol::measurements::tests::test_measurements_serialization ... ok +test protocol::mechanism::tests::test_mechanism_roundtrip ... ok +test protocol::replay::tests::test_event_roundtrip ... ok +test protocol::replay::tests::test_lenient_release ... ok +test protocol::request::tests::test_cookies_invalid ... ok +test protocol::request::tests::test_cookies_array ... ok +test protocol::request::tests::test_cookies_object ... ok +test protocol::request::tests::test_cookies_parsing ... ok +test protocol::request::tests::test_header_normalization ... ok +test protocol::request::tests::test_header_from_sequence ... ok +test protocol::request::tests::test_headers_lenient_value ... ok +test protocol::request::tests::test_headers_multiple_values ... ok +test protocol::request::tests::test_query_invalid ... ok +test protocol::request::tests::test_query_string ... ok +test protocol::request::tests::test_query_string_legacy_nested ... ok +test protocol::request::tests::test_querystring_without_value ... ok +test protocol::request::tests::test_request_roundtrip ... ok +test protocol::security_report::tests::test_csp_coerce_blocked_uri_if_missing ... ok +test protocol::security_report::tests::test_csp_culprit_0 ... ok +test protocol::security_report::tests::test_csp_culprit_2 ... ok +test protocol::security_report::tests::test_csp_culprit_1 ... ok +test protocol::security_report::tests::test_csp_culprit_5 ... ok +test protocol::security_report::tests::test_csp_get_message_0 ... ok +test protocol::security_report::tests::test_csp_culprit_uri_without_scheme ... ok +test protocol::security_report::tests::test_csp_culprit_4 ... ok +test protocol::security_report::tests::test_csp_basic ... ok +test protocol::security_report::tests::test_csp_culprit_3 ... ok +test protocol::security_report::tests::test_csp_get_message_1 ... ok +test protocol::security_report::tests::test_csp_get_message_2 ... ok +test protocol::security_report::tests::test_csp_get_message_3 ... ok +test protocol::security_report::tests::test_csp_get_message_4 ... ok +test protocol::security_report::tests::test_csp_get_message_5 ... ok +test protocol::security_report::tests::test_csp_get_message_6 ... ok +test protocol::security_report::tests::test_csp_get_message_7 ... ok +test protocol::security_report::tests::test_csp_get_message_8 ... ok +test protocol::security_report::tests::test_csp_get_message_9 ... ok +test protocol::security_report::tests::test_effective_directive_from_violated_directive_single ... ok +test protocol::security_report::tests::test_csp_tags_stripe ... ok +test protocol::security_report::tests::test_csp_msdn ... ok +test protocol::security_report::tests::test_expectct_invalid ... ok +test protocol::security_report::tests::test_extract_effective_directive_from_long_form ... ok +test protocol::security_report::tests::test_csp_real ... ok +test protocol::security_report::tests::test_normalize_uri ... ok +test protocol::security_report::tests::test_expectct_basic ... ok +test protocol::security_report::tests::test_security_report_type_deserializer_recognizes_csp_reports ... ok +test protocol::security_report::tests::test_security_report_type_deserializer_recognizes_expect_ct_reports ... ok +test protocol::security_report::tests::test_security_report_type_deserializer_recognizes_expect_staple_reports ... ok +test protocol::security_report::tests::test_expectstaple_basic ... ok +test protocol::security_report::tests::test_hpkp_basic ... ok +test protocol::security_report::tests::test_security_report_type_deserializer_recognizes_hpkp_reports ... ok +test protocol::security_report::tests::test_unsplit_uri ... ok +test protocol::session::tests::test_session_abnormal_mechanism ... ok +test protocol::session::tests::test_session_default_timestamp_and_sid ... ok +test protocol::session::tests::test_session_invalid_abnormal_mechanism ... ok +test protocol::session::tests::test_session_default_values ... ok +test protocol::session::tests::test_session_ip_addr_auto ... ok +test protocol::session::tests::test_session_null_abnormal_mechanism ... ok +test protocol::session::tests::test_sessionstatus_unknown ... ok +test protocol::session::tests::test_session_roundtrip ... ok +test protocol::span::tests::test_getter_span_data ... ok +test protocol::span::tests::test_span_serialization ... ok +test protocol::stacktrace::tests::test_frame_default_values ... ok +test protocol::stacktrace::tests::test_frame_empty_context_lines ... ok +test protocol::stacktrace::tests::test_frame_vars_empty_annotated_is_serialized ... ok +test protocol::stacktrace::tests::test_frame_vars_null_preserved ... ok +test protocol::stacktrace::tests::test_frame_roundtrip ... ok +test protocol::stacktrace::tests::test_stacktrace_default_values ... ok +test protocol::stacktrace::tests::test_php_frame_vars ... ok +test protocol::tags::tests::test_tags_from_object ... ok +test protocol::stacktrace::tests::test_stacktrace_roundtrip ... ok +test protocol::templateinfo::tests::test_template_default_values ... ok +test protocol::tags::tests::test_tags_from_array ... ok +test protocol::templateinfo::tests::test_template_roundtrip ... ok +test protocol::thread::tests::test_thread_default_values ... ok +test protocol::thread::tests::test_thread_id ... ok +test protocol::thread::tests::test_thread_roundtrip ... ok +test protocol::transaction::tests::test_other_source_roundtrip ... ok +test protocol::thread::tests::test_thread_lock_reason_roundtrip ... ok +test protocol::types::tests::test_hex_deserialization ... ok +test protocol::transaction::tests::test_transaction_info_roundtrip ... ok +test protocol::types::tests::test_hex_from_string ... ok +test protocol::types::tests::test_hex_serialization ... ok +test protocol::types::tests::test_hex_to_string ... ok +test protocol::types::tests::test_level ... ok +test protocol::types::tests::test_ip_addr ... ok +test protocol::types::tests::test_timestamp_completely_out_of_range ... ok +test protocol::types::tests::test_timestamp_year_out_of_range ... ok +test protocol::types::tests::test_values_deserialization ... ok +test protocol::types::tests::test_values_serialization ... ok +test protocol::user::tests::test_explicit_none ... ok +test protocol::user::tests::test_geo_default_values ... ok +test protocol::user::tests::test_geo_roundtrip ... ok +test protocol::user::tests::test_user_invalid_id ... ok +test protocol::user::tests::test_user_lenient_id ... ok +test protocol::user::tests::test_user_roundtrip ... ok + +test result: ok. 170 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.04s + + +running 2 tests +test tests::test_enums_processor_calls ... ok +test tests::test_simple_newtype ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 4 tests +test test_error ... ok +test test_ok ... ok +test test_panics ... ok +test test_unit ... ok + +test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 48 tests +test client_ips::tests::test_should_filter_blacklisted_ips ... ok +test browser_extensions::tests::test_dont_filter_when_disabled ... ok +test csp::tests::test_does_not_filter_benign_source_files ... ok +test csp::tests::test_does_not_filter_benign_uris ... ok +test csp::tests::test_does_not_filter_non_csp_messages ... ok +test csp::tests::test_filters_known_blocked_source_files ... ok +test csp::tests::test_filters_known_blocked_uris ... ok +test csp::tests::test_filters_known_document_uris ... ok +test csp::tests::test_scheme_domain_port ... ok +test csp::tests::test_matches_any_origin ... ok +test csp::tests::test_sentry_csp_filter_compatibility_bad_reports ... ok +test csp::tests::test_sentry_csp_filter_compatibility_good_reports ... ok +test error_messages::tests::test_filter_hydration_error ... ok +test error_messages::tests::test_should_filter_exception ... ok +test browser_extensions::tests::test_dont_filter_unkown_browser_extension ... ok +test browser_extensions::tests::test_filter_known_browser_extension_source ... ok +test legacy_browsers::tests::test_dont_filter_when_disabled ... ok +test browser_extensions::tests::test_filter_known_browser_extension_values ... ok +test config::tests::test_regression_legacy_browser_missing_options ... ok +test config::tests::test_empty_config ... ok +test config::tests::test_serialize_empty ... ok +test config::tests::test_serialize_full ... ok +test localhost::tests::test_dont_filter_missing_ip_or_domains ... ok +test localhost::tests::test_dont_filter_non_file_urls ... ok +test localhost::tests::test_dont_filter_non_local_domains ... ok +test localhost::tests::test_dont_filter_non_local_ip ... ok +test localhost::tests::test_dont_filter_when_disabled ... ok +test localhost::tests::test_filter_file_urls ... ok +test localhost::tests::test_filter_local_ip ... ok +test localhost::tests::test_filter_local_domains ... ok +test transaction_name::tests::test_does_not_filter_when_disabled ... ok +test transaction_name::tests::test_does_not_filter_when_disabled_with_flag ... ok +test transaction_name::tests::test_does_not_match_missing_transaction ... ok +test transaction_name::tests::test_does_not_filter_when_not_matching ... ok +test transaction_name::tests::test_filters_when_matching ... ok +test transaction_name::tests::test_does_not_match ... ok +test transaction_name::tests::test_matches ... ok +test transaction_name::tests::test_only_filters_transactions_not_anything_else ... ok +test web_crawlers::tests::test_filter_when_disabled ... ok +test releases::tests::test_release_filtering ... ok +test web_crawlers::tests::test_dont_filter_normal_user_agents ... ok +test web_crawlers::tests::test_filter_banned_user_agents ... ok +test legacy_browsers::tests::test_filter_default_browsers ... ok +test legacy_browsers::tests::test_dont_filter_default_above_minimum_versions ... ok +test legacy_browsers::tests::test_dont_filter_unconfigured_browsers ... ok +test legacy_browsers::tests::sentry_compatibility::test_dont_filter_sentry_allowed_user_agents ... ok +test legacy_browsers::tests::test_filter_configured_browsers ... ok +test legacy_browsers::tests::sentry_compatibility::test_filter_sentry_user_agents ... ok + +test result: ok. 48 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 2.37s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 1 test +test config::tests::test_kafka_config ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 65 tests +test aggregation::tests::test_bucket_value_cost ... ok +test aggregation::tests::test_bucket_key_cost ... ok +test aggregation::tests::test_aggregator_cost_enforcement_total ... ok +test aggregation::tests::test_aggregator_cost_tracking ... ok +test aggregation::tests::test_capped_iter_completeness_0 ... ok +test aggregation::tests::test_capped_iter_completeness_100 ... ok +test aggregation::tests::test_capped_iter_completeness_90 ... ok +test aggregation::tests::test_capped_iter_empty ... ok +test aggregation::tests::test_capped_iter_single ... ok +test aggregation::tests::test_capped_iter_split ... ok +test aggregation::tests::test_get_bucket_timestamp_multiple ... ok +test aggregation::tests::test_get_bucket_timestamp_non_multiple ... ok +test aggregation::tests::test_get_bucket_timestamp_overflow ... ok +test aggregation::tests::test_get_bucket_timestamp_zero ... ok +test aggregation::tests::test_parse_shift_key ... ok +test aggregation::tests::test_validate_bucket_key_chars ... ok +test aggregation::tests::test_validate_bucket_key_str_lens ... ok +test aggregation::tests::test_aggregator_mixed_projects ... ok +test bucket::tests::test_bucket_docs_roundtrip ... ok +test aggregation::tests::test_merge_back ... ok +test bucket::tests::test_bucket_value_merge_counter ... ok +test bucket::tests::test_bucket_value_merge_distribution ... ok +test bucket::tests::test_bucket_value_merge_gauge ... ok +test bucket::tests::test_bucket_value_merge_set ... ok +test aggregation::tests::test_aggregator_cost_enforcement_project ... ok +test bucket::tests::test_distribution_value_size ... ok +test bucket::tests::test_buckets_roundtrip ... ok +test bucket::tests::test_parse_all ... ok +test bucket::tests::test_parse_all_crlf ... ok +test bucket::tests::test_metrics_docs ... ok +test bucket::tests::test_parse_all_empty_lines ... ok +test bucket::tests::test_parse_all_trailing ... ok +test aggregation::tests::test_flush_bucket ... ok +test bucket::tests::test_parse_counter_packed ... ok +test aggregation::tests::test_validate_tag_values_special_chars ... ok +test bucket::tests::test_parse_distribution_packed ... ok +test bucket::tests::test_parse_empty_name ... ok +test bucket::tests::test_parse_garbage ... ok +test aggregation::tests::test_bucket_partitioning_128 ... ok +test aggregation::tests::test_bucket_partitioning_dummy ... ok +test bucket::tests::test_parse_bucket_defaults ... ok +test bucket::tests::test_parse_distribution ... ok +test bucket::tests::test_parse_counter ... ok +test bucket::tests::test_parse_buckets ... ok +test aggregation::tests::test_aggregator_merge_counters ... ok +test aggregation::tests::test_aggregator_merge_timestamps ... ok +test bucket::tests::test_parse_gauge ... ok +test bucket::tests::test_parse_gauge_packed ... ok +test bucket::tests::test_parse_histogram ... ok +test bucket::tests::test_parse_invalid_name ... ok +test aggregation::tests::test_cost_tracker ... ok +test bucket::tests::test_parse_invalid_name_with_leading_digit ... ok +test bucket::tests::test_parse_implicit_namespace ... ok +test bucket::tests::test_parse_sample_rate ... ok +test bucket::tests::test_parse_set ... ok +test bucket::tests::test_parse_set_hashed ... ok +test bucket::tests::test_parse_set_hashed_packed ... ok +test bucket::tests::test_parse_set_packed ... ok +test bucket::tests::test_parse_timestamp ... ok +test bucket::tests::test_parse_tags ... ok +test bucket::tests::test_parse_unit ... ok +test bucket::tests::test_parse_unit_regression ... ok +test bucket::tests::test_set_docs ... ok +test protocol::tests::test_sizeof_unit ... ok +test router::tests::condition_roundtrip ... ok + +test result: ok. 65 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.04s + + +running 7 tests +test tests::truncate_basic ... ok +test tests::process_invalid_environment ... ok +test tests::process_empty_slug ... ok +test tests::process_with_upsert_short ... ok +test tests::process_with_upsert_interval ... ok +test tests::process_json_roundtrip ... ok +test tests::process_with_upsert_full ... ok + +test result: ok. 7 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 148 tests +test attachments::tests::test_fill_content_wstr ... ok +test attachments::tests::test_fill_content_wstr_panic - should panic ... ok +test attachments::tests::test_get_wstr_match ... ok +test attachments::tests::test_bytes_regexes ... ok +test attachments::tests::test_segments_all_data ... ok +test attachments::tests::test_segments_end_aligned ... ok +test attachments::tests::test_segments_garbage ... ok +test attachments::tests::test_segments_middle_2_byte_aligned ... ok +test attachments::tests::test_segments_middle_2_byte_aligned_mutation ... ok +test attachments::tests::test_segments_middle_unaligned ... ok +test attachments::tests::test_segments_multiple ... ok +test attachments::tests::test_segments_too_short ... ok +test attachments::tests::test_selectors ... ok +test attachments::tests::test_swap_content_wstr ... ok +test attachments::tests::test_swap_content_wstr_panic - should panic ... ok +test builtin::tests::test_builtin_rules_completeness ... ok +test attachments::tests::test_all_the_bytes ... ok +test builtin::tests::test_email ... ok +test builtin::tests::test_iban_different_rules ... ok +test builtin::tests::test_creditcard ... ok +test builtin::tests::test_iban_scrubbing_word_boundaries ... ok +test builtin::tests::test_invalid_iban_codes ... ok +test builtin::tests::test_imei ... ok +test attachments::tests::test_ip_masking ... ok +test attachments::tests::test_ip_replace_padding ... ok +test builtin::tests::test_mac ... ok +test builtin::tests::test_ipv6 ... ok +test builtin::tests::test_ipv4 ... ok +test builtin::tests::test_urlauth ... ok +test attachments::tests::test_ip_hash_trunchating ... ok +test attachments::tests::test_ip_replace_padding_utf16 ... ok +test attachments::tests::test_ip_removing ... ok +test builtin::tests::test_usssn ... ok +test builtin::tests::test_pemkey ... ok +test attachments::tests::test_ip_removing_utf16 ... ok +test attachments::tests::test_ip_hash_trunchating_utf16 ... ok +test builtin::tests::test_userpath ... ok +test builtin::tests::test_uuid ... ok +test attachments::tests::test_ip_masking_utf16 ... ok +test builtin::tests::test_valid_iban_codes ... ok +test convert::tests::test_convert_empty_sensitive_field ... ok +test convert::tests::test_convert_exclude_field ... ok +test convert::tests::test_datascrubbing_default ... ok +test convert::tests::test_convert_scrub_ip_only ... ok +test convert::tests::test_convert_default_pii_config ... ok +test convert::tests::test_convert_sensitive_fields ... ok +test convert::tests::test_csp_blocked_uri ... ok +test convert::tests::test_contexts ... ok +test convert::tests::test_authorization_scrubbing ... ok +test convert::tests::test_debug_meta_files_not_strippable ... ok +test convert::tests::test_does_not_fail_on_non_string ... ok +test convert::tests::test_does_not_sanitize_timestamp_looks_like_card ... ok +test convert::tests::test_doesnt_scrub_not_scrubbed ... ok +test convert::tests::test_does_sanitize_social_security_number ... ok +test convert::tests::test_does_sanitize_encrypted_private_key ... ok +test convert::tests::test_does_sanitize_public_key ... ok +test convert::tests::test_does_sanitize_private_key ... ok +test convert::tests::test_breadcrumb_message ... ok +test convert::tests::test_does_sanitize_rsa_private_key ... ok +test convert::tests::test_empty_field ... ok +test convert::tests::test_event_message_not_strippable ... ok +test convert::tests::test_exclude_fields_on_field_value ... ok +test convert::tests::test_exclude_fields_on_field_name ... ok +test convert::tests::test_ip_stripped ... ok +test convert::tests::test_extra ... ok +test convert::tests::test_http_remote_addr_stripped ... ok +test convert::tests::test_regression_more_odd_keys ... ok +test convert::tests::test_http ... ok +test convert::tests::test_querystring_as_pairlist ... ok +test convert::tests::test_explicit_fields_case_insensitive ... ok +test convert::tests::test_explicit_fields ... ok +test convert::tests::test_no_scrub_object_with_safe_fields ... ok +test convert::tests::test_querystring_as_pairlist_with_partials ... ok +test convert::tests::test_querystring_as_string ... ok +test convert::tests::test_querystring_as_string_with_partials ... ok +test convert::tests::test_odd_keys ... ok +test convert::tests::test_safe_fields_for_token ... ok +test convert::tests::test_sanitize_credit_card_discover ... ok +test convert::tests::test_sanitize_credit_card_amex ... ok +test convert::tests::test_sanitize_credit_card ... ok +test convert::tests::test_sanitize_credit_card_visa ... ok +test convert::tests::test_sanitize_credit_card_within_value_1 ... ok +test convert::tests::test_sanitize_credit_card_mastercard ... ok +test convert::tests::test_sanitize_http_body ... ok +test convert::tests::test_sanitize_credit_card_within_value_2 ... ok +test convert::tests::test_sanitize_url_2 ... ok +test convert::tests::test_sanitize_url_1 ... ok +test convert::tests::test_sanitize_url_4 ... ok +test convert::tests::test_sanitize_additional_sensitive_fields ... ok +test convert::tests::test_sanitize_url_3 ... ok +test convert::tests::test_sanitize_url_6 ... ok +test convert::tests::test_sanitize_url_5 ... ok +test convert::tests::test_sanitize_url_7 ... ok +test convert::tests::test_should_have_mysql_pwd_as_a_default_2 ... ok +test convert::tests::test_should_have_mysql_pwd_as_a_default_1 ... ok +test generate_selectors::tests::test_empty ... ok +test convert::tests::test_scrub_object ... ok +test convert::tests::test_stacktrace_paths_not_strippable ... ok +test convert::tests::test_stacktrace ... ok +test minidumps::tests::test_module_list_removed_lin ... ok +test minidumps::tests::test_module_list_selectors ... ok +test convert::tests::test_user ... ok +test minidumps::tests::test_linux_environ_valuetype ... ok +test generate_selectors::tests::test_full ... ok +test minidumps::tests::test_module_list_removed_win ... ok +test minidumps::tests::test_stack_scrubbing_deep_wildcard ... ok +test processor::tests::test_anything_hash_on_container ... ok +test minidumps::tests::test_module_list_removed_mac ... ok +test minidumps::tests::test_stack_scrubbing_binary_not_stack ... ok +test minidumps::tests::test_stack_scrubbing_backwards_compatible_selector ... ok +test minidumps::tests::test_stack_scrubbing_valuetype_not_fully_qualified ... ok +test processor::tests::test_anything_hash_on_string ... ok +test minidumps::tests::test_stack_scrubbing_valuetype_selector - should panic ... ok +test processor::tests::test_debugmeta_path_not_addressible_with_wildcard_selector ... ok +test processor::tests::test_ip_address_hashing ... ok +test processor::tests::test_ip_address_hashing_does_not_overwrite_id ... ok +test processor::tests::test_logentry_value_types ... ok +test convert::tests::test_sensitive_cookies ... ok +test processor::tests::test_hash_debugmeta_path ... ok +test minidumps::tests::test_stack_scrubbing_path_item_selector ... ok +test processor::tests::test_quoted_keys ... ok +test processor::tests::test_no_field_upsert ... ok +test processor::tests::test_redact_containers ... ok +test processor::tests::test_remove_debugmeta_path ... ok +test processor::tests::test_redact_custom_pattern ... ok +test processor::tests::test_replace_replaced_text_anything ... ok +test processor::tests::test_replace_replaced_text ... ok +test processor::tests::test_replace_debugmeta_path ... ok +test processor::tests::test_scrub_breadcrumb_data_http_not_scrubbed ... ok +test minidumps::tests::test_stack_scrubbing_wildcard - should panic ... ok +test processor::tests::test_scrub_breadcrumb_data_http_strings_are_scrubbed ... ok +test processor::tests::test_scrub_breadcrumb_data_untyped_props_are_scrubbed ... ok +test processor::tests::test_scrub_breadcrumb_data_http_objects_are_scrubbed ... ok +test processor::tests::test_does_not_scrub_if_no_graphql ... ok +test processor::tests::test_basic_stripping ... ok +test redactions::tests::test_redaction_deser_method ... ok +test redactions::tests::test_redaction_deser_other ... ok +test processor::tests::test_scrub_span_data_http_not_scrubbed ... ok +test processor::tests::test_scrub_graphql_response_data_with_variables ... ok +test selector::tests::test_invalid ... ok +test selector::tests::test_roundtrip ... ok +test selector::tests::test_attachments_matching ... ok +test processor::tests::test_scrub_original_value ... ok +test processor::tests::test_scrub_graphql_response_data_without_variables ... ok +test processor::tests::test_scrub_span_data_untyped_props_are_scrubbed ... ok +test processor::tests::test_scrub_span_data_http_strings_are_scrubbed ... ok +test processor::tests::test_scrub_span_data_http_objects_are_scrubbed ... ok +test selector::tests::test_matching ... ok + +test result: ok. 148 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.07s + + +running 1 test +test test_scrub_pii_from_annotated_replay ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + + +running 30 tests +test measurements::tests::test_value_as_float ... ok +test measurements::tests::test_value_as_string ... ok +test native_debug_image::tests::test_native_debug_image_compatibility ... ok +test sample::tests::test_expand_with_all_samples_outside_transaction ... ok +test sample::tests::test_copying_transaction ... ok +test sample::tests::test_expand ... ok +test sample::tests::test_filter_samples ... ok +test sample::tests::test_expand_with_samples_inclusive ... ok +test sample::tests::test_extract_transaction_tags ... ok +test sample::tests::test_keep_profile_under_max_duration ... ok +test android::tests::test_remove_invalid_events ... ok +test sample::tests::test_parse_profile_with_all_samples_filtered ... ok +test sample::tests::test_parse_with_no_transaction ... ok +test sample::tests::test_profile_cleanup_metadata ... ok +test sample::tests::test_reject_profile_over_max_duration ... ok +test sample::tests::test_profile_remove_idle_samples_at_start_and_end ... ok +test sample::tests::test_roundtrip ... ok +test tests::test_minimal_profile_with_version ... ok +test tests::test_minimal_profile_without_version ... ok +test transaction_metadata::tests::test_invalid_transaction_metadata ... ok +test tests::test_expand_profile_with_version ... ok +test transaction_metadata::tests::test_valid_transaction_metadata ... ok +test transaction_metadata::tests::test_valid_transaction_metadata_without_relative_timestamp ... ok +test android::tests::test_no_transaction ... ok +test extract_from_transaction::tests::test_extract_transaction_metadata ... ok +test android::tests::test_transactions_to_top_level ... ok +test tests::test_expand_profile_without_version ... ok +test android::tests::test_extract_transaction_metadata ... ok +test android::tests::test_timestamp ... ok +test android::tests::test_roundtrip_android ... ok + +test result: ok. 30 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.51s + + +running 11 tests +test macros::tests::get_path ... ok +test macros::tests::get_path_array ... ok +test macros::tests::get_path_empty ... ok +test macros::tests::get_path_combined ... ok +test macros::tests::get_path_object ... ok +test macros::tests::get_value ... ok +test macros::tests::get_value_array ... ok +test macros::tests::get_value_combined ... ok +test macros::tests::get_value_empty ... ok +test macros::tests::get_value_object ... ok +test size::tests::test_estimate_size ... ok + +test result: ok. 11 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 1 test +test test_annotated_deserialize_with_meta ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 20 tests +test test_signed_integers ... ok +test test_floats ... ok +test test_skip_array_never ... ok +test test_skip_array_null ... ok +test test_skip_array_empty ... ok +test test_skip_array_null_deep ... ok +test test_skip_array_empty_deep ... ok +test test_skip_object_empty ... ok +test test_skip_object_never ... ok +test test_skip_object_empty_deep ... ok +test test_skip_object_null ... ok +test test_skip_serialization_on_regular_structs ... ok +test test_skip_object_null_deep ... ok +test test_skip_tuple_empty_deep ... ok +test test_skip_tuple_empty ... ok +test test_skip_tuple_never ... ok +test test_skip_tuple_null ... ok +test test_skip_tuple_null_deep ... ok +test test_wrapper_structs_and_skip_serialization ... ok +test test_unsigned_integers ... ok + +test result: ok. 20 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 44 tests +test quota::tests::test_quota_invalid_limited_mixed ... ok +test quota::tests::test_quota_invalid_only_unknown ... ok +test quota::tests::test_quota_invalid_unlimited_mixed ... ok +test quota::tests::test_quota_matches_key_scope ... ok +test quota::tests::test_quota_matches_multiple_categores ... ok +test quota::tests::test_quota_matches_no_categories ... ok +test quota::tests::test_quota_matches_no_invalid_scope ... ok +test quota::tests::test_quota_matches_organization_scope ... ok +test quota::tests::test_quota_matches_unknown_category ... ok +test quota::tests::test_quota_matches_project_scope ... ok +test quota::tests::test_quota_valid_reject_all ... ok +test quota::tests::test_quota_valid_reject_all_mixed ... ok +test rate_limit::tests::test_rate_limit_matches_categories ... ok +test rate_limit::tests::test_parse_retry_after ... ok +test rate_limit::tests::test_rate_limit_matches_organization ... ok +test rate_limit::tests::test_rate_limit_matches_key ... ok +test rate_limit::tests::test_rate_limit_matches_project ... ok +test quota::tests::test_parse_quota_limited ... ok +test quota::tests::test_parse_quota_reject_all ... ok +test quota::tests::test_parse_quota_unlimited ... ok +test quota::tests::test_parse_quota_project ... ok +test quota::tests::test_parse_quota_key ... ok +test quota::tests::test_parse_quota_unknown_variants ... ok +test rate_limit::tests::test_rate_limits_add_replacement ... ok +test quota::tests::test_parse_quota_reject_transactions ... ok +test quota::tests::test_parse_quota_project_large ... ok +test rate_limit::tests::test_rate_limits_add_buckets ... ok +test rate_limit::tests::test_rate_limits_add_shadowing ... ok +test rate_limit::tests::test_rate_limits_check ... ok +test rate_limit::tests::test_rate_limits_check_quotas ... ok +test rate_limit::tests::test_rate_limits_clean_expired ... ok +test rate_limit::tests::test_rate_limits_longest ... ok +test redis::tests::test_get_redis_key_scoped ... ok +test rate_limit::tests::test_rate_limits_merge ... ok +test redis::tests::test_get_redis_key_unscoped ... ok +test redis::tests::test_large_redis_limit_large ... ok +test redis::tests::test_bails_immediately_without_any_quota ... ok +test redis::tests::test_zero_size_quotas ... ok +test redis::tests::test_limited_with_unlimited_quota ... ok +test redis::tests::test_quantity_0 ... ok +test redis::tests::test_quota_go_over ... ok +test redis::tests::test_quota_with_quantity ... ok +test redis::tests::test_simple_quota ... ok +test redis::tests::test_is_rate_limited_script ... ok + +test result: ok. 44 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.08s + + +running 4 tests +test config::tests::test_redis_single ... ok +test config::tests::test_redis_single_opts_default ... ok +test config::tests::test_redis_cluster_nodes_opts ... ok +test config::tests::test_redis_single_opts ... ok + +test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 15 tests +test recording::tests::test_pii_credit_card_removal ... ignored, type 3 nodes are not supported +test recording::tests::test_pii_ip_address_removal ... ignored, type 3 nodes are not supported +test recording::tests::test_scrub_pii_full_snapshot_event ... ignored, type 2 nodes are not supported +test recording::tests::test_scrub_pii_incremental_snapshot_event ... ignored, type 3 nodes are not supported +test recording::tests::test_scrub_at_path ... ok +test recording::tests::test_process_recording_no_contents ... ok +test recording::tests::test_process_recording_no_headers ... ok +test recording::tests::test_process_recording_no_body_data ... ok +test recording::tests::test_process_recording_bad_body_data ... ok +test recording::tests::test_process_recording_end_to_end ... ok +test recording::tests::test_scrub_pii_navigation ... ok +test recording::tests::test_scrub_pii_resource ... ok +test recording::tests::test_scrub_pii_key_based ... ok +test recording::tests::test_scrub_pii_custom_event ... ok +test recording::tests::test_scrub_pii_key_based_edge_cases ... ok + +test result: ok. 11 passed; 0 failed; 4 ignored; 0 measured; 0 filtered out; finished in 0.07s + + +running 45 tests +test condition::tests::test_and_combinator ... ok +test condition::tests::test_not_combinator ... ok +test condition::tests::test_or_combinator ... ok +test condition::tests::unsupported_rule_deserialize ... ok +test config::tests::test_sample_rate_returns_same_samplingvalue_variant ... ok +test config::tests::config_deserialize ... ok +test config::tests::test_non_decaying_sampling_rule_deserialization_with_factor ... ok +test config::tests::test_sample_rate_valid_time_range ... ok +test config::tests::test_sample_rate_with_constant_decayingfn ... ok +test config::tests::test_non_decaying_sampling_rule_deserialization ... ok +test config::tests::test_sample_rate_with_linear_decay ... ok +test config::tests::test_sampling_config_with_rules_and_rules_v2_deserialization ... ok +test config::tests::test_sampling_rule_with_constant_decaying_function_deserialization ... ok +test config::tests::test_sampling_config_with_rules_and_rules_v2_serialization ... ok +test config::tests::test_supported ... ok +test config::tests::test_sampling_rule_with_linear_decaying_function_deserialization ... ok +test dsc::tests::getter_filled ... ok +test config::tests::test_unsupported_rule_type ... ok +test dsc::tests::getter_empty ... ok +test dsc::tests::parse_full ... ok +test dsc::tests::parse_user ... ok +test condition::tests::test_does_not_match ... ok +test dsc::tests::test_parse_sampled_with_incoming_boolean ... ok +test dsc::tests::test_parse_sample_rate_bogus ... ok +test dsc::tests::test_parse_sample_rate_number ... ok +test dsc::tests::test_parse_sample_rate_negative ... ok +test dsc::tests::test_parse_sampled_with_incoming_boolean_as_string ... ok +test dsc::tests::test_parse_sampled_with_incoming_invalid_boolean_as_string ... ok +test dsc::tests::test_parse_sampled_with_incoming_null_value ... ok +test evaluation::tests::matched_rule_ids_display ... ok +test evaluation::tests::matched_rule_ids_parse ... ok +test evaluation::tests::test_adjust_by_client_sample_rate ... ok +test evaluation::tests::test_adjust_sample_rate ... ok +test evaluation::tests::test_expired_rules ... ok +test evaluation::tests::test_get_sampling_match_result_with_no_match ... ok +test evaluation::tests::test_matches_reservoir ... ok +test evaluation::tests::test_repeatable_seed ... ok +test evaluation::tests::test_reservoir_evaluator_limit ... ok +test evaluation::tests::test_sample_rate_compounding ... ok +test evaluation::tests::test_condition_matching ... ok +test condition::tests::test_matches ... ok +test dsc::tests::test_parse_sample_rate ... ok +test dsc::tests::test_parse_sample_rate_scientific_notation ... ok +test dsc::tests::test_parse_user_partial ... ok +test condition::tests::deserialize ... ok + +test result: ok. 45 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.04s + + +running 197 tests +test actors::processor::tests::test_breadcrumbs_reversed_with_none ... ok +test actors::processor::tests::test_breadcrumbs_truncation ... ok +test actors::processor::tests::test_breadcrumbs_order_with_none ... ok +test actors::processor::tests::test_breadcrumbs_file1 ... ok +test actors::processor::tests::test_breadcrumbs_file2 ... ok +test actors::processor::tests::test_compute_sampling_decision_matching ... ok +test actors::processor::tests::test_client_sample_rate ... ok +test actors::processor::tests::test_empty_breadcrumbs_item ... ok +test actors::processor::tests::test_error_is_tagged_correctly_if_trace_sampling_result_is_none ... ok +test actors::processor::tests::test_error_is_not_tagged_if_already_tagged ... ok +test actors::processor::tests::test_from_outcome_type_client_discard ... ok +test actors::processor::tests::test_from_outcome_type_filtered ... ok +test actors::processor::tests::test_from_outcome_type_rate_limited ... ok +test actors::processor::tests::test_from_outcome_type_sampled ... ok +test actors::processor::tests::test_it_keeps_or_drops_transactions ... ok +test actors::processor::tests::test_error_is_tagged_correctly_if_trace_sampling_result_is_some ... ok +test actors::processor::tests::test_client_report_forwarding ... ok +test actors::processor::tests::test_client_report_removal_in_processing ... ok +test actors::processor::tests::test_dsc_respects_metrics_extracted ... ok +test actors::processor::tests::test_client_report_removal ... ok +test actors::processor::tests::test_matching_with_unsupported_rule ... ok +test actors::processor::tests::test_mri_overhead_constant ... ok +test actors::processor::tests::test_process_session_invalid_json ... ok +test actors::global_config::tests::proxy_relay_does_not_make_upstream_request ... ok +test actors::processor::tests::test_process_session_invalid_timestamp ... ok +test actors::global_config::tests::managed_relay_makes_upstream_request - should panic ... ok +test actors::global_config::tests::shutdown_service ... ok +test actors::processor::tests::test_process_session_sequence_overflow ... ok +test actors::processor::tests::test_unprintable_fields ... ok +test actors::processor::tests::test_processor_panics ... ok +test actors::processor::tests::test_process_session_keep_item ... ok +test actors::processor::tests::test_process_session_metrics_extracted ... ok +test actors::project::tests::get_state_expired ... ok +test actors::project::tests::test_rate_limit_incoming_buckets ... ok +test actors::project::tests::test_rate_limit_incoming_metrics ... ok +test actors::project::tests::test_rate_limit_incoming_buckets_no_quota ... ok +test actors::processor::tests::test_user_report_invalid ... ok +test actors::project::tests::test_rate_limit_incoming_metrics_no_quota ... ok +test actors::project::tests::test_stale_cache ... ok +test actors::project_redis::tests::test_parse_redis_response ... ok +test actors::project_redis::tests::test_parse_redis_response_compressed ... ok +test actors::project_local::tests::test_multi_pub_static_config ... ok +test actors::project_local::tests::test_symlinked_projects ... ok +test actors::processor::tests::test_log_transaction_metrics_no_match ... ok +test actors::processor::tests::test_log_transaction_metrics_none ... ok +test actors::processor::tests::test_log_transaction_metrics_rule ... ok +test actors::processor::tests::test_log_transaction_metrics_pattern ... ok +test actors::processor::tests::test_log_transaction_metrics_both ... ok +test actors::store::tests::test_return_attachments_when_missing_event_item ... ok +test actors::spooler::tests::metrics_work ... ok +test actors::store::tests::test_send_standalone_attachments_when_transaction ... ok +test actors::store::tests::test_store_attachment_in_event_when_not_a_transaction ... ok +test endpoints::common::tests::test_minimal_empty_event ... ok +test endpoints::common::tests::test_minimal_event_id ... ok +test endpoints::common::tests::test_minimal_event_invalid_type ... ok +test endpoints::common::tests::test_minimal_event_type ... ok +test endpoints::minidump::tests::test_validate_minidump ... ok +test envelope::tests::test_deserialize_envelope_empty ... ok +test envelope::tests::test_deserialize_envelope_empty_item_eof ... ok +test envelope::tests::test_deserialize_envelope_empty_item_newline ... ok +test envelope::tests::test_deserialize_envelope_implicit_length ... ok +test envelope::tests::test_deserialize_envelope_empty_newline ... ok +test envelope::tests::test_deserialize_envelope_implicit_length_empty_eof ... ok +test envelope::tests::test_deserialize_envelope_implicit_length_eof ... ok +test envelope::tests::test_deserialize_envelope_replay_recording ... ok +test envelope::tests::test_deserialize_envelope_multiple_items ... ok +test envelope::tests::test_deserialize_envelope_unknown_item ... ok +test envelope::tests::test_deserialize_envelope_view_hierarchy ... ok +test envelope::tests::test_envelope_add_item ... ok +test envelope::tests::test_deserialize_request_meta ... ok +test envelope::tests::test_envelope_empty ... ok +test envelope::tests::test_item_empty ... ok +test envelope::tests::test_envelope_take_item ... ok +test envelope::tests::test_item_set_header ... ok +test envelope::tests::test_item_set_payload ... ok +test envelope::tests::test_parse_request_envelope ... ok +test envelope::tests::test_parse_request_no_dsn ... ok +test envelope::tests::test_parse_request_no_origin ... ok +test envelope::tests::test_parse_request_sent_at ... ok +test envelope::tests::test_parse_request_sent_at_null ... ok +test envelope::tests::test_parse_request_validate_key - should panic ... ok +test envelope::tests::test_parse_request_validate_origin - should panic ... ok +test envelope::tests::test_parse_request_validate_project - should panic ... ok +test envelope::tests::test_serialize_envelope_attachments ... ok +test envelope::tests::test_serialize_envelope_empty ... ok +test envelope::tests::test_split_envelope_all ... ok +test envelope::tests::test_split_envelope_none ... ok +test envelope::tests::test_split_envelope_some ... ok +test extractors::forwarded_for::tests::test_fall_back_on_forwarded_for_header ... ok +test extractors::forwarded_for::tests::test_get_empty_string_if_invalid_header ... ok +test extractors::forwarded_for::tests::test_prefer_vercel_forwarded ... ok +test extractors::start_time::tests::start_time_from_timestamp ... ok +test extractors::request_meta::tests::test_request_meta_roundtrip ... ok +test metrics_extraction::generic::tests::extract_counter ... ok +test metrics_extraction::generic::tests::extract_set ... ok +test metrics_extraction::generic::tests::extract_distribution ... ok +test metrics_extraction::generic::tests::extract_tag_precedence ... ok +test metrics_extraction::generic::tests::extract_tag_conditions ... ok +test metrics_extraction::sessions::tests::test_extract_session_metrics ... ok +test metrics_extraction::sessions::tests::test_extract_session_metrics_abnormal ... ok +test metrics_extraction::sessions::tests::test_extract_session_metrics_errored ... ok +test metrics_extraction::sessions::tests::test_extract_session_metrics_aggregate ... ok +test metrics_extraction::sessions::tests::test_extract_session_metrics_fatal ... ok +test metrics_extraction::sessions::tests::test_extract_session_metrics_ok ... ok +test metrics_extraction::sessions::tests::test_nil_to_none ... ok +test metrics_extraction::generic::tests::extract_tag_precedence_multiple_rules ... ok +test metrics_extraction::transactions::tests::test_any_client_route ... ok +test metrics_extraction::transactions::tests::test_computed_metrics ... ok +test metrics_extraction::transactions::tests::test_conditional_tagging ... ok +test metrics_extraction::transactions::tests::test_custom_measurements ... ok +test metrics_extraction::transactions::tests::test_express ... ok +test metrics_extraction::transactions::tests::test_get_eventuser_tag ... ok +test metrics_extraction::transactions::tests::test_express_options ... ok +test metrics_extraction::transactions::tests::test_js_url_strict ... ok +test metrics_extraction::transactions::tests::test_legacy_js_does_not_look_like_url ... ok +test metrics_extraction::transactions::tests::test_legacy_js_looks_like_url ... ok +test metrics_extraction::transactions::tests::test_metric_measurement_unit_overrides ... ok +test metrics_extraction::transactions::tests::test_metric_measurement_units ... ok +test metrics_extraction::transactions::tests::test_other_client_unknown ... ok +test metrics_extraction::transactions::tests::test_parse_transaction_name_strategy ... ok +test metrics_extraction::transactions::tests::test_other_client_url ... ok +test metrics_extraction::transactions::tests::test_extract_transaction_metrics ... ok +test metrics_extraction::transactions::tests::test_python_200 ... ok +test metrics_extraction::transactions::tests::test_python_404 ... ok +test metrics_extraction::event::tests::test_extract_span_metrics_mobile ... ok +test metrics_extraction::transactions::tests::test_root_counter_keep ... ok +test metrics_extraction::transactions::tests::test_span_tags ... ok +test metrics_extraction::transactions::tests::test_transaction_duration ... ok +test metrics_extraction::transactions::tests::test_unknown_transaction_status ... ok +test metrics_extraction::transactions::tests::test_unknown_transaction_status_no_trace_context ... ok +test middlewares::normalize_path::tests::root ... ok +test middlewares::normalize_path::tests::query_and_fragment ... ok +test middlewares::normalize_path::tests::no_trailing_slash ... ok +test middlewares::normalize_path::tests::path ... ok +test utils::dynamic_sampling::tests::test_is_trace_fully_sampled_with_invalid_inputs ... ok +test utils::dynamic_sampling::tests::test_is_trace_fully_sampled_return_true_with_unsupported_rules ... ok +test utils::dynamic_sampling::tests::test_is_trace_fully_sampled_with_valid_dsc_and_sampling_config ... ok +test utils::dynamic_sampling::tests::test_match_rules_return_drop_with_match_and_0_sample_rate ... ok +test utils::dynamic_sampling::tests::test_match_rules_return_keep_with_match_and_100_sample_rate ... ok +test utils::dynamic_sampling::tests::test_match_rules_return_keep_with_no_match ... ok +test utils::dynamic_sampling::tests::test_match_rules_with_traces_rules_return_keep_when_match ... ok +test utils::metrics_rate_limits::tests::profiles_limits_are_reported ... ok +test utils::garbage::tests::test_garbage_disposal ... ok +test utils::metrics_rate_limits::tests::profiles_quota_is_enforced ... ok +test utils::multipart::tests::test_empty_formdata ... ok +test utils::multipart::tests::test_formdata ... ok +test utils::multipart::tests::test_get_boundary ... ok +test utils::param_parser::tests::test_aggregator_base_0 ... ok +test utils::multipart::tests::missing_trailing_newline ... ok +test utils::param_parser::tests::test_aggregator_base_1 ... ok +test utils::param_parser::tests::test_aggregator_empty ... ok +test utils::param_parser::tests::test_aggregator_holes ... ok +test utils::param_parser::tests::test_aggregator_override ... ok +test utils::param_parser::tests::test_aggregator_reversed ... ok +test utils::param_parser::tests::test_chunk_index ... ok +test utils::param_parser::tests::test_index_parser ... ok +test utils::param_parser::tests::test_merge_vals ... ok +test utils::rate_limits::tests::test_enforce_event_metrics_extracted ... ok +test utils::param_parser::tests::test_update_value ... ok +test utils::rate_limits::tests::test_enforce_event_metrics_extracted_no_indexing_quota ... ok +test utils::rate_limits::tests::test_enforce_limit_assumed_attachments ... ok +test utils::rate_limits::tests::test_enforce_limit_assumed_event ... ok +test utils::rate_limits::tests::test_enforce_limit_attachments ... ok +test utils::rate_limits::tests::test_enforce_limit_error_event ... ok +test utils::rate_limits::tests::test_enforce_limit_error_with_attachments ... ok +test utils::rate_limits::tests::test_enforce_limit_minidump ... ok +test utils::rate_limits::tests::test_enforce_limit_monitor_checkins ... ok +test utils::rate_limits::tests::test_enforce_limit_profiles ... ok +test utils::rate_limits::tests::test_enforce_limit_replays ... ok +test utils::rate_limits::tests::test_enforce_limit_sessions ... ok +test utils::rate_limits::tests::test_enforce_pass_empty ... ok +test utils::rate_limits::tests::test_enforce_pass_minidump ... ok +test utils::rate_limits::tests::test_enforce_pass_sessions ... ok +test utils::rate_limits::tests::test_enforce_skip_rate_limited ... ok +test utils::rate_limits::tests::test_enforce_transaction_attachment_enforced ... ok +test utils::rate_limits::tests::test_enforce_transaction_attachment_enforced_metrics_extracted_indexing_quota ... ok +test utils::rate_limits::tests::test_enforce_transaction_no_indexing_quota ... ok +test utils::rate_limits::tests::test_enforce_transaction_no_metrics_extracted ... ok +test utils::rate_limits::tests::test_enforce_transaction_profile_enforced ... ok +test utils::rate_limits::tests::test_format_rate_limits ... ok +test utils::rate_limits::tests::test_parse_invalid_rate_limits ... ok +test utils::rate_limits::tests::test_parse_rate_limits ... ok +test utils::rate_limits::tests::test_parse_rate_limits_only_unknown ... ok +test utils::semaphore::tests::test_empty ... ok +test utils::semaphore::tests::test_single_thread ... ok +test utils::semaphore::tests::test_multi_thread ... ok +test utils::unreal::tests::test_merge_unreal_context_is_assert_level_error ... ok +test utils::unreal::tests::test_merge_unreal_context_is_esure_level_warning ... ok +test utils::unreal::tests::test_merge_unreal_context ... ok +test utils::unreal::tests::test_merge_unreal_logs ... ok +test utils::unreal::tests::test_merge_unreal_context_event ... ok +test metrics_extraction::event::tests::test_extract_span_metrics_all_modules ... ok +test metrics_extraction::event::tests::test_extract_span_metrics ... ok +test actors::spooler::tests::dequeue_waits_for_permits ... ok +test actors::spooler::tests::ensure_start_time_restore ... ok +test actors::processor::tests::test_browser_version_extraction_with_pii_like_data ... ok +test actors::project_cache::tests::always_spools ... ok + +test result: ok. 197 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 2.87s + + +running 12 tests +test legacy_python::test_processing ... ok +test cordova::test_processing ... ok +test legacy_node_exception::test_processing ... ok +test dotnet::test_processing ... ok +test test_event_schema_snapshot ... ok +test unity_windows::test_processing ... ok +test unity_macos::test_processing ... ok +test unity_android::test_processing ... ok +test unity_ios::test_processing ... ok +test unity_linux::test_processing ... ok +test android::test_processing ... ok +test cocoa::test_processing ... ok + +test result: ok. 12 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.06s + + +running 1 test +test test_reponse_context_pii ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.06s + + +running 2 tests +test tests::current_client_is_global_client ... ok +test tests::test_capturing_client ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 1 test +test service::tests::test_backpressure_metrics ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 1 test +test relay-auth/src/lib.rs - (line 14) ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.92s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 5 tests +test relay-common/src/macros.rs - macros::derive_fromstr_and_display (line 80) ... ok +test relay-common/src/time.rs - time::UnixTimestamp::to_instant (line 147) ... ok +test relay-common/src/time.rs - time::chrono_to_positive_millis (line 53) ... ok +test relay-common/src/time.rs - time::chrono_to_positive_millis (line 43) ... ok +test relay-common/src/time.rs - time::duration_to_millis (line 25) ... ok + +test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.81s + + +running 2 tests +test relay-config/src/byte_size.rs - byte_size::ByteSize (line 24) ... ok +test relay-config/src/byte_size.rs - byte_size::ByteSize (line 33) ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.62s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 2 tests +test relay-event-schema/src/processor/attrs.rs - processor::attrs::BoxCow (line 378) ... ignored +test relay-event-schema/src/processor/chunks.rs - processor::chunks (line 8) ... ok + +test result: ok. 1 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.28s + + +running 8 tests +test relay-ffi/src/lib.rs - set_panic_hook (line 282) ... ok +test relay-ffi/src/lib.rs - (line 13) ... ok +test relay-ffi/src/lib.rs - (line 79) ... ok +test relay-ffi/src/lib.rs - (line 58) ... ok +test relay-ffi/src/lib.rs - take_last_error (line 188) ... ok +test relay-ffi/src/lib.rs - (line 41) ... ok +test relay-ffi/src/lib.rs - with_last_error (line 161) ... ok +test relay-ffi/src/lib.rs - Panic (line 212) ... ok + +test result: ok. 8 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.65s + + +running 1 test +test relay-ffi-macros/src/lib.rs - catch_unwind (line 52) ... ignored + +test result: ok. 0 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 2 tests +test relay-kafka/src/config.rs - config::Sharded (line 196) ... ignored +test relay-kafka/src/lib.rs - (line 9) - compile fail ... ok + +test result: ok. 1 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.10s + + +running 9 tests +test relay-log/src/lib.rs - (line 101) ... ok +test relay-log/src/utils.rs - utils::backtrace_enabled (line 10) ... ok +test relay-log/src/lib.rs - (line 72) ... ok +test relay-log/src/lib.rs - (line 46) ... ok +test relay-log/src/test.rs - test::init_test (line 31) ... ok +test relay-log/src/lib.rs - (line 58) ... ok +test relay-log/src/setup.rs - setup::init (line 206) ... ok +test relay-log/src/lib.rs - (line 88) ... ok +test relay-log/src/lib.rs - (line 9) ... ok + +test result: ok. 9 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 1.35s + + +running 5 tests +test relay-metrics/src/bucket.rs - bucket::Bucket::parse (line 665) ... ok +test relay-metrics/src/protocol.rs - protocol::MetricResourceIdentifier (line 211) ... ok +test relay-metrics/src/bucket.rs - bucket::Bucket::parse_all (line 687) ... ok +test relay-metrics/src/bucket.rs - bucket::dist (line 116) ... ok +test relay-metrics/src/bucket.rs - bucket::DistributionValue (line 82) ... ok + +test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.73s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 4 tests +test relay-protocol/src/traits.rs - traits::SkipSerialization (line 35) ... ignored +test relay-protocol/src/traits.rs - traits::Getter (line 161) ... ok +test relay-protocol/src/macros.rs - macros::get_path (line 32) ... ok +test relay-protocol/src/macros.rs - macros::get_value (line 105) ... ok + +test result: ok. 3 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.31s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 3 tests +test relay-replays/src/transform.rs - transform::Deserializer (line 159) ... ignored +test relay-replays/src/transform.rs - transform::Transform (line 33) ... ignored +test relay-replays/src/recording.rs - recording::RecordingScrubber (line 278) ... ok + +test result: ok. 1 passed; 0 failed; 2 ignored; 0 measured; 0 filtered out; finished in 0.52s + + +running 20 tests +test relay-sampling/src/condition.rs - condition::RuleCondition::Or (line 411) ... ok +test relay-sampling/src/condition.rs - condition::RuleCondition (line 328) ... ok +test relay-sampling/src/condition.rs - condition::RuleCondition::Lte (line 367) ... ok +test relay-sampling/src/condition.rs - condition::RuleCondition::Glob (line 400) ... ok +test relay-sampling/src/condition.rs - condition::RuleCondition::And (line 423) ... ok +test relay-sampling/src/condition.rs - condition::RuleCondition::Gt (line 378) ... ok +test relay-sampling/src/condition.rs - condition::RuleCondition::Eq (line 345) ... ok +test relay-sampling/src/condition.rs - condition::RuleCondition::Lt (line 389) ... ok +test relay-sampling/src/condition.rs - condition::RuleCondition::Not (line 435) ... ok +test relay-sampling/src/condition.rs - condition::RuleCondition::Gte (line 356) ... ok +test relay-sampling/src/condition.rs - condition::RuleCondition::and (line 575) ... ok +test relay-sampling/src/condition.rs - condition::RuleCondition::eq (line 467) ... ok +test relay-sampling/src/condition.rs - condition::RuleCondition::eq_ignore_case (line 487) ... ok +test relay-sampling/src/condition.rs - condition::RuleCondition::glob (line 504) ... ok +test relay-sampling/src/condition.rs - condition::RuleCondition::lt (line 547) ... ok +test relay-sampling/src/condition.rs - condition::RuleCondition::gt (line 521) ... ok +test relay-sampling/src/condition.rs - condition::RuleCondition::gte (line 534) ... ok +test relay-sampling/src/condition.rs - condition::RuleCondition::lte (line 560) ... ok +test relay-sampling/src/condition.rs - condition::RuleCondition::negate (line 621) ... ok +test relay-sampling/src/condition.rs - condition::RuleCondition::or (line 598) ... ok + +test result: ok. 20 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 1.86s + + +running 2 tests +test relay-server/src/actors/mod.rs - actors (line 24) ... ignored +test relay-server/src/utils/multipart.rs - utils::multipart::get_multipart_boundary (line 134) ... ignored + +test result: ok. 0 passed; 0 failed; 2 ignored; 0 measured; 0 filtered out; finished in 0.00s + + +running 8 tests +test relay-statsd/src/lib.rs - (line 23) - compile ... ok +test relay-statsd/src/lib.rs - (line 50) ... ok +test relay-statsd/src/lib.rs - GaugeMetric (line 492) ... ok +test relay-statsd/src/lib.rs - HistogramMetric (line 411) ... ok +test relay-statsd/src/lib.rs - (line 34) ... ok +test relay-statsd/src/lib.rs - CounterMetric (line 358) ... ok +test relay-statsd/src/lib.rs - SetMetric (line 448) ... ok +test relay-statsd/src/lib.rs - TimerMetric (line 297) ... ok + +test result: ok. 8 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.84s + + +running 16 tests +test relay-system/src/service.rs - service::FromMessage (line 572) ... ok +test relay-system/src/service.rs - service::BroadcastChannel (line 314) ... ok +test relay-system/src/service.rs - service::BroadcastSender::into_channel (line 505) ... ok +test relay-system/src/service.rs - service::FromMessage (line 598) ... ok +test relay-system/src/service.rs - service::BroadcastChannel::is_attached (line 408) ... ok +test relay-system/src/service.rs - service::BroadcastChannel::attach (line 363) ... ok +test relay-system/src/service.rs - service::Service (line 897) - compile ... ok +test relay-system/src/service.rs - service::BroadcastSender::send (line 480) ... ok +test relay-system/src/service.rs - service::BroadcastChannel::send (line 384) ... ok +test relay-system/src/service.rs - service::BroadcastSender (line 445) ... ok +test relay-system/src/controller.rs - controller::Controller (line 118) ... ok +test relay-system/src/service.rs - service::FromMessage (line 626) ... ok +test relay-system/src/service.rs - service::Interface (line 34) ... ok +test relay-system/src/service.rs - service::Interface (line 78) ... ok +test relay-system/src/service.rs - service::Interface (line 54) ... ok +test relay-system/src/service.rs - service::Service (line 937) ... ok + +test result: ok. 16 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 1.38s + + +running 1 test +test relay-test/src/lib.rs - (line 11) ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.29s + + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + diff --git a/relay-sampling/src/evaluation.rs b/relay-sampling/src/evaluation.rs index df7cfe982b..dd68a66102 100644 --- a/relay-sampling/src/evaluation.rs +++ b/relay-sampling/src/evaluation.rs @@ -450,6 +450,15 @@ mod tests { matched_rule_ids == sampling_match.matched_rules } + fn get_matched_rules( + sampling_evaluator: &ControlFlow, + ) -> Vec { + match sampling_evaluator { + ControlFlow::Continue(_) => panic!(), + ControlFlow::Break(m) => m.matched_rules.0.iter().map(|rule_id| rule_id.0).collect(), + } + } + /// Helper function to create a dsc with the provided getter-values set. fn mocked_dsc_with_getter_values( paths_and_values: Vec<(&str, &str)>, @@ -575,6 +584,50 @@ mod tests { vec } + /// Tests that reservoir rules override the other rules. + /// + /// Here all 3 rules are a match. But when the reservoir + /// rule (id = 1) has not yet reached its limit of "2" matches, the + /// previous rule(s) will not be present in the matched rules output. + /// After the limit has been reached, the reservoir rule is ignored + /// and the output is the two other rules (id = 0, id = 2). + #[test] + fn test_reservoir_override() { + let dsc = mocked_dsc_with_getter_values(vec![]); + let rules = simple_sampling_rules(vec![ + (RuleCondition::all(), SamplingValue::Factor { value: 0.5 }), + // The reservoir has a limit of 2, meaning it should be sampled twice + // before it is ignored. + (RuleCondition::all(), SamplingValue::Reservoir { limit: 2 }), + ( + RuleCondition::all(), + SamplingValue::SampleRate { value: 0.5 }, + ), + ]); + + // The reservoir keeps the counter state behind a mutex, which is how it + // shares state among multiple evaluator instances. + let reservoir = mock_reservoir_evaluator(vec![]); + + let evaluator = SamplingEvaluator::new(Utc::now()).set_reservoir(&reservoir); + let matched_rules = + get_matched_rules(&evaluator.match_rules(Uuid::default(), &dsc, rules.iter())); + // Reservoir rule overrides 0 and 2. + assert_eq!(&matched_rules, &[1]); + + let evaluator = SamplingEvaluator::new(Utc::now()).set_reservoir(&reservoir); + let matched_rules = + get_matched_rules(&evaluator.match_rules(Uuid::default(), &dsc, rules.iter())); + // Reservoir rule overrides 0 and 2. + assert_eq!(&matched_rules, &[1]); + + let evaluator = SamplingEvaluator::new(Utc::now()).set_reservoir(&reservoir); + let matched_rules = + get_matched_rules(&evaluator.match_rules(Uuid::default(), &dsc, rules.iter())); + // Reservoir rule reached its limit, rule 0 and 2 are now matched instead. + assert_eq!(&matched_rules, &[0, 2]); + } + /// Checks that rules don't match if the time is outside the time range. #[test] fn test_expired_rules() { From 27ab4120af761e2bf4ccc9e8f1fd4f519158f025 Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Thu, 28 Sep 2023 21:45:06 +0200 Subject: [PATCH 28/30] add test --- bat | 1838 ----------------------------------------------------------- 1 file changed, 1838 deletions(-) delete mode 100644 bat diff --git a/bat b/bat deleted file mode 100644 index e7a8bfde71..0000000000 --- a/bat +++ /dev/null @@ -1,1838 +0,0 @@ -cargo test --workspace --all-features - -running 1 test -test tests::test_parse_metrics ... ok - -test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.04s - - -running 7 tests -test tests::test_find_rs_files ... ok -test tests::test_single_type ... ok -test tests::test_pii_false ... ok -test tests::test_scoped_paths ... ok -test tests::test_pii_true ... ok -test tests::test_pii_all ... ok -test tests::test_pii_retain_additional_properties_truth_table ... ok - -test result: ok. 7 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.04s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 12 tests -test tests::test_deserialize_old_response ... ok -test tests::test_relay_version_any_supported ... ok -test tests::test_relay_version_current ... ok -test tests::test_relay_version_from_str ... ok -test tests::test_relay_version_oldest_supported ... ok -test tests::test_relay_version_oldest ... ok -test tests::test_relay_version_parse ... ok -test tests::test_keys ... ok -test tests::test_serializing ... ok -test tests::test_signatures ... ok -test tests::test_generate_strings_for_test_auth_py ... ok -test tests::test_registration ... ok - -test result: ok. 12 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 1 test -test metrics::tests::test_custom_unit_parse ... ok - -test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 3 tests -test processing::pii_config_validation_invalid_regex ... ok -test processing::pii_config_validation_valid_regex ... ok -test codeowners::tests::test_translate_codeowners_pattern ... ok - -test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 25 tests -test glob3::tests::test_match_negative ... ok -test glob3::tests::test_match_neg_unsupported ... ok -test glob3::tests::test_match_literal ... ok -test glob3::tests::test_match_newline ... ok -test glob3::tests::test_match_newline_inner ... ok -test glob3::tests::test_match_inner ... ok -test glob3::tests::test_match_prefix ... ok -test glob3::tests::test_match_newline_pattern ... ok -test glob3::tests::test_match_utf8 ... ok -test glob3::tests::test_match_range ... ok -test time::tests::test_parse_datetime_bogus ... ok -test glob3::tests::test_match_suffix ... ok -test glob3::tests::test_match_range_neg ... ok -test time::tests::test_parse_timestamp_float ... ok -test time::tests::test_parse_timestamp_int ... ok -test time::tests::test_parse_timestamp_large_float ... ok -test glob2::tests::test_do_not_replace ... ok -test time::tests::test_parse_timestamp_neg_float ... ok -test time::tests::test_parse_timestamp_neg_int ... ok -test time::tests::test_parse_timestamp_other ... ok -test time::tests::test_parse_timestamp_str ... ok -test glob2::tests::test_glob_matcher ... ok -test glob2::tests::test_glob_replace ... ok -test glob2::tests::test_glob ... ok -test glob::tests::test_globs ... ok - -test result: ok. 25 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.04s - - -running 10 tests -test byte_size::tests::test_as_bytes ... ok -test byte_size::tests::test_infer ... ok -test config::tests::test_emit_outcomes_invalid ... ok -test byte_size::tests::test_parse ... ok -test byte_size::tests::test_serde_number ... ok -test config::tests::test_emit_outcomes ... ok -test byte_size::tests::test_serde_string ... ok -test upstream::test::test_from_dsn ... ok -test upstream::test::test_basic_parsing ... ok -test config::tests::test_event_buffer_size ... ok - -test result: ok. 10 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 39 tests -test native::bindgen_test_layout___darwin_arm_cpmu_state64 ... ok -test native::bindgen_test_layout___arm_legacy_debug_state ... ok -test native::bindgen_test_layout___darwin_arm_debug_state32 ... ok -test native::bindgen_test_layout___darwin_arm_debug_state64 ... ok -test native::bindgen_test_layout___arm_pagein_state ... ok -test native::bindgen_test_layout___darwin_arm_exception_state64 ... ok -test native::bindgen_test_layout___darwin_arm_exception_state ... ok -test native::bindgen_test_layout___darwin_arm_neon_state ... ok -test native::bindgen_test_layout___darwin_arm_neon_state64 ... ok -test native::bindgen_test_layout___darwin_arm_thread_state ... ok -test native::bindgen_test_layout___darwin_arm_thread_state64 ... ok -test native::bindgen_test_layout___darwin_arm_vfp_state ... ok -test native::bindgen_test_layout___darwin_mcontext32 ... ok -test native::bindgen_test_layout___darwin_mcontext64 ... ok -test native::bindgen_test_layout___darwin_pthread_handler_rec ... ok -test native::bindgen_test_layout___darwin_sigaltstack ... ok -test native::bindgen_test_layout___darwin_ucontext ... ok -test native::bindgen_test_layout___mbstate_t ... ok -test native::bindgen_test_layout___sigaction ... ok -test native::bindgen_test_layout___sigaction_u ... ok -test native::bindgen_test_layout___siginfo ... ok -test native::bindgen_test_layout__opaque_pthread_attr_t ... ok -test native::bindgen_test_layout__opaque_pthread_cond_t ... ok -test native::bindgen_test_layout__opaque_pthread_condattr_t ... ok -test native::bindgen_test_layout__opaque_pthread_mutex_t ... ok -test native::bindgen_test_layout__opaque_pthread_mutexattr_t ... ok -test native::bindgen_test_layout__opaque_pthread_once_t ... ok -test native::bindgen_test_layout__opaque_pthread_rwlock_t ... ok -test native::bindgen_test_layout__opaque_pthread_t ... ok -test native::bindgen_test_layout__opaque_pthread_rwlockattr_t ... ok -test native::bindgen_test_layout_sentry_ucontext_s ... ok -test native::bindgen_test_layout_imaxdiv_t ... ok -test native::bindgen_test_layout_sentry_uuid_s ... ok -test native::bindgen_test_layout_sentry_value_u ... ok -test native::bindgen_test_layout_sigaction ... ok -test native::bindgen_test_layout_sigevent ... ok -test native::bindgen_test_layout_sigvec ... ok -test native::bindgen_test_layout_sigval ... ok -test native::bindgen_test_layout_sigstack ... ok - -test result: ok. 39 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 1 test -test stats::tests::parses_metric ... ok - -test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 12 tests -test error_boundary::tests::test_deserialize_ok ... ok -test error_boundary::tests::test_deserialize_err ... ok -test error_boundary::tests::test_deserialize_syntax_err ... ok -test error_boundary::tests::test_serialize_err ... ok -test error_boundary::tests::test_serialize_ok ... ok -test feature::tests::roundtrip ... ok -test metrics::tests::parse_tag_spec_field ... ok -test metrics::tests::parse_tag_spec_unsupported ... ok -test global::tests::test_global_config_roundtrip ... ok -test metrics::tests::parse_tag_spec_value ... ok -test utils::tests::test_validate_json ... ok -test metrics::tests::parse_tag_mapping ... ok - -test result: ok. 12 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 423 tests -test clock_drift::tests::test_clock_drift_unix ... ok -test clock_drift::tests::test_process_datetime ... ok -test clock_drift::tests::test_clock_drift_lower_bound ... ok -test clock_drift::tests::test_no_sent_at ... ok -test event_error::tests::test_no_errors ... ok -test event_error::tests::test_nested_errors ... ok -test event_error::tests::test_original_value ... ok -test event_error::tests::test_errors_in_other ... ok -test event_error::tests::test_multiple_errors ... ok -test clock_drift::tests::test_no_clock_drift ... ok -test clock_drift::tests::test_clock_drift_from_past ... ok -test clock_drift::tests::test_clock_drift_from_future ... ok -test event_error::tests::test_top_level_errors ... ok -test normalize::breakdowns::tests::test_noop_breakdowns_with_empty_config ... ok -test normalize::breakdowns::tests::test_skip_with_empty_breakdowns_config ... ok -test normalize::breakdowns::tests::test_emit_ops_breakdown ... ok -test normalize::contexts::tests::test_broken_json_with_fallback ... ok -test normalize::contexts::tests::test_broken_json_without_fallback ... ok -test normalize::contexts::tests::test_infer_json ... ok -test normalize::contexts::tests::test_dotnet_core ... ok -test normalize::contexts::tests::test_dotnet_framework_48_without_build_id ... ok -test normalize::contexts::tests::test_dotnet_framework_472 ... ok -test normalize::contexts::tests::test_dotnet_framework_future_version ... ok -test normalize::contexts::tests::test_dotnet_native ... ok -test normalize::contexts::tests::test_macos_with_build ... ok -test normalize::contexts::tests::test_macos_without_build ... ok -test normalize::contexts::tests::test_name_not_overwritten ... ok -test normalize::contexts::tests::test_no_name ... ok -test normalize::contexts::tests::test_unity_android_api_version ... ok -test normalize::contexts::tests::test_unity_mac_os ... ok -test normalize::contexts::tests::test_linux_5_11 ... ok -test normalize::contexts::tests::test_unity_windows_os ... ok -test normalize::contexts::tests::test_version_not_overwritten ... ok -test normalize::contexts::tests::test_windows_10 ... ok -test normalize::contexts::tests::test_windows_11 ... ok -test normalize::contexts::tests::test_windows_11_future1 ... ok -test normalize::contexts::tests::test_windows_11_future2 ... ok -test normalize::contexts::tests::test_windows_7_or_server_2008 ... ok -test normalize::contexts::tests::test_windows_8_or_server_2012_or_later ... ok -test normalize::logentry::tests::test_empty_logentry ... ok -test normalize::logentry::tests::test_empty_missing_message ... ok -test normalize::contexts::tests::test_wsl_ubuntu ... ok -test normalize::contexts::tests::test_centos_runtime_info ... ok -test normalize::contexts::tests::test_unity_android_os ... ok -test normalize::contexts::tests::test_android_4_4_2 ... ok -test normalize::contexts::tests::test_centos_os_version ... ok -test normalize::contexts::tests::test_macos_runtime ... ok -test normalize::contexts::tests::test_ios_15_0 ... ok -test normalize::contexts::tests::test_macos_os_version ... ok -test normalize::logentry::tests::test_format_no_params ... ok -test normalize::logentry::tests::test_message_formatted_equal ... ok -test normalize::logentry::tests::test_only_message ... ok -test normalize::mechanism::tests::test_normalize_errno ... ok -test normalize::mechanism::tests::test_normalize_errno_fail ... ok -test normalize::mechanism::tests::test_normalize_errno_override ... ok -test normalize::mechanism::tests::test_normalize_mach_fail ... ok -test normalize::mechanism::tests::test_normalize_mach ... ok -test normalize::mechanism::tests::test_normalize_missing ... ok -test normalize::mechanism::tests::test_normalize_mach_override ... ok -test normalize::mechanism::tests::test_normalize_partial_signal ... ok -test normalize::mechanism::tests::test_normalize_signal ... ok -test normalize::mechanism::tests::test_normalize_signal_fail ... ok -test normalize::mechanism::tests::test_normalize_signal_override ... ok -test normalize::request::tests::test_broken_json_with_fallback ... ok -test normalize::request::tests::test_broken_json_without_fallback ... ok -test normalize::request::tests::test_cookies_in_header ... ok -test normalize::request::tests::test_cookies_in_header_dont_override_cookies ... ok -test normalize::request::tests::test_infer_binary ... ok -test normalize::request::tests::test_infer_json ... ok -test normalize::request::tests::test_infer_url_encoded_base64 ... ok -test normalize::request::tests::test_infer_url_encoded ... ok -test normalize::request::tests::test_infer_xml ... ok -test normalize::request::tests::test_infer_url_false_positive ... ok -test normalize::request::tests::test_query_string_empty_value ... ok -test normalize::request::tests::test_url_only_path ... ok -test normalize::request::tests::test_url_precedence ... ok -test normalize::request::tests::test_method_invalid ... ok -test normalize::request::tests::test_url_punycoded ... ok -test normalize::request::tests::test_method_valid ... ok -test normalize::request::tests::test_url_truncation ... ok -test normalize::request::tests::test_url_truncation_reversed ... ok -test normalize::request::tests::test_url_with_ellipsis ... ok -test normalize::request::tests::test_url_with_qs_and_fragment ... ok -test normalize::span::attributes::tests::test_child_spans_consumes_all_of_parent ... ok -test normalize::span::attributes::tests::test_child_spans_dont_intersect_parent ... ok -test normalize::span::attributes::tests::test_child_spans_extend_beyond_parent ... ok -test normalize::span::attributes::tests::test_childless_spans ... ok -test normalize::span::attributes::tests::test_nested_spans ... ok -test normalize::logentry::tests::test_format_python ... ok -test normalize::span::attributes::tests::test_only_immediate_child_spans_affect_calculation ... ok -test normalize::span::attributes::tests::test_overlapping_child_spans ... ok -test normalize::span::attributes::tests::test_skip_exclusive_time ... ok -test normalize::span::description::sql::tests::activerecord ... ok -test normalize::logentry::tests::test_format_python_named ... ok -test normalize::span::description::sql::parser::tests::parse_deep_expression ... ok -test normalize::span::description::sql::tests::boolean_not_in_mid_tablename_true ... ok -test normalize::span::description::sql::tests::already_scrubbed ... ok -test normalize::span::description::sql::tests::boolean_not_in_tablename_true ... ok -test normalize::span::description::sql::tests::boolean_not_in_tablename_false ... ok -test normalize::span::description::sql::tests::boolean_not_in_mid_tablename_false ... ok -test normalize::logentry::tests::test_format_java ... ok -test normalize::logentry::tests::test_format_dotnet ... ok -test normalize::span::description::sql::tests::bytesa ... ok -test normalize::span::description::sql::tests::boolean_where_false ... ok -test normalize::span::description::sql::tests::case_when ... ok -test normalize::span::description::sql::tests::boolean_where_true ... ok -test normalize::span::description::sql::tests::case_when_nested ... ok -test normalize::span::description::sql::tests::boolean_where_bool_insensitive ... ok -test normalize::span::description::sql::tests::close_cursor ... ok -test normalize::span::description::sql::tests::collapse_columns_distinct ... ok -test normalize::span::description::sql::tests::collapse_columns_nested ... ok -test normalize::span::description::sql::tests::collapse_columns ... ok -test normalize::span::description::sql::tests::collapse_columns_with_as ... ok -test normalize::span::description::sql::tests::collapse_partial_column_lists ... ok -test normalize::span::description::sql::tests::collapse_columns_with_as_without_quotes ... ok -test normalize::span::description::sql::tests::declare_cursor ... ok -test normalize::span::description::sql::tests::declare_cursor_advanced ... ok -test normalize::span::description::sql::tests::collapse_partial_column_lists_2 ... ok -test normalize::span::description::sql::tests::do_not_collapse_single_column ... ok -test normalize::span::description::sql::tests::digits_in_compound_table_name ... ok -test normalize::span::description::sql::tests::dont_scrub_double_quoted_strings_format_mysql ... ok -test normalize::span::description::sql::tests::dont_scrub_double_quoted_strings_format_postgres ... ok -test normalize::span::description::sql::tests::digits_in_table_name ... ok -test normalize::span::description::sql::tests::fetch_cursor ... ok -test normalize::span::description::sql::tests::dont_scrub_nulls ... ok -test normalize::span::description::sql::tests::jsonb ... ok -test normalize::span::description::sql::tests::not_a_comment ... ok -test normalize::span::description::sql::tests::multiple_statements ... ok -test normalize::span::description::sql::tests::mixed ... ok -test normalize::span::description::sql::tests::num_e_where ... ok -test normalize::span::description::sql::tests::num_negative_where ... ok -test normalize::span::description::sql::tests::num_limit ... ok -test normalize::span::description::sql::tests::parameters_values ... ok -test normalize::span::description::sql::tests::num_where ... ok -test normalize::span::description::sql::tests::parameters_in ... ok -test normalize::span::description::sql::tests::php_placeholders ... ok -test normalize::span::description::sql::tests::parameters_values_with_quotes ... ok -test normalize::span::description::sql::tests::quotes_in_cast ... ok -test normalize::span::description::sql::tests::qualified_wildcard ... ok -test normalize::span::description::sql::tests::quotes_in_function ... ok -test normalize::span::description::sql::tests::savepoint_lowercase ... ok -test normalize::span::description::sql::tests::quotes_in_join ... ok -test normalize::span::description::sql::tests::savepoint_quoted ... ok -test normalize::span::description::sql::tests::savepoint_uppercase ... ok -test normalize::span::description::sql::tests::savepoint_uppercase_semicolon ... ok -test normalize::span::description::sql::tests::single_digit_in_table_name ... ok -test normalize::span::description::sql::tests::single_quoted_string ... ok -test normalize::span::description::sql::tests::mysql_comment_generic ... ok -test normalize::span::description::sql::tests::activerecord_truncated ... ok -test normalize::span::description::sql::tests::mysql_comment ... ok -test normalize::span::description::sql::tests::savepoint_quoted_backtick ... ok -test normalize::span::description::sql::tests::named_placeholders ... ok -test normalize::span::description::sql::tests::strip_prefixes ... ok -test normalize::span::description::sql::tests::strip_prefixes_mysql ... ok -test normalize::span::description::sql::tests::single_quoted_string_finished ... ok -test normalize::span::description::sql::tests::strip_prefixes_ansi ... ok -test normalize::span::description::sql::tests::single_quoted_string_unfinished ... ok -test normalize::span::description::sql::tests::clickhouse ... ok -test normalize::span::description::sql::tests::unique_alias ... ok -test normalize::span::description::sql::tests::unparameterized_ins_nvarchar ... ok -test normalize::span::description::sql::tests::type_casts ... ok -test normalize::span::description::sql::tests::update_multiple ... ok -test normalize::span::description::sql::tests::strip_prefixes_truncated ... ok -test normalize::span::description::sql::tests::uuid_in_table_name ... ok -test normalize::span::description::sql::tests::unparameterized_ins_uppercase ... ok -test normalize::span::description::sql::tests::values_multi ... ok -test normalize::span::description::sql::tests::various_parameterized_ins_lowercase ... ok -test normalize::span::description::sql::tests::various_parameterized_questionmarks ... ok -test normalize::contexts::tests::test_get_product_name ... ok -test normalize::span::description::sql::tests::various_parameterized_ins_percentage ... ok -test normalize::span::description::sql::tests::various_parameterized_ins_dollar ... ok -test normalize::span::description::tests::active_record ... ok -test normalize::span::description::sql::tests::whitespace_and_comments ... ok -test normalize::span::description::tests::active_record_with_db_system ... ok -test normalize::span::description::sql::tests::various_parameterized_strings ... ok -test normalize::span::description::tests::informed_sql_parser ... ok -test normalize::span::description::tests::span_description_scrub_empty ... ok -test normalize::span::description::tests::span_description_scrub_hex ... ok -test normalize::span::description::tests::span_description_scrub_nothing_cache ... ok -test normalize::span::description::tests::span_description_scrub_cache ... ok -test normalize::span::description::tests::span_description_scrub_only_urllike_on_http_ops ... ok -test normalize::span::description::sql::tests::strip_prefixes_mysql_generic ... ok -test normalize::span::description::tests::span_description_scrub_only_dblike_on_db_ops ... ok -test normalize::span::description::tests::span_description_scrub_only_domain ... ok -test normalize::span::description::tests::span_description_scrub_path_ids_end ... ok -test normalize::span::description::tests::span_description_scrub_path_md5_hashes ... ok -test normalize::span::description::tests::span_description_scrub_path_multiple_ids ... ok -test normalize::span::description::tests::span_description_scrub_path_uuids ... ok -test normalize::span::description::tests::span_description_scrub_path_ids_middle ... ok -test normalize::span::description::tests::span_description_scrub_redis_invalid ... ok -test normalize::span::description::tests::span_description_scrub_nothing_in_resource ... ok -test normalize::span::description::tests::span_description_scrub_redis_long_command ... ok -test normalize::span::description::tests::span_description_scrub_redis_no_args ... ok -test normalize::span::description::tests::span_description_scrub_redis_set ... ok -test normalize::span::description::tests::span_description_scrub_path_sha_hashes ... ok -test normalize::span::description::tests::span_description_scrub_redis_set_quoted ... ok -test normalize::span::description::tests::span_description_scrub_redis_whitespace ... ok -test normalize::span::description::sql::tests::various_parameterized_cutoff ... ok -test normalize::span::description::tests::span_description_scrub_resource_css ... ok -test normalize::span::description::tests::span_description_scrub_resource_script_numeric_filename ... ok -test normalize::span::description::tests::span_description_scrub_ui_load ... ok -test normalize::span::description::sql::tests::update_single ... ok -test normalize::span::description::tests::span_description_scrub_resource_script ... ok -test normalize::span::tag_extraction::tests::extract_table_insert ... ok -test normalize::span::tag_extraction::tests::extract_table_delete ... ok -test normalize::span::tag_extraction::tests::extract_table_multiple_mysql ... ok -test normalize::span::tag_extraction::tests::extract_table_multiple ... ok -test normalize::span::tag_extraction::tests::extract_table_multiple_advanced ... ok -test normalize::span::description::sql::tests::unparameterized_ins_odbc_escape_sequence ... ok -test normalize::span::tag_extraction::tests::extract_table_select ... ok -test normalize::span::tag_extraction::tests::extract_table_select_nested ... ok -test normalize::span::tag_extraction::tests::extract_table_update ... ok -test normalize::span::tag_extraction::tests::test_truncate_string_no_panic ... ok -test normalize::stacktrace::tests::test_coerce_empty_filename ... ok -test normalize::stacktrace::tests::test_does_not_overwrite_filename ... ok -test normalize::stacktrace::tests::test_coerces_url_filenames ... ok -test normalize::stacktrace::tests::test_ignores_results_with_empty_path ... ok -test normalize::stacktrace::tests::test_is_url ... ok -test normalize::span::tag_extraction::tests::test_http_method_context ... ok -test normalize::span::tag_extraction::tests::test_http_method_request_prioritized ... ok -test normalize::span::tag_extraction::tests::test_http_method_txname ... ok -test normalize::stacktrace::tests::test_ignores_results_with_slash_path ... ok -test normalize::tests::test_context_line_default ... ok -test normalize::tests::test_context_line_retain ... ok -test normalize::span::tag_extraction::tests::extract_sql_action ... ok -test normalize::tests::test_drops_measurements_with_invalid_characters ... ok -test normalize::tests::test_discards_received ... ok -test normalize::tests::test_drops_too_long_measurement_names ... ok -test normalize::tests::test_empty_environment_is_removed ... ok -test normalize::tests::test_empty_environment_is_removed_and_overwritten_with_tag ... ok -test normalize::tests::test_empty_tags_removed ... ok -test normalize::tests::test_environment_tag_is_moved ... ok -test normalize::tests::test_event_level_defaulted ... ok -test normalize::tests::test_exception_invalid ... ok -test normalize::tests::test_frame_null_context_lines ... ok -test normalize::mechanism::tests::test_normalize_http_url ... ok -test normalize::tests::test_filter_custom_measurements ... ok -test normalize::tests::test_future_timestamp ... ok -test normalize::tests::test_android_medium_device_class ... ok -test normalize::tests::test_apple_high_device_class ... ok -test normalize::tests::test_apple_low_device_class ... ok -test normalize::tests::test_computed_measurements ... ok -test normalize::tests::test_android_high_device_class ... ok -test normalize::tests::test_apple_medium_device_class ... ok -test normalize::tests::test_geo_from_ip_address ... ok -test normalize::tests::test_android_low_device_class ... ok -test normalize::tests::test_internal_tags_removed ... ok -test normalize::tests::test_handles_type_in_value ... ok -test normalize::tests::test_invalid_release_removed ... ok -test normalize::tests::test_keeps_valid_measurement ... ok -test normalize::tests::test_json_value ... ok -test normalize::tests::test_grouping_config ... ok -test normalize::tests::test_light_normalization_respects_is_renormalize ... ok -test normalize::tests::test_max_custom_measurement ... ok -test normalize::tests::test_light_normalize_validates_spans ... ok -test normalize::tests::test_merge_builtin_measurement_keys ... ok -test normalize::tests::test_no_device_class ... ok -test normalize::tests::test_none_environment_errors ... ok -test normalize::tests::test_light_normalization_is_idempotent ... ok -test normalize::tests::test_logentry_error ... ok -test normalize::tests::test_normalize_app_start_cold_spans_for_react_native ... ok -test normalize::tests::test_normalize_app_start_spans_only_for_react_native_3_to_4_4 ... ok -test normalize::tests::test_normalize_app_start_measurements_does_not_add_measurements ... ok -test normalize::tests::test_normalize_dist_empty ... ok -test normalize::tests::test_normalize_dist_none ... ok -test normalize::tests::test_normalize_dist_trim ... ok -test normalize::tests::test_normalize_dist_whitespace ... ok -test normalize::tests::test_normalize_app_start_warm_spans_for_react_native ... ok -test normalize::tests::test_normalize_app_start_warm_measurements ... ok -test normalize::tests::test_normalize_app_start_cold_measurements ... ok -test normalize::tests::test_normalize_security_report ... ok -test normalize::tests::test_normalize_logger_exact_length ... ok -test normalize::tests::test_normalize_logger_empty ... ok -test normalize::tests::test_normalize_units ... ok -test normalize::tests::test_parses_sdk_info_from_header ... ok -test normalize::tests::test_normalize_logger_trimmed ... ok -test normalize::tests::test_normalize_logger_word_leading_dots ... ok -test normalize::tests::test_normalize_logger_short_no_trimming ... ok -test normalize::tests::test_normalize_logger_too_long_single_word ... ok -test normalize::tests::test_regression_backfills_abs_path_even_when_moving_stacktrace ... ok -test normalize::tests::test_rejects_empty_exception_fields ... ok -test normalize::tests::test_normalize_logger_word_trimmed_at_max ... ok -test normalize::tests::test_normalize_logger_word_trimmed_before_max ... ok -test normalize::tests::test_past_timestamp ... ok -test normalize::tests::test_replay_id_added_from_dsc ... ok -test normalize::tests::test_tags_deduplicated ... ok -test normalize::tests::test_too_long_distribution ... ok -test normalize::tests::test_top_level_keys_moved_into_tags ... ok -test normalize::tests::test_too_long_tags ... ok -test normalize::tests::test_transaction_level_untouched ... ok -test normalize::tests::test_transaction_status_defaulted_to_unknown ... ok -test normalize::tests::test_user_data_moved ... ok -test normalize::tests::test_unknown_debug_image ... ok -test normalize::tests::test_user_ip_from_client_ip_without_auto ... ok -test normalize::tests::test_user_ip_from_client_ip_without_appropriate_platform ... ok -test normalize::tests::test_user_ip_from_invalid_remote_addr ... ok -test normalize::tests::test_user_ip_from_remote_addr ... ok -test normalize::tests::test_user_ip_from_client_ip_with_auto ... ok -test normalize::user_agent::tests::test_choose_client_hints_for_os_context ... ok -test normalize::user_agent::tests::test_default_empty ... ok -test normalize::user_agent::tests::test_client_hint_parser ... ok -test normalize::user_agent::tests::test_client_hints_detected ... ok -test normalize::user_agent::tests::test_client_hints_with_unknown_browser ... ok -test normalize::user_agent::tests::test_ignore_empty_device ... ok -test normalize::user_agent::tests::test_ignore_empty_os ... ok -test normalize::user_agent::tests::test_ignore_empty_browser ... ok -test normalize::user_agent::tests::test_keep_empty_os_version ... ok -test normalize::tests::test_geo_in_light_normalize ... ok -test normalize::user_agent::tests::test_indicate_frozen_os_windows ... ok -test normalize::user_agent::tests::test_indicate_frozen_os_mac ... ok -test normalize::user_agent::tests::test_fallback_on_ua_string_for_os ... ok -test normalize::user_agent::tests::test_skip_no_user_agent ... ok -test normalize::user_agent::tests::test_strip_quotes ... ok -test normalize::user_agent::tests::test_strip_whitespace_and_quotes ... ok -test normalize::user_agent::tests::test_use_client_hints_for_device ... ok -test normalize::user_agent::tests::test_user_agent_does_not_override_prefilled ... ok -test normalize::user_agent::tests::test_verison_missing_minor ... ok -test normalize::user_agent::tests::test_version_major ... ok -test normalize::user_agent::tests::test_version_major_minor ... ok -test normalize::user_agent::tests::test_version_major_minor_patch ... ok -test normalize::user_agent::tests::test_version_none ... ok -test remove_other::tests::test_breadcrumb_errors ... ok -test remove_other::tests::test_remove_legacy_attributes ... ok -test remove_other::tests::test_remove_nested_other ... ok -test remove_other::tests::test_remove_unknown_attributes ... ok -test remove_other::tests::test_retain_context_other ... ok -test replay::tests::test_capped_values ... ok -test replay::tests::test_event_roundtrip ... ok -test replay::tests::test_lenient_release ... ok -test normalize::user_agent::tests::test_skip_unrecognizable_user_agent ... ok -test normalize::user_agent::tests::fallback_on_ua_string_when_missing_browser_field ... ok -test normalize::user_agent::tests::test_all_contexts ... ok -test normalize::user_agent::tests::test_device_context ... ok -test normalize::user_agent::tests::test_fallback_to_ua_if_no_client_hints ... ok -test replay::tests::test_truncated_list_less_than_limit ... ok -test replay::tests::test_validate_u16_segment_id ... ok -test schema::tests::test_client_sdk_missing_attribute ... ok -test schema::tests::test_invalid_email ... ok -test schema::tests::test_mechanism_missing_attributes ... ok -test schema::tests::test_newlines_release ... ok -test schema::tests::test_stacktrace_missing_attribute ... ok -test transactions::processor::tests::test_allows_transaction_event_with_empty_span_list ... ok -test transactions::processor::tests::test_allows_transaction_event_with_null_span_list ... ok -test transactions::processor::tests::test_allows_transaction_event_without_span_list ... ok -test transactions::processor::tests::test_default_transaction_source_unknown ... ok -test transactions::processor::tests::test_allows_valid_transaction_event_with_spans ... ok -test transactions::processor::tests::test_defaults_missing_op_in_context ... ok -test transactions::processor::tests::test_defaults_transaction_event_with_span_with_missing_op ... ok -test transactions::processor::tests::test_defaults_transaction_name_when_empty ... ok -test transactions::processor::tests::test_discards_on_missing_context ... ok -test transactions::processor::tests::test_discards_on_missing_contexts_map ... ok -test transactions::processor::tests::test_discards_on_missing_span_id_in_context ... ok -test transactions::processor::tests::test_discards_on_missing_trace_id_in_context ... ok -test transactions::processor::tests::test_defaults_transaction_name_when_missing ... ok -test transactions::processor::tests::test_discards_on_null_context ... ok -test transactions::processor::tests::test_discards_transaction_event_with_nulled_out_span ... ok -test transactions::processor::tests::test_discards_transaction_event_with_span_with_missing_span_id ... ok -test transactions::processor::tests::test_discards_transaction_event_with_span_with_missing_start_timestamp ... ok -test transactions::processor::tests::test_discards_transaction_event_with_span_with_missing_trace_id ... ok -test transactions::processor::tests::test_discards_when_missing_start_timestamp ... ok -test transactions::processor::tests::test_discards_when_missing_timestamp ... ok -test transactions::processor::tests::test_discards_when_timestamp_out_of_range ... ok -test transactions::processor::tests::test_is_high_cardinality_sdk_ruby_error ... ok -test transactions::processor::tests::test_is_high_cardinality_sdk_ruby_ok ... ok -test transactions::processor::tests::test_normalize_legacy_javascript ... ok -test transactions::processor::tests::test_normalize_legacy_python ... ok -test transactions::processor::tests::test_no_sanitized_if_no_rules ... ok -test transactions::processor::tests::test_normalize_twice ... ok -test transactions::processor::tests::test_replace_missing_timestamp ... ok -test normalize::user_agent::tests::test_os_context_short_version ... ok -test normalize::user_agent::tests::test_os_context ... ok -test transactions::processor::tests::test_skips_non_transaction_events ... ok -test transactions::processor::tests::test_normalize_transaction_names ... ok -test transactions::processor::tests::test_scrub_identifiers_and_apply_rules ... ok -test transactions::processor::tests::test_scrub_identifiers_before_rules ... ok -test transactions::processor::tests::test_transaction_name_normalize ... ok -test transactions::processor::tests::test_transaction_name_normalize_hex ... ok -test transactions::processor::tests::test_transaction_name_normalize_in_segments_1 ... ok -test transactions::processor::tests::test_transaction_name_normalize_in_segments_3 ... ok -test replay::tests::test_set_ip_address_missing_user_ip_address ... ok -test transactions::processor::tests::test_transaction_name_normalize_id ... ok -test transactions::processor::tests::test_transaction_name_normalize_in_segments_2 ... ok -test transactions::processor::tests::test_transaction_name_normalize_mark_as_sanitized ... ok -test transactions::processor::tests::test_transaction_name_normalize_in_segments_4 ... ok -test transactions::processor::tests::test_transaction_name_normalize_in_segments_5 ... ok -test transactions::processor::tests::test_transaction_name_normalize_mark_as_sanitized_when_ready ... ok -test transactions::processor::tests::test_transaction_name_normalize_url_encode_1 ... ok -test transactions::processor::tests::test_transaction_name_normalize_url_encode_5 ... ok -test transactions::processor::tests::test_transaction_name_normalize_url_encode_4 ... ok -test transactions::processor::tests::test_transaction_name_normalize_sha ... ok -test transactions::processor::tests::test_transaction_name_normalize_url_encode_6 ... ok -test transactions::processor::tests::test_transaction_name_normalize_url_encode_7 ... ok -test transactions::processor::tests::test_transaction_name_normalize_url_encode_3 ... ok -test transactions::processor::tests::test_transaction_name_normalize_url_encode_2 ... ok -test transactions::processor::tests::test_transaction_name_normalize_windows_path ... ok -test transactions::processor::tests::test_transaction_name_skip_original_value ... ok -test transactions::processor::tests::test_transaction_name_rename_end_slash ... ok -test transactions::processor::tests::test_transaction_name_unsupported_source ... ok -test transactions::rules::tests::test_rule_format ... ok -test normalize::user_agent::tests::test_os_context_full_version ... ok -test transactions::processor::tests::test_transaction_name_skip_replace_all ... ok -test transactions::rules::tests::test_rule_format_defaults ... ok -test transactions::rules::tests::test_rule_format_roundtrip ... ok -test transactions::rules::tests::test_rule_format_unsupported_reduction ... ok -test trimming::tests::test_basic_trimming ... ok -test transactions::processor::tests::test_transaction_name_skip_replace_all2 ... ok -test trimming::tests::test_custom_context_trimming ... ok -test transactions::processor::tests::test_transaction_name_normalize_uuid ... ok -test trimming::tests::test_databag_stripping ... ok -test trimming::tests::test_frame_hard_limit ... ok -test trimming::tests::test_frameqty_equals_limit ... ok -test trimming::tests::test_slim_frame_data_over_max ... ok -test transactions::processor::tests::test_transaction_name_rename_with_rules ... ok -test trimming::tests::test_slim_frame_data_under_max ... ok -test trimming::tests::test_string_trimming ... ok -test trimming::tests::test_tags_stripping ... ok -test trimming::tests::test_databag_state_leak ... ok -test replay::tests::test_set_user_agent_meta ... ok -test replay::tests::test_loose_type_requirements ... ok -test trimming::tests::test_extra_trimming_long_arrays ... ok -test trimming::tests::test_databag_array_stripping ... ok -test normalize::user_agent::tests::test_browser_context ... ok -test replay::tests::test_missing_user ... ok - -test result: ok. 423 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 2.68s - - -running 170 tests -test processor::chunks::tests::test_chunk_split ... ok -test protocol::breadcrumb::tests::test_python_ty_regression ... ok -test protocol::breadcrumb::tests::test_breadcrumb_default_values ... ok -test protocol::clientsdk::tests::test_client_sdk_default_values ... ok -test protocol::client_report::tests::test_client_report_roundtrip ... ok -test protocol::breadcrumb::tests::test_breadcrumb_roundtrip ... ok -test protocol::clientsdk::tests::test_client_sdk_roundtrip ... ok -test protocol::contexts::browser::tests::test_browser_context_roundtrip ... ok -test protocol::contexts::cloud_resource::tests::test_cloud_resource_context_roundtrip ... ok -test protocol::contexts::app::tests::test_app_context_roundtrip ... ok -test protocol::contexts::os::tests::test_os_context_roundtrip ... ok -test protocol::contexts::profile::tests::test_trace_context_normalization ... ok -test protocol::contexts::profile::tests::test_trace_context_roundtrip ... ok -test protocol::contexts::otel::tests::test_otel_context_roundtrip ... ok -test protocol::contexts::replay::tests::test_replay_context_normalization ... ok -test protocol::contexts::replay::tests::test_trace_context_roundtrip ... ok -test protocol::contexts::device::tests::test_device_context_roundtrip ... ok -test protocol::contexts::reprocessing::tests::test_reprocessing_context_roundtrip ... ok -test protocol::contexts::runtime::tests::test_runtime_context_roundtrip ... ok -test protocol::contexts::response::tests::test_response_context_roundtrip ... ok -test protocol::contexts::tests::test_other_context_roundtrip ... ok -test protocol::contexts::tests::test_multiple_contexts_roundtrip ... ok -test protocol::contexts::tests::test_untagged_context_deserialize ... ok -test protocol::contexts::tests::test_context_processing ... ok -test protocol::contexts::trace::tests::test_trace_context_normalization ... ok -test protocol::contexts::trace::tests::test_trace_context_roundtrip ... ok -test protocol::debugmeta::tests::test_debug_image_apple_default_values ... ok -test protocol::debugmeta::tests::test_debug_image_apple_roundtrip ... ok -test protocol::debugmeta::tests::test_debug_image_elf_roundtrip ... ok -test protocol::debugmeta::tests::test_debug_image_jvm_based_roundtrip ... ok -test protocol::debugmeta::tests::test_debug_image_macho_roundtrip ... ok -test protocol::debugmeta::tests::test_debug_image_other_roundtrip ... ok -test protocol::debugmeta::tests::test_debug_image_pe_dotnet_roundtrip ... ok -test protocol::debugmeta::tests::test_debug_image_proguard_roundtrip ... ok -test protocol::debugmeta::tests::test_debug_image_pe_roundtrip ... ok -test protocol::debugmeta::tests::test_debug_image_symbolic_default_values ... ok -test protocol::debugmeta::tests::test_debug_image_symbolic_legacy ... ok -test protocol::debugmeta::tests::test_debug_image_untagged_roundtrip ... ok -test protocol::debugmeta::tests::test_debug_image_symbolic_roundtrip ... ok -test protocol::debugmeta::tests::test_debug_meta_default_values ... ok -test protocol::debugmeta::tests::test_debug_meta_roundtrip ... ok -test protocol::debugmeta::tests::test_source_map_image_roundtrip ... ok -test protocol::event::tests::test_empty_threads ... ok -test protocol::event::tests::test_event_default_values ... ok -test protocol::event::tests::test_event_type ... ok -test protocol::event::tests::test_field_value_provider_event_empty ... ok -test protocol::event::tests::test_extra_at ... ok -test protocol::event::tests::test_event_default_values_with_meta ... ok -test protocol::event::tests::test_field_value_provider_event_filled ... ok -test protocol::event::tests::test_fingerprint_empty_string ... ok -test protocol::event::tests::test_event_roundtrip ... ok -test protocol::event::tests::test_fingerprint_null_values ... ok -test protocol::event::tests::test_lenient_release ... ok -test protocol::exception::tests::test_coerces_object_value_to_string ... ok -test protocol::exception::tests::test_exception_default_values ... ok -test protocol::exception::tests::test_exception_empty_fields ... ok -test protocol::exception::tests::test_exception_roundtrip ... ok -test protocol::exception::tests::test_explicit_none ... ok -test protocol::fingerprint::tests::test_fingerprint_bool ... ok -test protocol::fingerprint::tests::test_fingerprint_empty ... ok -test protocol::fingerprint::tests::test_fingerprint_float ... ok -test protocol::fingerprint::tests::test_fingerprint_float_bounds ... ok -test protocol::fingerprint::tests::test_fingerprint_float_strip ... ok -test protocol::fingerprint::tests::test_fingerprint_float_trunc ... ok -test protocol::fingerprint::tests::test_fingerprint_invalid_fallback ... ok -test protocol::fingerprint::tests::test_fingerprint_number ... ok -test protocol::fingerprint::tests::test_fingerprint_string ... ok -test protocol::logentry::tests::test_logentry_empty_params ... ok -test protocol::logentry::tests::test_logentry_from_message ... ok -test protocol::logentry::tests::test_logentry_invalid_params ... ok -test protocol::logentry::tests::test_logentry_named_params ... ok -test protocol::logentry::tests::test_logentry_roundtrip ... ok -test protocol::mechanism::tests::test_mechanism_default_values ... ok -test protocol::mechanism::tests::test_mechanism_empty ... ok -test protocol::mechanism::tests::test_mechanism_legacy_conversion ... ok -test protocol::measurements::tests::test_measurements_serialization ... ok -test protocol::mechanism::tests::test_mechanism_roundtrip ... ok -test protocol::replay::tests::test_event_roundtrip ... ok -test protocol::replay::tests::test_lenient_release ... ok -test protocol::request::tests::test_cookies_invalid ... ok -test protocol::request::tests::test_cookies_array ... ok -test protocol::request::tests::test_cookies_object ... ok -test protocol::request::tests::test_cookies_parsing ... ok -test protocol::request::tests::test_header_normalization ... ok -test protocol::request::tests::test_header_from_sequence ... ok -test protocol::request::tests::test_headers_lenient_value ... ok -test protocol::request::tests::test_headers_multiple_values ... ok -test protocol::request::tests::test_query_invalid ... ok -test protocol::request::tests::test_query_string ... ok -test protocol::request::tests::test_query_string_legacy_nested ... ok -test protocol::request::tests::test_querystring_without_value ... ok -test protocol::request::tests::test_request_roundtrip ... ok -test protocol::security_report::tests::test_csp_coerce_blocked_uri_if_missing ... ok -test protocol::security_report::tests::test_csp_culprit_0 ... ok -test protocol::security_report::tests::test_csp_culprit_2 ... ok -test protocol::security_report::tests::test_csp_culprit_1 ... ok -test protocol::security_report::tests::test_csp_culprit_5 ... ok -test protocol::security_report::tests::test_csp_get_message_0 ... ok -test protocol::security_report::tests::test_csp_culprit_uri_without_scheme ... ok -test protocol::security_report::tests::test_csp_culprit_4 ... ok -test protocol::security_report::tests::test_csp_basic ... ok -test protocol::security_report::tests::test_csp_culprit_3 ... ok -test protocol::security_report::tests::test_csp_get_message_1 ... ok -test protocol::security_report::tests::test_csp_get_message_2 ... ok -test protocol::security_report::tests::test_csp_get_message_3 ... ok -test protocol::security_report::tests::test_csp_get_message_4 ... ok -test protocol::security_report::tests::test_csp_get_message_5 ... ok -test protocol::security_report::tests::test_csp_get_message_6 ... ok -test protocol::security_report::tests::test_csp_get_message_7 ... ok -test protocol::security_report::tests::test_csp_get_message_8 ... ok -test protocol::security_report::tests::test_csp_get_message_9 ... ok -test protocol::security_report::tests::test_effective_directive_from_violated_directive_single ... ok -test protocol::security_report::tests::test_csp_tags_stripe ... ok -test protocol::security_report::tests::test_csp_msdn ... ok -test protocol::security_report::tests::test_expectct_invalid ... ok -test protocol::security_report::tests::test_extract_effective_directive_from_long_form ... ok -test protocol::security_report::tests::test_csp_real ... ok -test protocol::security_report::tests::test_normalize_uri ... ok -test protocol::security_report::tests::test_expectct_basic ... ok -test protocol::security_report::tests::test_security_report_type_deserializer_recognizes_csp_reports ... ok -test protocol::security_report::tests::test_security_report_type_deserializer_recognizes_expect_ct_reports ... ok -test protocol::security_report::tests::test_security_report_type_deserializer_recognizes_expect_staple_reports ... ok -test protocol::security_report::tests::test_expectstaple_basic ... ok -test protocol::security_report::tests::test_hpkp_basic ... ok -test protocol::security_report::tests::test_security_report_type_deserializer_recognizes_hpkp_reports ... ok -test protocol::security_report::tests::test_unsplit_uri ... ok -test protocol::session::tests::test_session_abnormal_mechanism ... ok -test protocol::session::tests::test_session_default_timestamp_and_sid ... ok -test protocol::session::tests::test_session_invalid_abnormal_mechanism ... ok -test protocol::session::tests::test_session_default_values ... ok -test protocol::session::tests::test_session_ip_addr_auto ... ok -test protocol::session::tests::test_session_null_abnormal_mechanism ... ok -test protocol::session::tests::test_sessionstatus_unknown ... ok -test protocol::session::tests::test_session_roundtrip ... ok -test protocol::span::tests::test_getter_span_data ... ok -test protocol::span::tests::test_span_serialization ... ok -test protocol::stacktrace::tests::test_frame_default_values ... ok -test protocol::stacktrace::tests::test_frame_empty_context_lines ... ok -test protocol::stacktrace::tests::test_frame_vars_empty_annotated_is_serialized ... ok -test protocol::stacktrace::tests::test_frame_vars_null_preserved ... ok -test protocol::stacktrace::tests::test_frame_roundtrip ... ok -test protocol::stacktrace::tests::test_stacktrace_default_values ... ok -test protocol::stacktrace::tests::test_php_frame_vars ... ok -test protocol::tags::tests::test_tags_from_object ... ok -test protocol::stacktrace::tests::test_stacktrace_roundtrip ... ok -test protocol::templateinfo::tests::test_template_default_values ... ok -test protocol::tags::tests::test_tags_from_array ... ok -test protocol::templateinfo::tests::test_template_roundtrip ... ok -test protocol::thread::tests::test_thread_default_values ... ok -test protocol::thread::tests::test_thread_id ... ok -test protocol::thread::tests::test_thread_roundtrip ... ok -test protocol::transaction::tests::test_other_source_roundtrip ... ok -test protocol::thread::tests::test_thread_lock_reason_roundtrip ... ok -test protocol::types::tests::test_hex_deserialization ... ok -test protocol::transaction::tests::test_transaction_info_roundtrip ... ok -test protocol::types::tests::test_hex_from_string ... ok -test protocol::types::tests::test_hex_serialization ... ok -test protocol::types::tests::test_hex_to_string ... ok -test protocol::types::tests::test_level ... ok -test protocol::types::tests::test_ip_addr ... ok -test protocol::types::tests::test_timestamp_completely_out_of_range ... ok -test protocol::types::tests::test_timestamp_year_out_of_range ... ok -test protocol::types::tests::test_values_deserialization ... ok -test protocol::types::tests::test_values_serialization ... ok -test protocol::user::tests::test_explicit_none ... ok -test protocol::user::tests::test_geo_default_values ... ok -test protocol::user::tests::test_geo_roundtrip ... ok -test protocol::user::tests::test_user_invalid_id ... ok -test protocol::user::tests::test_user_lenient_id ... ok -test protocol::user::tests::test_user_roundtrip ... ok - -test result: ok. 170 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.04s - - -running 2 tests -test tests::test_enums_processor_calls ... ok -test tests::test_simple_newtype ... ok - -test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 4 tests -test test_error ... ok -test test_ok ... ok -test test_panics ... ok -test test_unit ... ok - -test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 48 tests -test client_ips::tests::test_should_filter_blacklisted_ips ... ok -test browser_extensions::tests::test_dont_filter_when_disabled ... ok -test csp::tests::test_does_not_filter_benign_source_files ... ok -test csp::tests::test_does_not_filter_benign_uris ... ok -test csp::tests::test_does_not_filter_non_csp_messages ... ok -test csp::tests::test_filters_known_blocked_source_files ... ok -test csp::tests::test_filters_known_blocked_uris ... ok -test csp::tests::test_filters_known_document_uris ... ok -test csp::tests::test_scheme_domain_port ... ok -test csp::tests::test_matches_any_origin ... ok -test csp::tests::test_sentry_csp_filter_compatibility_bad_reports ... ok -test csp::tests::test_sentry_csp_filter_compatibility_good_reports ... ok -test error_messages::tests::test_filter_hydration_error ... ok -test error_messages::tests::test_should_filter_exception ... ok -test browser_extensions::tests::test_dont_filter_unkown_browser_extension ... ok -test browser_extensions::tests::test_filter_known_browser_extension_source ... ok -test legacy_browsers::tests::test_dont_filter_when_disabled ... ok -test browser_extensions::tests::test_filter_known_browser_extension_values ... ok -test config::tests::test_regression_legacy_browser_missing_options ... ok -test config::tests::test_empty_config ... ok -test config::tests::test_serialize_empty ... ok -test config::tests::test_serialize_full ... ok -test localhost::tests::test_dont_filter_missing_ip_or_domains ... ok -test localhost::tests::test_dont_filter_non_file_urls ... ok -test localhost::tests::test_dont_filter_non_local_domains ... ok -test localhost::tests::test_dont_filter_non_local_ip ... ok -test localhost::tests::test_dont_filter_when_disabled ... ok -test localhost::tests::test_filter_file_urls ... ok -test localhost::tests::test_filter_local_ip ... ok -test localhost::tests::test_filter_local_domains ... ok -test transaction_name::tests::test_does_not_filter_when_disabled ... ok -test transaction_name::tests::test_does_not_filter_when_disabled_with_flag ... ok -test transaction_name::tests::test_does_not_match_missing_transaction ... ok -test transaction_name::tests::test_does_not_filter_when_not_matching ... ok -test transaction_name::tests::test_filters_when_matching ... ok -test transaction_name::tests::test_does_not_match ... ok -test transaction_name::tests::test_matches ... ok -test transaction_name::tests::test_only_filters_transactions_not_anything_else ... ok -test web_crawlers::tests::test_filter_when_disabled ... ok -test releases::tests::test_release_filtering ... ok -test web_crawlers::tests::test_dont_filter_normal_user_agents ... ok -test web_crawlers::tests::test_filter_banned_user_agents ... ok -test legacy_browsers::tests::test_filter_default_browsers ... ok -test legacy_browsers::tests::test_dont_filter_default_above_minimum_versions ... ok -test legacy_browsers::tests::test_dont_filter_unconfigured_browsers ... ok -test legacy_browsers::tests::sentry_compatibility::test_dont_filter_sentry_allowed_user_agents ... ok -test legacy_browsers::tests::test_filter_configured_browsers ... ok -test legacy_browsers::tests::sentry_compatibility::test_filter_sentry_user_agents ... ok - -test result: ok. 48 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 2.37s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 1 test -test config::tests::test_kafka_config ... ok - -test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 65 tests -test aggregation::tests::test_bucket_value_cost ... ok -test aggregation::tests::test_bucket_key_cost ... ok -test aggregation::tests::test_aggregator_cost_enforcement_total ... ok -test aggregation::tests::test_aggregator_cost_tracking ... ok -test aggregation::tests::test_capped_iter_completeness_0 ... ok -test aggregation::tests::test_capped_iter_completeness_100 ... ok -test aggregation::tests::test_capped_iter_completeness_90 ... ok -test aggregation::tests::test_capped_iter_empty ... ok -test aggregation::tests::test_capped_iter_single ... ok -test aggregation::tests::test_capped_iter_split ... ok -test aggregation::tests::test_get_bucket_timestamp_multiple ... ok -test aggregation::tests::test_get_bucket_timestamp_non_multiple ... ok -test aggregation::tests::test_get_bucket_timestamp_overflow ... ok -test aggregation::tests::test_get_bucket_timestamp_zero ... ok -test aggregation::tests::test_parse_shift_key ... ok -test aggregation::tests::test_validate_bucket_key_chars ... ok -test aggregation::tests::test_validate_bucket_key_str_lens ... ok -test aggregation::tests::test_aggregator_mixed_projects ... ok -test bucket::tests::test_bucket_docs_roundtrip ... ok -test aggregation::tests::test_merge_back ... ok -test bucket::tests::test_bucket_value_merge_counter ... ok -test bucket::tests::test_bucket_value_merge_distribution ... ok -test bucket::tests::test_bucket_value_merge_gauge ... ok -test bucket::tests::test_bucket_value_merge_set ... ok -test aggregation::tests::test_aggregator_cost_enforcement_project ... ok -test bucket::tests::test_distribution_value_size ... ok -test bucket::tests::test_buckets_roundtrip ... ok -test bucket::tests::test_parse_all ... ok -test bucket::tests::test_parse_all_crlf ... ok -test bucket::tests::test_metrics_docs ... ok -test bucket::tests::test_parse_all_empty_lines ... ok -test bucket::tests::test_parse_all_trailing ... ok -test aggregation::tests::test_flush_bucket ... ok -test bucket::tests::test_parse_counter_packed ... ok -test aggregation::tests::test_validate_tag_values_special_chars ... ok -test bucket::tests::test_parse_distribution_packed ... ok -test bucket::tests::test_parse_empty_name ... ok -test bucket::tests::test_parse_garbage ... ok -test aggregation::tests::test_bucket_partitioning_128 ... ok -test aggregation::tests::test_bucket_partitioning_dummy ... ok -test bucket::tests::test_parse_bucket_defaults ... ok -test bucket::tests::test_parse_distribution ... ok -test bucket::tests::test_parse_counter ... ok -test bucket::tests::test_parse_buckets ... ok -test aggregation::tests::test_aggregator_merge_counters ... ok -test aggregation::tests::test_aggregator_merge_timestamps ... ok -test bucket::tests::test_parse_gauge ... ok -test bucket::tests::test_parse_gauge_packed ... ok -test bucket::tests::test_parse_histogram ... ok -test bucket::tests::test_parse_invalid_name ... ok -test aggregation::tests::test_cost_tracker ... ok -test bucket::tests::test_parse_invalid_name_with_leading_digit ... ok -test bucket::tests::test_parse_implicit_namespace ... ok -test bucket::tests::test_parse_sample_rate ... ok -test bucket::tests::test_parse_set ... ok -test bucket::tests::test_parse_set_hashed ... ok -test bucket::tests::test_parse_set_hashed_packed ... ok -test bucket::tests::test_parse_set_packed ... ok -test bucket::tests::test_parse_timestamp ... ok -test bucket::tests::test_parse_tags ... ok -test bucket::tests::test_parse_unit ... ok -test bucket::tests::test_parse_unit_regression ... ok -test bucket::tests::test_set_docs ... ok -test protocol::tests::test_sizeof_unit ... ok -test router::tests::condition_roundtrip ... ok - -test result: ok. 65 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.04s - - -running 7 tests -test tests::truncate_basic ... ok -test tests::process_invalid_environment ... ok -test tests::process_empty_slug ... ok -test tests::process_with_upsert_short ... ok -test tests::process_with_upsert_interval ... ok -test tests::process_json_roundtrip ... ok -test tests::process_with_upsert_full ... ok - -test result: ok. 7 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 148 tests -test attachments::tests::test_fill_content_wstr ... ok -test attachments::tests::test_fill_content_wstr_panic - should panic ... ok -test attachments::tests::test_get_wstr_match ... ok -test attachments::tests::test_bytes_regexes ... ok -test attachments::tests::test_segments_all_data ... ok -test attachments::tests::test_segments_end_aligned ... ok -test attachments::tests::test_segments_garbage ... ok -test attachments::tests::test_segments_middle_2_byte_aligned ... ok -test attachments::tests::test_segments_middle_2_byte_aligned_mutation ... ok -test attachments::tests::test_segments_middle_unaligned ... ok -test attachments::tests::test_segments_multiple ... ok -test attachments::tests::test_segments_too_short ... ok -test attachments::tests::test_selectors ... ok -test attachments::tests::test_swap_content_wstr ... ok -test attachments::tests::test_swap_content_wstr_panic - should panic ... ok -test builtin::tests::test_builtin_rules_completeness ... ok -test attachments::tests::test_all_the_bytes ... ok -test builtin::tests::test_email ... ok -test builtin::tests::test_iban_different_rules ... ok -test builtin::tests::test_creditcard ... ok -test builtin::tests::test_iban_scrubbing_word_boundaries ... ok -test builtin::tests::test_invalid_iban_codes ... ok -test builtin::tests::test_imei ... ok -test attachments::tests::test_ip_masking ... ok -test attachments::tests::test_ip_replace_padding ... ok -test builtin::tests::test_mac ... ok -test builtin::tests::test_ipv6 ... ok -test builtin::tests::test_ipv4 ... ok -test builtin::tests::test_urlauth ... ok -test attachments::tests::test_ip_hash_trunchating ... ok -test attachments::tests::test_ip_replace_padding_utf16 ... ok -test attachments::tests::test_ip_removing ... ok -test builtin::tests::test_usssn ... ok -test builtin::tests::test_pemkey ... ok -test attachments::tests::test_ip_removing_utf16 ... ok -test attachments::tests::test_ip_hash_trunchating_utf16 ... ok -test builtin::tests::test_userpath ... ok -test builtin::tests::test_uuid ... ok -test attachments::tests::test_ip_masking_utf16 ... ok -test builtin::tests::test_valid_iban_codes ... ok -test convert::tests::test_convert_empty_sensitive_field ... ok -test convert::tests::test_convert_exclude_field ... ok -test convert::tests::test_datascrubbing_default ... ok -test convert::tests::test_convert_scrub_ip_only ... ok -test convert::tests::test_convert_default_pii_config ... ok -test convert::tests::test_convert_sensitive_fields ... ok -test convert::tests::test_csp_blocked_uri ... ok -test convert::tests::test_contexts ... ok -test convert::tests::test_authorization_scrubbing ... ok -test convert::tests::test_debug_meta_files_not_strippable ... ok -test convert::tests::test_does_not_fail_on_non_string ... ok -test convert::tests::test_does_not_sanitize_timestamp_looks_like_card ... ok -test convert::tests::test_doesnt_scrub_not_scrubbed ... ok -test convert::tests::test_does_sanitize_social_security_number ... ok -test convert::tests::test_does_sanitize_encrypted_private_key ... ok -test convert::tests::test_does_sanitize_public_key ... ok -test convert::tests::test_does_sanitize_private_key ... ok -test convert::tests::test_breadcrumb_message ... ok -test convert::tests::test_does_sanitize_rsa_private_key ... ok -test convert::tests::test_empty_field ... ok -test convert::tests::test_event_message_not_strippable ... ok -test convert::tests::test_exclude_fields_on_field_value ... ok -test convert::tests::test_exclude_fields_on_field_name ... ok -test convert::tests::test_ip_stripped ... ok -test convert::tests::test_extra ... ok -test convert::tests::test_http_remote_addr_stripped ... ok -test convert::tests::test_regression_more_odd_keys ... ok -test convert::tests::test_http ... ok -test convert::tests::test_querystring_as_pairlist ... ok -test convert::tests::test_explicit_fields_case_insensitive ... ok -test convert::tests::test_explicit_fields ... ok -test convert::tests::test_no_scrub_object_with_safe_fields ... ok -test convert::tests::test_querystring_as_pairlist_with_partials ... ok -test convert::tests::test_querystring_as_string ... ok -test convert::tests::test_querystring_as_string_with_partials ... ok -test convert::tests::test_odd_keys ... ok -test convert::tests::test_safe_fields_for_token ... ok -test convert::tests::test_sanitize_credit_card_discover ... ok -test convert::tests::test_sanitize_credit_card_amex ... ok -test convert::tests::test_sanitize_credit_card ... ok -test convert::tests::test_sanitize_credit_card_visa ... ok -test convert::tests::test_sanitize_credit_card_within_value_1 ... ok -test convert::tests::test_sanitize_credit_card_mastercard ... ok -test convert::tests::test_sanitize_http_body ... ok -test convert::tests::test_sanitize_credit_card_within_value_2 ... ok -test convert::tests::test_sanitize_url_2 ... ok -test convert::tests::test_sanitize_url_1 ... ok -test convert::tests::test_sanitize_url_4 ... ok -test convert::tests::test_sanitize_additional_sensitive_fields ... ok -test convert::tests::test_sanitize_url_3 ... ok -test convert::tests::test_sanitize_url_6 ... ok -test convert::tests::test_sanitize_url_5 ... ok -test convert::tests::test_sanitize_url_7 ... ok -test convert::tests::test_should_have_mysql_pwd_as_a_default_2 ... ok -test convert::tests::test_should_have_mysql_pwd_as_a_default_1 ... ok -test generate_selectors::tests::test_empty ... ok -test convert::tests::test_scrub_object ... ok -test convert::tests::test_stacktrace_paths_not_strippable ... ok -test convert::tests::test_stacktrace ... ok -test minidumps::tests::test_module_list_removed_lin ... ok -test minidumps::tests::test_module_list_selectors ... ok -test convert::tests::test_user ... ok -test minidumps::tests::test_linux_environ_valuetype ... ok -test generate_selectors::tests::test_full ... ok -test minidumps::tests::test_module_list_removed_win ... ok -test minidumps::tests::test_stack_scrubbing_deep_wildcard ... ok -test processor::tests::test_anything_hash_on_container ... ok -test minidumps::tests::test_module_list_removed_mac ... ok -test minidumps::tests::test_stack_scrubbing_binary_not_stack ... ok -test minidumps::tests::test_stack_scrubbing_backwards_compatible_selector ... ok -test minidumps::tests::test_stack_scrubbing_valuetype_not_fully_qualified ... ok -test processor::tests::test_anything_hash_on_string ... ok -test minidumps::tests::test_stack_scrubbing_valuetype_selector - should panic ... ok -test processor::tests::test_debugmeta_path_not_addressible_with_wildcard_selector ... ok -test processor::tests::test_ip_address_hashing ... ok -test processor::tests::test_ip_address_hashing_does_not_overwrite_id ... ok -test processor::tests::test_logentry_value_types ... ok -test convert::tests::test_sensitive_cookies ... ok -test processor::tests::test_hash_debugmeta_path ... ok -test minidumps::tests::test_stack_scrubbing_path_item_selector ... ok -test processor::tests::test_quoted_keys ... ok -test processor::tests::test_no_field_upsert ... ok -test processor::tests::test_redact_containers ... ok -test processor::tests::test_remove_debugmeta_path ... ok -test processor::tests::test_redact_custom_pattern ... ok -test processor::tests::test_replace_replaced_text_anything ... ok -test processor::tests::test_replace_replaced_text ... ok -test processor::tests::test_replace_debugmeta_path ... ok -test processor::tests::test_scrub_breadcrumb_data_http_not_scrubbed ... ok -test minidumps::tests::test_stack_scrubbing_wildcard - should panic ... ok -test processor::tests::test_scrub_breadcrumb_data_http_strings_are_scrubbed ... ok -test processor::tests::test_scrub_breadcrumb_data_untyped_props_are_scrubbed ... ok -test processor::tests::test_scrub_breadcrumb_data_http_objects_are_scrubbed ... ok -test processor::tests::test_does_not_scrub_if_no_graphql ... ok -test processor::tests::test_basic_stripping ... ok -test redactions::tests::test_redaction_deser_method ... ok -test redactions::tests::test_redaction_deser_other ... ok -test processor::tests::test_scrub_span_data_http_not_scrubbed ... ok -test processor::tests::test_scrub_graphql_response_data_with_variables ... ok -test selector::tests::test_invalid ... ok -test selector::tests::test_roundtrip ... ok -test selector::tests::test_attachments_matching ... ok -test processor::tests::test_scrub_original_value ... ok -test processor::tests::test_scrub_graphql_response_data_without_variables ... ok -test processor::tests::test_scrub_span_data_untyped_props_are_scrubbed ... ok -test processor::tests::test_scrub_span_data_http_strings_are_scrubbed ... ok -test processor::tests::test_scrub_span_data_http_objects_are_scrubbed ... ok -test selector::tests::test_matching ... ok - -test result: ok. 148 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.07s - - -running 1 test -test test_scrub_pii_from_annotated_replay ... ok - -test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s - - -running 30 tests -test measurements::tests::test_value_as_float ... ok -test measurements::tests::test_value_as_string ... ok -test native_debug_image::tests::test_native_debug_image_compatibility ... ok -test sample::tests::test_expand_with_all_samples_outside_transaction ... ok -test sample::tests::test_copying_transaction ... ok -test sample::tests::test_expand ... ok -test sample::tests::test_filter_samples ... ok -test sample::tests::test_expand_with_samples_inclusive ... ok -test sample::tests::test_extract_transaction_tags ... ok -test sample::tests::test_keep_profile_under_max_duration ... ok -test android::tests::test_remove_invalid_events ... ok -test sample::tests::test_parse_profile_with_all_samples_filtered ... ok -test sample::tests::test_parse_with_no_transaction ... ok -test sample::tests::test_profile_cleanup_metadata ... ok -test sample::tests::test_reject_profile_over_max_duration ... ok -test sample::tests::test_profile_remove_idle_samples_at_start_and_end ... ok -test sample::tests::test_roundtrip ... ok -test tests::test_minimal_profile_with_version ... ok -test tests::test_minimal_profile_without_version ... ok -test transaction_metadata::tests::test_invalid_transaction_metadata ... ok -test tests::test_expand_profile_with_version ... ok -test transaction_metadata::tests::test_valid_transaction_metadata ... ok -test transaction_metadata::tests::test_valid_transaction_metadata_without_relative_timestamp ... ok -test android::tests::test_no_transaction ... ok -test extract_from_transaction::tests::test_extract_transaction_metadata ... ok -test android::tests::test_transactions_to_top_level ... ok -test tests::test_expand_profile_without_version ... ok -test android::tests::test_extract_transaction_metadata ... ok -test android::tests::test_timestamp ... ok -test android::tests::test_roundtrip_android ... ok - -test result: ok. 30 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.51s - - -running 11 tests -test macros::tests::get_path ... ok -test macros::tests::get_path_array ... ok -test macros::tests::get_path_empty ... ok -test macros::tests::get_path_combined ... ok -test macros::tests::get_path_object ... ok -test macros::tests::get_value ... ok -test macros::tests::get_value_array ... ok -test macros::tests::get_value_combined ... ok -test macros::tests::get_value_empty ... ok -test macros::tests::get_value_object ... ok -test size::tests::test_estimate_size ... ok - -test result: ok. 11 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 1 test -test test_annotated_deserialize_with_meta ... ok - -test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 20 tests -test test_signed_integers ... ok -test test_floats ... ok -test test_skip_array_never ... ok -test test_skip_array_null ... ok -test test_skip_array_empty ... ok -test test_skip_array_null_deep ... ok -test test_skip_array_empty_deep ... ok -test test_skip_object_empty ... ok -test test_skip_object_never ... ok -test test_skip_object_empty_deep ... ok -test test_skip_object_null ... ok -test test_skip_serialization_on_regular_structs ... ok -test test_skip_object_null_deep ... ok -test test_skip_tuple_empty_deep ... ok -test test_skip_tuple_empty ... ok -test test_skip_tuple_never ... ok -test test_skip_tuple_null ... ok -test test_skip_tuple_null_deep ... ok -test test_wrapper_structs_and_skip_serialization ... ok -test test_unsigned_integers ... ok - -test result: ok. 20 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 44 tests -test quota::tests::test_quota_invalid_limited_mixed ... ok -test quota::tests::test_quota_invalid_only_unknown ... ok -test quota::tests::test_quota_invalid_unlimited_mixed ... ok -test quota::tests::test_quota_matches_key_scope ... ok -test quota::tests::test_quota_matches_multiple_categores ... ok -test quota::tests::test_quota_matches_no_categories ... ok -test quota::tests::test_quota_matches_no_invalid_scope ... ok -test quota::tests::test_quota_matches_organization_scope ... ok -test quota::tests::test_quota_matches_unknown_category ... ok -test quota::tests::test_quota_matches_project_scope ... ok -test quota::tests::test_quota_valid_reject_all ... ok -test quota::tests::test_quota_valid_reject_all_mixed ... ok -test rate_limit::tests::test_rate_limit_matches_categories ... ok -test rate_limit::tests::test_parse_retry_after ... ok -test rate_limit::tests::test_rate_limit_matches_organization ... ok -test rate_limit::tests::test_rate_limit_matches_key ... ok -test rate_limit::tests::test_rate_limit_matches_project ... ok -test quota::tests::test_parse_quota_limited ... ok -test quota::tests::test_parse_quota_reject_all ... ok -test quota::tests::test_parse_quota_unlimited ... ok -test quota::tests::test_parse_quota_project ... ok -test quota::tests::test_parse_quota_key ... ok -test quota::tests::test_parse_quota_unknown_variants ... ok -test rate_limit::tests::test_rate_limits_add_replacement ... ok -test quota::tests::test_parse_quota_reject_transactions ... ok -test quota::tests::test_parse_quota_project_large ... ok -test rate_limit::tests::test_rate_limits_add_buckets ... ok -test rate_limit::tests::test_rate_limits_add_shadowing ... ok -test rate_limit::tests::test_rate_limits_check ... ok -test rate_limit::tests::test_rate_limits_check_quotas ... ok -test rate_limit::tests::test_rate_limits_clean_expired ... ok -test rate_limit::tests::test_rate_limits_longest ... ok -test redis::tests::test_get_redis_key_scoped ... ok -test rate_limit::tests::test_rate_limits_merge ... ok -test redis::tests::test_get_redis_key_unscoped ... ok -test redis::tests::test_large_redis_limit_large ... ok -test redis::tests::test_bails_immediately_without_any_quota ... ok -test redis::tests::test_zero_size_quotas ... ok -test redis::tests::test_limited_with_unlimited_quota ... ok -test redis::tests::test_quantity_0 ... ok -test redis::tests::test_quota_go_over ... ok -test redis::tests::test_quota_with_quantity ... ok -test redis::tests::test_simple_quota ... ok -test redis::tests::test_is_rate_limited_script ... ok - -test result: ok. 44 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.08s - - -running 4 tests -test config::tests::test_redis_single ... ok -test config::tests::test_redis_single_opts_default ... ok -test config::tests::test_redis_cluster_nodes_opts ... ok -test config::tests::test_redis_single_opts ... ok - -test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 15 tests -test recording::tests::test_pii_credit_card_removal ... ignored, type 3 nodes are not supported -test recording::tests::test_pii_ip_address_removal ... ignored, type 3 nodes are not supported -test recording::tests::test_scrub_pii_full_snapshot_event ... ignored, type 2 nodes are not supported -test recording::tests::test_scrub_pii_incremental_snapshot_event ... ignored, type 3 nodes are not supported -test recording::tests::test_scrub_at_path ... ok -test recording::tests::test_process_recording_no_contents ... ok -test recording::tests::test_process_recording_no_headers ... ok -test recording::tests::test_process_recording_no_body_data ... ok -test recording::tests::test_process_recording_bad_body_data ... ok -test recording::tests::test_process_recording_end_to_end ... ok -test recording::tests::test_scrub_pii_navigation ... ok -test recording::tests::test_scrub_pii_resource ... ok -test recording::tests::test_scrub_pii_key_based ... ok -test recording::tests::test_scrub_pii_custom_event ... ok -test recording::tests::test_scrub_pii_key_based_edge_cases ... ok - -test result: ok. 11 passed; 0 failed; 4 ignored; 0 measured; 0 filtered out; finished in 0.07s - - -running 45 tests -test condition::tests::test_and_combinator ... ok -test condition::tests::test_not_combinator ... ok -test condition::tests::test_or_combinator ... ok -test condition::tests::unsupported_rule_deserialize ... ok -test config::tests::test_sample_rate_returns_same_samplingvalue_variant ... ok -test config::tests::config_deserialize ... ok -test config::tests::test_non_decaying_sampling_rule_deserialization_with_factor ... ok -test config::tests::test_sample_rate_valid_time_range ... ok -test config::tests::test_sample_rate_with_constant_decayingfn ... ok -test config::tests::test_non_decaying_sampling_rule_deserialization ... ok -test config::tests::test_sample_rate_with_linear_decay ... ok -test config::tests::test_sampling_config_with_rules_and_rules_v2_deserialization ... ok -test config::tests::test_sampling_rule_with_constant_decaying_function_deserialization ... ok -test config::tests::test_sampling_config_with_rules_and_rules_v2_serialization ... ok -test config::tests::test_supported ... ok -test config::tests::test_sampling_rule_with_linear_decaying_function_deserialization ... ok -test dsc::tests::getter_filled ... ok -test config::tests::test_unsupported_rule_type ... ok -test dsc::tests::getter_empty ... ok -test dsc::tests::parse_full ... ok -test dsc::tests::parse_user ... ok -test condition::tests::test_does_not_match ... ok -test dsc::tests::test_parse_sampled_with_incoming_boolean ... ok -test dsc::tests::test_parse_sample_rate_bogus ... ok -test dsc::tests::test_parse_sample_rate_number ... ok -test dsc::tests::test_parse_sample_rate_negative ... ok -test dsc::tests::test_parse_sampled_with_incoming_boolean_as_string ... ok -test dsc::tests::test_parse_sampled_with_incoming_invalid_boolean_as_string ... ok -test dsc::tests::test_parse_sampled_with_incoming_null_value ... ok -test evaluation::tests::matched_rule_ids_display ... ok -test evaluation::tests::matched_rule_ids_parse ... ok -test evaluation::tests::test_adjust_by_client_sample_rate ... ok -test evaluation::tests::test_adjust_sample_rate ... ok -test evaluation::tests::test_expired_rules ... ok -test evaluation::tests::test_get_sampling_match_result_with_no_match ... ok -test evaluation::tests::test_matches_reservoir ... ok -test evaluation::tests::test_repeatable_seed ... ok -test evaluation::tests::test_reservoir_evaluator_limit ... ok -test evaluation::tests::test_sample_rate_compounding ... ok -test evaluation::tests::test_condition_matching ... ok -test condition::tests::test_matches ... ok -test dsc::tests::test_parse_sample_rate ... ok -test dsc::tests::test_parse_sample_rate_scientific_notation ... ok -test dsc::tests::test_parse_user_partial ... ok -test condition::tests::deserialize ... ok - -test result: ok. 45 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.04s - - -running 197 tests -test actors::processor::tests::test_breadcrumbs_reversed_with_none ... ok -test actors::processor::tests::test_breadcrumbs_truncation ... ok -test actors::processor::tests::test_breadcrumbs_order_with_none ... ok -test actors::processor::tests::test_breadcrumbs_file1 ... ok -test actors::processor::tests::test_breadcrumbs_file2 ... ok -test actors::processor::tests::test_compute_sampling_decision_matching ... ok -test actors::processor::tests::test_client_sample_rate ... ok -test actors::processor::tests::test_empty_breadcrumbs_item ... ok -test actors::processor::tests::test_error_is_tagged_correctly_if_trace_sampling_result_is_none ... ok -test actors::processor::tests::test_error_is_not_tagged_if_already_tagged ... ok -test actors::processor::tests::test_from_outcome_type_client_discard ... ok -test actors::processor::tests::test_from_outcome_type_filtered ... ok -test actors::processor::tests::test_from_outcome_type_rate_limited ... ok -test actors::processor::tests::test_from_outcome_type_sampled ... ok -test actors::processor::tests::test_it_keeps_or_drops_transactions ... ok -test actors::processor::tests::test_error_is_tagged_correctly_if_trace_sampling_result_is_some ... ok -test actors::processor::tests::test_client_report_forwarding ... ok -test actors::processor::tests::test_client_report_removal_in_processing ... ok -test actors::processor::tests::test_dsc_respects_metrics_extracted ... ok -test actors::processor::tests::test_client_report_removal ... ok -test actors::processor::tests::test_matching_with_unsupported_rule ... ok -test actors::processor::tests::test_mri_overhead_constant ... ok -test actors::processor::tests::test_process_session_invalid_json ... ok -test actors::global_config::tests::proxy_relay_does_not_make_upstream_request ... ok -test actors::processor::tests::test_process_session_invalid_timestamp ... ok -test actors::global_config::tests::managed_relay_makes_upstream_request - should panic ... ok -test actors::global_config::tests::shutdown_service ... ok -test actors::processor::tests::test_process_session_sequence_overflow ... ok -test actors::processor::tests::test_unprintable_fields ... ok -test actors::processor::tests::test_processor_panics ... ok -test actors::processor::tests::test_process_session_keep_item ... ok -test actors::processor::tests::test_process_session_metrics_extracted ... ok -test actors::project::tests::get_state_expired ... ok -test actors::project::tests::test_rate_limit_incoming_buckets ... ok -test actors::project::tests::test_rate_limit_incoming_metrics ... ok -test actors::project::tests::test_rate_limit_incoming_buckets_no_quota ... ok -test actors::processor::tests::test_user_report_invalid ... ok -test actors::project::tests::test_rate_limit_incoming_metrics_no_quota ... ok -test actors::project::tests::test_stale_cache ... ok -test actors::project_redis::tests::test_parse_redis_response ... ok -test actors::project_redis::tests::test_parse_redis_response_compressed ... ok -test actors::project_local::tests::test_multi_pub_static_config ... ok -test actors::project_local::tests::test_symlinked_projects ... ok -test actors::processor::tests::test_log_transaction_metrics_no_match ... ok -test actors::processor::tests::test_log_transaction_metrics_none ... ok -test actors::processor::tests::test_log_transaction_metrics_rule ... ok -test actors::processor::tests::test_log_transaction_metrics_pattern ... ok -test actors::processor::tests::test_log_transaction_metrics_both ... ok -test actors::store::tests::test_return_attachments_when_missing_event_item ... ok -test actors::spooler::tests::metrics_work ... ok -test actors::store::tests::test_send_standalone_attachments_when_transaction ... ok -test actors::store::tests::test_store_attachment_in_event_when_not_a_transaction ... ok -test endpoints::common::tests::test_minimal_empty_event ... ok -test endpoints::common::tests::test_minimal_event_id ... ok -test endpoints::common::tests::test_minimal_event_invalid_type ... ok -test endpoints::common::tests::test_minimal_event_type ... ok -test endpoints::minidump::tests::test_validate_minidump ... ok -test envelope::tests::test_deserialize_envelope_empty ... ok -test envelope::tests::test_deserialize_envelope_empty_item_eof ... ok -test envelope::tests::test_deserialize_envelope_empty_item_newline ... ok -test envelope::tests::test_deserialize_envelope_implicit_length ... ok -test envelope::tests::test_deserialize_envelope_empty_newline ... ok -test envelope::tests::test_deserialize_envelope_implicit_length_empty_eof ... ok -test envelope::tests::test_deserialize_envelope_implicit_length_eof ... ok -test envelope::tests::test_deserialize_envelope_replay_recording ... ok -test envelope::tests::test_deserialize_envelope_multiple_items ... ok -test envelope::tests::test_deserialize_envelope_unknown_item ... ok -test envelope::tests::test_deserialize_envelope_view_hierarchy ... ok -test envelope::tests::test_envelope_add_item ... ok -test envelope::tests::test_deserialize_request_meta ... ok -test envelope::tests::test_envelope_empty ... ok -test envelope::tests::test_item_empty ... ok -test envelope::tests::test_envelope_take_item ... ok -test envelope::tests::test_item_set_header ... ok -test envelope::tests::test_item_set_payload ... ok -test envelope::tests::test_parse_request_envelope ... ok -test envelope::tests::test_parse_request_no_dsn ... ok -test envelope::tests::test_parse_request_no_origin ... ok -test envelope::tests::test_parse_request_sent_at ... ok -test envelope::tests::test_parse_request_sent_at_null ... ok -test envelope::tests::test_parse_request_validate_key - should panic ... ok -test envelope::tests::test_parse_request_validate_origin - should panic ... ok -test envelope::tests::test_parse_request_validate_project - should panic ... ok -test envelope::tests::test_serialize_envelope_attachments ... ok -test envelope::tests::test_serialize_envelope_empty ... ok -test envelope::tests::test_split_envelope_all ... ok -test envelope::tests::test_split_envelope_none ... ok -test envelope::tests::test_split_envelope_some ... ok -test extractors::forwarded_for::tests::test_fall_back_on_forwarded_for_header ... ok -test extractors::forwarded_for::tests::test_get_empty_string_if_invalid_header ... ok -test extractors::forwarded_for::tests::test_prefer_vercel_forwarded ... ok -test extractors::start_time::tests::start_time_from_timestamp ... ok -test extractors::request_meta::tests::test_request_meta_roundtrip ... ok -test metrics_extraction::generic::tests::extract_counter ... ok -test metrics_extraction::generic::tests::extract_set ... ok -test metrics_extraction::generic::tests::extract_distribution ... ok -test metrics_extraction::generic::tests::extract_tag_precedence ... ok -test metrics_extraction::generic::tests::extract_tag_conditions ... ok -test metrics_extraction::sessions::tests::test_extract_session_metrics ... ok -test metrics_extraction::sessions::tests::test_extract_session_metrics_abnormal ... ok -test metrics_extraction::sessions::tests::test_extract_session_metrics_errored ... ok -test metrics_extraction::sessions::tests::test_extract_session_metrics_aggregate ... ok -test metrics_extraction::sessions::tests::test_extract_session_metrics_fatal ... ok -test metrics_extraction::sessions::tests::test_extract_session_metrics_ok ... ok -test metrics_extraction::sessions::tests::test_nil_to_none ... ok -test metrics_extraction::generic::tests::extract_tag_precedence_multiple_rules ... ok -test metrics_extraction::transactions::tests::test_any_client_route ... ok -test metrics_extraction::transactions::tests::test_computed_metrics ... ok -test metrics_extraction::transactions::tests::test_conditional_tagging ... ok -test metrics_extraction::transactions::tests::test_custom_measurements ... ok -test metrics_extraction::transactions::tests::test_express ... ok -test metrics_extraction::transactions::tests::test_get_eventuser_tag ... ok -test metrics_extraction::transactions::tests::test_express_options ... ok -test metrics_extraction::transactions::tests::test_js_url_strict ... ok -test metrics_extraction::transactions::tests::test_legacy_js_does_not_look_like_url ... ok -test metrics_extraction::transactions::tests::test_legacy_js_looks_like_url ... ok -test metrics_extraction::transactions::tests::test_metric_measurement_unit_overrides ... ok -test metrics_extraction::transactions::tests::test_metric_measurement_units ... ok -test metrics_extraction::transactions::tests::test_other_client_unknown ... ok -test metrics_extraction::transactions::tests::test_parse_transaction_name_strategy ... ok -test metrics_extraction::transactions::tests::test_other_client_url ... ok -test metrics_extraction::transactions::tests::test_extract_transaction_metrics ... ok -test metrics_extraction::transactions::tests::test_python_200 ... ok -test metrics_extraction::transactions::tests::test_python_404 ... ok -test metrics_extraction::event::tests::test_extract_span_metrics_mobile ... ok -test metrics_extraction::transactions::tests::test_root_counter_keep ... ok -test metrics_extraction::transactions::tests::test_span_tags ... ok -test metrics_extraction::transactions::tests::test_transaction_duration ... ok -test metrics_extraction::transactions::tests::test_unknown_transaction_status ... ok -test metrics_extraction::transactions::tests::test_unknown_transaction_status_no_trace_context ... ok -test middlewares::normalize_path::tests::root ... ok -test middlewares::normalize_path::tests::query_and_fragment ... ok -test middlewares::normalize_path::tests::no_trailing_slash ... ok -test middlewares::normalize_path::tests::path ... ok -test utils::dynamic_sampling::tests::test_is_trace_fully_sampled_with_invalid_inputs ... ok -test utils::dynamic_sampling::tests::test_is_trace_fully_sampled_return_true_with_unsupported_rules ... ok -test utils::dynamic_sampling::tests::test_is_trace_fully_sampled_with_valid_dsc_and_sampling_config ... ok -test utils::dynamic_sampling::tests::test_match_rules_return_drop_with_match_and_0_sample_rate ... ok -test utils::dynamic_sampling::tests::test_match_rules_return_keep_with_match_and_100_sample_rate ... ok -test utils::dynamic_sampling::tests::test_match_rules_return_keep_with_no_match ... ok -test utils::dynamic_sampling::tests::test_match_rules_with_traces_rules_return_keep_when_match ... ok -test utils::metrics_rate_limits::tests::profiles_limits_are_reported ... ok -test utils::garbage::tests::test_garbage_disposal ... ok -test utils::metrics_rate_limits::tests::profiles_quota_is_enforced ... ok -test utils::multipart::tests::test_empty_formdata ... ok -test utils::multipart::tests::test_formdata ... ok -test utils::multipart::tests::test_get_boundary ... ok -test utils::param_parser::tests::test_aggregator_base_0 ... ok -test utils::multipart::tests::missing_trailing_newline ... ok -test utils::param_parser::tests::test_aggregator_base_1 ... ok -test utils::param_parser::tests::test_aggregator_empty ... ok -test utils::param_parser::tests::test_aggregator_holes ... ok -test utils::param_parser::tests::test_aggregator_override ... ok -test utils::param_parser::tests::test_aggregator_reversed ... ok -test utils::param_parser::tests::test_chunk_index ... ok -test utils::param_parser::tests::test_index_parser ... ok -test utils::param_parser::tests::test_merge_vals ... ok -test utils::rate_limits::tests::test_enforce_event_metrics_extracted ... ok -test utils::param_parser::tests::test_update_value ... ok -test utils::rate_limits::tests::test_enforce_event_metrics_extracted_no_indexing_quota ... ok -test utils::rate_limits::tests::test_enforce_limit_assumed_attachments ... ok -test utils::rate_limits::tests::test_enforce_limit_assumed_event ... ok -test utils::rate_limits::tests::test_enforce_limit_attachments ... ok -test utils::rate_limits::tests::test_enforce_limit_error_event ... ok -test utils::rate_limits::tests::test_enforce_limit_error_with_attachments ... ok -test utils::rate_limits::tests::test_enforce_limit_minidump ... ok -test utils::rate_limits::tests::test_enforce_limit_monitor_checkins ... ok -test utils::rate_limits::tests::test_enforce_limit_profiles ... ok -test utils::rate_limits::tests::test_enforce_limit_replays ... ok -test utils::rate_limits::tests::test_enforce_limit_sessions ... ok -test utils::rate_limits::tests::test_enforce_pass_empty ... ok -test utils::rate_limits::tests::test_enforce_pass_minidump ... ok -test utils::rate_limits::tests::test_enforce_pass_sessions ... ok -test utils::rate_limits::tests::test_enforce_skip_rate_limited ... ok -test utils::rate_limits::tests::test_enforce_transaction_attachment_enforced ... ok -test utils::rate_limits::tests::test_enforce_transaction_attachment_enforced_metrics_extracted_indexing_quota ... ok -test utils::rate_limits::tests::test_enforce_transaction_no_indexing_quota ... ok -test utils::rate_limits::tests::test_enforce_transaction_no_metrics_extracted ... ok -test utils::rate_limits::tests::test_enforce_transaction_profile_enforced ... ok -test utils::rate_limits::tests::test_format_rate_limits ... ok -test utils::rate_limits::tests::test_parse_invalid_rate_limits ... ok -test utils::rate_limits::tests::test_parse_rate_limits ... ok -test utils::rate_limits::tests::test_parse_rate_limits_only_unknown ... ok -test utils::semaphore::tests::test_empty ... ok -test utils::semaphore::tests::test_single_thread ... ok -test utils::semaphore::tests::test_multi_thread ... ok -test utils::unreal::tests::test_merge_unreal_context_is_assert_level_error ... ok -test utils::unreal::tests::test_merge_unreal_context_is_esure_level_warning ... ok -test utils::unreal::tests::test_merge_unreal_context ... ok -test utils::unreal::tests::test_merge_unreal_logs ... ok -test utils::unreal::tests::test_merge_unreal_context_event ... ok -test metrics_extraction::event::tests::test_extract_span_metrics_all_modules ... ok -test metrics_extraction::event::tests::test_extract_span_metrics ... ok -test actors::spooler::tests::dequeue_waits_for_permits ... ok -test actors::spooler::tests::ensure_start_time_restore ... ok -test actors::processor::tests::test_browser_version_extraction_with_pii_like_data ... ok -test actors::project_cache::tests::always_spools ... ok - -test result: ok. 197 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 2.87s - - -running 12 tests -test legacy_python::test_processing ... ok -test cordova::test_processing ... ok -test legacy_node_exception::test_processing ... ok -test dotnet::test_processing ... ok -test test_event_schema_snapshot ... ok -test unity_windows::test_processing ... ok -test unity_macos::test_processing ... ok -test unity_android::test_processing ... ok -test unity_ios::test_processing ... ok -test unity_linux::test_processing ... ok -test android::test_processing ... ok -test cocoa::test_processing ... ok - -test result: ok. 12 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.06s - - -running 1 test -test test_reponse_context_pii ... ok - -test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.06s - - -running 2 tests -test tests::current_client_is_global_client ... ok -test tests::test_capturing_client ... ok - -test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 1 test -test service::tests::test_backpressure_metrics ... ok - -test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 1 test -test relay-auth/src/lib.rs - (line 14) ... ok - -test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.92s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 5 tests -test relay-common/src/macros.rs - macros::derive_fromstr_and_display (line 80) ... ok -test relay-common/src/time.rs - time::UnixTimestamp::to_instant (line 147) ... ok -test relay-common/src/time.rs - time::chrono_to_positive_millis (line 53) ... ok -test relay-common/src/time.rs - time::chrono_to_positive_millis (line 43) ... ok -test relay-common/src/time.rs - time::duration_to_millis (line 25) ... ok - -test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.81s - - -running 2 tests -test relay-config/src/byte_size.rs - byte_size::ByteSize (line 24) ... ok -test relay-config/src/byte_size.rs - byte_size::ByteSize (line 33) ... ok - -test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.62s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 2 tests -test relay-event-schema/src/processor/attrs.rs - processor::attrs::BoxCow (line 378) ... ignored -test relay-event-schema/src/processor/chunks.rs - processor::chunks (line 8) ... ok - -test result: ok. 1 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.28s - - -running 8 tests -test relay-ffi/src/lib.rs - set_panic_hook (line 282) ... ok -test relay-ffi/src/lib.rs - (line 13) ... ok -test relay-ffi/src/lib.rs - (line 79) ... ok -test relay-ffi/src/lib.rs - (line 58) ... ok -test relay-ffi/src/lib.rs - take_last_error (line 188) ... ok -test relay-ffi/src/lib.rs - (line 41) ... ok -test relay-ffi/src/lib.rs - with_last_error (line 161) ... ok -test relay-ffi/src/lib.rs - Panic (line 212) ... ok - -test result: ok. 8 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.65s - - -running 1 test -test relay-ffi-macros/src/lib.rs - catch_unwind (line 52) ... ignored - -test result: ok. 0 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 2 tests -test relay-kafka/src/config.rs - config::Sharded (line 196) ... ignored -test relay-kafka/src/lib.rs - (line 9) - compile fail ... ok - -test result: ok. 1 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.10s - - -running 9 tests -test relay-log/src/lib.rs - (line 101) ... ok -test relay-log/src/utils.rs - utils::backtrace_enabled (line 10) ... ok -test relay-log/src/lib.rs - (line 72) ... ok -test relay-log/src/lib.rs - (line 46) ... ok -test relay-log/src/test.rs - test::init_test (line 31) ... ok -test relay-log/src/lib.rs - (line 58) ... ok -test relay-log/src/setup.rs - setup::init (line 206) ... ok -test relay-log/src/lib.rs - (line 88) ... ok -test relay-log/src/lib.rs - (line 9) ... ok - -test result: ok. 9 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 1.35s - - -running 5 tests -test relay-metrics/src/bucket.rs - bucket::Bucket::parse (line 665) ... ok -test relay-metrics/src/protocol.rs - protocol::MetricResourceIdentifier (line 211) ... ok -test relay-metrics/src/bucket.rs - bucket::Bucket::parse_all (line 687) ... ok -test relay-metrics/src/bucket.rs - bucket::dist (line 116) ... ok -test relay-metrics/src/bucket.rs - bucket::DistributionValue (line 82) ... ok - -test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.73s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 4 tests -test relay-protocol/src/traits.rs - traits::SkipSerialization (line 35) ... ignored -test relay-protocol/src/traits.rs - traits::Getter (line 161) ... ok -test relay-protocol/src/macros.rs - macros::get_path (line 32) ... ok -test relay-protocol/src/macros.rs - macros::get_value (line 105) ... ok - -test result: ok. 3 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.31s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 3 tests -test relay-replays/src/transform.rs - transform::Deserializer (line 159) ... ignored -test relay-replays/src/transform.rs - transform::Transform (line 33) ... ignored -test relay-replays/src/recording.rs - recording::RecordingScrubber (line 278) ... ok - -test result: ok. 1 passed; 0 failed; 2 ignored; 0 measured; 0 filtered out; finished in 0.52s - - -running 20 tests -test relay-sampling/src/condition.rs - condition::RuleCondition::Or (line 411) ... ok -test relay-sampling/src/condition.rs - condition::RuleCondition (line 328) ... ok -test relay-sampling/src/condition.rs - condition::RuleCondition::Lte (line 367) ... ok -test relay-sampling/src/condition.rs - condition::RuleCondition::Glob (line 400) ... ok -test relay-sampling/src/condition.rs - condition::RuleCondition::And (line 423) ... ok -test relay-sampling/src/condition.rs - condition::RuleCondition::Gt (line 378) ... ok -test relay-sampling/src/condition.rs - condition::RuleCondition::Eq (line 345) ... ok -test relay-sampling/src/condition.rs - condition::RuleCondition::Lt (line 389) ... ok -test relay-sampling/src/condition.rs - condition::RuleCondition::Not (line 435) ... ok -test relay-sampling/src/condition.rs - condition::RuleCondition::Gte (line 356) ... ok -test relay-sampling/src/condition.rs - condition::RuleCondition::and (line 575) ... ok -test relay-sampling/src/condition.rs - condition::RuleCondition::eq (line 467) ... ok -test relay-sampling/src/condition.rs - condition::RuleCondition::eq_ignore_case (line 487) ... ok -test relay-sampling/src/condition.rs - condition::RuleCondition::glob (line 504) ... ok -test relay-sampling/src/condition.rs - condition::RuleCondition::lt (line 547) ... ok -test relay-sampling/src/condition.rs - condition::RuleCondition::gt (line 521) ... ok -test relay-sampling/src/condition.rs - condition::RuleCondition::gte (line 534) ... ok -test relay-sampling/src/condition.rs - condition::RuleCondition::lte (line 560) ... ok -test relay-sampling/src/condition.rs - condition::RuleCondition::negate (line 621) ... ok -test relay-sampling/src/condition.rs - condition::RuleCondition::or (line 598) ... ok - -test result: ok. 20 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 1.86s - - -running 2 tests -test relay-server/src/actors/mod.rs - actors (line 24) ... ignored -test relay-server/src/utils/multipart.rs - utils::multipart::get_multipart_boundary (line 134) ... ignored - -test result: ok. 0 passed; 0 failed; 2 ignored; 0 measured; 0 filtered out; finished in 0.00s - - -running 8 tests -test relay-statsd/src/lib.rs - (line 23) - compile ... ok -test relay-statsd/src/lib.rs - (line 50) ... ok -test relay-statsd/src/lib.rs - GaugeMetric (line 492) ... ok -test relay-statsd/src/lib.rs - HistogramMetric (line 411) ... ok -test relay-statsd/src/lib.rs - (line 34) ... ok -test relay-statsd/src/lib.rs - CounterMetric (line 358) ... ok -test relay-statsd/src/lib.rs - SetMetric (line 448) ... ok -test relay-statsd/src/lib.rs - TimerMetric (line 297) ... ok - -test result: ok. 8 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.84s - - -running 16 tests -test relay-system/src/service.rs - service::FromMessage (line 572) ... ok -test relay-system/src/service.rs - service::BroadcastChannel (line 314) ... ok -test relay-system/src/service.rs - service::BroadcastSender::into_channel (line 505) ... ok -test relay-system/src/service.rs - service::FromMessage (line 598) ... ok -test relay-system/src/service.rs - service::BroadcastChannel::is_attached (line 408) ... ok -test relay-system/src/service.rs - service::BroadcastChannel::attach (line 363) ... ok -test relay-system/src/service.rs - service::Service (line 897) - compile ... ok -test relay-system/src/service.rs - service::BroadcastSender::send (line 480) ... ok -test relay-system/src/service.rs - service::BroadcastChannel::send (line 384) ... ok -test relay-system/src/service.rs - service::BroadcastSender (line 445) ... ok -test relay-system/src/controller.rs - controller::Controller (line 118) ... ok -test relay-system/src/service.rs - service::FromMessage (line 626) ... ok -test relay-system/src/service.rs - service::Interface (line 34) ... ok -test relay-system/src/service.rs - service::Interface (line 78) ... ok -test relay-system/src/service.rs - service::Interface (line 54) ... ok -test relay-system/src/service.rs - service::Service (line 937) ... ok - -test result: ok. 16 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 1.38s - - -running 1 test -test relay-test/src/lib.rs - (line 11) ... ok - -test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.29s - - -running 0 tests - -test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s - From 175a66eee79031a4cc36ce3f80bf96e630a29272 Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Thu, 28 Sep 2023 22:18:30 +0200 Subject: [PATCH 29/30] add test --- relay-sampling/src/config.rs | 1 - relay-sampling/src/evaluation.rs | 14 ++++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/relay-sampling/src/config.rs b/relay-sampling/src/config.rs index 352467fe94..9addfceb54 100644 --- a/relay-sampling/src/config.rs +++ b/relay-sampling/src/config.rs @@ -83,7 +83,6 @@ impl SamplingRule { } /// Returns the updated [`SamplingValue`] if it's valid. - /// todo(tor): Refactor this function. pub fn evaluate( &self, now: DateTime, diff --git a/relay-sampling/src/evaluation.rs b/relay-sampling/src/evaluation.rs index dd68a66102..7b81c481ba 100644 --- a/relay-sampling/src/evaluation.rs +++ b/relay-sampling/src/evaluation.rs @@ -53,7 +53,7 @@ pub struct ReservoirEvaluator<'a> { redis_pool: Option<&'a RedisPool>, #[cfg(feature = "redis")] org_id: Option, - // Using PhantomData because the lifetimes are behind a processing flag. + // Using PhantomData because the lifetimes are behind a feature flag. _phantom: std::marker::PhantomData<&'a ()>, } @@ -130,7 +130,7 @@ impl<'a> ReservoirEvaluator<'a> { match (*counter_value).cmp(&limit) { // Limit not yet reached. Eagerly incrementing to avoid an additional lock - // in the case where it doesn't get overrwritten by the redis count. + // in the case where it doesn't get overwritten by the redis count. Ordering::Less => { *counter_value += 1; Some(*counter_value) @@ -141,8 +141,10 @@ impl<'a> ReservoirEvaluator<'a> { } /// Evaluates a reservoir rule, returning `true` if it should be sampled. + /// + /// Both `local_count` and `redis_count` include the current rule in their count. pub fn evaluate(&self, rule: RuleId, limit: i64, _rule_expiry: Option<&DateTime>) -> bool { - let Some(incremented_local_count) = self.local_count(rule, limit) else { + let Some(local_count) = self.local_count(rule, limit) else { return false; }; @@ -154,17 +156,17 @@ impl<'a> ReservoirEvaluator<'a> { // We don't sample at all if we lost access to redis. // Therefore we revert the previous increment. // Seems inefficient, but this should be a rare occurence. - self.update_counter(rule, incremented_local_count - 1); + self.update_counter(rule, local_count - 1); return false; }; - if redis_count > incremented_local_count { + if redis_count > local_count { self.update_counter(rule, redis_count); return redis_count <= limit; }; } - incremented_local_count <= limit + local_count <= limit } } From 763f8bccf0f3b1312860696541f2b9543cd45771 Mon Sep 17 00:00:00 2001 From: Tor Berge Saebjoernsen Date: Fri, 29 Sep 2023 09:57:37 +0200 Subject: [PATCH 30/30] wip --- relay-sampling/Cargo.toml | 3 +- relay-sampling/src/evaluation.rs | 106 ++++++++++----------------- relay-sampling/src/redis_sampling.rs | 17 +++-- 3 files changed, 48 insertions(+), 78 deletions(-) diff --git a/relay-sampling/Cargo.toml b/relay-sampling/Cargo.toml index fcc04aef3d..4eb0665081 100644 --- a/relay-sampling/Cargo.toml +++ b/relay-sampling/Cargo.toml @@ -11,8 +11,7 @@ publish = false [features] default = [] -redis = ["dep:anyhow", "dep:relay-redis"] - +redis = ["dep:anyhow", "relay-redis/impl"] [dependencies] anyhow = { workspace = true, optional = true } diff --git a/relay-sampling/src/evaluation.rs b/relay-sampling/src/evaluation.rs index 7b81c481ba..0d414fa888 100644 --- a/relay-sampling/src/evaluation.rs +++ b/relay-sampling/src/evaluation.rs @@ -1,6 +1,5 @@ //! Evaluation of dynamic sampling rules. -use std::cmp::Ordering; use std::collections::BTreeMap; use std::fmt; use std::num::ParseIntError; @@ -50,9 +49,7 @@ pub type ReservoirCounters = Arc>>; pub struct ReservoirEvaluator<'a> { counters: ReservoirCounters, #[cfg(feature = "redis")] - redis_pool: Option<&'a RedisPool>, - #[cfg(feature = "redis")] - org_id: Option, + org_id_and_redis_pool: Option<(u64, &'a RedisPool)>, // Using PhantomData because the lifetimes are behind a feature flag. _phantom: std::marker::PhantomData<&'a ()>, } @@ -63,9 +60,7 @@ impl<'a> ReservoirEvaluator<'a> { Self { counters, #[cfg(feature = "redis")] - org_id: None, - #[cfg(feature = "redis")] - redis_pool: None, + org_id_and_redis_pool: None, _phantom: std::marker::PhantomData, } } @@ -75,12 +70,11 @@ impl<'a> ReservoirEvaluator<'a> { /// These values are needed to synchronize with Redis. #[cfg(feature = "redis")] pub fn set_redis(&mut self, org_id: u64, redis_pool: &'a RedisPool) { - self.org_id = Some(org_id); - self.redis_pool = Some(redis_pool); + self.org_id_and_redis_pool = Some((org_id, redis_pool)); } #[cfg(feature = "redis")] - fn redis_count( + fn redis_incr( &self, key: &ReservoirRuleKey, redis_pool: &RedisPool, @@ -89,84 +83,59 @@ impl<'a> ReservoirEvaluator<'a> { let mut redis_client = redis_pool.client()?; let mut redis_connection = redis_client.connection()?; - let val = match redis_sampling::increment_redis_reservoir_count(&mut redis_connection, key) - { - Ok(val) => val, - Err(e) => { - relay_log::error!("failed to increment redis value: {:?}", e); - return Err(e); - } - }; - - if let Err(e) = redis_sampling::set_redis_expiry(&mut redis_connection, key, rule_expiry) { - relay_log::error!("failed to set redis reservoir rule expiry"); - return Err(e); - } + let val = redis_sampling::increment_redis_reservoir_count(&mut redis_connection, key)?; + redis_sampling::set_redis_expiry(&mut redis_connection, key, rule_expiry)?; Ok(val) } - #[cfg(feature = "redis")] - fn update_counter(&self, rule: RuleId, new_value: i64) { - let Ok(mut map_guard) = self.counters.lock() else { - return; - }; - - match map_guard.get_mut(&rule) { - Some(value) => *value = new_value, - // Logging an error because at this point the value should definitively be here. - None => relay_log::error!("failed to retrieve counter entry"), - } - } - - /// Gets the local count of a reserovir rule and increments it, if the limit has yet to be reached - fn local_count(&self, rule: RuleId, limit: i64) -> Option { + /// Evaluates a reservoir rule, returning `true` if it should be sampled. + pub fn incr_local(&self, rule: RuleId, limit: i64) -> bool { let Ok(mut map_guard) = self.counters.lock() else { relay_log::error!("failed to lock reservoir counter mutex"); - return None; + return false; }; let counter_value = map_guard.entry(rule).or_insert(0); - match (*counter_value).cmp(&limit) { - // Limit not yet reached. Eagerly incrementing to avoid an additional lock - // in the case where it doesn't get overwritten by the redis count. - Ordering::Less => { - *counter_value += 1; - Some(*counter_value) - } - // Limit has already been reached. - Ordering::Equal | Ordering::Greater => None, + if *counter_value < limit { + *counter_value += 1; + true + } else { + false } } /// Evaluates a reservoir rule, returning `true` if it should be sampled. - /// - /// Both `local_count` and `redis_count` include the current rule in their count. pub fn evaluate(&self, rule: RuleId, limit: i64, _rule_expiry: Option<&DateTime>) -> bool { - let Some(local_count) = self.local_count(rule, limit) else { - return false; - }; - #[cfg(feature = "redis")] - if let (Some(org_id), Some(redis_pool)) = (self.org_id, self.redis_pool.as_ref()) { - let key = ReservoirRuleKey::new(org_id, rule); + if let Some((org_id, redis_pool)) = self.org_id_and_redis_pool { + if let Ok(guard) = self.counters.lock() { + if *guard.get(&rule).unwrap_or(&0) > limit { + return false; + } + } - let Ok(redis_count) = self.redis_count(&key, redis_pool, _rule_expiry) else { - // We don't sample at all if we lost access to redis. - // Therefore we revert the previous increment. - // Seems inefficient, but this should be a rare occurence. - self.update_counter(rule, local_count - 1); - return false; + let key = ReservoirRuleKey::new(org_id, rule); + let redis_count = match self.redis_incr(&key, redis_pool, _rule_expiry) { + Ok(redis_count) => redis_count, + Err(e) => { + relay_log::error!(error = &*e, "failed to increment reservoir rule"); + return false; + } }; - if redis_count > local_count { - self.update_counter(rule, redis_count); - return redis_count <= limit; - }; + if let Ok(mut map_guard) = self.counters.lock() { + // If the rule isn't present, it has just been cleaned up by a project state update. + // In that case, it is no longer relevant so we ignore it. + if let Some(value) = map_guard.get_mut(&rule) { + *value = redis_count.max(*value); + } + } + return redis_count <= limit; } - local_count <= limit + self.incr_local(rule, limit) } } @@ -452,11 +421,12 @@ mod tests { matched_rule_ids == sampling_match.matched_rules } + // Helper method to "unwrap" the sampling match. fn get_matched_rules( sampling_evaluator: &ControlFlow, ) -> Vec { match sampling_evaluator { - ControlFlow::Continue(_) => panic!(), + ControlFlow::Continue(_) => panic!("expected a sampling match"), ControlFlow::Break(m) => m.matched_rules.0.iter().map(|rule_id| rule_id.0).collect(), } } diff --git a/relay-sampling/src/redis_sampling.rs b/relay-sampling/src/redis_sampling.rs index a0da5421e0..d56c217dde 100644 --- a/relay-sampling/src/redis_sampling.rs +++ b/relay-sampling/src/redis_sampling.rs @@ -22,13 +22,14 @@ pub fn increment_redis_reservoir_count( redis_connection: &mut relay_redis::Connection, key: &ReservoirRuleKey, ) -> anyhow::Result { - let mut command = relay_redis::redis::cmd("INCR"); - command.arg(key.as_str()); - let val = command.query(redis_connection)?; + let val = relay_redis::redis::cmd("INCR") + .arg(key.as_str()) + .query(redis_connection)?; Ok(val) } +/// Sets the expiry time for a reservoir rule count. pub fn set_redis_expiry( redis_connection: &mut relay_redis::Connection, key: &ReservoirRuleKey, @@ -36,12 +37,12 @@ pub fn set_redis_expiry( ) -> anyhow::Result<()> { let now = Utc::now().timestamp(); let expiry_time = rule_expiry - .map(|rule_expiry| rule_expiry.timestamp()) + .map(|rule_expiry| rule_expiry.timestamp() + 60) .unwrap_or_else(|| now + 86400); - let ttl = expiry_time - now; - let mut expire_command = relay_redis::redis::cmd("EXPIRE"); - expire_command.arg(key.as_str()).arg(ttl); - expire_command.query(redis_connection)?; + relay_redis::redis::cmd("EXPIRE") + .arg(key.as_str()) + .arg(expiry_time - now) + .query(redis_connection)?; Ok(()) }