Implement Network Error Logging

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## Unreleased
+
+**Features**:
+
+- Added support for NEL (Network Error Logging) reports ingestion. ([#2421](https://github.com/getsentry/relay/pull/2421))
+
 ## 23.10.0
 
 **Features**:

py/CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Unreleased
 
 - Add `scraping_attempts` field to the event schema. ([#2575](https://github.com/getsentry/relay/pull/2575))
+- Added support for NEL (Network Error Logging) reports ingestion. ([#2421](https://github.com/getsentry/relay/pull/2421))
 
 ## 0.8.31
 

relay-base-schema/src/data_category.rs

@@ -152,7 +152,7 @@ impl FromStr for DataCategory {
 impl From<EventType> for DataCategory {
     fn from(ty: EventType) -> Self {
         match ty {
-            EventType::Default | EventType::Error => Self::Error,
+            EventType::Default | EventType::Error | EventType::Nel => Self::Error,
             EventType::Transaction => Self::Transaction,
             EventType::Csp | EventType::Hpkp | EventType::ExpectCt | EventType::ExpectStaple => {
                 Self::Security

relay-base-schema/src/events.rs

@@ -39,6 +39,8 @@ pub enum EventType {
     ExpectCt,
     /// An ExpectStaple violation payload.
     ExpectStaple,
+    /// Network Error Logging report.
+    Nel,
     /// Performance monitoring transactions carrying spans.
     Transaction,
     /// All events that do not qualify as any other type.
@@ -70,6 +72,7 @@ impl FromStr for EventType {
             "hpkp" => EventType::Hpkp,
             "expectct" => EventType::ExpectCt,
             "expectstaple" => EventType::ExpectStaple,
+            "nel" => EventType::Nel,
             "transaction" => EventType::Transaction,
             _ => return Err(ParseEventTypeError),
         })
@@ -85,6 +88,7 @@ impl fmt::Display for EventType {
             EventType::Hpkp => write!(f, "hpkp"),
             EventType::ExpectCt => write!(f, "expectct"),
             EventType::ExpectStaple => write!(f, "expectstaple"),
+            EventType::Nel => write!(f, "nel"),
             EventType::Transaction => write!(f, "transaction"),
         }
     }

relay-event-normalization/src/normalize/mod.rs

@@ -17,7 +17,7 @@ use relay_event_schema::processor::{
 use relay_event_schema::protocol::{
     AsPair, Breadcrumb, ClientSdkInfo, Context, ContextInner, Contexts, DebugImage, DeviceClass,
     Event, EventId, EventType, Exception, Frame, Headers, IpAddr, Level, LogEntry, Measurement,
-    Measurements, ReplayContext, Request, SpanAttribute, SpanStatus, Stacktrace, Tags,
+    Measurements, NelContext, ReplayContext, Request, SpanAttribute, SpanStatus, Stacktrace, Tags,
     TraceContext, User, VALID_PLATFORMS,
 };
 use relay_protocol::{
@@ -264,6 +264,8 @@ impl<'a> NormalizeProcessor<'a> {
             EventType::ExpectCt
         } else if event.expectstaple.value().is_some() {
             EventType::ExpectStaple
+        } else if event.context::<NelContext>().is_some() {
+            EventType::Nel
         } else {
             EventType::Default
         }
@@ -675,6 +677,18 @@ fn is_security_report(event: &Event) -> bool {
         || event.hpkp.value().is_some()
 }
 
+/// Backfills the client IP address on for the NEL reports.
+fn normalize_nel_report(event: &mut Event, client_ip: Option<&IpAddr>) {
+    if event.context::<NelContext>().is_none() {
+        return;
+    }
+
+    if let Some(client_ip) = client_ip {
+        let user = event.user.value_mut().get_or_insert_with(User::default);
+        user.ip_address = Annotated::new(client_ip.to_owned());
+    }
+}
+
 /// Backfills common security report attributes.
 fn normalize_security_report(
     event: &mut Event,
@@ -1024,6 +1038,9 @@ pub fn light_normalize_event(
         // Process security reports first to ensure all props.
         normalize_security_report(event, config.client_ip, &config.user_agent);
 
+        // Process NEL reports to ensure all props.
+        normalize_nel_report(event, config.client_ip);
+
         // Insert IP addrs before recursing, since geo lookup depends on it.
         normalize_ip_addresses(
             &mut event.request,

relay-event-schema/src/protocol/contexts/mod.rs

@@ -4,6 +4,7 @@ mod cloud_resource;
 mod device;
 mod gpu;
 mod monitor;
+mod nel;
 mod os;
 mod otel;
 mod profile;
@@ -19,6 +20,7 @@ pub use cloud_resource::*;
 pub use device::*;
 pub use gpu::*;
 pub use monitor::*;
+pub use nel::*;
 pub use os::*;
 pub use otel::*;
 pub use profile::*;
@@ -78,6 +80,8 @@ pub enum Context {
     Otel(Box<OtelContext>),
     /// Cloud resource information.
     CloudResource(Box<CloudResourceContext>),
+    /// Nel information.
+    Nel(Box<NelContext>),
     /// Additional arbitrary fields for forwards compatibility.
     #[metastructure(fallback_variant)]
     Other(#[metastructure(pii = "true")] Object<Value>),

relay-event-schema/src/protocol/contexts/nel.rs

@@ -0,0 +1,61 @@
+#[cfg(feature = "jsonschema")]
+use relay_jsonschema_derive::JsonSchema;
+use relay_protocol::{Annotated, Empty, FromValue, IntoValue, Object, Value};
+
+use crate::processor::ProcessValue;
+
+/// Contains the unique to NEL reports information.
+///
+/// NEL is a browser feature that allows reporting of failed network requests from the client side.
+/// W3C Editor's Draft: <https://w3c.github.io/network-error-logging/>
+/// MDN: <https://developer.mozilla.org/en-US/docs/Web/HTTP/Network_Error_Logging>
+#[derive(Clone, Debug, Default, PartialEq, Empty, FromValue, IntoValue, ProcessValue)]
+#[cfg_attr(feature = "jsonschema", derive(JsonSchema))]
+pub struct NelContext {
+    /// If request failed, the type of its network error. If request succeeded, "ok".
+    #[metastructure(field = "type")]
+    pub ty: Annotated<String>,
+    /// Server IP where the requests was sent to.
+    #[metastructure(pii = "true")]
+    pub server_ip: Annotated<String>,
+    /// The time between the start of the resource fetch and when it was completed or aborted.
+    pub elapsed_time: Annotated<u64>,
+    /// If request failed, the phase of its network error. If request succeeded, "application".
+    pub phase: Annotated<String>,
+    /// The sampling rate.
+    pub sampling_fraction: Annotated<f64>,
+    /// For forward compatibility.
+    #[metastructure(additional_properties, pii = "maybe")]
+    pub other: Object<Value>,
+}
+
+impl super::DefaultContext for NelContext {
+    fn default_key() -> &'static str {
+        "nel"
+    }
+
+    fn from_context(context: super::Context) -> Option<Self> {
+        match context {
+            super::Context::Nel(c) => Some(*c),
+            _ => None,
+        }
+    }
+
+    fn cast(context: &super::Context) -> Option<&Self> {
+        match context {
+            super::Context::Nel(c) => Some(c),
+            _ => None,
+        }
+    }
+
+    fn cast_mut(context: &mut super::Context) -> Option<&mut Self> {
+        match context {
+            super::Context::Nel(c) => Some(c),
+            _ => None,
+        }
+    }
+
+    fn into_context(self) -> super::Context {
+        super::Context::Nel(Box::new(self))
+    }
+}

relay-event-schema/src/protocol/mod.rs

@@ -16,6 +16,7 @@ mod logentry;
 mod measurements;
 mod mechanism;
 mod metrics;
+mod nel;
 mod relay_info;
 mod replay;
 mod request;
@@ -52,6 +53,7 @@ pub use self::logentry::*;
 pub use self::measurements::*;
 pub use self::mechanism::*;
 pub use self::metrics::*;
+pub use self::nel::*;
 pub use self::relay_info::*;
 pub use self::replay::*;
 pub use self::request::*;

relay-event-schema/src/protocol/nel.rs

@@ -0,0 +1,132 @@
+//! Contains definitions for the Network Error Logging (NEL) interface.
+//!
+//! NEL is a browser feature that allows reporting of failed network requests from the client side.
+//! W3C Editor's Draft: <https://w3c.github.io/network-error-logging/>
+//! MDN: <https://developer.mozilla.org/en-US/docs/Web/HTTP/Network_Error_Logging>
+
+#[cfg(feature = "jsonschema")]
+use relay_jsonschema_derive::JsonSchema;
+use relay_protocol::{Annotated, Empty, FromValue, IntoValue, Object, Value};
+use thiserror::Error;
+
+/// The NEL parsing errors.
+#[derive(Debug, Error)]
+pub enum NetworkReportError {
+    /// Unexpected format.
+    #[error("unexpected format")]
+    InvalidNel,
+    /// Incoming Json is unparsable.
+    #[error("incoming json is unparsable")]
+    InvalidJson(#[from] serde_json::Error),
+}
+
+/// Generated network error report (NEL).
+#[derive(Debug, Default, Clone, PartialEq, FromValue, IntoValue, Empty)]
+#[cfg_attr(feature = "jsonschema", derive(JsonSchema))]
+pub struct BodyRaw {
+    /// The time between the start of the resource fetch and when it was completed or aborted.
+    pub elapsed_time: Annotated<u64>,
+    /// HTTP method.
+    pub method: Annotated<String>,
+    /// If request failed, the phase of its network error. If request succeeded, "application".
+    pub phase: Annotated<String>,
+    /// The HTTP protocol and version.
+    pub protocol: Annotated<String>,
+    /// Request's referrer, as determined by the referrer policy associated with its client.
+    pub referrer: Annotated<String>,
+    /// The sampling rate.
+    pub sampling_fraction: Annotated<f64>,
+    /// The IP address of the server where the site is hosted.
+    pub server_ip: Annotated<String>,
+    /// HTTP status code.
+    pub status_code: Annotated<i64>,
+    /// If request failed, the type of its network error. If request succeeded, "ok".
+    #[metastructure(field = "type")]
+    pub ty: Annotated<String>,
+    /// For forward compatibility.
+    #[metastructure(additional_properties, pii = "maybe")]
+    pub other: Object<Value>,
+}
+
+/// Models the content of a NEL report.
+///
+/// NOTE: This is the structure used inside the Event (serialization is based on Annotated
+/// infrastructure). We also use a version of this structure to deserialize from raw JSON
+/// via serde.
+///
+/// See <https://w3c.github.io/network-error-logging/>
+#[derive(Debug, Default, Clone, PartialEq, FromValue, IntoValue, Empty)]
+#[cfg_attr(feature = "jsonschema", derive(JsonSchema))]
+pub struct NetworkReportRaw {
+    /// The age of the report since it got collected and before it got sent.
+    pub age: Annotated<i64>,
+    /// The type of the report.
+    #[metastructure(field = "type")]
+    pub ty: Annotated<String>,
+    /// The URL of the document in which the error occurred.
+    #[metastructure(pii = "true")]
+    pub url: Annotated<String>,
+    /// The User-Agent HTTP header.
+    pub user_agent: Annotated<String>,
+    /// The body of the NEL report.
+    pub body: Annotated<BodyRaw>,
+    /// For forward compatibility.
+    #[metastructure(additional_properties, pii = "maybe")]
+    pub other: Object<Value>,
+}
+
+impl NetworkReportRaw {
+    pub fn try_annotated_from(value: &[u8]) -> Result<Annotated<Self>, NetworkReportError> {
+        Annotated::from_json_bytes(value).map_err(NetworkReportError::InvalidJson)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use relay_protocol::assert_annotated_snapshot;
+
+    use crate::protocol::NetworkReportRaw;
+
+    #[test]
+    fn test_nel_raw_basic() {
+        let json = r#"{
+            "age": 31042,
+            "body": {
+                "elapsed_time": 0,
+                "method": "GET",
+                "phase": "connection",
+                "protocol": "http/1.1",
+                "referrer": "",
+                "sampling_fraction": 1.0,
+                "server_ip": "127.0.0.1",
+                "status_code": 0,
+                "type": "tcp.refused"
+            },
+            "type": "network-error",
+            "url": "http://example.com/",
+            "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36"
+        }"#;
+
+        let report = NetworkReportRaw::try_annotated_from(json.as_bytes()).unwrap();
+
+        assert_annotated_snapshot!(report, @r###"
+        {
+          "age": 31042,
+          "type": "network-error",
+          "url": "http://example.com/",
+          "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
+          "body": {
+            "elapsed_time": 0,
+            "method": "GET",
+            "phase": "connection",
+            "protocol": "http/1.1",
+            "referrer": "",
+            "sampling_fraction": 1.0,
+            "server_ip": "127.0.0.1",
+            "status_code": 0,
+            "type": "tcp.refused"
+          }
+        }
+        "###);
+    }
+}

relay-event-schema/src/protocol/request.rs

@@ -444,6 +444,9 @@ pub struct Request {
     /// HTTP request method.
     pub method: Annotated<String>,
 
+    /// HTTP protocol.
+    pub protocol: Annotated<String>,
+
     /// Request data in any format that makes sense.
     ///
     /// SDKs should discard large and binary bodies by default. Can be given as a string or
@@ -684,6 +687,7 @@ mod tests {
         let request = Annotated::new(Request {
             url: Annotated::new("https://google.com/search".to_string()),
             method: Annotated::new("GET".to_string()),
+            protocol: Annotated::empty(),
             data: {
                 let mut map = Object::new();
                 map.insert("some".to_string(), Annotated::new(Value::I64(1)));

relay-pii/src/snapshots/relay_pii__processor__tests__does_not_scrub_if_no_graphql.snap

@@ -28,6 +28,7 @@ Event {
     request: Request {
         url: ~,
         method: ~,
+        protocol: ~,
         data: Object(
             {
                 "query": String(

relay-pii/src/snapshots/relay_pii__processor__tests__scrub_graphql_response_data_with_variables.snap

@@ -28,6 +28,7 @@ Event {
     request: Request {
         url: ~,
         method: ~,
+        protocol: ~,
         data: Object(
             {
                 "query": String(

relay-pii/src/snapshots/relay_pii__processor__tests__scrub_graphql_response_data_without_variables.snap

@@ -28,6 +28,7 @@ Event {
     request: Request {
         url: ~,
         method: ~,
+        protocol: ~,
         data: Object(
             {
                 "query": String(