diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..0252e0f0 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true +indent_style = tab +indent_size = 4 +trim_trailing_whitespace = true + +[*.md] +indent_style = space + +[*.{yml,yaml}] +indent_style = space +indent_size = 2 diff --git a/data-formats/df-000-base.md b/data-formats/df-000-base.md index bf8ef922..27104e9f 100644 --- a/data-formats/df-000-base.md +++ b/data-formats/df-000-base.md @@ -89,7 +89,7 @@ experimental, research-oriented client `miniooni`. - `probe_asn` (`string`): AS Number of the probe (prefixed by AS, e.g., `"AS1234"`), or `"AS0"` if the user does not want to share their ASN. -- `probe_network_name` (`string`; optional; since `2020-04-22`): The organisation name corresponding to +- `probe_network_name` (`string`; optional; since `2020-04-22`): The organisation name corresponding to the AS of the probe. - `probe_cc` (`string`): two letter country code of the probe (e.g., diff --git a/data-formats/df-001-httpt.md b/data-formats/df-001-httpt.md index 76bb758d..90b14abe 100644 --- a/data-formats/df-001-httpt.md +++ b/data-formats/df-001-httpt.md @@ -14,7 +14,7 @@ code. See this directory's [README](README.md) for the basic concepts. { "agent": "redirect", "requests": [], - "socksproxy": "127.0.0.1:54321" + "socksproxy": null } ``` @@ -23,22 +23,37 @@ follow redirects, set to `redirect` otherwise. - `request` (`[]Transaction`): list of transaction objects. See below. -- `socksproxy` (`string`; optional): address of the SOCKS proxy being +- `socksproxy` (`string`; optional; legacy): address of the SOCKS proxy being used in a nettest. Note that this proxy is different from the one used to contact the backend. Omit or set to `null` if no SOCKS proxy is being used. The format -to be used is `1.2.3.4:54321` for IPv4 and `[::1234]:54321` for IPv6. +to be used is `1.2.3.4:54321` for IPv4 and `[::1234]:54321` for IPv6. Modern +probes always set this field to `null`. ## Transaction ```JSON { + "network": "tcp", + "address": "1.1.1.1:443", + "alpn": "h2", "failure": "dns_nxdomain_error", "request": {}, "response": {}, "response_length": 1234, + "t0": 0.9, + "t": 1.0, "transaction_id": 1 } ``` +- `network` (`string`; optional): if available, the network of the +underlying connection we are using: either `"tcp"` or `"quic"`. + +- `address` (`string`; optional): if available, the endpoint of the +underlying connection we are using (e.g., `"[::1]:443"`). + +- `alpn` (`string`; optional): if available and applicable, the ALPN +that was negotiated with the server, typically `"http/1.1"`, `"h2"` or `"h3"`. + - `failure` (`string`; nullable): if there was an error, this field is a string indicating the error, otherwise it MUST be `null`. @@ -46,15 +61,22 @@ a string indicating the error, otherwise it MUST be `null`. - `response` (`Response`): object describing the response. -- `response_length` (`int`; deprecated): this is a legacy field that +- `response_length` (`int`; optional; deprecated): this is a legacy field that contains the response length and is typically set to `null` or directly omitted by modern clients (e.g. from Measurement Kit onwards). -- `transaction_id` (`int`; optional; since 2020-01-11): unique ID of this -transaction. A zero transaction ID means "unspecified". The code SHOULD -NOT include the transaction ID in this case. This ID will be unique -within a single measurement session; do not assume it will be unique -for longer than that. +- `t0` (`float`): number of seconds elapsed since `measurement_start_time` +measured in the moment in which we started the operation (`t - t0` gives you +the amount of time spent performing the operation); + +- `t` (`float`): number of seconds elapsed since `measurement_start_time` +measured in the moment in which `failure` is determined (`t - t0` gives you +the amount of time spent performing the operation); + +- `transaction_id` (`int`; optional; since 2020-01-11): the set of operations +to which this event belongs to (typically an HTTP transaction or a DNS +round trip). A zero or missing value means we don't know the transaction +to which this code belongs to. ## Request @@ -96,7 +118,8 @@ are multiple values for the same header key. See below the definition of - `tor` (`TorInfo`): this is an object containing information on the instance of `tor` that we may be using for measuring. -- `x_transport` (`string`): indicates what transports was used for issuing the request. +- `x_transport` (`string`; deprecated): indicates what transports was used for +issuing the request. Typically `"tcp"` or `"quic"`. In case we have the following headers: @@ -199,65 +222,117 @@ not relevant to the HTTP data format: ```JSON { - "test_keys": { - "agent": "redirect", - "requests": [ - { - "failure": null, - "request": { - "body": "", - "body_is_truncated": false, - "headers_list": [[ - "Host", "149.154.171.5" - ], [ - "User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" - ], [ - "Content-Length", "0" - ], [ - "Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" - ], [ - "Accept-Language", "en-US;q=0.8,en;q=0.5" - ] - ], - "headers": { - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Accept-Language": "en-US;q=0.8,en;q=0.5", - "Content-Length": "0", - "Host": "149.154.171.5", - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" - }, - "method": "POST", - "tor": { - "exit_ip": null, - "exit_name": null, - "is_tor": false - }, - "url": "http://149.154.171.5/" - }, - "response": { - "body": "\r\n501 Not Implemented\r\n\r\n

501 Not Implemented

\r\n
nginx/0.3.33
\r\n\r\n\r\n", - "body_is_truncated": false, - "code": 501, - "headers_list": [[ - "Content-Length", "181" - ], [ - "Server", "nginx/0.3.33" - ], [ - "Date", "Fri, 10 Jan 2020 17:25:20 GMT" - ], [ - "Content-Type", "text/html" - ] - ], - "headers": { - "Content-Length": "181", - "Content-Type": "text/html", - "Date": "Fri, 10 Jan 2020 17:25:20 GMT", - "Server": "nginx/0.3.33" - } - }, - "transaction_id": 1 - } - ] - } + "network": "tcp", + "address": "93.184.216.34:443", + "alpn": "h2", + "failure": null, + "request": { + "body": "", + "body_is_truncated": false, + "headers_list": [ + [ + "Accept", + "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + ], + [ + "Accept-Language", + "en-US,en;q=0.9" + ], + [ + "Host", + "example.com" + ], + [ + "Referer", + "" + ], + [ + "User-Agent", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36" + ] + ], + "headers": { + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + "Host": "example.com", + "Referer": "", + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36" + }, + "method": "GET", + "tor": { + "exit_ip": null, + "exit_name": null, + "is_tor": false + }, + "x_transport": "tcp", + "url": "https://example.com" + }, + "response": { + "body": "\n\n\n Example Domain\n\n \n \n \n \n\n\n\n
\n

Example Domain

\n

This domain is for use in illustrative examples in documents. You may use this\n domain in literature without prior coordination or asking for permission.

\n

More information...

\n
\n\n\n", + "body_is_truncated": false, + "code": 200, + "headers_list": [ + [ + "Age", + "590050" + ], + [ + "Cache-Control", + "max-age=604800" + ], + [ + "Content-Length", + "1256" + ], + [ + "Content-Type", + "text/html; charset=UTF-8" + ], + [ + "Date", + "Thu, 08 Sep 2022 09:47:49 GMT" + ], + [ + "Etag", + "\"3147526947+ident\"" + ], + [ + "Expires", + "Thu, 15 Sep 2022 09:47:49 GMT" + ], + [ + "Last-Modified", + "Thu, 17 Oct 2019 07:18:26 GMT" + ], + [ + "Server", + "ECS (dcb/7F5B)" + ], + [ + "Vary", + "Accept-Encoding" + ], + [ + "X-Cache", + "HIT" + ] + ], + "headers": { + "Age": "590050", + "Cache-Control": "max-age=604800", + "Content-Length": "1256", + "Content-Type": "text/html; charset=UTF-8", + "Date": "Thu, 08 Sep 2022 09:47:49 GMT", + "Etag": "\"3147526947+ident\"", + "Expires": "Thu, 15 Sep 2022 09:47:49 GMT", + "Last-Modified": "Thu, 17 Oct 2019 07:18:26 GMT", + "Server": "ECS (dcb/7F5B)", + "Vary": "Accept-Encoding", + "X-Cache": "HIT" + } + }, + "t0": 0.900978, + "t": 1.051177, + "transaction_id": 4 } ``` diff --git a/data-formats/df-002-dnst.md b/data-formats/df-002-dnst.md index 38d38074..f3f85884 100644 --- a/data-formats/df-002-dnst.md +++ b/data-formats/df-002-dnst.md @@ -20,10 +20,10 @@ code. See this directory's [README](README.md) for the basic concepts. ## Query -```JSON +```JavaScript { + // fields currently used as 2022-09-08 "answers": [], - "dial_id": 177171, "engine": "udp", "failure": "dns_nxdomain_error", "getaddrinfo_error": 0, @@ -32,17 +32,20 @@ code. See this directory's [README](README.md) for the basic concepts. "raw_response": "", "rcode": 0, "resolver_address": "8.8.8.8:53", - "resolver_hostname": null, - "resolve_port": null, "t0": 1.007, "t": 1.114, - "transaction_id": 1 + "transaction_id": 1, + + // specified but unused or deprecated fields + "dial_id": 177171, + "resolver_hostname": null, + "resolve_port": null, } ``` - `answers` (`[]Answer`): list of answer objects. See below. -- `dial_id` (`int`; optional; since 2020-01-11): identifier of a dialing +- `dial_id` (`int`; optional; since 2020-01-11; deprecated): identifier of a dialing operation (i.e. name resolution followed by connect). The zero dial_id means that we don't know the real dial ID. Applications SHOULD NOT emit the dial_id when it is zero. Rest assured that the dial_id will @@ -86,19 +89,16 @@ used by ooni/probe-engine, which sets it to `null`. - `t0` (`float`): number of seconds elapsed since `measurement_start_time` measured in the moment in which we started the operation (`t - t0` gives you -the amount of time spent performing the given lookup); +the amount of time spent performing the operation); - `t` (`float`): number of seconds elapsed since `measurement_start_time` measured in the moment in which `failure` is determined (`t - t0` gives you -the amount of time spent performing the given lookup); +the amount of time spent performing the operation); -- `transaction_id` (`int`; optional; since 2020-01-11): if present and nonzero, -this is the ID of the transaction that caused this query. By grouping by transaction -ID, you could observe chains of events that are logically related. For example, a -DoH lookup includes ancillary DNS lookups, TCP connects, TLS handshakes, etc. If -zero, this field just indicates we don't know the transaction ID. Before 2022-08-26, -this field was not used in production and had a different definition where the -transaction was necessarily an HTTP transaction, which was too strict. +- `transaction_id` (`int`; optional; since 2020-01-11): the set of operations +to which this event belongs to (typically an HTTP transaction or a DNS +round trip). A zero or missing value means we don't know the transaction +to which this code belongs to. ### DNS resolver engines @@ -153,21 +153,24 @@ the name because we changed our way of representing getaddrinfo-like resolutions ## Answer -```JSON +```JavaScript { + // fields currently used as 2022-09-08 "answer_type": "AAAA", "asn": 13335 "as_org_name": "Cloudflare, Inc." - "expiration_limit": "", "hostname": "", "ipv4": "1.1.1.1", "ipv6": "", + "ttl": null, + + // specified but unused or deprecated fields + "expiration_limit": "", "minimum_ttl": "", "refresh_interval": "", "responsible_name": "", "retry_interval": "", "serial_number": 0, - "ttl": null, } ``` @@ -213,30 +216,23 @@ not relevant to the DNS data format: ```JSON { - "test_keys": { - "agent": "redirect", - "queries": [ - { - "answers": [ - { - "answer_type": "A", - "ipv4": "149.154.167.99", - "ttl": null - } - ], - "dial_id": 177171, - "engine": "system", - "failure": null, - "hostname": "web.telegram.org", - "query_type": "ANY", - "resolver_hostname": null, - "resolver_port": null, - "resolver_address": "", - "t0": 1.001, - "t": 1.114, - "transaction_id": 1 - } - ] - } -} + "answers": [{ + "asn": 15133, + "as_org_name": "Edgecast Inc.", + "answer_type": "A", + "ipv4": "93.184.216.34", + "ttl": null + }], + "engine": "udp", + "failure": null, + "hostname": "example.com", + "query_type": "A", + "raw_response": "dUuBgAABAAEAAAAAB2V4YW1wbGUDY29tAAABAAHADAABAAEAAE2IAARduNgi", + "resolver_hostname": null, + "resolver_port": null, + "resolver_address": "8.8.4.4:53", + "t0": 0.001145, + "t": 0.06544, + "transaction_id": 2 +}, ``` diff --git a/data-formats/df-005-tcpconnect.md b/data-formats/df-005-tcpconnect.md index c08af283..2041247f 100644 --- a/data-formats/df-005-tcpconnect.md +++ b/data-formats/df-005-tcpconnect.md @@ -21,22 +21,26 @@ basic concepts. ## TCPConnect -```JSON +```JavaScript { - "conn_id": 141, - "dial_id": 177171, + // fields currently used as of 2022-09-08 "ip": "149.154.171.5", "port": 80, "status": {}, + "t0": 1.011, "t": 1.114, - "transaction_id": 1 + "transaction_id": 1, + + // deprecated or unused fields + "conn_id": 141, + "dial_id": 177171, } ``` -- `conn_id` (`int`; optional; since 2020-01-11): identifier of the connection. See +- `conn_id` (`int`; optional; since 2020-01-11; deprecated): identifier of the connection. See the discussion in `df-008-netevents.md`. -- `dial_id` (`int`; optional; since 2020-01-11): identifier of a dialing +- `dial_id` (`int`; optional; since 2020-01-11; deprecated): identifier of a dialing operation (i.e. name resolution followed by connect). See the discussion in `df-002-dnst.md`. @@ -46,21 +50,34 @@ discussion in `df-002-dnst.md`. - `status` (`Status`): object describing the results. +- `t0` (`float`): number of seconds elapsed since `measurement_start_time` +measured in the moment in which we started the operation (`t - t0` gives you +the amount of time spent performing the operation); + - `t` (`float`): number of seconds elapsed since `measurement_start_time` -measured when `connect` is complete. +measured in the moment in which `failure` is determined (`t - t0` gives you +the amount of time spent performing the operation); -- `transaction_id` (`int`; optional; since 2020-01-11): if present, this is the -ID of the HTTP transaction that caused this TCP connect. +- `transaction_id` (`int`; optional; since 2020-01-11): the set of operations +to which this event belongs to (typically an HTTP transaction or a DNS +round trip). A zero or missing value means we don't know the transaction +to which this code belongs to. ## Status -```JSON +```JavaScript { + "blocked": null, // only WebConnectivity, new nettests SHOULD NOT include it "failure": null, "success": true } ``` +- `blocked` (`string`; nullable; deprecated; optional): field used only by Web +Connectivity to indicate whether this endpoint is blocked. New experiments +SHOULD NOT use this field and SHOULD instead use distinct keys to represent +network observations and the probe's analysis. + - `failure` (`string`; nullable): if there was an error, this field is a string indicating the error, otherwise it MUST be `null`. @@ -73,21 +90,15 @@ not relevant to the HTTP data format: ```JSON { - "test_keys": { - "tcp_connect": [ - { - "conn_id": 5, - "dial_id": 555, - "ip": "149.154.171.5", - "port": 80, - "status": { - "failure": null, - "success": true - }, - "t": 1.114, - "transaction_id": 1 - } - ] - } + "ip": "93.184.216.34", + "port": 443, + "status": { + "blocked": false, + "failure": null, + "success": true + }, + "t0": 0.450831, + "t": 0.595157, + "transaction_id": 4 } ``` diff --git a/data-formats/df-006-tlshandshake.md b/data-formats/df-006-tlshandshake.md index 7dd7d954..74530e48 100644 --- a/data-formats/df-006-tlshandshake.md +++ b/data-formats/df-006-tlshandshake.md @@ -30,19 +30,22 @@ code. See this directory's [README](README.md) for the basic concepts. "negotiated_protocol": "", "no_tls_verify": false, "peer_certificates": [], + "t0": 1.001, "t": 1.11, "tags": [], "tls_version": "", "transaction_id": 1 } ``` -- `network` (`string`; optional): The network for the handshake (`tls` or `quic`) +- `network` (`string`; optional): The network for the handshake (`"tcp"` or `"quic"`). Until +2022-09-08, we incorrectly documented that this could have been either `"tls"` or `"quic"` but +the code always used `"tcp"`, so we adapted the spec to the code. - `address` (`string`): The endpoint IP address (host:port) with which the handshake is performed. - `cipher_suite` (`string`): the negotiated cipher suite, if any. -- `conn_id` (`int`; optional; since 2020-01-11): identifier of the connection. See +- `conn_id` (`int`; optional; since 2020-01-11; deprecated): identifier of the connection. See the discussion in `df-008-netevents.md`. - `failure` (`string`; nullable): if there was an error, this field is @@ -50,23 +53,30 @@ a string indicating the error, otherwise it MUST be `null`. - `negotiated_protocol`: (`string`): the protocol negotiated with ALPN, if any. -- `no_tls_verify`: (`bool`): indicates if we have bypassed certificate chain validation when performing +- `no_tls_verify`: (`bool`): indicates if we have bypassed certificate chain validation when performing the TLS handshake. - `peer_certificates`: (`[]BinaryData`): list of peer certificates in ASN.1 DER format represented using the `BinaryData` object described in `df-001-httpt.md`. +- `t0` (`float`): number of seconds elapsed since `measurement_start_time` +measured in the moment in which we started the operation (`t - t0` gives you +the amount of time spent performing the operation); + - `t` (`float`): number of seconds elapsed since `measurement_start_time` -measured in the moment in which `failure` is determined. +measured in the moment in which `failure` is determined (`t - t0` gives you +the amount of time spent performing the operation); -- `tags` (`[]string`): list of tags for this event. This is useful to +- `tags` (`[]string`; optional): list of tags for this event. This is useful to understand what part of a complex measurement generated an event. - `tls_version` (`string`): the negotiated TLS version, if any. -- `transaction_id` (`int`; optional; since 2020-01-11): if present, this is the -ID of the HTTP transaction that caused this TLS handshake. +- `transaction_id` (`int`; optional; since 2020-01-11): the set of operations +to which this event belongs to (typically an HTTP transaction or a DNS +round trip). A zero or missing value means we don't know the transaction +to which this code belongs to. ## Example @@ -75,36 +85,27 @@ not relevant to the TLS data format: ```JSON { - "test_keys": { - "tls_handshakes": [ - { - "address": "142.250.193.14:443" - "cipher_suite": "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", - "failure": null, - "negotiated_protocol": "http/1.1", - "peer_certificates": [ - { - "data": "MIIFPDCCBCSgAwIBAgIJAJnXpwlYmeKBMA0GCSqGSIb3DQEBCwUAMIG0MQswCQYDVQQGEwJVUzEQMA4GA1UECBMHQXJpem9uYTETMBEGA1UEBxMKU2NvdHRzZGFsZTEaMBgGA1UEChMRR29EYWRkeS5jb20sIEluYy4xLTArBgNVBAsTJGh0dHA6Ly9jZXJ0cy5nb2RhZGR5LmNvbS9yZXBvc2l0b3J5LzEzMDEGA1UEAxMqR28gRGFkZHkgU2VjdXJlIENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTE3MTAyMzEyMjkwMFoXDTIwMTAyNzE2MDQzN1owQDEhMB8GA1UECxMYRG9tYWluIENvbnRyb2wgVmFsaWRhdGVkMRswGQYDVQQDDBIqLndlYi50ZWxlZ3JhbS5vcmcwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDXBGS7HQY5gJe8Qp5AOA+B0qB9e9evyiZRSP24mHT5w53yx95XIFrFd+Fk0ABaomc6uRg9PES+Qc9BruNL9pV3cytQMxJE0Ybou5hY4aMYjvTRMGs9jl/rgClXAczOc/qvbvPrCS3JbkfOgS7+1T5eN4ud4y9dP2Q7d51fm/3Z0EufupGWSKMdOiXTymuSv2FbVY7Rq2PlaZ3IK4Ro9s+a7TwZ/b3KATWXWzfIkGM90VxxQH+pHruRaSdOiWeNT5AfwebgFFCy7Wzt5GYOK4k0zYDO2nX+fms4YeeynMwH1gCmlvRoOJSWNP6sfwgBJCdFkoBOJfrp4SRjxvn453krAgMBAAGjggHCMIIBvjAMBgNVHRMBAf8EAjAAMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAOBgNVHQ8BAf8EBAMCBaAwNwYDVR0fBDAwLjAsoCqgKIYmaHR0cDovL2NybC5nb2RhZGR5LmNvbS9nZGlnMnMxLTc1OS5jcmwwXQYDVR0gBFYwVDBIBgtghkgBhv1tAQcXATA5MDcGCCsGAQUFBwIBFitodHRwOi8vY2VydGlmaWNhdGVzLmdvZGFkZHkuY29tL3JlcG9zaXRvcnkvMAgGBmeBDAECATB2BggrBgEFBQcBAQRqMGgwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmdvZGFkZHkuY29tLzBABggrBgEFBQcwAoY0aHR0cDovL2NlcnRpZmljYXRlcy5nb2RhZGR5LmNvbS9yZXBvc2l0b3J5L2dkaWcyLmNydDAfBgNVHSMEGDAWgBRAwr0njsw0gzCiM9f7bLPwtCyAzjAvBgNVHREEKDAmghIqLndlYi50ZWxlZ3JhbS5vcmeCEHdlYi50ZWxlZ3JhbS5vcmcwHQYDVR0OBBYEFEeHj2e6FBgMjFEmTOgZSLFtI8eZMA0GCSqGSIb3DQEBCwUAA4IBAQCvRBAnnq02614m5Xgroam1i7JB2bMZVMrwCejwY9otLkM3tnjE0q6ZFQYtawImkfBgJZ7nWxho4JOU1DTXDsjjr9VulbsTo8YhUAWC+rMcygkCmEjzRv7mNOmQ/hsLSLNZejLqAPRCuzX7dfTAZ+f4gJqbeA0FOuN50NQiJEFPWu2bkQsC9G1bHAQs6e3lwP6RvCzsQxUocx+Q36TiwIH1Jm81oxkQQetQQ01nlcExV0knLAQ3ZyDyVO/5Off+j3viquUQas9HtudFbyIPPsbvCD+hs2KO+PLjqLLTiSammhN1qZy9R89GEd8CdY4U8tdfuqhvKrabIdiNwBsqPo3f", - "format": "base64" - }, - { - "data": "MIIE0DCCA7igAwIBAgIBBzANBgkqhkiG9w0BAQsFADCBgzELMAkGA1UEBhMCVVMxEDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxGjAYBgNVBAoTEUdvRGFkZHkuY29tLCBJbmMuMTEwLwYDVQQDEyhHbyBEYWRkeSBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcyMB4XDTExMDUwMzA3MDAwMFoXDTMxMDUwMzA3MDAwMFowgbQxCzAJBgNVBAYTAlVTMRAwDgYDVQQIEwdBcml6b25hMRMwEQYDVQQHEwpTY290dHNkYWxlMRowGAYDVQQKExFHb0RhZGR5LmNvbSwgSW5jLjEtMCsGA1UECxMkaHR0cDovL2NlcnRzLmdvZGFkZHkuY29tL3JlcG9zaXRvcnkvMTMwMQYDVQQDEypHbyBEYWRkeSBTZWN1cmUgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IC0gRzIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC54MsQ1K92vdSTYuswZLiBCGzDBNliF44v/z5lz4/OYuY8UhzaFkVLVat4a2ODYpDOD2lsmcgaFItMzEUz6ojcnqOvK/6AYZ15V8TPLvQ/MDxdR/yaFrzDN5ZBUY4RS1T4KL7QjL7wMDge87Am+GZHY23ecSZHjzhHU9FGHbTj3ADqRay9vHHZqm8A29vNMDp5T19MR/gd71vCxJ1gO7GyQ5HYpDNO6rPWJ0+tJYqlxvTV0KaudAVkV4i1RFXULSo6Pvi4vekyCgKUZMQWOlDxSq7neTOvDCAHf+jfBDnCaQJsY1L6d8EbyHSHyLmTGFBUNUtpTrw700kuH9zB0lL7AgMBAAGjggEaMIIBFjAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUQMK9J47MNIMwojPX+2yz8LQsgM4wHwYDVR0jBBgwFoAUOpqFBxBnKLbv9r0FQW4gwZTaD94wNAYIKwYBBQUHAQEEKDAmMCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5nb2RhZGR5LmNvbS8wNQYDVR0fBC4wLDAqoCigJoYkaHR0cDovL2NybC5nb2RhZGR5LmNvbS9nZHJvb3QtZzIuY3JsMEYGA1UdIAQ/MD0wOwYEVR0gADAzMDEGCCsGAQUFBwIBFiVodHRwczovL2NlcnRzLmdvZGFkZHkuY29tL3JlcG9zaXRvcnkvMA0GCSqGSIb3DQEBCwUAA4IBAQAIfmyTEMg4uJapkEv/oV9PBO9sPpyIBslQj6Zz91cxG7685C/b+LrTW+C05+Z5Yg4MotdqY3MxtfWoSKQ7CC2iXZDXtHwlTxFWMMS2RJ17LJ3lXubvDGGqv+QqG+6EnriDfcFDzkSnE3ANkR/0yBOtg2DZ2HKocyQetawiDsoXiWJYRBuriSUBAA/NxBti21G00w9RKpv0vHP8ds42pM3Z2Czqrpv1KrKQ0U11GIo/ikGQI31bS/6kA1ibRrLDYGCD+H1QQc7CoZDDu+8CL9IVVO5EFdkKrqeKM+2xLXY2JtwE65/3YR8V3Idv7kaWKK2hJn0KCacuBKONvPi8BDAB", - "format": "base64" - }, - { - "data": "MIIEfTCCA2WgAwIBAgIDG+cVMA0GCSqGSIb3DQEBCwUAMGMxCzAJBgNVBAYTAlVTMSEwHwYDVQQKExhUaGUgR28gRGFkZHkgR3JvdXAsIEluYy4xMTAvBgNVBAsTKEdvIERhZGR5IENsYXNzIDIgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTQwMTAxMDcwMDAwWhcNMzEwNTMwMDcwMDAwWjCBgzELMAkGA1UEBhMCVVMxEDAOBgNVBAgTB0FyaXpvbmExEzARBgNVBAcTClNjb3R0c2RhbGUxGjAYBgNVBAoTEUdvRGFkZHkuY29tLCBJbmMuMTEwLwYDVQQDEyhHbyBEYWRkeSBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAtIEcyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAv3FiCPH6WTT3G8kYo/eASVjpIoMTpsUgQwE7hPHmhUmfJ+r2hBtOoLTbcJjHMgGxBT4HTu70+k8vWTAi56sZVmvigAf88xZ1gDlRe+X5NbZ0TqmNghPktj+pA4P6or6KFWp/3gvDthkUBcrqw6gElDtGfDIN8wBmIsiNaW02jBEYt9OyHGC0OPoCjM7T3UYH3go+6118yHz7sCtTpJJiaVElBWEaRIGMLKlDliPfrDqBmg4pxRyp6V0etp6eMAo5zvGIgPtLXcwy7IViQyU0AlYnAZG0O3AqP26x6JyIAX2f1PnbU21gnb8s51iruF9G/M7EGwM8CetJMVxpRrPgRwIDAQABo4IBFzCCARMwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFDqahQcQZyi27/a9BUFuIMGU2g/eMB8GA1UdIwQYMBaAFNLEsNKR1EwRcbNhyz2h/t2oatTjMDQGCCsGAQUFBwEBBCgwJjAkBggrBgEFBQcwAYYYaHR0cDovL29jc3AuZ29kYWRkeS5jb20vMDIGA1UdHwQrMCkwJ6AloCOGIWh0dHA6Ly9jcmwuZ29kYWRkeS5jb20vZ2Ryb290LmNybDBGBgNVHSAEPzA9MDsGBFUdIAAwMzAxBggrBgEFBQcCARYlaHR0cHM6Ly9jZXJ0cy5nb2RhZGR5LmNvbS9yZXBvc2l0b3J5LzANBgkqhkiG9w0BAQsFAAOCAQEAWQtTvZKGEacke+1bMc8dH2xwxbhuvk679r6XUOEwf7ooXGKUwuN+M/f7QnaF25UcjCJYdQkMiGVnOQoWCcWgOJekxSOTP7QYpgEGRJHjp2kntFolfzq3Ms3dhP8qOCkzpN1nsoX+oYggHFCJyNwq9kIDN0zmiN/VryTyscPfzLXs4Jlet0lUIDyUGAzHHFIYSaRt4bNYC8nY7NmuHDKOKHAN4v6mF56ED71XcLNa6R+ghlO773z/aQvgSMO3kwvIClTErF0UZzdsyqUvMQg3qm5vjLyb4lddJIGvl5echK1srDdMZvNhkREg5L4wn3qkKQmw4TRfZHcYQFHfjDCmrw==", - "format": "base64" - }, - { - "data": "MIIEADCCAuigAwIBAgIBADANBgkqhkiG9w0BAQUFADBjMQswCQYDVQQGEwJVUzEhMB8GA1UEChMYVGhlIEdvIERhZGR5IEdyb3VwLCBJbmMuMTEwLwYDVQQLEyhHbyBEYWRkeSBDbGFzcyAyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5MB4XDTA0MDYyOTE3MDYyMFoXDTM0MDYyOTE3MDYyMFowYzELMAkGA1UEBhMCVVMxITAfBgNVBAoTGFRoZSBHbyBEYWRkeSBHcm91cCwgSW5jLjExMC8GA1UECxMoR28gRGFkZHkgQ2xhc3MgMiBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTCCASAwDQYJKoZIhvcNAQEBBQADggENADCCAQgCggEBAN6d1+pXGEmhW+vXX0iG6r7d/+TvZxz0ZWizV3GgXne77ZtJ6XCAPVYYYwhv2vLM0D9/AlQiVBDYsoHUwHU9S3/Hd8M+eKsaA7Ugay9qK7HFiH7Eux6wwdhFJ2+qN1j3hybX2C32qRe3H3I2TqYXP2WYktsqbl2i/ojgC95/5Y0V4evLOtXiEqITLdiOr18SPaAIBQi2XKVlOARFmR6jYGB0xUGlcmIbYsUfb18aQr4CUWWoriMYavx4A6lNf4DD+qta/KFApMoZFv6yyO9ecw3ud72a9nmYvLEHZ6IVDd2gWMZEewo+YihfukEHU1jPEX44dMX4/7VpkI+EdOqXG68CAQOjgcAwgb0wHQYDVR0OBBYEFNLEsNKR1EwRcbNhyz2h/t2oatTjMIGNBgNVHSMEgYUwgYKAFNLEsNKR1EwRcbNhyz2h/t2oatTjoWekZTBjMQswCQYDVQQGEwJVUzEhMB8GA1UEChMYVGhlIEdvIERhZGR5IEdyb3VwLCBJbmMuMTEwLwYDVQQLEyhHbyBEYWRkeSBDbGFzcyAyIENlcnRpZmljYXRpb24gQXV0aG9yaXR5ggEAMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBADJL87LKPpH8EsahB4yOd6AzBhRckB4Y9wimPQoZ+YeAEW5p5JYXMP80kWNyOO7MHAGjHZQopDH2esRU1/blMVgDoszOYtuURXO1v0XJJLXVggKtI3lpjbi2Tc7PTMozI+gciKqdi0FuFskg5YmezTvacPd+mSYgFFQlq25zheabIZ0KbIIOqPjCDPoQHmyW74cNxA9hi63ugyuV+I6ShHI56yDqg+2DzZduCLzrTia2cyvk0/ZM/iZx4mERdEr/VxqHD3VILs9RaRegAhJhldXRQLIQTO7ErBBDpqWeCtWVYpoNz4iCxTIM5CufReYNnyicsbkqWletNw+vHX/bvZ8=", - "format": "base64" - } - ], - "t": 2.603595, - "tls_version": "TLSv1.2", - "transaction_id": 1 - } - ] - }, + "network": "tcp", + "address": "93.184.216.34:443", + "cipher_suite": "TLS_AES_256_GCM_SHA384", + "failure": null, + "negotiated_protocol": "h2", + "no_tls_verify": false, + "peer_certificates": [ + { + "data": "MIIHRzCCBi+gAwIBAgIQD6pjEJMHvD1BSJJkDM1NmjANBgkqhkiG9w0BAQsFADBPMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMSkwJwYDVQQDEyBEaWdpQ2VydCBUTFMgUlNBIFNIQTI1NiAyMDIwIENBMTAeFw0yMjAzMTQwMDAwMDBaFw0yMzAzMTQyMzU5NTlaMIGWMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEUMBIGA1UEBxMLTG9zIEFuZ2VsZXMxQjBABgNVBAoMOUludGVybmV0wqBDb3Jwb3JhdGlvbsKgZm9ywqBBc3NpZ25lZMKgTmFtZXPCoGFuZMKgTnVtYmVyczEYMBYGA1UEAxMPd3d3LmV4YW1wbGUub3JnMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAlV2WY5rlGn1fpwvuBhj0nVBcNxCxkHUG/pJG4HvaJen7YIZ1mLc7/P4snOJZiEfwWFTikHNbcUCcYiKG8JkFebZOYMc1U9PiEtVWGU4kuYuxiXpD8oMPin1B0SgrF7gKfO1//I2weJdAUjgZuXBCPAlhz2EnHddzXUtwm9XuOLO/Y6LATVMsbp8/lXnfo/bX0UgJ7C0aVqOu07A0Vr6OkPxwWmOvF3cRKhVCM7U4B51KK+IsWRLm8cVW1IaXjwhGzW7BR6EI3sxCQ4Wnc6HVPSgmomLWWWkIGFPAwcWUB4NC12yhCO5iW/dxNMWNLMRVtnZAyq6FpZ8wFK6j4OMwMwIDAQABo4ID1TCCA9EwHwYDVR0jBBgwFoAUt2ui6qiqhIx56rTaD5iyxZV2ufQwHQYDVR0OBBYEFPcqCdAkWxFx7rq+9D4cPVYSiBa7MIGBBgNVHREEejB4gg93d3cuZXhhbXBsZS5vcmeCC2V4YW1wbGUubmV0ggtleGFtcGxlLmVkdYILZXhhbXBsZS5jb22CC2V4YW1wbGUub3Jngg93d3cuZXhhbXBsZS5jb22CD3d3dy5leGFtcGxlLmVkdYIPd3d3LmV4YW1wbGUubmV0MA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwgY8GA1UdHwSBhzCBhDBAoD6gPIY6aHR0cDovL2NybDMuZGlnaWNlcnQuY29tL0RpZ2lDZXJ0VExTUlNBU0hBMjU2MjAyMENBMS00LmNybDBAoD6gPIY6aHR0cDovL2NybDQuZGlnaWNlcnQuY29tL0RpZ2lDZXJ0VExTUlNBU0hBMjU2MjAyMENBMS00LmNybDA+BgNVHSAENzA1MDMGBmeBDAECAjApMCcGCCsGAQUFBwIBFhtodHRwOi8vd3d3LmRpZ2ljZXJ0LmNvbS9DUFMwfwYIKwYBBQUHAQEEczBxMCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2VydC5jb20wSQYIKwYBBQUHMAKGPWh0dHA6Ly9jYWNlcnRzLmRpZ2ljZXJ0LmNvbS9EaWdpQ2VydFRMU1JTQVNIQTI1NjIwMjBDQTEtMS5jcnQwCQYDVR0TBAIwADCCAXwGCisGAQQB1nkCBAIEggFsBIIBaAFmAHUA6D7Q2j71BjUy51covIlryQPTy9ERa+zraeF3fW0GvW4AAAF/ip6hdQAABAMARjBEAiAxePNT60Z/vTJTPVryiGzXrLxCNJQqteULkguBEMbG/gIgR3QwvILJIWAUfvSfJQ/zMmqr2JDanWE8uzbC4EWbcwAAdQA1zxkbv7FsV78PrUxtQsu7ticgJlHqP+Eq76gDwzvWTAAAAX+KnqF8AAAEAwBGMEQCIDspTxwkUBpEoeA+IolNYwOKl9Yxmwk816yd0O2IJPZcAiAV8TWhoOLiiqGKnY02CdcGXOzAzC7tT6m7OtLAku2+WAB2ALNzdwfhhFD4Y4bWBancEQlKeS2xZwwLh9zwAw55NqWaAAABf4qeoYcAAAQDAEcwRQIgKR7qwPLQb6UT2+S7w7uQsbsDZfZVX/g8FkBtAltaTpACIQDLdtedRNGNhuzYpB6gmBBydhtSQi5YZLspFvaVHpeW1zANBgkqhkiG9w0BAQsFAAOCAQEAqp++XZEbreROTsyPB2RENbStOxM/wSnYtKvzQlFJRjvWzx5Bg+ELVy+DaXllB29ZA4xRlIkYED4eXO26PY5PGhSS0yv/1JjLp5MOvLcbk6RCQkbZ5bEaa2gqmy5IqS8dKrDj+CCUVIFQLu7X4CB6ey5n+/rYF6Rb3MoAYu8jr3pY8Hp0DL1NQ/GMAofc464J0vf6NzzSS6sE5UOl0lURDkGHXzio5XpeTEa4tvo/w0vNQDX/4KRxdArBIIvjVEeE1Ri9UZtAXd1CMBLROqVjmq+QCNYb0XELBnGQ666tr7pfx9trHniitNEGI6dj87VD+laMUBd7HBtOEGsiDoRSlA==", + "format": "base64" + }, + { + "data": "MIIEvjCCA6agAwIBAgIQBtjZBNVYQ0b2ii+nVCJ+xDANBgkqhkiG9w0BAQsFADBhMQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBDQTAeFw0yMTA0MTQwMDAwMDBaFw0zMTA0MTMyMzU5NTlaME8xCzAJBgNVBAYTAlVTMRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxKTAnBgNVBAMTIERpZ2lDZXJ0IFRMUyBSU0EgU0hBMjU2IDIwMjAgQ0ExMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwUuzZUdwvN1PWNvsnO3DZuUfMRNUrUpmRh8sCuxkB+Uu3Ny5CiDt3+PE0J6aqXodgojlEVbbHp9YwlHnLDQNLtKS4VbL8Xlfs7uHyiUDe5pSQWYQYE9XE0nw6Ddng9/n00tnTCJRpt8OmRDtV1F0JuJ9x8piLhMbfyOIJVNvwTRYAIuE//i+p1hJInuWraKImxW8oHzf6VGo1bDtN+I2tIJLYrVJmuzHZ9bjPvXj1hJeRPG/cUJ9WIQDgLGBAfr5yjK7tI4nhyfFK3TUqNaX3sNk+crOU6JWvHgXjkkDKa77SU+kFbnO8lwZV21reacroicgE7XQPUDTITAHk+qZ9QIDAQABo4IBgjCCAX4wEgYDVR0TAQH/BAgwBgEB/wIBADAdBgNVHQ4EFgQUt2ui6qiqhIx56rTaD5iyxZV2ufQwHwYDVR0jBBgwFoAUA95QNVbRTLtm8KPiGxvDl7I90VUwDgYDVR0PAQH/BAQDAgGGMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjB2BggrBgEFBQcBAQRqMGgwJAYIKwYBBQUHMAGGGGh0dHA6Ly9vY3NwLmRpZ2ljZXJ0LmNvbTBABggrBgEFBQcwAoY0aHR0cDovL2NhY2VydHMuZGlnaWNlcnQuY29tL0RpZ2lDZXJ0R2xvYmFsUm9vdENBLmNydDBCBgNVHR8EOzA5MDegNaAzhjFodHRwOi8vY3JsMy5kaWdpY2VydC5jb20vRGlnaUNlcnRHbG9iYWxSb290Q0EuY3JsMD0GA1UdIAQ2MDQwCwYJYIZIAYb9bAIBMAcGBWeBDAEBMAgGBmeBDAECATAIBgZngQwBAgIwCAYGZ4EMAQIDMA0GCSqGSIb3DQEBCwUAA4IBAQCAMs5eC91uWg0Kr+HWhMvAjvqFcO3aXbMM9yt1QP6FCvrzMXi3cEsaiVi6gL3zax3pfs8LulicWdSQ0/1s/dCYbbdxglvPbQtaCdB73sRD2Cqk3p5BJl+7j5nL3a7hqG+fh/50tx8bIKuxT8b1Z11dmzzp/2n3YWzW2fP9NsarA4h20ksudYbj/NhVfSbCEXffPgK2fPOre3qGNm+499iTcc+G33Mw+nur7SpZyEKEOxEXGlLzyQ4UfaJbcme6ce1XR2bFuAJKZTRei9AqPCCcUZlM51Ke92sRKw2Sfh3oius2FkOH6ipjv3U/697EA7sKPPcw7+uvTPyLNhBzPvOk", + "format": "base64" + } + ], + "server_name": "example.com", + "t0": 0.601812, + "t": 0.900702, + "tags": [], + "tls_version": "TLSv1.3", + "transaction_id": 4 } ``` diff --git a/data-formats/df-008-netevents.md b/data-formats/df-008-netevents.md index cb741f37..2bfffef9 100644 --- a/data-formats/df-008-netevents.md +++ b/data-formats/df-008-netevents.md @@ -20,27 +20,31 @@ MAY use to include network-level events. See this directory's ## Event -```JSON +```JavaScript { + // fields currently used as of 2022-09-08 "address": "1.1.1.1:443", - "conn_id": 11, - "dial_id": 4, "failure": "connection_reset", "num_bytes": 4114, "operation": "read", "proto": "tcp", + "t0": 1.001, "t": 1.174, "tags": [], - "transaction_id": 1 + "transaction_id": 1, + + // deprecated or unused fields + "conn_id": 11, + "dial_id": 4, } ``` - `address` (`string`; optional): address for `connect`. -- `conn_id` (`int`; optional): identifier of this connection (see below). When +- `conn_id` (`int`; optional; deprecated): identifier of this connection (see below). When zero, it means we don't know the conn ID. SHOULD be omitted when zero. -- `dial_id` (`int`; optional; since 2020-01-11): identifier of a dialing +- `dial_id` (`int`; optional; since 2020-01-11; deprecated): identifier of a dialing operation (i.e. name resolution followed by connect). See the discussion in `df-002-dnst.md`. @@ -54,16 +58,23 @@ a string indicating the error, otherwise it MUST be `null`. - `proto` (`string`; optional): protocol for `connect` (`tcp` or `udp`). +- `t0` (`float`): number of seconds elapsed since `measurement_start_time` +measured in the moment in which we started the operation (`t - t0` gives you +the amount of time spent performing the operation); + - `t` (`float`): number of seconds elapsed since `measurement_start_time` -measured when `operation` is complete. +measured in the moment in which `failure` is determined (`t - t0` gives you +the amount of time spent performing the operation); - `tags` (`[]string`): list of tags for this event. This is useful to understand what part of a complex measurement generated an event. -- `transaction_id` (`int`; optional; since 2020-01-11): if present, this is the -ID of the HTTP transaction that caused this TCP connect. +- `transaction_id` (`int`; optional; since 2020-01-11): the set of operations +to which this event belongs to (typically an HTTP transaction or a DNS +round trip). A zero or missing value means we don't know the transaction +to which this code belongs to. -## Connection ID and Life Cycle +## Connection ID and Life Cycle (deprecated) When a connection is created you see a `connect` event with a specific `conn_id` and no failure. Subsequently you should see @@ -83,42 +94,13 @@ not relevant to the netevents data format: ```JSON { - "test_keys": { - "network_events": [{ - "address": "1.1.1.1:444", - "conn_id": 11, - "dial_id": 4, - "failure": "connection_refused", - "operation": "connect", - "proto": "tcp", - "t": 0.11, - "tags": ["tcptls_experiment"], - "transaction_id": 1 - }, { - "address": "1.1.1.1:443", - "conn_id": 12, - "dial_id": 4, - "failure": null, - "operation": "connect", - "proto": "tcp", - "t": 0.16, - "tags": ["tcptls_experiment"], - "transaction_id": 1 - }, { - "conn_id": 12, - "failure": null, - "num_bytes": 1024, - "operation": "write", - "t": 0.17, - "tags": ["tcptls_experiment"] - }, { - "conn_id": 12, - "failure": null, - "num_bytes": 5110, - "operation": "read", - "t": 0.44, - "tags": ["tcptls_experiment"] - }] - } + "address": "93.184.216.34:443", + "failure": null, + "num_bytes": 99, + "operation": "read", + "proto": "tcp", + "t0": 0.602109, + "t": 0.746866, + "transaction_id": 4 } ```