diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1677c9827..87255aceb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,5 +11,5 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v3 with: - go-version: "1.20.6" + go-version: "1.20.8" - run: go build ./... diff --git a/UPSTREAM b/UPSTREAM index 3cc22e3b6..7b2ea994d 100644 --- a/UPSTREAM +++ b/UPSTREAM @@ -1 +1 @@ -v3.18.0-alpha-86-g2051a2d3 +v3.19.0-alpha-1-gca2019be diff --git a/go.mod b/go.mod index 0648d5545..b6b3fc12b 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,7 @@ go 1.20 require ( filippo.io/age v1.1.1 - git.torproject.org/pluggable-transports/goptlib.git v1.3.0 - git.torproject.org/pluggable-transports/snowflake.git/v2 v2.5.1 - github.com/Psiphon-Labs/psiphon-tunnel-core v1.0.11-0.20230418182520-830177ebde85 + github.com/Psiphon-Labs/psiphon-tunnel-core v1.0.11-0.20230822172011-3f91b1b804b1 github.com/apex/log v1.9.0 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/cloudflare/circl v1.3.3 @@ -16,68 +14,65 @@ require ( github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/gorilla/websocket v1.5.0 github.com/hexops/gotextdiff v1.0.3 - github.com/iancoleman/strcase v0.3.0 - github.com/miekg/dns v1.1.55 + github.com/miekg/dns v1.1.56 github.com/montanaflynn/stats v0.7.1 github.com/ooni/go-libtor v1.1.8 - github.com/ooni/netem v0.0.0-20230906091637-85d962536ff3 - github.com/ooni/oocrypto v0.5.3 - github.com/ooni/oohttp v0.6.3 - github.com/ooni/probe-assets v0.18.0 + github.com/ooni/netem v0.0.0-20230920215742-15f3ffec0107 + github.com/ooni/oocrypto v0.5.4 + github.com/ooni/oohttp v0.6.4 + github.com/ooni/probe-assets v0.19.0 github.com/pborman/getopt/v2 v2.1.0 github.com/pion/stun v0.6.1 github.com/pkg/errors v0.9.1 - github.com/quic-go/quic-go v0.33.0 + github.com/quic-go/quic-go v0.39.0 github.com/rogpeppe/go-internal v1.11.0 - github.com/rubenv/sql-migrate v1.5.1 + github.com/rubenv/sql-migrate v1.5.2 github.com/schollz/progressbar/v3 v3.13.1 github.com/upper/db/v4 v4.6.0 - gitlab.com/yawning/obfs4.git v0.0.0-20230519154740-645026c2ada4 + gitlab.com/yawning/obfs4.git v0.0.0-20231005123604-19f5a37fe427 gitlab.com/yawning/utls.git v0.0.12-1 - golang.org/x/crypto v0.11.0 - golang.org/x/net v0.12.0 - golang.org/x/sys v0.10.0 + gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/goptlib v1.5.0 + gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2 v2.6.1 + golang.org/x/crypto v0.14.0 + golang.org/x/net v0.16.0 + golang.org/x/sys v0.13.0 ) require ( - github.com/BurntSushi/toml v1.3.0 // indirect github.com/Psiphon-Labs/bolt v0.0.0-20200624191537-23cedaef7ad7 // indirect github.com/Psiphon-Labs/goptlib v0.0.0-20200406165125-c0e32a7a3464 // indirect - github.com/Psiphon-Labs/qtls-go1-18 v0.0.0-20221014170512-3bdc7291c091 // indirect - github.com/Psiphon-Labs/qtls-go1-19 v0.0.0-20230515185100-099bac32c181 // indirect - github.com/Psiphon-Labs/quic-go v0.0.0-20230215230806-9b1ddbf778cc // indirect - github.com/Psiphon-Labs/tls-tris v0.0.0-20210713133851-676a693d51ad // indirect + github.com/Psiphon-Labs/qtls-go1-19 v0.0.0-20230608213623-d58aa73e519a // indirect + github.com/Psiphon-Labs/qtls-go1-20 v0.0.0-20230608214729-dd57d6787acf // indirect + github.com/Psiphon-Labs/quic-go v0.0.0-20230626192210-73f29effc9da // indirect + github.com/Psiphon-Labs/tls-tris v0.0.0-20230824155421-58bf6d336a9a // indirect github.com/andybalholm/brotli v1.0.5 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/fatih/color v1.15.0 // indirect - github.com/gaukas/godicttls v0.0.3 // indirect - github.com/go-redis/redis/v8 v8.11.5 // indirect - github.com/golang/mock v1.6.0 // indirect + github.com/gaukas/godicttls v0.0.4 // indirect github.com/google/btree v1.1.2 // indirect - github.com/google/martian/v3 v3.3.2 // indirect - github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect + github.com/google/uuid v1.3.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/klauspost/compress v1.16.5 // indirect + github.com/klauspost/compress v1.17.0 // indirect + github.com/libp2p/go-reuseport v0.4.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-runewidth v0.0.14 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect - github.com/mroth/weightedrand v1.0.0 // indirect - github.com/onsi/ginkgo/v2 v2.9.7 // indirect - github.com/pebbe/zmq4 v1.2.9 // indirect - github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pion/transport/v2 v2.2.1 // indirect + github.com/onsi/ginkgo/v2 v2.12.1 // indirect + github.com/pion/transport/v2 v2.2.4 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/quic-go/qpack v0.4.0 // indirect - github.com/quic-go/qtls-go1-19 v0.3.2 // indirect - github.com/quic-go/qtls-go1-20 v0.2.2 // indirect - github.com/refraction-networking/conjure v0.4.0 // indirect + github.com/quic-go/qtls-go1-20 v0.3.4 // indirect + github.com/refraction-networking/conjure v0.7.4 // indirect + github.com/refraction-networking/ed25519 v0.1.2 // indirect + github.com/refraction-networking/obfs4 v0.1.2 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/segmentio/fasthash v1.0.3 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/testify v1.8.4 // indirect - golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect + gitlab.com/yawning/edwards25519-extra v0.0.0-20231005122941-2149dcafc266 // indirect + go.uber.org/mock v0.3.0 // indirect + golang.org/x/exp v0.0.0-20231005195138-3e424a577f31 // indirect golang.org/x/exp/typeparams v0.0.0-20230522175609-2e198f4a06a1 // indirect golang.org/x/time v0.3.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect @@ -100,37 +95,37 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect - github.com/golang/glog v1.1.1 // indirect + github.com/golang/glog v1.1.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/grafov/m3u8 v0.12.0 // indirect - github.com/hashicorp/golang-lru v0.6.0 // indirect + github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/juju/ratelimit v1.0.2 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect - github.com/klauspost/reedsolomon v1.11.7 // indirect + github.com/klauspost/reedsolomon v1.11.8 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-sqlite3 v1.14.17 github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/oschwald/maxminddb-golang v1.11.0 + github.com/oschwald/maxminddb-golang v1.12.0 github.com/pion/datachannel v1.5.5 // indirect github.com/pion/dtls/v2 v2.2.7 // indirect - github.com/pion/ice/v2 v2.3.6 // indirect - github.com/pion/interceptor v0.1.17 // indirect + github.com/pion/ice/v2 v2.3.11 // indirect + github.com/pion/interceptor v0.1.21 // indirect github.com/pion/logging v0.2.2 // indirect - github.com/pion/mdns v0.0.7 // indirect + github.com/pion/mdns v0.0.9 // indirect github.com/pion/randutil v0.1.0 // indirect github.com/pion/rtcp v1.2.10 // indirect - github.com/pion/rtp v1.7.13 // indirect - github.com/pion/sctp v1.8.7 // indirect + github.com/pion/rtp v1.8.2 // indirect + github.com/pion/sctp v1.8.9 // indirect github.com/pion/sdp/v3 v3.0.6 // indirect - github.com/pion/srtp/v2 v2.0.15 // indirect - github.com/pion/turn/v2 v2.1.0 // indirect - github.com/pion/webrtc/v3 v3.2.9 // indirect - github.com/prometheus/client_golang v1.16.0 - github.com/prometheus/client_model v0.4.0 // indirect + github.com/pion/srtp/v2 v2.0.17 // indirect + github.com/pion/turn/v2 v2.1.4 // indirect + github.com/pion/webrtc/v3 v3.2.21 // indirect + github.com/prometheus/client_golang v1.17.0 + github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.10.1 // indirect - github.com/refraction-networking/gotapdance v1.5.0 // indirect - github.com/refraction-networking/utls v1.3.2 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + github.com/refraction-networking/gotapdance v1.7.4 // indirect + github.com/refraction-networking/utls v1.3.3 // indirect github.com/sergeyfrolov/bsbuffer v0.0.0-20180903213811-94e85abb8507 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/cobra v1.7.0 @@ -142,12 +137,10 @@ require ( github.com/wader/filtertransport v0.0.0-20200316221534-bdd9e61eee78 // indirect github.com/xtaci/kcp-go/v5 v5.6.2 // indirect github.com/xtaci/smux v1.5.24 // indirect - github.com/zach-klippenstein/goregen v0.0.0-20160303162051-795b5e3961ea // indirect gitlab.com/yawning/bsaes.git v0.0.0-20190805113838-0a714cd429ec // indirect - gitlab.com/yawning/edwards25519-extra.git v0.0.0-20220726154925-def713fd18e4 // indirect - golang.org/x/mod v0.10.0 // indirect - golang.org/x/term v0.10.0 // indirect - golang.org/x/text v0.11.0 // indirect - golang.org/x/tools v0.9.3 // indirect - google.golang.org/protobuf v1.30.0 // indirect + golang.org/x/mod v0.12.0 // indirect + golang.org/x/term v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect + golang.org/x/tools v0.13.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect ) diff --git a/go.sum b/go.sum index e0b39e4dc..57ef91a74 100644 --- a/go.sum +++ b/go.sum @@ -1,20 +1,12 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= filippo.io/age v1.1.1 h1:pIpO7l151hCnQ4BdyBujnGP2YlUo0uj6sAVNHGBvXHg= filippo.io/age v1.1.1/go.mod h1:l03SrzDUrBkdBx8+IILdnn2KZysqQdbEBUQ4p3sqEQE= -filippo.io/edwards25519 v1.0.0-rc.1.0.20210721174708-390f27c3be20/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns= filippo.io/edwards25519 v1.0.0 h1:0wAIcmJUqRdI8IJ/3eGi5/HwXZWPujYXXlkrQogz0Ek= filippo.io/edwards25519 v1.0.0/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns= -git.torproject.org/pluggable-transports/goptlib.git v1.3.0 h1:G+iuRUblCCC2xnO+0ag1/4+aaM98D5mjWP1M0v9s8a0= -git.torproject.org/pluggable-transports/goptlib.git v1.3.0/go.mod h1:4PBMl1dg7/3vMWSoWb46eGWlrxkUyn/CAJmxhDLAlDs= -git.torproject.org/pluggable-transports/snowflake.git/v2 v2.5.1 h1:hs6eyZ/zJzXBfNCgBK73TLbsEZtYRp2fC1anrNc2Vsw= -git.torproject.org/pluggable-transports/snowflake.git/v2 v2.5.1/go.mod h1:KzHdsnwfBugy6JLtsk5XcHgajV3eib1tOxlhD7t5dvo= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.3.0 h1:Ws8e5YmnrGEHzZEzg0YvK/7COGYtTC5PbaH9oSSbgfA= -github.com/BurntSushi/toml v1.3.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Psiphon-Inc/rotate-safe-writer v0.0.0-20210303140923-464a7a37606e h1:NPfqIbzmijrl0VclX2t8eO5EPBhqe47LLGKpRrcVjXk= @@ -22,57 +14,34 @@ github.com/Psiphon-Labs/bolt v0.0.0-20200624191537-23cedaef7ad7 h1:Hx/NCZTnvoKZu github.com/Psiphon-Labs/bolt v0.0.0-20200624191537-23cedaef7ad7/go.mod h1:alTtZBo3j4AWFvUrAH6F5ZaHcTj4G5Y01nHz8dkU6vU= github.com/Psiphon-Labs/goptlib v0.0.0-20200406165125-c0e32a7a3464 h1:VmnMMMheFXwLV0noxYhbJbLmkV4iaVW3xNnj6xcCNHo= github.com/Psiphon-Labs/goptlib v0.0.0-20200406165125-c0e32a7a3464/go.mod h1:Pe5BqN2DdIdChorAXl6bDaQd/wghpCleJfid2NoSli0= -github.com/Psiphon-Labs/psiphon-tunnel-core v1.0.11-0.20230418182520-830177ebde85 h1:jrSZ81X/M3nZCtoSBqZX/WogKTXESsPWzCQFrNOfPxY= -github.com/Psiphon-Labs/psiphon-tunnel-core v1.0.11-0.20230418182520-830177ebde85/go.mod h1:ZqtRlpd0qHKS8bN3NzZR39si6RuHyVEeZUL67lMAE8w= -github.com/Psiphon-Labs/qtls-go1-18 v0.0.0-20221014170512-3bdc7291c091 h1:Kv0LQQ3joUp8s2z36aigpNgNyiLiExT/OS9KOC/L/gI= -github.com/Psiphon-Labs/qtls-go1-18 v0.0.0-20221014170512-3bdc7291c091/go.mod h1:0IvfcPDkLvBkir+WGq3E0shsx+TLasdcl8ojVWWTflE= -github.com/Psiphon-Labs/qtls-go1-19 v0.0.0-20230515185100-099bac32c181 h1:+rhvNaRVcVr6OXDPJx3lOaSccBhCxgcKlG/OVU/uvGc= -github.com/Psiphon-Labs/qtls-go1-19 v0.0.0-20230515185100-099bac32c181/go.mod h1:mHM/QFYc02W9MKJ/Ux5XGOKP4OImosPeQUO7XAaXs0E= -github.com/Psiphon-Labs/quic-go v0.0.0-20230215230806-9b1ddbf778cc h1:FUmGSvMiMbf1tFXWbK0+N7+5zBhOol8CHQdpB4ZQlDg= -github.com/Psiphon-Labs/quic-go v0.0.0-20230215230806-9b1ddbf778cc/go.mod h1:cu4yhfHkyt+uQ9FFFjTpjCjcQYf52ntEAyoV4Zg0+fg= -github.com/Psiphon-Labs/tls-tris v0.0.0-20210713133851-676a693d51ad h1:m6HS84+b5xDPLj7D/ya1CeixyaHOCZoMbBilJ48y+Ts= -github.com/Psiphon-Labs/tls-tris v0.0.0-20210713133851-676a693d51ad/go.mod h1:v3y9GXFo9Sf2mO6auD2ExGG7oDgrK8TI7eb49ZnUxrE= -github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= -github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/Psiphon-Labs/psiphon-tunnel-core v1.0.11-0.20230822172011-3f91b1b804b1 h1:yHV1QsdiZw355X+vq+qbLcx5APdiZUdvLtLcn1d/Gko= +github.com/Psiphon-Labs/psiphon-tunnel-core v1.0.11-0.20230822172011-3f91b1b804b1/go.mod h1:675zphn0MOOnCjdC9NTJt+F4ItH+Nq4f9c4OvVj0O3M= +github.com/Psiphon-Labs/qtls-go1-19 v0.0.0-20230608213623-d58aa73e519a h1:O8D+GcEoZwutcERaABP2AM3RDvswBVtNmBWvlBn5wiw= +github.com/Psiphon-Labs/qtls-go1-19 v0.0.0-20230608213623-d58aa73e519a/go.mod h1:81bbD3bvEvi3BSamZb30PgvPvqwSLfEPqwwmq5sx7fc= +github.com/Psiphon-Labs/qtls-go1-20 v0.0.0-20230608214729-dd57d6787acf h1:bGS+WxWdHHuf42hn3M1GFSJbzCgtKNVTuiRqwCo3zyc= +github.com/Psiphon-Labs/qtls-go1-20 v0.0.0-20230608214729-dd57d6787acf/go.mod h1:wUiSd0qyefymNlikc99B2rRC01YPN1uUvDMytMOGmF8= +github.com/Psiphon-Labs/quic-go v0.0.0-20230626192210-73f29effc9da h1:TI2+ExyFR3A0kPrFHfaM6y3RybP0HGfP9N1R8hfZzfk= +github.com/Psiphon-Labs/quic-go v0.0.0-20230626192210-73f29effc9da/go.mod h1:wTIxqsKVrEQIxVIIYOEHuscY+PM3h6Wz79u5aF60fo0= +github.com/Psiphon-Labs/tls-tris v0.0.0-20230824155421-58bf6d336a9a h1:BOfU6ghaMsT/c40sWHmf3PXNwIendYXzL6tRv6NbPog= +github.com/Psiphon-Labs/tls-tris v0.0.0-20230824155421-58bf6d336a9a/go.mod h1:v3y9GXFo9Sf2mO6auD2ExGG7oDgrK8TI7eb49ZnUxrE= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA= github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-proxyproto v0.0.0-20210323213023-7e956b284f0a h1:AP/vsCIvJZ129pdm9Ek7bH7yutN3hByqsMoNrWAxRQc= github.com/armon/go-proxyproto v0.0.0-20210323213023-7e956b284f0a/go.mod h1:QmP9hvJ91BbJmGVGSbutW19IC0Q9phDCLGaomwTJbgU= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= -github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bifurcation/mint v0.0.0-20180306135233-198357931e61 h1:BU+NxuoaYPIvvp8NNkNlLr8aA0utGyuunf4Q3LJ0bh0= github.com/bifurcation/mint v0.0.0-20180306135233-198357931e61/go.mod h1:zVt7zX3K/aDCk9Tj+VM7YymsX66ERvzCJzw8rFCX2JU= -github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -80,27 +49,19 @@ github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= -github.com/clarkduvall/hyperloglog v0.0.0-20171127014514-a0107a5d8004/go.mod h1:drodPoQNro6QBO6TJ/MpMZbz8Bn2eSDtRN6jpG4VGw8= -github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/cognusion/go-cache-lru v0.0.0-20170419142635-f73e2280ecea h1:9C2rdYRp8Vzwhm3sbFX0yYfB+70zKFRjn7cnPCucHSw= github.com/cognusion/go-cache-lru v0.0.0-20170419142635-f73e2280ecea/go.mod h1:MdyNkAe06D7xmJsf+MsLvbZKYNXuOHLKJrvw+x4LlcQ= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/cretz/bine v0.1.0/go.mod h1:6PF6fWAvYtwjRGkAuDEJeWNOv3a2hUouSP/yRYXmvHw= @@ -118,55 +79,36 @@ github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrV github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= -github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elazarl/goproxy v0.0.0-20200809112317-0581fc3aee2d h1:rtM8HsT3NG37YPjz8sYSbUSdElP9lUsQENYzJDZDUBE= github.com/elazarl/goproxy/ext v0.0.0-20200809112317-0581fc3aee2d h1:st1tmvy+4duoRj+RaeeJoECWCWM015fBtf/4aR+hhqk= -github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/florianl/go-nfqueue v1.1.1-0.20200829120558-a2f196e98ab0 h1:7ZJyJV4KiWBijCCzUPvVaqxsDxO36+KD0XKBdEN3I+8= -github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= -github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/gaukas/godicttls v0.0.3 h1:YNDIf0d9adcxOijiLrEzpfZGAkNwLRzPaG6OjU7EITk= -github.com/gaukas/godicttls v0.0.3/go.mod h1:l6EenT4TLWgTdwslVb4sEMOCf7Bv0JAK67deKr9/NCI= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gaukas/godicttls v0.0.4 h1:NlRaXb3J6hAnTmWdsEKb9bcSBD6BvcIjdGdeb0zfXbk= +github.com/gaukas/godicttls v0.0.4/go.mod h1:l6EenT4TLWgTdwslVb4sEMOCf7Bv0JAK67deKr9/NCI= github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs= github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= -github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -178,20 +120,13 @@ github.com/gobuffalo/packd v1.0.1 h1:U2wXfRr4E9DH8IdsDLlRFwTZTK7hLfq9qT/QHXGVe/0 github.com/gobuffalo/packr/v2 v2.8.3 h1:xE1yzvnO56cUC0sTpKR3DIbxZgB54AftTFMhB2XEWlY= github.com/gobwas/glob v0.2.4-0.20180402141543-f00a7392b439 h1:T6zlOdzrYuHf6HUKujm9bzkzbZ5Iv/xf6rs8BHZDpoI= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.1.1 h1:jxpi2eWoU84wbX9iIEyAeeoac3FLuifZpY9tcNUD9kw= -github.com/golang/glog v1.1.1/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= +github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -201,91 +136,45 @@ github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:x github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= -github.com/google/gxui v0.0.0-20151028112939-f85e0a97b3a4 h1:OL2d27ueTKnlQJoqLW2fc9pWYulFnJYLWzomGV7HqZo= -github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= -github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= -github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 h1:hR7/MlvK23p6+lIw9SN1TigNLn9ZnF3W4SYRKq2gAHs= -github.com/google/pprof v0.0.0-20230602150820-91b7bce49751/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= +github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 h1:pUa4ghanp6q4IJHwE9RwLgmVFfReJN+KbQ8ExNEUUoQ= +github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grafov/m3u8 v0.12.0 h1:T6iTwTsSEtMcwkayef+FJO8kj+Sglr4Lh81Zj8Ked/4= github.com/grafov/m3u8 v0.12.0/go.mod h1:nqzOkfBiZJENr52zTVd/Dcl03yzphIMbJqkXGu+u080= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= -github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4= -github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= +github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= -github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= -github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= @@ -327,40 +216,26 @@ github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0f github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/josharian/native v1.0.0 h1:Ts/E8zCSEsG17dUqv7joXJFybuMLjQfWE04tsBODTxk= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/ratelimit v1.0.2 h1:sRxmtRiajbvrcLQT7S+JbqU0ntsb9W2yhSdNN8tWfaI= github.com/juju/ratelimit v1.0.2/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= -github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= +github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/cpuid v1.2.4/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= github.com/klauspost/cpuid/v2 v2.0.14/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/klauspost/reedsolomon v1.9.9/go.mod h1:O7yFFHiQwDR6b2t63KPUpccPtNdp5ADgh1gg4fd12wo= github.com/klauspost/reedsolomon v1.10.0/go.mod h1:qHMIzMkuZUWqIh8mS/GruPdo3u0qwX2jk/LH440ON7Y= -github.com/klauspost/reedsolomon v1.11.7 h1:9uaHU0slncktTEEg4+7Vl7q7XUNMBUOK4R9gnKhMjAU= -github.com/klauspost/reedsolomon v1.11.7/go.mod h1:4bXRN+cVzMdml6ti7qLouuYi32KHJ5MGv0Qd8a47h6A= +github.com/klauspost/reedsolomon v1.11.8 h1:s8RpUW5TK4hjr+djiOpbZJB4ksx+TdYbRH7vHQpwPOY= +github.com/klauspost/reedsolomon v1.11.8/go.mod h1:4bXRN+cVzMdml6ti7qLouuYi32KHJ5MGv0Qd8a47h6A= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -375,22 +250,18 @@ github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= -github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= -github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= +github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQscQm2s= +github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/markbates/errx v1.1.0 h1:QDFeR+UP95dO12JgW+tgi2UVfo0V8YBHiUIOaeBPiEI= github.com/markbates/oncer v1.0.0 h1:E83IaVAHygyndzPimgUYJjbshhDTALZyXxvk9FOlQRY= github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI= github.com/marusama/semaphore v0.0.0-20171214154724-565ffd8e868a h1:6SRny9FLB1eWasPyDUqBQnMi9NhXU01XIlB0ao89YoI= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -399,225 +270,140 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mdlayher/netlink v1.4.2-0.20210930205308-a81a8c23d40a h1:yk5OmRew64lWdeNanQ3l0hDgUt1E8MfipPhh/GO9Tuw= github.com/mdlayher/socket v0.0.0-20210624160740-9dbe287ded84 h1:L1jnQ6o+K3M574eez7eTxbsia6H1SfJaVpaXY33L37Q= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo= -github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/miekg/dns v1.1.56 h1:5imZaSeoRNvpM9SzWNhEcP9QliKiz20/dA2QabIGVnE= +github.com/miekg/dns v1.1.56/go.mod h1:cRm6Oo2C8TY9ZS/TqsSrseAcncm74lfK5G+ikN2SWWY= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mmcloughlin/avo v0.0.0-20200803215136-443f81d77104/go.mod h1:wqKykBG2QzQDJEzvRkcS8x6MiSJkF52hXZsXcjaB3ls= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/mroth/weightedrand v1.0.0 h1:V8JeHChvl2MP1sAoXq4brElOcza+jxLkRuwvtQu8L3E= -github.com/mroth/weightedrand v1.0.0/go.mod h1:3p2SIcC8al1YMzGhAIoXD+r9olo/g/cdJgAD905gyNE= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= -github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= -github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= -github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= -github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= -github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.9.7 h1:06xGQy5www2oN160RtEZoTvnP2sPhEfePYmCDc2szss= -github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/ginkgo/v2 v2.12.1 h1:uHNEO1RP2SpuZApSkel9nEh1/Mu+hmQe7Q+Pepg5OYA= +github.com/onsi/ginkgo/v2 v2.12.1/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/ooni/go-libtor v1.1.8 h1:Wo3V3DVTxl5vZdxtQakqYP+DAHx7pPtAFSl1bnAa08w= github.com/ooni/go-libtor v1.1.8/go.mod h1:q1YyLwRD9GeMyeerVvwc0vJ2YgwDLTp2bdVcrh/JXyI= -github.com/ooni/netem v0.0.0-20230906091637-85d962536ff3 h1:zpTbzNzpo00cKbjLLnWMKjZeGLdoNC81vMiBDiur7NU= -github.com/ooni/netem v0.0.0-20230906091637-85d962536ff3/go.mod h1:3LJOzTIu2O4ADDJN2ILG4ViJOqyH/u9fKY8QT2Rma8Y= -github.com/ooni/oocrypto v0.5.3 h1:CAb0Ze6q/EWD1PRGl9KqpzMfkut4O3XMaiKYsyxrWOs= -github.com/ooni/oocrypto v0.5.3/go.mod h1:HjEQ5pQBl6btcWgAsKKq1tFo8CfBrZu63C/vPAUGIDk= -github.com/ooni/oohttp v0.6.3 h1:MHydpeAPU/LSDSI/hIFJwZm4afBhd2Yo+rNxxFdeMCY= -github.com/ooni/oohttp v0.6.3/go.mod h1:Zk8frCRZeVn2x68SI6rc0PvAJrK5unPLczIhCU/STAk= -github.com/ooni/probe-assets v0.18.0 h1:/Ui1mvGzPKu+IB9sBWGeVDJPuIlWGAfKb7nPOhQfgGQ= -github.com/ooni/probe-assets v0.18.0/go.mod h1:m0k2FFzcLfFm7dhgyYkLCUR3R0CoRPr0jcjctDS2+gU= -github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= -github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= -github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= -github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= -github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/oschwald/maxminddb-golang v1.11.0 h1:aSXMqYR/EPNjGE8epgqwDay+P30hCBZIveY0WZbAWh0= -github.com/oschwald/maxminddb-golang v1.11.0/go.mod h1:YmVI+H0zh3ySFR3w+oz8PCfglAFj3PuCmui13+P9zDg= -github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/ooni/netem v0.0.0-20230920215742-15f3ffec0107 h1:PktaCPQ1NYZOaK+J8pQGYiPCYFkGR5H3ZURg9zPkQsI= +github.com/ooni/netem v0.0.0-20230920215742-15f3ffec0107/go.mod h1:5X3Lk4+cnrwrQiYgRlCWXgV33IMDgLaO5s1x0DD/fO0= +github.com/ooni/oocrypto v0.5.4 h1:/AkVZd+aq54+OXgOtWEmK8xgZsFQtlmtPf2VgY20YWw= +github.com/ooni/oocrypto v0.5.4/go.mod h1:HjEQ5pQBl6btcWgAsKKq1tFo8CfBrZu63C/vPAUGIDk= +github.com/ooni/oohttp v0.6.4 h1:QZyOO4e88AzLOHGTgapXmsjtn1EVR7Wl+BtHd8okIf4= +github.com/ooni/oohttp v0.6.4/go.mod h1:RipdYAUiw1UTnpm0ISd0r1Kiv/CGaRUgn08xbK1JgVo= +github.com/ooni/probe-assets v0.19.0 h1:XloDJQt6uxn6EYVwfWCOnlgsJZbmzO7VPFsJ8RPW8Ns= +github.com/ooni/probe-assets v0.19.0/go.mod h1:m0k2FFzcLfFm7dhgyYkLCUR3R0CoRPr0jcjctDS2+gU= +github.com/oschwald/geoip2-golang v1.9.0 h1:uvD3O6fXAXs+usU+UGExshpdP13GAqp4GBrzN7IgKZc= +github.com/oschwald/maxminddb-golang v1.12.0 h1:9FnTOD0YOhP7DGxGsq4glzpGy5+w7pq50AS6wALUMYs= +github.com/oschwald/maxminddb-golang v1.12.0/go.mod h1:q0Nob5lTCqyQ8WT6FYgS1L7PXKVVbgiymefNwIjPzgY= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/pborman/getopt/v2 v2.1.0 h1:eNfR+r+dWLdWmV8g5OlpyrTYHkhVNxHBdN2cCrJmOEA= github.com/pborman/getopt/v2 v2.1.0/go.mod h1:4NtW75ny4eBw9fO1bhtNdYTlZKYX5/tBLtsOpwKIKd0= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pebbe/zmq4 v1.2.9 h1:JlHcdgq6zpppNR1tH0wXJq0XK03pRUc4lBlHTD7aj/4= -github.com/pebbe/zmq4 v1.2.9/go.mod h1:nqnPueOapVhE2wItZ0uOErngczsJdLOGkebMxaO8r48= +github.com/pebbe/zmq4 v1.2.10 h1:wQkqRZ3CZeABIeidr3e8uQZMMH5YAykA/WN0L5zkd1c= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= -github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= -github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pion/datachannel v1.5.2/go.mod h1:FTGQWaHrdCwIJ1rw6xBIfZVkslikjShim5yr05XFuCQ= github.com/pion/datachannel v1.5.5 h1:10ef4kwdjije+M9d7Xm9im2Y3O6A6ccQb0zcqZcJew8= github.com/pion/datachannel v1.5.5/go.mod h1:iMz+lECmfdCMqFRhXhcA/219B0SQlbpoR2V118yimL0= -github.com/pion/dtls/v2 v2.1.3/go.mod h1:o6+WvyLDAlXF7YiPB/RlskRoeK+/JtuaZa5emwQcWus= -github.com/pion/dtls/v2 v2.1.5/go.mod h1:BqCE7xPZbPSubGasRoDFJeTsyJtdD1FanJYL0JGheqY= github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8= github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= -github.com/pion/ice/v2 v2.2.6/go.mod h1:SWuHiOGP17lGromHTFadUe1EuPgFh/oCU6FCMZHooVE= -github.com/pion/ice/v2 v2.3.6 h1:Jgqw36cAud47iD+N6rNX225uHvrgWtAlHfVyOQc3Heg= -github.com/pion/ice/v2 v2.3.6/go.mod h1:9/TzKDRwBVAPsC+YOrKH/e3xDrubeTRACU9/sHQarsU= -github.com/pion/interceptor v0.1.11/go.mod h1:tbtKjZY14awXd7Bq0mmWvgtHB5MDaRN7HV3OZ/uy7s8= -github.com/pion/interceptor v0.1.17 h1:prJtgwFh/gB8zMqGZoOgJPHivOwVAp61i2aG61Du/1w= -github.com/pion/interceptor v0.1.17/go.mod h1:SY8kpmfVBvrbUzvj2bsXz7OJt5JvmVNZ+4Kjq7FcwrI= +github.com/pion/ice/v2 v2.3.11 h1:rZjVmUwyT55cmN8ySMpL7rsS8KYsJERsrxJLLxpKhdw= +github.com/pion/ice/v2 v2.3.11/go.mod h1:hPcLC3kxMa+JGRzMHqQzjoSj3xtE9F+eoncmXLlCL4E= +github.com/pion/interceptor v0.1.18/go.mod h1:tpvvF4cPM6NGxFA1DUMbhabzQBxdWMATDGEUYOR9x6I= +github.com/pion/interceptor v0.1.21 h1:owpNzUHITYK5IqP83LoPECO5Rq6uK4io7dGUx1SQJoo= +github.com/pion/interceptor v0.1.21/go.mod h1:wkbPYAak5zKsfpVDYMtEfWEy8D4zL+rpxCxPImLOg3Y= github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= -github.com/pion/mdns v0.0.5/go.mod h1:UgssrvdD3mxpi8tMxAXbsppL3vJ4Jipw1mTCW+al01g= -github.com/pion/mdns v0.0.7 h1:P0UB4Sr6xDWEox0kTVxF0LmQihtCbSAdW0H2nEgkA3U= -github.com/pion/mdns v0.0.7/go.mod h1:4iP2UbeFhLI/vWju/bw6ZfwjJzk0z8DNValjGxR/dD8= +github.com/pion/mdns v0.0.8/go.mod h1:hYE72WX8WDveIhg7fmXgMKivD3Puklk0Ymzog0lSyaI= +github.com/pion/mdns v0.0.9 h1:7Ue5KZsqq8EuqStnpPWV33vYYEH0+skdDN5L7EiEsI4= +github.com/pion/mdns v0.0.9/go.mod h1:2JA5exfxwzXiCihmxpTKgFUpiQws2MnipoPK09vecIc= github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= -github.com/pion/rtcp v1.2.9/go.mod h1:qVPhiCzAm4D/rxb6XzKeyZiQK69yJpbUDJSF7TgrqNo= github.com/pion/rtcp v1.2.10 h1:nkr3uj+8Sp97zyItdN60tE/S6vk4al5CPRR6Gejsdjc= github.com/pion/rtcp v1.2.10/go.mod h1:ztfEwXZNLGyF1oQDttz/ZKIBaeeg/oWbRYqzBM9TL1I= -github.com/pion/rtp v1.7.13 h1:qcHwlmtiI50t1XivvoawdCGTP4Uiypzfrsap+bijcoA= -github.com/pion/rtp v1.7.13/go.mod h1:bDb5n+BFZxXx0Ea7E5qe+klMuqiBrP+w8XSjiWtCUko= -github.com/pion/sctp v1.8.0/go.mod h1:xFe9cLMZ5Vj6eOzpyiKjT9SwGM4KpK/8Jbw5//jc+0s= -github.com/pion/sctp v1.8.2/go.mod h1:xFe9cLMZ5Vj6eOzpyiKjT9SwGM4KpK/8Jbw5//jc+0s= +github.com/pion/rtp v1.8.1/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= +github.com/pion/rtp v1.8.2 h1:oKMM0K1/QYQ5b5qH+ikqDSZRipP5mIxPJcgcvw5sH0w= +github.com/pion/rtp v1.8.2/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= github.com/pion/sctp v1.8.5/go.mod h1:SUFFfDpViyKejTAdwD1d/HQsCu+V/40cCs2nZIvC3s0= -github.com/pion/sctp v1.8.7 h1:JnABvFakZueGAn4KU/4PSKg+GWbF6QWbKTWZOSGJjXw= -github.com/pion/sctp v1.8.7/go.mod h1:g1Ul+ARqZq5JEmoFy87Q/4CePtKnTJ1QCL9dBBdN6AU= -github.com/pion/sdp/v3 v3.0.5/go.mod h1:iiFWFpQO8Fy3S5ldclBkpXqmWy02ns78NOKoLLL0YQw= +github.com/pion/sctp v1.8.8/go.mod h1:igF9nZBrjh5AtmKc7U30jXltsFHicFCXSmWA2GWRaWs= +github.com/pion/sctp v1.8.9 h1:TP5ZVxV5J7rz7uZmbyvnUvsn7EJ2x/5q9uhsTtXbI3g= +github.com/pion/sctp v1.8.9/go.mod h1:cMLT45jqw3+jiJCrtHVwfQLnfR0MGZ4rgOJwUOIqLkI= github.com/pion/sdp/v3 v3.0.6 h1:WuDLhtuFUUVpTfus9ILC4HRyHsW6TdugjEX/QY9OiUw= github.com/pion/sdp/v3 v3.0.6/go.mod h1:iiFWFpQO8Fy3S5ldclBkpXqmWy02ns78NOKoLLL0YQw= -github.com/pion/srtp/v2 v2.0.9/go.mod h1:5TtM9yw6lsH0ppNCehB/EjEUli7VkUgKSPJqWVqbhQ4= -github.com/pion/srtp/v2 v2.0.15 h1:+tqRtXGsGwHC0G0IUIAzRmdkHvriF79IHVfZGfHrQoA= -github.com/pion/srtp/v2 v2.0.15/go.mod h1:b/pQOlDrbB0HEH5EUAQXzSYxikFbNcNuKmF8tM0hCtw= -github.com/pion/stun v0.3.5/go.mod h1:gDMim+47EeEtfWogA37n6qXZS88L5V6LqFcf+DZA2UA= -github.com/pion/stun v0.4.0/go.mod h1:QPsh1/SbXASntw3zkkrIk3ZJVKz4saBY2G7S10P3wCw= -github.com/pion/stun v0.6.0/go.mod h1:HPqcfoeqQn9cuaet7AOmB5e5xkObu9DwBdurwLKO9oA= +github.com/pion/srtp/v2 v2.0.17 h1:ECuOk+7uIpY6HUlTb0nXhfvu4REG2hjtC4ronYFCZE4= +github.com/pion/srtp/v2 v2.0.17/go.mod h1:y5WSHcJY4YfNB/5r7ca5YjHeIr1H3LM1rKArGGs8jMc= github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/8= -github.com/pion/transport v0.12.2/go.mod h1:N3+vZQD9HlDP5GWkZ85LohxNsDcNgofQmyL6ojX5d8Q= -github.com/pion/transport v0.12.3/go.mod h1:OViWW9SP2peE/HbwBvARicmAVnesphkNkCVZIWJ6q9A= -github.com/pion/transport v0.13.0/go.mod h1:yxm9uXpK9bpBBWkITk13cLo1y5/ur5VQpG22ny6EP7g= github.com/pion/transport v0.14.1 h1:XSM6olwW+o8J4SCmOBb/BpwZypkHeyM0PGFCxNQBr40= github.com/pion/transport v0.14.1/go.mod h1:4tGmbk00NeYA3rUa9+n+dzCCoKkcy3YlYb99Jn2fNnI= -github.com/pion/transport/v2 v2.0.0/go.mod h1:HS2MEBJTwD+1ZI2eSXSvHJx/HnzQqRy2/LXxt6eVMHc= -github.com/pion/transport/v2 v2.1.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= -github.com/pion/transport/v2 v2.2.0/go.mod h1:AdSw4YBZVDkZm8fpoz+fclXyQwANWmZAlDuQdctTThQ= -github.com/pion/transport/v2 v2.2.1 h1:7qYnCBlpgSJNYMbLCKuSY9KbQdBFoETvPNETv0y4N7c= github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= -github.com/pion/turn/v2 v2.0.8/go.mod h1:+y7xl719J8bAEVpSXBXvTxStjJv3hbz9YFflvkpcGPw= -github.com/pion/turn/v2 v2.1.0 h1:5wGHSgGhJhP/RpabkUb/T9PdsAjkGLS6toYz5HNzoSI= -github.com/pion/turn/v2 v2.1.0/go.mod h1:yrT5XbXSGX1VFSF31A3c1kCNB5bBZgk/uu5LET162qs= -github.com/pion/udp v0.1.1/go.mod h1:6AFo+CMdKQm7UiA0eUPA8/eVCTx8jBIITLZHc9DWX5M= -github.com/pion/webrtc/v3 v3.1.41/go.mod h1:sUcW9SFPEWerDqGOBmdYEMfRvbdd7rgwo4bNzfsXww4= -github.com/pion/webrtc/v3 v3.2.9 h1:U8NSjQDlZZ+Iy/hg42Q/u6mhEVSXYvKrOIZiZwYTfLc= -github.com/pion/webrtc/v3 v3.2.9/go.mod h1:gjQLMZeyN3jXBGdxGmUYCyKjOuYX/c99BDjGqmadq0A= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pion/transport/v2 v2.2.2/go.mod h1:OJg3ojoBJopjEeECq2yJdXH9YVrUJ1uQ++NjXLOUorc= +github.com/pion/transport/v2 v2.2.3/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= +github.com/pion/transport/v2 v2.2.4 h1:41JJK6DZQYSeVLxILA2+F4ZkKb4Xd/tFJZRFZQ9QAlo= +github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= +github.com/pion/transport/v3 v3.0.1 h1:gDTlPJwROfSfz6QfSi0ZmeCSkFcnWWiiR9ES0ouANiM= +github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0= +github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= +github.com/pion/turn/v2 v2.1.4 h1:2xn8rduI5W6sCZQkEnIUDAkrBQNl2eYIBCHMZ3QMmP8= +github.com/pion/turn/v2 v2.1.4/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= +github.com/pion/webrtc/v3 v3.2.21 h1:c8fy5JcqJkAQBwwy3Sk9huQLTBUSqaggyRlv9Lnh2zY= +github.com/pion/webrtc/v3 v3.2.21/go.mod h1:vVURQTBOG5BpWKOJz3nlr23NfTDeyKVmubRNqzQp+Tg= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= -github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= -github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= -github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= -github.com/quic-go/qtls-go1-19 v0.3.2 h1:tFxjCFcTQzK+oMxG6Zcvp4Dq8dx4yD3dDiIiyc86Z5U= -github.com/quic-go/qtls-go1-19 v0.3.2/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI= -github.com/quic-go/qtls-go1-20 v0.2.2 h1:WLOPx6OY/hxtTxKV1Zrq20FtXtDEkeY00CGQm8GEa3E= -github.com/quic-go/qtls-go1-20 v0.2.2/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM= -github.com/quic-go/quic-go v0.33.0 h1:ItNoTDN/Fm/zBlq769lLJc8ECe9gYaW40veHCCco7y0= -github.com/quic-go/quic-go v0.33.0/go.mod h1:YMuhaAV9/jIu0XclDXwZPAsP/2Kgr5yMYhe9oxhhOFA= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/refraction-networking/conjure v0.4.0 h1:YObJ+6L/MnupE82N2+3m93GMzlHjHSHe1ysIYDOpqug= -github.com/refraction-networking/conjure v0.4.0/go.mod h1:7na+SFWdaTOToNoNrsJRIj4CIMwdGFZ4kwb/CO5Z0rQ= -github.com/refraction-networking/gotapdance v1.5.0 h1:F28CrhFhyQLAgk2diKxSn7EIp8XfKFPmI6GlSjimb5k= -github.com/refraction-networking/gotapdance v1.5.0/go.mod h1:MrslNp4kScP2RX23nmVf4kegVsKCL5xqToqN1YEWCZc= -github.com/refraction-networking/utls v1.0.0/go.mod h1:tz9gX959MEFfFN5whTIocCLUG57WiILqtdVxI8c6Wj0= -github.com/refraction-networking/utls v1.3.2 h1:o+AkWB57mkcoW36ET7uJ002CpBWHu0KPxi6vzxvPnv8= -github.com/refraction-networking/utls v1.3.2/go.mod h1:fmoaOww2bxzzEpIKOebIsnBvjQpqP7L2vcm/9KUfm/E= +github.com/quic-go/qtls-go1-20 v0.3.4 h1:MfFAPULvst4yoMgY9QmtpYmfij/em7O8UUi+bNVm7Cg= +github.com/quic-go/qtls-go1-20 v0.3.4/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= +github.com/quic-go/quic-go v0.39.0 h1:AgP40iThFMY0bj8jGxROhw3S0FMGa8ryqsmi9tBH3So= +github.com/quic-go/quic-go v0.39.0/go.mod h1:T09QsDQWjLiQ74ZmacDfqZmhY/NLnw5BC40MANNNZ1Q= +github.com/refraction-networking/conjure v0.7.4 h1:D6e0DFd1B7VuiTcKDLQ5oAz1w0OtL9VNegabogZUZzU= +github.com/refraction-networking/conjure v0.7.4/go.mod h1:ptOjgA+P0wJ41CCobva75vpZurUiXz1ZXbKrr2kYRZA= +github.com/refraction-networking/ed25519 v0.1.2 h1:08kJZUkAlY7a7cZGosl1teGytV+QEoNxPO7NnRvAB+g= +github.com/refraction-networking/ed25519 v0.1.2/go.mod h1:nxYLUAYt/hmNpAh64PNSQ/tQ9gTIB89wCaGKJlRtZ9I= +github.com/refraction-networking/gotapdance v1.7.4 h1:HSdgJajosy9dwwrEiqZntTHf8apBMpB5gi6hY9OxfIk= +github.com/refraction-networking/gotapdance v1.7.4/go.mod h1:uUZDICZjk37aExgntA92OPL5Q7LrZgvpQncYNrO4sKg= +github.com/refraction-networking/obfs4 v0.1.2 h1:J842O4fGSkd2W8ogYj0KN6gqVVY+Cpqodw9qFGL7wVU= +github.com/refraction-networking/obfs4 v0.1.2/go.mod h1:wAl/+gWiLsrcykJA3nKJHx89f5/gXGM8UKvty7+mvbM= +github.com/refraction-networking/utls v1.3.3 h1:f/TBLX7KBciRyFH3bwupp+CE4fzoYKCirhdRcC490sw= +github.com/refraction-networking/utls v1.3.3/go.mod h1:DlecWW1LMlMJu+9qpzzQqdHDT/C2LAe03EdpLUz/RL8= github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= @@ -625,19 +411,15 @@ github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUz github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= -github.com/rubenv/sql-migrate v1.5.1 h1:WsZo4jPQfjmddDTh/suANP2aKPA7/ekN0LzuuajgQEo= -github.com/rubenv/sql-migrate v1.5.1/go.mod h1:H38GW8Vqf8F0Su5XignRyaRcbXbJunSWxs+kmzlg0Is= +github.com/rubenv/sql-migrate v1.5.2 h1:bMDqOnrJVV/6JQgQ/MxOpU+AdO8uzYYA/TxFUBzFtS0= +github.com/rubenv/sql-migrate v1.5.2/go.mod h1:H38GW8Vqf8F0Su5XignRyaRcbXbJunSWxs+kmzlg0Is= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735 h1:7YvPJVmEeFHR1Tj9sZEYsmarJEQfMVYpd/Vyy/A8dqE= -github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/schollz/progressbar/v3 v3.13.1 h1:o8rySDYiQ59Mwzy2FELeHY5ZARXZTVJC7iHD6PEFUiE= github.com/schollz/progressbar/v3 v3.13.1/go.mod h1:xvrbki8kfT1fzWzBT/UZd9L6GA+jdL7HAgq2RFnO6fQ= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= github.com/sergeyfrolov/bsbuffer v0.0.0-20180903213811-94e85abb8507 h1:ML7ZNtcln5UBo5Wv7RIv9Xg3Pr5VuRCWLFXEwda54Y4= @@ -645,40 +427,28 @@ github.com/sergeyfrolov/bsbuffer v0.0.0-20180903213811-94e85abb8507/go.mod h1:Db github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= @@ -693,7 +463,6 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= @@ -702,7 +471,6 @@ github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a h1:SJy1Pu0eH1C29XwJucQo73FrleVK6t4kYz4NVhp34Yw= github.com/tailscale/hujson v0.0.0-20221223112325-20486734a56a/go.mod h1:DFSS3NAGHthKo1gTlmEcSBiZrRJXi28rLNd/1udP1c8= github.com/templexxx/cpu v0.0.1/go.mod h1:w7Tb+7qgcAlIyX4NhLuDKt78AHA5SzPmq0Wj6HiEnnk= -github.com/templexxx/cpu v0.0.7/go.mod h1:w7Tb+7qgcAlIyX4NhLuDKt78AHA5SzPmq0Wj6HiEnnk= github.com/templexxx/cpu v0.0.9/go.mod h1:w7Tb+7qgcAlIyX4NhLuDKt78AHA5SzPmq0Wj6HiEnnk= github.com/templexxx/cpu v0.1.0 h1:wVM+WIJP2nYaxVxqgHPD4wGA2aJ9rvrQRV8CvFzNb40= github.com/templexxx/cpu v0.1.0/go.mod h1:w7Tb+7qgcAlIyX4NhLuDKt78AHA5SzPmq0Wj6HiEnnk= @@ -716,54 +484,42 @@ github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= -github.com/tjfoc/gmsm v1.3.2/go.mod h1:HaUcFuY0auTiaHB9MHFGCPx5IaLhTUd2atbCFBQXn9w= github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho= github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/upper/db/v4 v4.6.0 h1:0VmASnqrl/XN8Ehoq++HBgZ4zRD5j3GXygW8FhP0C5I= github.com/upper/db/v4 v4.6.0/go.mod h1:2mnRcPf+RcCXmVcD+o04LYlyu3UuF7ubamJia7CkN6s= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/wader/filtertransport v0.0.0-20200316221534-bdd9e61eee78 h1:9sreu9e9KOihf2Y0NbpyfWhd1XFDcL4GTkPYL4IvMrg= github.com/wader/filtertransport v0.0.0-20200316221534-bdd9e61eee78/go.mod h1:HazXTRLhXFyq80TQp7PUXi6BKE6mS+ydEdzEqNBKopQ= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/xtaci/kcp-go/v5 v5.6.1/go.mod h1:W3kVPyNYwZ06p79dNwFWQOVFrdcBpDBsdyvK8moQrYo= github.com/xtaci/kcp-go/v5 v5.6.2 h1:pSXMa5MOsb+EIZKe4sDBqlTExu2A/2Z+DFhoX2qtt2A= github.com/xtaci/kcp-go/v5 v5.6.2/go.mod h1:LsinWoru+lWWJHb+EM9HeuqYxV6bb9rNcK12v67jYzQ= github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae h1:J0GxkO96kL4WF+AIT3M4mfUVinOCPgf2uUWYFUzN0sM= github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae/go.mod h1:gXtu8J62kEgmN++bm9BVICuT/e8yiLI2KFobd/TRFsE= -github.com/xtaci/smux v1.5.15/go.mod h1:OMlQbT5vcgl2gb49mFkYo6SMf+zP3rcjcwQz7ZU7IGY= github.com/xtaci/smux v1.5.24 h1:77emW9dtnOxxOQ5ltR+8BbsX1kzcOxQ5gB+aaV9hXOY= github.com/xtaci/smux v1.5.24/go.mod h1:OMlQbT5vcgl2gb49mFkYo6SMf+zP3rcjcwQz7ZU7IGY= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/zach-klippenstein/goregen v0.0.0-20160303162051-795b5e3961ea h1:CyhwejzVGvZ3Q2PSbQ4NRRYn+ZWv5eS1vlaEusT+bAI= -github.com/zach-klippenstein/goregen v0.0.0-20160303162051-795b5e3961ea/go.mod h1:eNr558nEUjP8acGw8FFjTeWvSgU1stO7FAO6eknhHe4= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= gitlab.com/yawning/bsaes.git v0.0.0-20190805113838-0a714cd429ec h1:FpfFs4EhNehiVfzQttTuxanPIT43FtkkCFypIod8LHo= gitlab.com/yawning/bsaes.git v0.0.0-20190805113838-0a714cd429ec/go.mod h1:BZ1RAoRPbCxum9Grlv5aeksu2H8BiKehBYooU2LFiOQ= -gitlab.com/yawning/edwards25519-extra.git v0.0.0-20220726154925-def713fd18e4 h1:LeXiZggivkDGgmkl7+r+m/2xj3rd+K/30/0obRKayAU= -gitlab.com/yawning/edwards25519-extra.git v0.0.0-20220726154925-def713fd18e4/go.mod h1:gvdJuZuO/tPZyhEV8K3Hmoxv/DWud5L4qEQxfYjEUTo= -gitlab.com/yawning/obfs4.git v0.0.0-20230519154740-645026c2ada4 h1:fSa6jhoahWuEV8rFHpqQlok+soFJ0b9DRys+W4wFjXg= -gitlab.com/yawning/obfs4.git v0.0.0-20230519154740-645026c2ada4/go.mod h1:DZ9ywgfitWCCPQjfhfoK/SXfESZvuMKqV8khzX3Rg5s= +gitlab.com/yawning/edwards25519-extra v0.0.0-20231005122941-2149dcafc266 h1:IvjshROr8z24+UCiOe/90cUWt3QDr8Rt+VkUjZsn+i0= +gitlab.com/yawning/edwards25519-extra v0.0.0-20231005122941-2149dcafc266/go.mod h1:K/3SQWdJL6udzwInHk1gaYaECYxMp9dDayniPq6gCSo= +gitlab.com/yawning/obfs4.git v0.0.0-20231005123604-19f5a37fe427 h1:moySt+kfvAGnbqCi7ZWA6JjwSiutDAE90fxn9cHn4hE= +gitlab.com/yawning/obfs4.git v0.0.0-20231005123604-19f5a37fe427/go.mod h1:IxDhHI4UQk/m78DYvsYsQaNhJS07b7HNzwQ8BEX2G04= gitlab.com/yawning/utls.git v0.0.12-1 h1:RL6O0MP2YI0KghuEU/uGN6+8b4183eqNWoYgx7CXD0U= gitlab.com/yawning/utls.git v0.0.12-1/go.mod h1:3ONKiSFR9Im/c3t5RKmMJTVdmZN496FNyk3mjrY1dyo= -gitlab.torproject.org/tpo/anti-censorship/geoip v0.0.0-20210928150955-7ce4b3d98d01/go.mod h1:K3LOI4H8fa6j+7E10ViHeGEQV10304FG4j94ypmKLjY= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= -go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/goptlib v1.5.0 h1:rzdY78Ox2T+VlXcxGxELF+6VyUXlZBhmRqZu5etLm+c= +gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/goptlib v1.5.0/go.mod h1:70bhd4JKW/+1HLfm+TMrgHJsUHG4coelMWwiVEJ2gAg= +gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2 v2.6.1 h1:PenLil49Ka399yxO9CfVpLFFsOLjwLCKMc/uMFTVGo4= +gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2 v2.6.1/go.mod h1:Edotm7eSJgyaVDc0aQq3W7/cNNhWyWajm4DQgTKC5yI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo= +go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= @@ -771,9 +527,6 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9E go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -golang.org/x/arch v0.0.0-20190909030613-46d78d1859ac/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -781,153 +534,106 @@ golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaE golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191219195013-becbf705a915/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220516162934-403b01795ae8/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= -golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20181106170214-d68db9428509/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= -golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/exp v0.0.0-20231005195138-3e424a577f31 h1:9k5exFQKQglLo+RoP+4zMjOFE14P6+vyR0baDAi0Rcs= +golang.org/x/exp v0.0.0-20231005195138-3e424a577f31/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/exp/typeparams v0.0.0-20230522175609-2e198f4a06a1 h1:pnP8r+W8Fm7XJ8CWtXi4S9oJmPBTrkfYN/dNbaPj6Y4= golang.org/x/exp/typeparams v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190328230028-74de082e2cca/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201201195509-5d6afe98e0b7/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211201190559-0a0e4e1bb54c/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220401154927-543a649e0bdd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos= +golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200808120158-1030fc2bf1d9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -936,24 +642,30 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= +golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -961,24 +673,19 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -988,80 +695,49 @@ golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200425043458-8463f397d07c/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200808161706-5bf02b21f123/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= -golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= -google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.53.0-dev.0.20230123225046-4075ef07c5d5 h1:qq9WB3Dez2tMAKtZTVtZsZSmTkDgPeXx+FRPt5kLEkM= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/grpc v1.58.0 h1:32JY8YpPMSR45K+c3o6b8VL73V+rR8k+DeMIr4vRH8o= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 h1:VpOs+IwYnYBaFnrNAeB8UUWtL3vEUnzSCL1nVjPhqrw= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -1071,7 +747,6 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gvisor.dev/gvisor v0.0.0-20230603040744-5c9219dedd33 h1:64QentohifmKGeTgJCHilDgfmQVuYE45fsaS9psJ3zY= gvisor.dev/gvisor v0.0.0-20230603040744-5c9219dedd33/go.mod h1:sQuqOkxbfJq/GS2uSnqHphtXclHyk/ZrAGhZBxxsq6g= -honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= @@ -1094,6 +769,3 @@ modernc.org/ql v1.4.0/go.mod h1:q4c29Bgdx+iAtxx47ODW5Xo2X0PDkjSCK9NdQl6KFxc= modernc.org/sortutil v1.1.0/go.mod h1:ZyL98OQHJgH9IEfN71VsamvJgrtRX9Dj2gX+vH86L1k= modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= modernc.org/zappy v1.0.3/go.mod h1:w/Akq8ipfols/xZJdR5IYiQNOqC80qz2mVvsEwEbkiI= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= diff --git a/pkg/cmd/apitool/main.go b/pkg/cmd/apitool/main.go index d57ef59f0..9f7f172b0 100644 --- a/pkg/cmd/apitool/main.go +++ b/pkg/cmd/apitool/main.go @@ -26,6 +26,8 @@ import ( ) func newclient() probeservices.Client { + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPTransportStdlib has QUIRKS but we + // don't actually care about those QUIRKS in this context txp := netxlite.NewHTTPTransportStdlib(log.Log) ua := fmt.Sprintf("apitool/%s ooniprobe-engine/%s", version.Version, version.Version) return probeservices.Client{ diff --git a/pkg/cmd/buildtool/android_test.go b/pkg/cmd/buildtool/android_test.go index f7736e6ba..231c4758a 100644 --- a/pkg/cmd/buildtool/android_test.go +++ b/pkg/cmd/buildtool/android_test.go @@ -702,12 +702,12 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) { expect: []buildtooltest.ExecExpectations{{ Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.1.2.tar.gz", + "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.1.3.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "openssl-3.1.2.tar.gz", + "tar", "-xf", "openssl-3.1.3.tar.gz", }, }, { Env: []string{}, @@ -763,12 +763,12 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) { }, { Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.1.2.tar.gz", + "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.1.3.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "openssl-3.1.2.tar.gz", + "tar", "-xf", "openssl-3.1.3.tar.gz", }, }, { Env: []string{}, @@ -824,12 +824,12 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) { }, { Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.1.2.tar.gz", + "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.1.3.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "openssl-3.1.2.tar.gz", + "tar", "-xf", "openssl-3.1.3.tar.gz", }, }, { Env: []string{}, @@ -885,12 +885,12 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) { }, { Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.1.2.tar.gz", + "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.1.3.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "openssl-3.1.2.tar.gz", + "tar", "-xf", "openssl-3.1.3.tar.gz", }, }, { Env: []string{}, @@ -1646,12 +1646,12 @@ func TestAndroidBuildCdepsTor(t *testing.T) { expect: []buildtooltest.ExecExpectations{{ Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.7.14.tar.gz", + "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.7.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "tor-0.4.7.14.tar.gz", + "tar", "-xf", "tor-0.4.8.7.tar.gz", }, }, { Env: []string{}, @@ -1722,12 +1722,12 @@ func TestAndroidBuildCdepsTor(t *testing.T) { }, { Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.7.14.tar.gz", + "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.7.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "tor-0.4.7.14.tar.gz", + "tar", "-xf", "tor-0.4.8.7.tar.gz", }, }, { Env: []string{}, @@ -1798,12 +1798,12 @@ func TestAndroidBuildCdepsTor(t *testing.T) { }, { Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.7.14.tar.gz", + "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.7.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "tor-0.4.7.14.tar.gz", + "tar", "-xf", "tor-0.4.8.7.tar.gz", }, }, { Env: []string{}, @@ -1874,12 +1874,12 @@ func TestAndroidBuildCdepsTor(t *testing.T) { }, { Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.7.14.tar.gz", + "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.7.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "tor-0.4.7.14.tar.gz", + "tar", "-xf", "tor-0.4.8.7.tar.gz", }, }, { Env: []string{}, diff --git a/pkg/cmd/buildtool/cdepsopenssl.go b/pkg/cmd/buildtool/cdepsopenssl.go index 426fbf398..bb9fd1937 100644 --- a/pkg/cmd/buildtool/cdepsopenssl.go +++ b/pkg/cmd/buildtool/cdepsopenssl.go @@ -26,14 +26,14 @@ func cdepsOpenSSLBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependencie restore := cdepsMustChdir(work) defer restore() - // See https://github.com/Homebrew/homebrew-core/blob/master/Formula/openssl@3.rb - cdepsMustFetch("https://www.openssl.org/source/openssl-3.1.2.tar.gz") + // See https://github.com/Homebrew/homebrew-core/blob/master/Formula/o/openssl@3.rb + cdepsMustFetch("https://www.openssl.org/source/openssl-3.1.3.tar.gz") deps.VerifySHA256( // must be mockable - "a0ce69b8b97ea6a35b96875235aa453b966ba3cba8af2de23657d8b6767d6539", - "openssl-3.1.2.tar.gz", + "f0316a2ebd89e7f2352976445458689f80302093788c466692fb2a188b2eacf6", + "openssl-3.1.3.tar.gz", ) - must.Run(log.Log, "tar", "-xf", "openssl-3.1.2.tar.gz") - _ = deps.MustChdir("openssl-3.1.2") // must be mockable + must.Run(log.Log, "tar", "-xf", "openssl-3.1.3.tar.gz") + _ = deps.MustChdir("openssl-3.1.3") // must be mockable mydir := filepath.Join(topdir, "CDEPS", "openssl") for _, patch := range cdepsMustListPatches(mydir) { diff --git a/pkg/cmd/buildtool/cdepstor.go b/pkg/cmd/buildtool/cdepstor.go index 2db445225..52a7883bd 100644 --- a/pkg/cmd/buildtool/cdepstor.go +++ b/pkg/cmd/buildtool/cdepstor.go @@ -27,13 +27,13 @@ func cdepsTorBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependencies) { defer restore() // See https://github.com/Homebrew/homebrew-core/blob/master/Formula/t/tor.rb - cdepsMustFetch("https://www.torproject.org/dist/tor-0.4.7.14.tar.gz") + cdepsMustFetch("https://www.torproject.org/dist/tor-0.4.8.7.tar.gz") deps.VerifySHA256( // must be mockable - "a5ac67f6466380fc05e8043d01c581e4e8a2b22fe09430013473e71065e65df8", - "tor-0.4.7.14.tar.gz", + "b20d2b9c74db28a00c07f090ee5b0241b2b684f3afdecccc6b8008931c557491", + "tor-0.4.8.7.tar.gz", ) - must.Run(log.Log, "tar", "-xf", "tor-0.4.7.14.tar.gz") - _ = deps.MustChdir("tor-0.4.7.14") // must be mockable + must.Run(log.Log, "tar", "-xf", "tor-0.4.8.7.tar.gz") + _ = deps.MustChdir("tor-0.4.8.7") // must be mockable mydir := filepath.Join(topdir, "CDEPS", "tor") for _, patch := range cdepsMustListPatches(mydir) { diff --git a/pkg/cmd/buildtool/linuxcdeps_test.go b/pkg/cmd/buildtool/linuxcdeps_test.go index a54e07de2..3b934d6e7 100644 --- a/pkg/cmd/buildtool/linuxcdeps_test.go +++ b/pkg/cmd/buildtool/linuxcdeps_test.go @@ -92,12 +92,12 @@ func TestLinuxCdepsBuildMain(t *testing.T) { expect: []buildtooltest.ExecExpectations{{ Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.1.2.tar.gz", + "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.1.3.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "openssl-3.1.2.tar.gz", + "tar", "-xf", "openssl-3.1.3.tar.gz", }, }, { Env: []string{}, @@ -299,12 +299,12 @@ func TestLinuxCdepsBuildMain(t *testing.T) { expect: []buildtooltest.ExecExpectations{{ Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.7.14.tar.gz", + "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.7.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "tor-0.4.7.14.tar.gz", + "tar", "-xf", "tor-0.4.8.7.tar.gz", }, }, { Env: []string{}, diff --git a/pkg/cmd/gardener/internal/dnsreport/dnsreport.go b/pkg/cmd/gardener/internal/dnsreport/dnsreport.go index 93313f045..126672fcb 100644 --- a/pkg/cmd/gardener/internal/dnsreport/dnsreport.go +++ b/pkg/cmd/gardener/internal/dnsreport/dnsreport.go @@ -8,7 +8,6 @@ import ( "encoding/json" "fmt" "net" - "net/http" "net/url" "os" "path/filepath" @@ -254,19 +253,8 @@ func (s *Subcommand) dnsLookupHost(domain string) ([]string, error) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - // create DNS transport using HTTP default client - dnsTransport := netxlite.WrapDNSTransport(&netxlite.DNSOverHTTPSTransport{ - Client: http.DefaultClient, - Decoder: &netxlite.DNSDecoderMiekg{}, - URL: s.DNSOverHTTPSServerURL, - HostOverride: "", - }) - - // create DNS resolver - dnsResolver := netxlite.WrapResolver( - log.Log, - netxlite.NewUnwrappedParallelResolver(dnsTransport), - ) + dnsResolver := netxlite.NewParallelDNSOverHTTPSResolver(log.Log, s.DNSOverHTTPSServerURL) + defer dnsResolver.CloseIdleConnections() // lookup for both A and AAAA entries return dnsResolver.LookupHost(ctx, domain) diff --git a/pkg/cmd/oohelper/oohelper.go b/pkg/cmd/oohelper/oohelper.go index d29402a43..bb55b8d2e 100644 --- a/pkg/cmd/oohelper/oohelper.go +++ b/pkg/cmd/oohelper/oohelper.go @@ -29,6 +29,7 @@ func init() { // puzzling https://github.com/ooni/probe/issues/1409 issue. const resolverURL = "https://8.8.8.8/dns-query" resolver = netxlite.NewParallelDNSOverHTTPSResolver(log.Log, resolverURL) + // TODO(https://github.com/ooni/probe/issues/2534): the NewHTTPClientWithResolver func has QUIRKS but we don't care. httpClient = netxlite.NewHTTPClientWithResolver(log.Log, resolver) } diff --git a/pkg/cmd/oohelperd/main_test.go b/pkg/cmd/oohelperd/main_test.go index 10992cd9f..f5c52622c 100644 --- a/pkg/cmd/oohelperd/main_test.go +++ b/pkg/cmd/oohelperd/main_test.go @@ -21,6 +21,10 @@ type ( ) func TestMainRunServerWorkingAsIntended(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + // let the kernel pick a random free port *apiEndpoint = "127.0.0.1:0" diff --git a/pkg/database/actions_test.go b/pkg/database/actions_test.go index f665cad95..8d20c609a 100644 --- a/pkg/database/actions_test.go +++ b/pkg/database/actions_test.go @@ -367,6 +367,10 @@ func TestPerformanceTestKeys(t *testing.T) { } func TestGetMeasurementJSON(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + tmpfile, err := ioutil.TempFile("", "dbtest") if err != nil { t.Fatal(err) diff --git a/pkg/dslx/dns.go b/pkg/dslx/dns.go index 4a426f956..951028380 100644 --- a/pkg/dslx/dns.go +++ b/pkg/dslx/dns.go @@ -9,6 +9,7 @@ import ( "sync/atomic" "time" + "github.com/ooni/probe-engine/pkg/logx" "github.com/ooni/probe-engine/pkg/measurexlite" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" @@ -146,7 +147,7 @@ func (f *dnsLookupGetaddrinfoFunc) Apply( trace := measurexlite.NewTrace(input.IDGenerator.Add(1), input.ZeroTime, input.Tags...) // start the operation logger - ol := measurexlite.NewOperationLogger( + ol := logx.NewOperationLogger( input.Logger, "[#%d] DNSLookup[getaddrinfo] %s", trace.Index, @@ -209,7 +210,7 @@ func (f *dnsLookupUDPFunc) Apply( trace := measurexlite.NewTrace(input.IDGenerator.Add(1), input.ZeroTime, input.Tags...) // start the operation logger - ol := measurexlite.NewOperationLogger( + ol := logx.NewOperationLogger( input.Logger, "[#%d] DNSLookup[%s/udp] %s", trace.Index, diff --git a/pkg/dslx/httpcore.go b/pkg/dslx/httpcore.go index fcfe9ce3d..0bba35301 100644 --- a/pkg/dslx/httpcore.go +++ b/pkg/dslx/httpcore.go @@ -13,6 +13,7 @@ import ( "sync/atomic" "time" + "github.com/ooni/probe-engine/pkg/logx" "github.com/ooni/probe-engine/pkg/measurexlite" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" @@ -160,7 +161,7 @@ func (f *httpRequestFunc) Apply( if err == nil { // start the operation logger - ol := measurexlite.NewOperationLogger( + ol := logx.NewOperationLogger( input.Logger, "[#%d] HTTPRequest %s with %s/%s host=%s", input.Trace.Index, diff --git a/pkg/dslx/httptcp.go b/pkg/dslx/httptcp.go index 8967d2d15..83b15367d 100644 --- a/pkg/dslx/httptcp.go +++ b/pkg/dslx/httptcp.go @@ -26,6 +26,9 @@ type httpTransportTCPFunc struct{} // Apply implements Func func (f *httpTransportTCPFunc) Apply( ctx context.Context, input *TCPConnection) *Maybe[*HTTPTransport] { + // TODO(https://github.com/ooni/probe/issues/2534): here we're using the QUIRKY netxlite.NewHTTPTransport + // function, but we can probably avoid using it, given that this code is + // not using tracing and does not care about those quirks. httpTransport := netxlite.NewHTTPTransport( input.Logger, netxlite.NewSingleUseDialer(input.Conn), diff --git a/pkg/dslx/httptls.go b/pkg/dslx/httptls.go index 380d916cd..47df01dd1 100644 --- a/pkg/dslx/httptls.go +++ b/pkg/dslx/httptls.go @@ -26,6 +26,9 @@ type httpTransportTLSFunc struct{} // Apply implements Func. func (f *httpTransportTLSFunc) Apply( ctx context.Context, input *TLSConnection) *Maybe[*HTTPTransport] { + // TODO(https://github.com/ooni/probe/issues/2534): here we're using the QUIRKY netxlite.NewHTTPTransport + // function, but we can probably avoid using it, given that this code is + // not using tracing and does not care about those quirks. httpTransport := netxlite.NewHTTPTransport( input.Logger, netxlite.NewNullDialer(), diff --git a/pkg/dslx/quic.go b/pkg/dslx/quic.go index 45a4379df..e56fcca26 100644 --- a/pkg/dslx/quic.go +++ b/pkg/dslx/quic.go @@ -13,6 +13,7 @@ import ( "sync/atomic" "time" + "github.com/ooni/probe-engine/pkg/logx" "github.com/ooni/probe-engine/pkg/measurexlite" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" @@ -88,7 +89,7 @@ func (f *quicHandshakeFunc) Apply( serverName := f.serverName(input) // start the operation logger - ol := measurexlite.NewOperationLogger( + ol := logx.NewOperationLogger( input.Logger, "[#%d] QUICHandshake with %s SNI=%s", trace.Index, @@ -97,10 +98,10 @@ func (f *quicHandshakeFunc) Apply( ) // setup - quicListener := netxlite.NewQUICListener() + udpListener := netxlite.NewUDPListener() quicDialer := f.dialer if quicDialer == nil { - quicDialer = trace.NewQUICDialerWithoutResolver(quicListener, input.Logger) + quicDialer = trace.NewQUICDialerWithoutResolver(udpListener, input.Logger) } config := &tls.Config{ NextProtos: []string{"h3"}, @@ -119,7 +120,7 @@ func (f *quicHandshakeFunc) Apply( var tlsState tls.ConnectionState if quicConn != nil { closerConn = &quicCloserConn{quicConn} - tlsState = quicConn.ConnectionState().TLS.ConnectionState // only quicConn can be nil + tlsState = quicConn.ConnectionState().TLS // only quicConn can be nil } // possibly track established conn for late close diff --git a/pkg/dslx/tcp.go b/pkg/dslx/tcp.go index d913c7048..8fb5971de 100644 --- a/pkg/dslx/tcp.go +++ b/pkg/dslx/tcp.go @@ -10,6 +10,7 @@ import ( "sync/atomic" "time" + "github.com/ooni/probe-engine/pkg/logx" "github.com/ooni/probe-engine/pkg/measurexlite" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" @@ -35,7 +36,7 @@ func (f *tcpConnectFunc) Apply( trace := measurexlite.NewTrace(input.IDGenerator.Add(1), input.ZeroTime, input.Tags...) // start the operation logger - ol := measurexlite.NewOperationLogger( + ol := logx.NewOperationLogger( input.Logger, "[#%d] TCPConnect %s", trace.Index, diff --git a/pkg/dslx/tls.go b/pkg/dslx/tls.go index f8ca47eeb..af5043831 100644 --- a/pkg/dslx/tls.go +++ b/pkg/dslx/tls.go @@ -12,6 +12,7 @@ import ( "sync/atomic" "time" + "github.com/ooni/probe-engine/pkg/logx" "github.com/ooni/probe-engine/pkg/measurexlite" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" @@ -99,7 +100,7 @@ func (f *tlsHandshakeFunc) Apply( nextProto := f.nextProto() // start the operation logger - ol := measurexlite.NewOperationLogger( + ol := logx.NewOperationLogger( input.Logger, "[#%d] TLSHandshake with %s SNI=%s ALPN=%v", trace.Index, @@ -123,7 +124,7 @@ func (f *tlsHandshakeFunc) Apply( defer cancel() // handshake - conn, tlsState, err := handshaker.Handshake(ctx, input.Conn, config) + conn, err := handshaker.Handshake(ctx, input.Conn, config) // possibly register established conn for late close f.Pool.MaybeTrack(conn) @@ -131,19 +132,14 @@ func (f *tlsHandshakeFunc) Apply( // stop the operation logger ol.Stop(err) - var tlsConn netxlite.TLSConn - if conn != nil { - tlsConn = conn.(netxlite.TLSConn) // guaranteed to work - } - state := &TLSConnection{ Address: input.Address, - Conn: tlsConn, // possibly nil + Conn: conn, // possibly nil Domain: input.Domain, IDGenerator: input.IDGenerator, Logger: input.Logger, Network: input.Network, - TLSState: tlsState, + TLSState: netxlite.MaybeTLSConnectionState(conn), Trace: trace, ZeroTime: input.ZeroTime, } diff --git a/pkg/dslx/tls_test.go b/pkg/dslx/tls_test.go index cedebfb1e..29fb740b2 100644 --- a/pkg/dslx/tls_test.go +++ b/pkg/dslx/tls_test.go @@ -70,17 +70,22 @@ func TestTLSHandshake(t *testing.T) { return nil }, } - tlsConn := &mocks.TLSConn{Conn: tcpConn} + tlsConn := &mocks.TLSConn{ + Conn: tcpConn, + MockConnectionState: func() tls.ConnectionState { + return tls.ConnectionState{} + }, + } eofHandshaker := &mocks.TLSHandshaker{ - MockHandshake: func(ctx context.Context, conn net.Conn, config *tls.Config) (net.Conn, tls.ConnectionState, error) { - return nil, tls.ConnectionState{}, io.EOF + MockHandshake: func(ctx context.Context, conn net.Conn, config *tls.Config) (model.TLSConn, error) { + return nil, io.EOF }, } goodHandshaker := &mocks.TLSHandshaker{ - MockHandshake: func(ctx context.Context, conn net.Conn, config *tls.Config) (net.Conn, tls.ConnectionState, error) { - return tlsConn, tls.ConnectionState{}, nil + MockHandshake: func(ctx context.Context, conn net.Conn, config *tls.Config) (model.TLSConn, error) { + return tlsConn, nil }, } diff --git a/pkg/engine/experiment.go b/pkg/engine/experiment.go index 8afab6a50..1eb8c9d8a 100644 --- a/pkg/engine/experiment.go +++ b/pkg/engine/experiment.go @@ -232,7 +232,7 @@ func (e *experiment) OpenReportContext(ctx context.Context) error { // use custom client to have proper byte accounting httpClient := &http.Client{ Transport: bytecounter.WrapHTTPTransport( - e.session.httpDefaultTransport, // proxy is OK + e.session.network.HTTPTransport(), e.byteCounter, ), } diff --git a/pkg/engine/experiment_test.go b/pkg/engine/experiment_test.go index 67bc31adf..8d441f05f 100644 --- a/pkg/engine/experiment_test.go +++ b/pkg/engine/experiment_test.go @@ -3,12 +3,12 @@ package engine import ( "testing" - "github.com/ooni/probe-engine/pkg/geolocate" + "github.com/ooni/probe-engine/pkg/enginelocate" "github.com/ooni/probe-engine/pkg/model" ) func TestExperimentHonoursSharingDefaults(t *testing.T) { - measure := func(info *geolocate.Results) *model.Measurement { + measure := func(info *enginelocate.Results) *model.Measurement { sess := &Session{location: info} builder, err := sess.NewExperimentBuilder("example") if err != nil { @@ -19,48 +19,48 @@ func TestExperimentHonoursSharingDefaults(t *testing.T) { } type spec struct { name string - locationInfo *geolocate.Results + locationInfo *enginelocate.Results expect func(*model.Measurement) bool } allspecs := []spec{{ name: "probeIP", - locationInfo: &geolocate.Results{ProbeIP: "8.8.8.8"}, + locationInfo: &enginelocate.Results{ProbeIP: "8.8.8.8"}, expect: func(m *model.Measurement) bool { return m.ProbeIP == model.DefaultProbeIP }, }, { name: "probeASN", - locationInfo: &geolocate.Results{ASN: 30722}, + locationInfo: &enginelocate.Results{ASN: 30722}, expect: func(m *model.Measurement) bool { return m.ProbeASN == "AS30722" }, }, { name: "probeCC", - locationInfo: &geolocate.Results{CountryCode: "IT"}, + locationInfo: &enginelocate.Results{CountryCode: "IT"}, expect: func(m *model.Measurement) bool { return m.ProbeCC == "IT" }, }, { name: "probeNetworkName", - locationInfo: &geolocate.Results{NetworkName: "Vodafone Italia"}, + locationInfo: &enginelocate.Results{NetworkName: "Vodafone Italia"}, expect: func(m *model.Measurement) bool { return m.ProbeNetworkName == "Vodafone Italia" }, }, { name: "resolverIP", - locationInfo: &geolocate.Results{ResolverIP: "9.9.9.9"}, + locationInfo: &enginelocate.Results{ResolverIP: "9.9.9.9"}, expect: func(m *model.Measurement) bool { return m.ResolverIP == "9.9.9.9" }, }, { name: "resolverASN", - locationInfo: &geolocate.Results{ResolverASN: 44}, + locationInfo: &enginelocate.Results{ResolverASN: 44}, expect: func(m *model.Measurement) bool { return m.ResolverASN == "AS44" }, }, { name: "resolverNetworkName", - locationInfo: &geolocate.Results{ResolverNetworkName: "Google LLC"}, + locationInfo: &enginelocate.Results{ResolverNetworkName: "Google LLC"}, expect: func(m *model.Measurement) bool { return m.ResolverNetworkName == "Google LLC" }, diff --git a/pkg/engine/session.go b/pkg/engine/session.go index 27e522a26..cd95689cb 100644 --- a/pkg/engine/session.go +++ b/pkg/engine/session.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io/ioutil" - "net/http" "net/url" "os" "sync" @@ -13,15 +12,15 @@ import ( "github.com/ooni/probe-engine/pkg/bytecounter" "github.com/ooni/probe-engine/pkg/checkincache" - "github.com/ooni/probe-engine/pkg/geolocate" + "github.com/ooni/probe-engine/pkg/enginelocate" + "github.com/ooni/probe-engine/pkg/enginenetx" + "github.com/ooni/probe-engine/pkg/engineresolver" "github.com/ooni/probe-engine/pkg/kvstore" "github.com/ooni/probe-engine/pkg/model" - "github.com/ooni/probe-engine/pkg/netxlite" "github.com/ooni/probe-engine/pkg/platform" "github.com/ooni/probe-engine/pkg/probeservices" "github.com/ooni/probe-engine/pkg/registry" "github.com/ooni/probe-engine/pkg/runtimex" - "github.com/ooni/probe-engine/pkg/sessionresolver" "github.com/ooni/probe-engine/pkg/tunnel" "github.com/ooni/probe-engine/pkg/version" ) @@ -58,13 +57,13 @@ type Session struct { availableProbeServices []model.OOAPIService availableTestHelpers map[string][]model.OOAPIService byteCounter *bytecounter.Counter - httpDefaultTransport model.HTTPTransport + network *enginenetx.Network kvStore model.KeyValueStore - location *geolocate.Results + location *enginelocate.Results logger model.Logger proxyURL *url.URL queryProbeServicesCount *atomic.Int64 - resolver *sessionresolver.Resolver + resolver *engineresolver.Resolver selectedProbeServiceHook func(*model.OOAPIService) selectedProbeService *model.OOAPIService softwareName string @@ -79,7 +78,7 @@ type Session struct { // testLookupLocationContext is a an optional hook for testing // allowing us to mock LookupLocationContext. - testLookupLocationContext func(ctx context.Context) (*geolocate.Results, error) + testLookupLocationContext func(ctx context.Context) (*enginelocate.Results, error) // testMaybeLookupBackendsContext is an optional hook for testing // allowing us to mock MaybeLookupBackendsContext. @@ -132,8 +131,8 @@ type sessionProbeServicesClientForCheckIn interface { // // 5. Create a compound resolver for the session that will attempt // to use a bunch of DoT/DoH servers before falling back to the system -// resolver if nothing else works (see the sessionresolver pkg). This -// sessionresolver will be using the configured proxy, if any. +// resolver if nothing else works (see the engineresolver pkg). This +// engineresolver will be using the configured proxy, if any. // // 6. Create the default HTTP transport that we should be using when // we communicate with the OONI backends. This transport will be @@ -208,17 +207,19 @@ func NewSession(ctx context.Context, config SessionConfig) (*Session, error) { } } sess.proxyURL = proxyURL - sess.resolver = &sessionresolver.Resolver{ + sess.resolver = &engineresolver.Resolver{ ByteCounter: sess.byteCounter, KVStore: config.KVStore, Logger: sess.logger, ProxyURL: proxyURL, } - txp := netxlite.NewHTTPTransportWithLoggerResolverAndOptionalProxyURL( - sess.logger, sess.resolver, sess.proxyURL, + sess.network = enginenetx.NewNetwork( + sess.byteCounter, + config.KVStore, + sess.logger, + proxyURL, + sess.resolver, ) - txp = bytecounter.WrapHTTPTransport(txp, sess.byteCounter) - sess.httpDefaultTransport = txp return sess, nil } @@ -341,7 +342,9 @@ func (s *Session) Close() error { // doClose implements Close. This function is called just once. func (s *Session) doClose() { - s.httpDefaultTransport.CloseIdleConnections() + // make sure we close open connections and persist stats to the key-value store + s.network.Close() + s.resolver.CloseIdleConnections() if s.tunnel != nil { s.tunnel.Stop() @@ -360,7 +363,7 @@ func (s *Session) GetTestHelpersByName(name string) ([]model.OOAPIService, bool) // DefaultHTTPClient returns the session's default HTTP client. func (s *Session) DefaultHTTPClient() model.HTTPClient { - return &http.Client{Transport: s.httpDefaultTransport} + return s.network.NewHTTPClient() } // FetchTorTargets fetches tor targets from the API. @@ -676,8 +679,8 @@ func (s *Session) MaybeLookupBackendsContext(ctx context.Context) error { // LookupLocationContext performs a location lookup. If you want memoisation // of the results, you should use MaybeLookupLocationContext. -func (s *Session) LookupLocationContext(ctx context.Context) (*geolocate.Results, error) { - task := geolocate.NewTask(geolocate.Config{ +func (s *Session) LookupLocationContext(ctx context.Context) (*enginelocate.Results, error) { + task := enginelocate.NewTask(enginelocate.Config{ Logger: s.Logger(), Resolver: s.resolver, UserAgent: s.UserAgent(), @@ -687,7 +690,7 @@ func (s *Session) LookupLocationContext(ctx context.Context) (*geolocate.Results // lookupLocationContext calls testLookupLocationContext if set and // otherwise calls LookupLocationContext. -func (s *Session) lookupLocationContext(ctx context.Context) (*geolocate.Results, error) { +func (s *Session) lookupLocationContext(ctx context.Context) (*enginelocate.Results, error) { if s.testLookupLocationContext != nil { return s.testLookupLocationContext(ctx) } diff --git a/pkg/engine/session_integration_test.go b/pkg/engine/session_integration_test.go index 30037a298..642b25acd 100644 --- a/pkg/engine/session_integration_test.go +++ b/pkg/engine/session_integration_test.go @@ -492,6 +492,10 @@ func TestNewOrchestraClientProbeServicesNewClientFailure(t *testing.T) { } func TestSessionNewSubmitterReturnsNonNilSubmitter(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + sess := newSessionForTesting(t) subm, err := sess.NewSubmitter(context.Background()) if err != nil { diff --git a/pkg/engine/session_internal_test.go b/pkg/engine/session_internal_test.go index b4a97b4ed..9241f1e68 100644 --- a/pkg/engine/session_internal_test.go +++ b/pkg/engine/session_internal_test.go @@ -11,9 +11,9 @@ import ( "github.com/apex/log" "github.com/google/go-cmp/cmp" "github.com/ooni/probe-engine/pkg/checkincache" + "github.com/ooni/probe-engine/pkg/enginelocate" "github.com/ooni/probe-engine/pkg/experiment/webconnectivity" "github.com/ooni/probe-engine/pkg/experiment/webconnectivitylte" - "github.com/ooni/probe-engine/pkg/geolocate" "github.com/ooni/probe-engine/pkg/kvstore" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/registry" @@ -85,7 +85,7 @@ func TestSessionCheckInSuccessful(t *testing.T) { }, } s := &Session{ - location: &geolocate.Results{ + location: &enginelocate.Results{ ASN: 137, CountryCode: "IT", }, @@ -136,7 +136,7 @@ func TestSessionCheckInNetworkError(t *testing.T) { Error: expect, } s := &Session{ - location: &geolocate.Results{ + location: &enginelocate.Results{ ASN: 137, CountryCode: "IT", }, @@ -178,7 +178,7 @@ func TestSessionCheckInCannotLookupLocation(t *testing.T) { func TestSessionCheckInCannotCreateProbeServicesClient(t *testing.T) { errMocked := errors.New("mocked error") s := &Session{ - location: &geolocate.Results{ + location: &enginelocate.Results{ ASN: 137, CountryCode: "IT", }, @@ -225,6 +225,10 @@ func TestNewProbeServicesClientForCheckIn(t *testing.T) { } func TestSessionNewSubmitterWithCancelledContext(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + sess := newSessionForTesting(t) ctx, cancel := context.WithCancel(context.Background()) cancel() // fail immediately @@ -240,7 +244,7 @@ func TestSessionNewSubmitterWithCancelledContext(t *testing.T) { func TestSessionMaybeLookupLocationContextLookupLocationContextFailure(t *testing.T) { errMocked := errors.New("mocked error") sess := newSessionForTestingNoLookups(t) - sess.testLookupLocationContext = func(ctx context.Context) (*geolocate.Results, error) { + sess.testLookupLocationContext = func(ctx context.Context) (*enginelocate.Results, error) { return nil, errMocked } err := sess.MaybeLookupLocationContext(context.Background()) diff --git a/pkg/geolocate/cloudflare.go b/pkg/enginelocate/cloudflare.go similarity index 97% rename from pkg/geolocate/cloudflare.go rename to pkg/enginelocate/cloudflare.go index f91330eaa..8cf5fa957 100644 --- a/pkg/geolocate/cloudflare.go +++ b/pkg/enginelocate/cloudflare.go @@ -1,4 +1,4 @@ -package geolocate +package enginelocate import ( "context" diff --git a/pkg/geolocate/cloudflare_test.go b/pkg/enginelocate/cloudflare_test.go similarity index 85% rename from pkg/geolocate/cloudflare_test.go rename to pkg/enginelocate/cloudflare_test.go index e259ee227..01eb091f1 100644 --- a/pkg/geolocate/cloudflare_test.go +++ b/pkg/enginelocate/cloudflare_test.go @@ -1,4 +1,4 @@ -package geolocate +package enginelocate import ( "context" @@ -12,6 +12,10 @@ import ( ) func TestIPLookupWorksUsingcloudlflare(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + ip, err := cloudflareIPLookup( context.Background(), http.DefaultClient, diff --git a/pkg/geolocate/fake_test.go b/pkg/enginelocate/fake_test.go similarity index 96% rename from pkg/geolocate/fake_test.go rename to pkg/enginelocate/fake_test.go index 9c89dee46..a15382010 100644 --- a/pkg/geolocate/fake_test.go +++ b/pkg/enginelocate/fake_test.go @@ -1,4 +1,4 @@ -package geolocate +package enginelocate import ( "net/http" diff --git a/pkg/geolocate/geolocate.go b/pkg/enginelocate/geolocate.go similarity index 97% rename from pkg/geolocate/geolocate.go rename to pkg/enginelocate/geolocate.go index 1175aa489..ad3d82af1 100644 --- a/pkg/geolocate/geolocate.go +++ b/pkg/enginelocate/geolocate.go @@ -1,5 +1,5 @@ -// Package geolocate implements IP lookup, resolver lookup, and geolocation. -package geolocate +// Package enginelocate implements IP lookup, resolver lookup, and geolocation. +package enginelocate import ( "context" diff --git a/pkg/geolocate/geolocate_test.go b/pkg/enginelocate/geolocate_test.go similarity index 98% rename from pkg/geolocate/geolocate_test.go rename to pkg/enginelocate/geolocate_test.go index 08de826a9..311c5b1e4 100644 --- a/pkg/geolocate/geolocate_test.go +++ b/pkg/enginelocate/geolocate_test.go @@ -1,4 +1,4 @@ -package geolocate +package enginelocate import ( "context" @@ -266,6 +266,10 @@ func TestLocationLookupSuccessWithResolverLookup(t *testing.T) { } func TestSmoke(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + config := Config{} task := NewTask(config) result, err := task.Run(context.Background()) diff --git a/pkg/geolocate/invalid_test.go b/pkg/enginelocate/invalid_test.go similarity index 92% rename from pkg/geolocate/invalid_test.go rename to pkg/enginelocate/invalid_test.go index 474e21d73..241a28d5a 100644 --- a/pkg/geolocate/invalid_test.go +++ b/pkg/enginelocate/invalid_test.go @@ -1,4 +1,4 @@ -package geolocate +package enginelocate import ( "context" diff --git a/pkg/geolocate/iplookup.go b/pkg/enginelocate/iplookup.go similarity index 78% rename from pkg/geolocate/iplookup.go rename to pkg/enginelocate/iplookup.go index d1a28bd27..8d90e3a88 100644 --- a/pkg/geolocate/iplookup.go +++ b/pkg/enginelocate/iplookup.go @@ -1,4 +1,4 @@ -package geolocate +package enginelocate import ( "context" @@ -77,16 +77,26 @@ func makeSlice() []method { return ret } +func contextForIPLookupWithTimeout(ctx context.Context) (context.Context, context.CancelFunc) { + // TODO(https://github.com/ooni/probe/issues/2551): we must enforce a timeout this + // large to ensure we give all resolvers a chance to run. We set this value as part of + // an hotfix. The above mentioned issue explains how to improve the situation and + // avoid the need of setting such large timeouts here. + const timeout = 45 * time.Second + return context.WithTimeout(ctx, timeout) +} + func (c ipLookupClient) doWithCustomFunc( ctx context.Context, fn lookupFunc, ) (string, error) { - // Reliability fix: let these mechanisms timeout earlier. - const timeout = 7 * time.Second - ctx, cancel := context.WithTimeout(ctx, timeout) + ctx, cancel := contextForIPLookupWithTimeout(ctx) defer cancel() + // Implementation note: we MUST use an HTTP client that we're // sure IS NOT using any proxy. To this end, we construct a // client ourself that we know is not proxied. + // TODO(https://github.com/ooni/probe/issues/2534): the NewHTTPTransportWithResolver has QUIRKS but + // we don't care about them in this context txp := netxlite.NewHTTPTransportWithResolver(c.Logger, c.Resolver) clnt := &http.Client{Transport: txp} defer clnt.CloseIdleConnections() diff --git a/pkg/geolocate/iplookup_test.go b/pkg/enginelocate/iplookup_test.go similarity index 70% rename from pkg/geolocate/iplookup_test.go rename to pkg/enginelocate/iplookup_test.go index 62cc1fe38..05348e103 100644 --- a/pkg/geolocate/iplookup_test.go +++ b/pkg/enginelocate/iplookup_test.go @@ -1,10 +1,11 @@ -package geolocate +package enginelocate import ( "context" "errors" "net" "testing" + "time" "github.com/apex/log" "github.com/ooni/probe-engine/pkg/model" @@ -12,6 +13,10 @@ import ( ) func TestIPLookupGood(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + ip, err := (ipLookupClient{ Logger: log.Log, Resolver: netxlite.NewStdlibResolver(model.DiscardLogger), @@ -55,3 +60,19 @@ func TestIPLookupInvalidIP(t *testing.T) { t.Fatal("expected the default IP here") } } + +func TestContextForIPLookupWithTimeout(t *testing.T) { + now := time.Now() + ctx, cancel := contextForIPLookupWithTimeout(context.Background()) + defer cancel() + deadline, okay := ctx.Deadline() + if !okay { + t.Fatal("the context does not have a deadline") + } + delta := deadline.Sub(now) + // Note: super conservative check. Assume it may take up to five seconds + // for the code to create a context, which is totally unrealistic. + if delta < 40*time.Second { + t.Fatal("the deadline is too short") + } +} diff --git a/pkg/geolocate/mmdblookup.go b/pkg/enginelocate/mmdblookup.go similarity index 92% rename from pkg/geolocate/mmdblookup.go rename to pkg/enginelocate/mmdblookup.go index fe38c8bc8..3b87c6c7f 100644 --- a/pkg/geolocate/mmdblookup.go +++ b/pkg/enginelocate/mmdblookup.go @@ -1,4 +1,4 @@ -package geolocate +package enginelocate import ( "github.com/ooni/probe-engine/pkg/geoipx" diff --git a/pkg/geolocate/resolverlookup.go b/pkg/enginelocate/resolverlookup.go similarity index 97% rename from pkg/geolocate/resolverlookup.go rename to pkg/enginelocate/resolverlookup.go index db33b6639..0cf513046 100644 --- a/pkg/geolocate/resolverlookup.go +++ b/pkg/enginelocate/resolverlookup.go @@ -1,4 +1,4 @@ -package geolocate +package enginelocate import ( "context" diff --git a/pkg/geolocate/resolverlookup_test.go b/pkg/enginelocate/resolverlookup_test.go similarity index 90% rename from pkg/geolocate/resolverlookup_test.go rename to pkg/enginelocate/resolverlookup_test.go index 22be2d747..725a5d686 100644 --- a/pkg/geolocate/resolverlookup_test.go +++ b/pkg/enginelocate/resolverlookup_test.go @@ -1,4 +1,4 @@ -package geolocate +package enginelocate import ( "context" @@ -9,6 +9,10 @@ import ( ) func TestLookupResolverIPSuccess(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + rlc := resolverLookupClient{ Logger: model.DiscardLogger, } diff --git a/pkg/geolocate/stun.go b/pkg/enginelocate/stun.go similarity index 99% rename from pkg/geolocate/stun.go rename to pkg/enginelocate/stun.go index ce247a557..ef17985f6 100644 --- a/pkg/geolocate/stun.go +++ b/pkg/enginelocate/stun.go @@ -1,4 +1,4 @@ -package geolocate +package enginelocate import ( "context" @@ -29,6 +29,7 @@ func stunNewClient(conn net.Conn) (stunClient, error) { func stunIPLookup(ctx context.Context, config stunConfig) (string, error) { config.Logger.Debugf("STUNIPLookup: start using %s", config.Endpoint) + ip, err := func() (string, error) { dialer := config.Dialer if dialer == nil { @@ -74,6 +75,7 @@ func stunIPLookup(ctx context.Context, config stunConfig) (string, error) { return model.DefaultProbeIP, ctx.Err() } }() + if err != nil { config.Logger.Debugf("STUNIPLookup: failure using %s: %+v", config.Endpoint, err) return model.DefaultProbeIP, err diff --git a/pkg/geolocate/stun_test.go b/pkg/enginelocate/stun_test.go similarity index 96% rename from pkg/geolocate/stun_test.go rename to pkg/enginelocate/stun_test.go index d9778966b..bd8037082 100644 --- a/pkg/geolocate/stun_test.go +++ b/pkg/enginelocate/stun_test.go @@ -1,4 +1,4 @@ -package geolocate +package enginelocate import ( "context" @@ -147,6 +147,10 @@ func TestSTUNIPLookupCannotDecodeMessage(t *testing.T) { } func TestIPLookupWorksUsingSTUNEkiga(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + ip, err := stunEkigaIPLookup( context.Background(), http.DefaultClient, @@ -163,6 +167,10 @@ func TestIPLookupWorksUsingSTUNEkiga(t *testing.T) { } func TestIPLookupWorksUsingSTUNGoogle(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + ip, err := stunGoogleIPLookup( context.Background(), http.DefaultClient, diff --git a/pkg/geolocate/ubuntu.go b/pkg/enginelocate/ubuntu.go similarity index 97% rename from pkg/geolocate/ubuntu.go rename to pkg/enginelocate/ubuntu.go index f2eab8686..2071932a3 100644 --- a/pkg/geolocate/ubuntu.go +++ b/pkg/enginelocate/ubuntu.go @@ -1,4 +1,4 @@ -package geolocate +package enginelocate import ( "context" diff --git a/pkg/geolocate/ubuntu_test.go b/pkg/enginelocate/ubuntu_test.go similarity index 92% rename from pkg/geolocate/ubuntu_test.go rename to pkg/enginelocate/ubuntu_test.go index cd3d0176f..e224383e9 100644 --- a/pkg/geolocate/ubuntu_test.go +++ b/pkg/enginelocate/ubuntu_test.go @@ -1,4 +1,4 @@ -package geolocate +package enginelocate import ( "context" @@ -35,6 +35,10 @@ func TestUbuntuParseError(t *testing.T) { } func TestIPLookupWorksUsingUbuntu(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + ip, err := ubuntuIPLookup( context.Background(), http.DefaultClient, diff --git a/pkg/enginenetx/bridgespolicy.go b/pkg/enginenetx/bridgespolicy.go new file mode 100644 index 000000000..0b282630d --- /dev/null +++ b/pkg/enginenetx/bridgespolicy.go @@ -0,0 +1,307 @@ +package enginenetx + +// +// bridges policy - a policy where we treat some IP addresses as special for +// some domains, bypassing DNS lookups and using custom SNIs +// + +import ( + "context" + "math/rand" + "time" +) + +// bridgesPolicy is a policy where we use bridges for communicating +// with the OONI backend, i.e., api.ooni.io. +// +// A bridge is an IP address that can route traffic from and to +// the OONI backend and accepts any SNI. +// +// The zero value is invalid; please, init MANDATORY fields. +type bridgesPolicy struct { + // Fallback is the MANDATORY fallback policy. + Fallback httpsDialerPolicy +} + +var _ httpsDialerPolicy = &bridgesPolicy{} + +// LookupTactics implements httpsDialerPolicy. +func (p *bridgesPolicy) LookupTactics(ctx context.Context, domain, port string) <-chan *httpsDialerTactic { + out := make(chan *httpsDialerTactic) + + go func() { + defer close(out) // tell the parent when we're done + index := 0 + + // emit bridges related tactics first which are empty if there are + // no bridges for the givend domain and port + for tx := range p.bridgesTacticsForDomain(domain, port) { + tx.InitialDelay = happyEyeballsDelay(index) + index += 1 + out <- tx + } + + // now fallback to get more tactics (typically here the fallback + // uses the DNS and obtains some extra tactics) + // + // we wrap whatever the underlying policy returns us with some + // extra logic for better communicating with test helpers + for tx := range p.maybeRewriteTestHelpersTactics(p.Fallback.LookupTactics(ctx, domain, port)) { + tx.InitialDelay = happyEyeballsDelay(index) + index += 1 + out <- tx + } + }() + + return out +} + +var bridgesPolicyTestHelpersDomains = []string{ + "0.th.ooni.org", + "1.th.ooni.org", + "2.th.ooni.org", + "3.th.ooni.org", + "d33d1gs9kpq1c5.cloudfront.net", +} + +// TODO(bassosimone): this would be slices.Contains when we'll use go1.21 +func bridgesPolicySlicesContains(slice []string, value string) bool { + for _, entry := range slice { + if value == entry { + return true + } + } + return false +} + +func (p *bridgesPolicy) maybeRewriteTestHelpersTactics(input <-chan *httpsDialerTactic) <-chan *httpsDialerTactic { + out := make(chan *httpsDialerTactic) + + go func() { + defer close(out) // tell the parent when we're done + + for tactic := range input { + // When we're not connecting to a TH, pass the policy down the chain unmodified + if !bridgesPolicySlicesContains(bridgesPolicyTestHelpersDomains, tactic.VerifyHostname) { + out <- tactic + continue + } + + // This is the case where we're connecting to a test helper. Let's try + // to produce policies hiding the SNI to censoring middleboxes. + for _, sni := range p.bridgesDomainsInRandomOrder() { + out <- &httpsDialerTactic{ + Address: tactic.Address, + InitialDelay: 0, + Port: tactic.Port, + SNI: sni, + VerifyHostname: tactic.VerifyHostname, + } + } + } + }() + + return out +} + +func (p *bridgesPolicy) bridgesTacticsForDomain(domain, port string) <-chan *httpsDialerTactic { + out := make(chan *httpsDialerTactic) + + go func() { + defer close(out) // tell the parent when we're done + + // we currently only have bridges for api.ooni.io + if domain != "api.ooni.io" { + return + } + + for _, ipAddr := range p.bridgesAddrs() { + for _, sni := range p.bridgesDomainsInRandomOrder() { + out <- &httpsDialerTactic{ + Address: ipAddr, + InitialDelay: 0, + Port: port, + SNI: sni, + VerifyHostname: domain, + } + } + } + }() + + return out +} + +func (p *bridgesPolicy) bridgesDomainsInRandomOrder() (out []string) { + out = p.bridgesDomains() + r := rand.New(rand.NewSource(time.Now().UnixNano())) + r.Shuffle(len(out), func(i, j int) { + out[i], out[j] = out[j], out[i] + }) + return +} + +func (p *bridgesPolicy) bridgesAddrs() (out []string) { + return append( + out, + "162.55.247.208", + ) +} + +func (p *bridgesPolicy) bridgesDomains() (out []string) { + // See https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/-/issues/40273 + return append( + out, + "adtm.spreadshirts.net", + "alb.reddit.com", + "a.loveholidays.com", + "api.giphy.com", + "api.nextgen.guardianapps.co.uk", + "api.trademe.co.nz", + "app.launchdarkly.com", + "apps.voxmedia.com", + "assets0.uswitch.com", + "assets.boots.com", + "assets.dunelm.com", + "assets.guim.co.uk", + "assets.hearstapps.com", + "assets-jpcust.jwpsrv.com", + "assets.nymag.com", + "assets.thecut.com", + "atreseries.atresmedia.com", + "cdn.bfldr.com", + "cdn.concert.io", + "cdn.contentful.com", + "cdn.ketchjs.com", + "cdn.laredoute.com", + "cdn.polyfill.io", + "cdn.speedcurve.com", + "cdn.sstatic.net", + "cdn.taboola.com", + "client.grubstreet.com", + "client.nymag.com", + "client-registry.mutinycdn.com", + "client.thecut.com", + "client.thestrategist.co.uk", + "client.vulture.com", + "compote.slate.com", + "concertads-configs.vox-cdn.com", + "contributions.guardianapis.com", + "display.bidder.taboola.com", + "edgemesh.webflow.io", + "embed.api.video", + "epsf.ticketmaster.com", + "fastly.com", + "fastly.jsdelivr.net", + "fast.ssqt.io", + "fast.wistia.com", + "fdyn.pubwise.io", + "fonts.nymag.com", + "foursquare.com", + "frend-assets.freetls.fastly.net", + "f.vimeocdn.com", + "github.githubassets.com", + "global.ketchcdn.com", + "helpersng.taboola.com", + "hips.hearstapps.com", + "i.guimcode.co.uk", + "i.guim.co.uk", + "i.insider.com", + "images.mutinycdn.com", + "images.taboola.com", + "interactive.guim.co.uk", + "i.vimeocdn.com", + "js-agent.newrelic.com", + "js.sentry-cdn.com", + "linktr.ee", + "login.nine.com.au", + "lux.speedcurve.com", + "martech.condenastdigital.com", + "media0.giphy.com", + "media1.giphy.com", + "media2.giphy.com", + "media3.giphy.com", + "media.giphy.com", + "media.newyorker.com", + "media.wired.com", + "mparticle.weather.com", + "mv.outbrain.com", + "newrelic.com", + "next.ticketmaster.com", + "nm.realtyninja.com", + "pingback.giphy.com", + "pips.taboola.com", + "pitchfork.com", + "pixel.condenastdigital.com", + "player.ex.co", + "pm-widget.taboola.com", + "polyfill.io", + "prd.jwpltx.com", + "pyxis.nymag.com", + "rapid-cdn.yottaa.com", + "rtd-tm.everesttech.net", + "s1.ticketm.net", + "s3-media0.fl.yelpcdn.com", + "slate.com", + "sourcepoint.theguardian.com", + "ssl.p.jwpcdn.com", + "sstc.dunelm.com", + "static.ads-twitter.com", + "static.filestackapi.com", + "static.klaviyo.com", + "static.theguardian.com", + "static-tracking.klaviyo.com", + "s.w-x.co", + "trademe.tmcdn.co.nz", + "trc.taboola.com", + "t.seenthis.se", + "uploads.guim.co.uk", + "video.seenthis.se", + "vidstat.taboola.com", + "vod.api.video", + "vulcan.condenastdigital.com", + "widget.perfectmarket.com", + "www.allure.com", + "www.amazeelabs.com", + "www.architecturaldigest.com", + "www.blackpepper.co.nz", + "www.bonappetit.com", + "www.cntraveler.com", + "www.drupal.org", + "www.dunelm.com", + "www.epicurious.com", + "www.fastly.com", + "www.filestack.com", + "www.giphy.com", + "www.glamour.com", + "www.gq.com", + "www.insider.com", + "www.jimdo.com", + "www.loveholidays.com", + "www.madeiramadeira.com.br", + "www.newrelic.com", + "www.newyorker.com", + "www.pronovias.com", + "www.redditstatic.com", + "www.rvu.co.uk", + "www.self.com", + "www.shazam.com", + "www.shondaland.com", + "www.split.io", + "www.spreadgroup.com", + "www.spreadshirt.com", + "www.taboola.com", + "www.teenvogue.com", + "www.thecut.com", + "www.theguardian.com", + "www.them.us", + "www.ticketmaster.com", + "www.trademe.co.nz", + "www.vanityfair.com", + "www.vogue.com", + "www.wikihow.com", + "www.wired.com", + "www.yelp.com", + "x.giphy.com", + "yelp.com", + ) +} diff --git a/pkg/enginenetx/bridgespolicy_test.go b/pkg/enginenetx/bridgespolicy_test.go new file mode 100644 index 000000000..ebf6e708b --- /dev/null +++ b/pkg/enginenetx/bridgespolicy_test.go @@ -0,0 +1,164 @@ +package enginenetx + +import ( + "context" + "errors" + "testing" + + "github.com/ooni/probe-engine/pkg/mocks" + "github.com/ooni/probe-engine/pkg/model" +) + +func TestBeaconsPolicy(t *testing.T) { + t.Run("for domains for which we don't have bridges and DNS failure", func(t *testing.T) { + expected := errors.New("mocked error") + p := &bridgesPolicy{ + Fallback: &dnsPolicy{ + Logger: model.DiscardLogger, + Resolver: &mocks.Resolver{ + MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { + return nil, expected + }, + }, + }, + } + + ctx := context.Background() + tactics := p.LookupTactics(ctx, "www.example.com", "443") + + var count int + for range tactics { + count++ + } + + if count != 0 { + t.Fatal("expected to see zero tactics") + } + }) + + t.Run("for domains for which we don't have bridges and DNS success", func(t *testing.T) { + p := &bridgesPolicy{ + Fallback: &dnsPolicy{ + Logger: model.DiscardLogger, + Resolver: &mocks.Resolver{ + MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { + return []string{"93.184.216.34"}, nil + }, + }, + }, + } + + ctx := context.Background() + tactics := p.LookupTactics(ctx, "www.example.com", "443") + + var count int + for tactic := range tactics { + count++ + + if tactic.Port != "443" { + t.Fatal("the port should always be 443") + } + if tactic.Address != "93.184.216.34" { + t.Fatal("the host should always be 93.184.216.34") + } + + if tactic.SNI != "www.example.com" { + t.Fatal("the SNI field should always be like `www.example.com`") + } + + if tactic.VerifyHostname != "www.example.com" { + t.Fatal("the VerifyHostname field should always be like `www.example.com`") + } + } + + if count != 1 { + t.Fatal("expected to see one tactic") + } + }) + + t.Run("for the api.ooni.io domain", func(t *testing.T) { + expected := errors.New("mocked error") + p := &bridgesPolicy{ + Fallback: &dnsPolicy{ + Logger: model.DiscardLogger, + Resolver: &mocks.Resolver{ + MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { + return nil, expected + }, + }, + }, + } + + ctx := context.Background() + tactics := p.LookupTactics(ctx, "api.ooni.io", "443") + + var count int + for tactic := range tactics { + count++ + + if tactic.Port != "443" { + t.Fatal("the port should always be 443") + } + if tactic.Address != "162.55.247.208" { + t.Fatal("the host should always be 162.55.247.208") + } + + if tactic.SNI == "api.ooni.io" { + t.Fatal("we should not see the `api.ooni.io` SNI on the wire") + } + + if tactic.VerifyHostname != "api.ooni.io" { + t.Fatal("the VerifyHostname field should always be like `api.ooni.io`") + } + } + + if count <= 0 { + t.Fatal("expected to see at least one tactic") + } + }) + + t.Run("for test helper domains", func(t *testing.T) { + for _, domain := range bridgesPolicyTestHelpersDomains { + t.Run(domain, func(t *testing.T) { + expectedAddrs := []string{"164.92.180.7"} + + p := &bridgesPolicy{ + Fallback: &dnsPolicy{ + Logger: model.DiscardLogger, + Resolver: &mocks.Resolver{ + MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { + return expectedAddrs, nil + }, + }, + }, + } + + ctx := context.Background() + index := 0 + for tactics := range p.LookupTactics(ctx, domain, "443") { + + if tactics.Address != "164.92.180.7" { + t.Fatal("unexpected .Address") + } + + if tactics.InitialDelay != happyEyeballsDelay(index) { + t.Fatal("unexpected .InitialDelay") + } + index++ + + if tactics.Port != "443" { + t.Fatal("unexpected .Port") + } + + if tactics.SNI == domain { + t.Fatal("unexpected .Domain") + } + + if tactics.VerifyHostname != domain { + t.Fatal("unexpected .VerifyHostname") + } + } + }) + } + }) +} diff --git a/pkg/enginenetx/dnspolicy.go b/pkg/enginenetx/dnspolicy.go new file mode 100644 index 000000000..9fd97fb8f --- /dev/null +++ b/pkg/enginenetx/dnspolicy.go @@ -0,0 +1,72 @@ +package enginenetx + +// +// HTTPS dialing policy where we generate tactics in the usual way +// by using a DNS resolver and using SNI == VerifyHostname +// + +import ( + "context" + + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/netxlite" +) + +// dnsPolicy is the default TLS dialing policy where we use the +// given resolver and the domain as the SNI. +// +// The zero value is invalid; please, init all MANDATORY fields. +// +// This policy uses an Happy-Eyeballs-like algorithm. +type dnsPolicy struct { + // Logger is the MANDATORY logger. + Logger model.Logger + + // Resolver is the MANDATORY resolver. + Resolver model.Resolver +} + +var _ httpsDialerPolicy = &dnsPolicy{} + +// LookupTactics implements httpsDialerPolicy. +func (p *dnsPolicy) LookupTactics( + ctx context.Context, domain, port string) <-chan *httpsDialerTactic { + out := make(chan *httpsDialerTactic) + + go func() { + // make sure we close the output channel when done + // so the reader knows that we're done + defer close(out) + + // Do not even start the DNS lookup if the context has already been canceled, which + // happens if some policy running before us had successfully connected + if err := ctx.Err(); err != nil { + p.Logger.Debugf("dnsPolicy: LookupTactics: %s", err.Error()) + return + } + + // See https://github.com/ooni/probe-cli/pull/1295#issuecomment-1731243994 for context + // on why here we MUST make sure we short-circuit IP addresses. + resoWithShortCircuit := &netxlite.ResolverShortCircuitIPAddr{Resolver: p.Resolver} + + addrs, err := resoWithShortCircuit.LookupHost(ctx, domain) + if err != nil { + p.Logger.Warnf("resoWithShortCircuit.LookupHost: %s", err.Error()) + return + } + + // The tactics we generate here have SNI == VerifyHostname == domain + for idx, addr := range addrs { + tactic := &httpsDialerTactic{ + Address: addr, + InitialDelay: happyEyeballsDelay(idx), + Port: port, + SNI: domain, + VerifyHostname: domain, + } + out <- tactic + } + }() + + return out +} diff --git a/pkg/enginenetx/dnspolicy_test.go b/pkg/enginenetx/dnspolicy_test.go new file mode 100644 index 000000000..a3fbc1113 --- /dev/null +++ b/pkg/enginenetx/dnspolicy_test.go @@ -0,0 +1,72 @@ +package enginenetx + +import ( + "context" + "testing" + + "github.com/ooni/probe-engine/pkg/mocks" + "github.com/ooni/probe-engine/pkg/model" +) + +func TestDNSPolicy(t *testing.T) { + t.Run("LookupTactics with canceled context", func(t *testing.T) { + var called int + + policy := &dnsPolicy{ + Logger: &mocks.Logger{ + MockDebugf: func(format string, v ...interface{}) { + called++ + }, + }, + Resolver: &mocks.Resolver{}, // empty so we crash if we hit the resolver + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // immediately cancel! + + tactics := policy.LookupTactics(ctx, "www.example.com", "443") + + var count int + for range tactics { + count++ + } + + if count != 0 { + t.Fatal("expected to see no tactic") + } + if called != 1 { + t.Fatal("did not call Debugf") + } + }) + + t.Run("we short circuit IP addresses", func(t *testing.T) { + policy := &dnsPolicy{ + Logger: model.DiscardLogger, + Resolver: &mocks.Resolver{}, // empty so we crash if we hit the resolver + } + + tactics := policy.LookupTactics(context.Background(), "130.192.91.211", "443") + + var count int + for tactic := range tactics { + count++ + + if tactic.Address != "130.192.91.211" { + t.Fatal("invalid endpoint address") + } + if tactic.Port != "443" { + t.Fatal("invalid endpoint port") + } + if tactic.SNI != "130.192.91.211" { + t.Fatal("invalid SNI") + } + if tactic.VerifyHostname != "130.192.91.211" { + t.Fatal("invalid VerifyHostname") + } + } + + if count != 1 { + t.Fatal("expected to see just one tactic") + } + }) +} diff --git a/pkg/enginenetx/doc.go b/pkg/enginenetx/doc.go new file mode 100644 index 000000000..15cdd9992 --- /dev/null +++ b/pkg/enginenetx/doc.go @@ -0,0 +1,2 @@ +// Package enginenetx contains engine-specific network-extensions. +package enginenetx diff --git a/pkg/enginenetx/happyeyeballs.go b/pkg/enginenetx/happyeyeballs.go new file mode 100644 index 000000000..75259b9aa --- /dev/null +++ b/pkg/enginenetx/happyeyeballs.go @@ -0,0 +1,33 @@ +package enginenetx + +import "time" + +// happyEyeballsDelay implements an happy-eyeballs like algorithm with a +// base delay of 1 second and the given index. The index is the attempt number +// and the first attempt should have zero as its index. +// +// The standard Go library uses a 300ms delay for connecting. Because a TCP +// connect is one round trip and the TLS handshake is two round trips (roughly), +// we use 1 second as the base delay increment here. +// +// The algorithm should emit 0 as the first delay, the base delay as the +// second delay, and then it should double the base delay at each attempt, +// until we reach the 8 seconds, after which the delay increments +// linearly spacing each subsequent attempts 8 seconds in the future. +// +// By doubling the base delay, we account for the case where there are +// actual issues inside the network. By using this algorithm, we are still +// able to overlap and pack more dialing attempts overall. +func happyEyeballsDelay(idx int) time.Duration { + const baseDelay = time.Second + switch { + case idx <= 0: + return 0 + case idx == 1: + return baseDelay + case idx <= 4: + return baseDelay << (idx - 1) + default: + return baseDelay << 3 * (time.Duration(idx) - 3) + } +} diff --git a/pkg/enginenetx/happyeyeballs_test.go b/pkg/enginenetx/happyeyeballs_test.go new file mode 100644 index 000000000..b1f7c4af5 --- /dev/null +++ b/pkg/enginenetx/happyeyeballs_test.go @@ -0,0 +1,39 @@ +package enginenetx + +import ( + "fmt" + "testing" + "time" +) + +func TestHappyEyeballsDelay(t *testing.T) { + type testcase struct { + idx int + expect time.Duration + } + + cases := []testcase{ + {-1, 0}, // make sure we gracefully handle negative numbers (i.e., we don't crash) + {0, 0}, + {1, time.Second}, + {2, 2 * time.Second}, + {3, 4 * time.Second}, + {4, 8 * time.Second}, + {5, 2 * 8 * time.Second}, + {6, 3 * 8 * time.Second}, + {7, 4 * 8 * time.Second}, + {8, 5 * 8 * time.Second}, + {9, 6 * 8 * time.Second}, + {10, 7 * 8 * time.Second}, + } + + for _, tc := range cases { + t.Run(fmt.Sprintf("tc.idx=%v", tc.idx), func(t *testing.T) { + got := happyEyeballsDelay(tc.idx) + if got != tc.expect { + t.Fatalf("with tc.idx=%v we got %v but expected %v", tc.idx, got, tc.expect) + } + t.Logf("with tc.idx=%v: got %v", tc.idx, got) + }) + } +} diff --git a/pkg/enginenetx/httpsdialer.go b/pkg/enginenetx/httpsdialer.go new file mode 100644 index 000000000..19f8f509d --- /dev/null +++ b/pkg/enginenetx/httpsdialer.go @@ -0,0 +1,446 @@ +package enginenetx + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "net" + "sync/atomic" + "time" + + "github.com/ooni/probe-engine/pkg/logx" + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/netxlite" + "github.com/ooni/probe-engine/pkg/runtimex" +) + +// httpsDialerTactic is a tactic to establish a TLS connection. +type httpsDialerTactic struct { + // Address is the IPv4/IPv6 address for dialing. + Address string + + // InitialDelay is the time in nanoseconds after which + // you would like to start this policy. + InitialDelay time.Duration + + // Port is the TCP port for dialing. + Port string + + // SNI is the TLS ServerName to send over the wire. + SNI string + + // VerifyHostname is the hostname using during + // the X.509 certificate verification. + VerifyHostname string +} + +var _ fmt.Stringer = &httpsDialerTactic{} + +// Clone makes a deep copy of this [httpsDialerTactic]. +func (dt *httpsDialerTactic) Clone() *httpsDialerTactic { + return &httpsDialerTactic{ + Address: dt.Address, + InitialDelay: dt.InitialDelay, + Port: dt.Port, + SNI: dt.SNI, + VerifyHostname: dt.VerifyHostname, + } +} + +// String implements fmt.Stringer. +func (dt *httpsDialerTactic) String() string { + return string(runtimex.Try1(json.Marshal(dt))) +} + +// tacticSummaryKey returns a string summarizing the tactic's features. +// +// The fields used to compute the summary are: +// +// - IPAddr +// +// - Port +// +// - SNI +// +// - VerifyHostname +// +// The returned string contains the above fields separated by space with +// `sni=` before the SNI and `verify=` before the verify hostname. +// +// We should be careful not to change this format unless we also change the +// format version used by user policies and by the state management. +func (dt *httpsDialerTactic) tacticSummaryKey() string { + return fmt.Sprintf( + "%v sni=%v verify=%v", + net.JoinHostPort(dt.Address, dt.Port), + dt.SNI, + dt.VerifyHostname, + ) +} + +// domainEndpointKey returns a string consisting of the domain endpoint only. +// +// We always use the VerifyHostname and the Port to construct the domain endpoint. +func (dt *httpsDialerTactic) domainEndpointKey() string { + return net.JoinHostPort(dt.VerifyHostname, dt.Port) +} + +// httpsDialerPolicy is a policy used by the [*httpsDialer]. +type httpsDialerPolicy interface { + // LookupTactics emits zero or more tactics for the given host and port + // through the returned channel, which is closed when done. + LookupTactics(ctx context.Context, domain, port string) <-chan *httpsDialerTactic +} + +// httpsDialerEventsHandler handles events occurring while we try dialing TLS. +type httpsDialerEventsHandler interface { + // These callbacks are invoked during the TLS handshake to inform this + // interface about events that occurred. A policy SHOULD keep track of which + // addresses, SNIs, etc. work and return them more frequently. + // + // Callbacks that take an error as argument also take a context as + // argument and MUST check whether the context has been canceled or + // its timeout has expired (i.e., using ctx.Err()) to determine + // whether the operation failed or was merely canceled. In the latter + // case, obviously, you MUST NOT consider the tactic failed. + OnStarting(tactic *httpsDialerTactic) + OnTCPConnectError(ctx context.Context, tactic *httpsDialerTactic, err error) + OnTLSHandshakeError(ctx context.Context, tactic *httpsDialerTactic, err error) + OnTLSVerifyError(tactic *httpsDialerTactic, err error) + OnSuccess(tactic *httpsDialerTactic) +} + +// httpsDialer is the [model.TLSDialer] used by the engine to dial HTTPS connections. +// +// The zero value of this struct is invalid; construct using [newHTTPSDialer]. +// +// This dialer MAY use an happy-eyeballs-like policy where we may try several IP addresses, +// including IPv4 and IPv6, and dialing tactics in parallel. +type httpsDialer struct { + // idGenerator is the ID generator. + idGenerator *atomic.Int64 + + // logger is the logger to use. + logger model.Logger + + // netx is the [*netxlite.Netx] to use. + netx *netxlite.Netx + + // policy defines the dialing policy to use. + policy httpsDialerPolicy + + // rootCAs contains the root certificate pool we should use. + rootCAs *x509.CertPool + + // stats tracks what happens while dialing. + stats httpsDialerEventsHandler +} + +// newHTTPSDialer constructs a new [*httpsDialer] instance. +// +// Arguments: +// +// - logger is the logger to use for logging; +// +// - netx is the [*netxlite.Netx] to use; +// +// - policy defines the dialer policy; +// +// - stats tracks what happens while we're dialing. +// +// The returned [*httpsDialer] would use the underlying network's +// DefaultCertPool to create and cache the cert pool to use. +func newHTTPSDialer( + logger model.Logger, + netx *netxlite.Netx, + policy httpsDialerPolicy, + stats httpsDialerEventsHandler, +) *httpsDialer { + return &httpsDialer{ + idGenerator: &atomic.Int64{}, + logger: &logx.PrefixLogger{ + Prefix: "httpsDialer: ", + Logger: logger, + }, + netx: netx, + policy: policy, + rootCAs: netx.MaybeCustomUnderlyingNetwork().Get().DefaultCertPool(), + stats: stats, + } +} + +var _ model.TLSDialer = &httpsDialer{} + +// CloseIdleConnections implements model.TLSDialer. +func (hd *httpsDialer) CloseIdleConnections() { + // nothing +} + +// httpsDialerErrorOrConn contains either an error or a valid conn. +type httpsDialerErrorOrConn struct { + // Conn is the established TLS conn or nil. + Conn model.TLSConn + + // Err is the error or nil. + Err error +} + +// errDNSNoAnswer is the error returned when we have no tactic to try +var errDNSNoAnswer = netxlite.NewErrWrapper( + netxlite.ClassifyResolverError, + netxlite.DNSRoundTripOperation, + netxlite.ErrOODNSNoAnswer, +) + +// DialTLSContext implements model.TLSDialer. +func (hd *httpsDialer) DialTLSContext(ctx context.Context, network string, endpoint string) (net.Conn, error) { + hostname, port, err := net.SplitHostPort(endpoint) + if err != nil { + return nil, err + } + + // We need a cancellable context to interrupt the tactics emitter early when we + // immediately get a valid response and we don't need to use other tactics. + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // The emitter will emit tactics and then close the channel when done. We spawn 16 workers + // that handle tactics in parallel and post results on the collector channel. + emitter := hd.policy.LookupTactics(ctx, hostname, port) + collector := make(chan *httpsDialerErrorOrConn) + joiner := make(chan any) + const parallelism = 16 + t0 := time.Now() + for idx := 0; idx < parallelism; idx++ { + go hd.worker(ctx, joiner, emitter, t0, collector) + } + + // wait until all goroutines have joined + var ( + connv = []model.TLSConn{} + errorv = []error{} + numJoined = 0 + ) + for numJoined < parallelism { + select { + case <-joiner: + numJoined++ + + case result := <-collector: + // If the goroutine failed, record the error and continue processing results + if result.Err != nil { + errorv = append(errorv, result.Err) + continue + } + + // Save the conn and tell goroutines to stop ASAP + connv = append(connv, result.Conn) + cancel() + } + } + + return httpsDialerReduceResult(connv, errorv) +} + +// httpsDialerReduceResult returns either an established conn or an error, using [errDNSNoAnswer] in +// case the list of connections and the list of errors are empty. +func httpsDialerReduceResult(connv []model.TLSConn, errorv []error) (model.TLSConn, error) { + switch { + case len(connv) >= 1: + for _, c := range connv[1:] { + c.Close() + } + return connv[0], nil + + case len(errorv) >= 1: + return nil, errors.Join(errorv...) + + default: + return nil, errDNSNoAnswer + } +} + +// worker attempts to establish a TLS connection and emits the result using +// a [*httpsDialerErrorOrConn] for each tactic, until there are no more tactics +// and the reader channel is closed. At which point it posts on joiner to let +// the parent know that this goroutine has done its job. +func (hd *httpsDialer) worker( + ctx context.Context, + joiner chan<- any, + reader <-chan *httpsDialerTactic, + t0 time.Time, + writer chan<- *httpsDialerErrorOrConn, +) { + // let the parent know that we terminated + defer func() { joiner <- true }() + + for tactic := range reader { + prefixLogger := &logx.PrefixLogger{ + Prefix: fmt.Sprintf("[#%d] ", hd.idGenerator.Add(1)), + Logger: hd.logger, + } + + // perform the actual dial + conn, err := hd.dialTLS(ctx, prefixLogger, t0, tactic) + + // send results to the parent + writer <- &httpsDialerErrorOrConn{Conn: conn, Err: err} + } +} + +// dialTLS performs the actual TLS dial. +func (hd *httpsDialer) dialTLS( + ctx context.Context, + logger model.Logger, + t0 time.Time, + tactic *httpsDialerTactic, +) (model.TLSConn, error) { + // honor happy-eyeballs delays and wait for the tactic to be ready to run + if err := httpsDialerTacticWaitReady(ctx, t0, tactic); err != nil { + return nil, err + } + + // tell the observer that we're starting + hd.stats.OnStarting(tactic) + + // create dialer and establish TCP connection + endpoint := net.JoinHostPort(tactic.Address, tactic.Port) + ol := logx.NewOperationLogger(logger, "TCPConnect %s", endpoint) + dialer := hd.netx.NewDialerWithoutResolver(logger) + tcpConn, err := dialer.DialContext(ctx, "tcp", endpoint) + ol.Stop(err) + + // handle a dialing error + if err != nil { + hd.stats.OnTCPConnectError(ctx, tactic, err) + return nil, err + } + + // create TLS configuration + tlsConfig := &tls.Config{ + InsecureSkipVerify: true, // Note: we're going to verify at the end of the func! + NextProtos: []string{"h2", "http/1.1"}, + RootCAs: hd.rootCAs, + ServerName: tactic.SNI, + } + + // create handshaker and establish a TLS connection + ol = logx.NewOperationLogger( + logger, + "TLSHandshake with %s SNI=%s ALPN=%v", + endpoint, + tlsConfig.ServerName, + tlsConfig.NextProtos, + ) + thx := hd.netx.NewTLSHandshakerStdlib(logger) + tlsConn, err := thx.Handshake(ctx, tcpConn, tlsConfig) + ol.Stop(err) + + // handle handshake error + if err != nil { + hd.stats.OnTLSHandshakeError(ctx, tactic, err) + tcpConn.Close() + return nil, err + } + + // verify the certificate chain + ol = logx.NewOperationLogger(logger, "TLSVerifyCertificateChain %s", tactic.VerifyHostname) + err = httpsDialerVerifyCertificateChain(tactic.VerifyHostname, tlsConn, hd.rootCAs) + ol.Stop(err) + + // handle verification error + if err != nil { + hd.stats.OnTLSVerifyError(tactic, err) + tlsConn.Close() + return nil, err + } + + // make sure the observer knows it worked + hd.stats.OnSuccess(tactic) + + return tlsConn, nil +} + +// httpsDialerWaitReady waits for the given delay to expire or the context to be canceled. If the +// delay is zero or negative, we immediately return nil. We also return nil when the delay expires. We +// return the context error if the context expires. +func httpsDialerTacticWaitReady( + ctx context.Context, + t0 time.Time, + tactic *httpsDialerTactic, +) error { + deadline := t0.Add(tactic.InitialDelay) + delta := time.Until(deadline) + if delta <= 0 { + return nil + } + + timer := time.NewTimer(delta) + defer timer.Stop() + + select { + case <-timer.C: + return nil + + case <-ctx.Done(): + return netxlite.NewTopLevelGenericErrWrapper(ctx.Err()) + } +} + +// errNoPeerCertificate is an internal error returned when we don't have any peer certificate. +var errNoPeerCertificate = errors.New("no peer certificate") + +// errEmptyVerifyHostname indicates there is no hostname to verify against +var errEmptyVerifyHostname = errors.New("empty VerifyHostname") + +// httpsDialerVerifyCertificateChain verifies the certificate chain with the given hostname. +func httpsDialerVerifyCertificateChain(hostname string, conn model.TLSConn, rootCAs *x509.CertPool) error { + // This code comes from the example in the Go source tree that shows + // how to override certificate verification and which is advertised + // as follows: + // + // VerifyConnection can be used to replace and customize connection + // verification. This example shows a VerifyConnection implementation that + // will be approximately equivalent to what crypto/tls does normally to + // verify the peer's certificate. + // + // See https://github.com/golang/go/blob/go1.21.0/src/crypto/tls/example_test.go#L186 + // + // As of go1.21.0, the code we're replacing has approximately the same + // implementation of the verification code we added below. + // + // See https://github.com/golang/go/blob/go1.21.0/src/crypto/tls/handshake_client.go#L962. + + // Protect against a programming or configuration error where the + // programmer or user has not set the hostname. + if hostname == "" { + return errEmptyVerifyHostname + } + + state := conn.ConnectionState() + opts := x509.VerifyOptions{ + DNSName: hostname, // note: here we're using the real hostname + Intermediates: x509.NewCertPool(), + Roots: rootCAs, + } + + // The following check is rather paranoid and it's not part of the Go codebase + // from which we copied it, but I think it's important to be defensive. + // + // Because of that, I don't want to just drop an assertion here. + if len(state.PeerCertificates) < 1 { + return errNoPeerCertificate + } + + for _, cert := range state.PeerCertificates[1:] { + opts.Intermediates.AddCert(cert) + } + + if _, err := state.PeerCertificates[0].Verify(opts); err != nil { + return netxlite.NewErrWrapper(netxlite.ClassifyTLSHandshakeError, netxlite.TopLevelOperation, err) + } + return nil +} diff --git a/pkg/enginenetx/httpsdialer_test.go b/pkg/enginenetx/httpsdialer_test.go new file mode 100644 index 000000000..cee656d4d --- /dev/null +++ b/pkg/enginenetx/httpsdialer_test.go @@ -0,0 +1,633 @@ +package enginenetx + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "net/url" + "testing" + "time" + + "github.com/apex/log" + "github.com/google/go-cmp/cmp" + "github.com/ooni/netem" + "github.com/ooni/probe-engine/pkg/mocks" + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/netemx" + "github.com/ooni/probe-engine/pkg/netxlite" + "github.com/ooni/probe-engine/pkg/runtimex" + "github.com/ooni/probe-engine/pkg/testingx" +) + +// Flags controlling when [httpsDialerCancelingContextStatsTracker] cancels the context +const ( + httpsDialerCancelingContextStatsTrackerOnStarting = 1 << iota + httpsDialerCancelingContextStatsTrackerOnSuccess +) + +// httpsDialerCancelingContextStatsTracker is an [httpsDialerEventsHandler] with a cancel +// function that causes the context to be canceled once we start dialing. +// +// This struct helps with testing [httpsDialer] is WAI when the context +// has been canceled and we correctly shutdown all goroutines. +type httpsDialerCancelingContextStatsTracker struct { + cancel context.CancelFunc + flags int +} + +var _ httpsDialerEventsHandler = &httpsDialerCancelingContextStatsTracker{} + +// OnStarting implements httpsDialerEventsHandler. +func (st *httpsDialerCancelingContextStatsTracker) OnStarting(tactic *httpsDialerTactic) { + if (st.flags & httpsDialerCancelingContextStatsTrackerOnStarting) != 0 { + st.cancel() + } +} + +// OnTCPConnectError implements httpsDialerEventsHandler. +func (*httpsDialerCancelingContextStatsTracker) OnTCPConnectError(ctx context.Context, tactic *httpsDialerTactic, err error) { + // nothing +} + +// OnTLSHandshakeError implements httpsDialerEventsHandler. +func (*httpsDialerCancelingContextStatsTracker) OnTLSHandshakeError(ctx context.Context, tactic *httpsDialerTactic, err error) { + // nothing +} + +// OnTLSVerifyError implements httpsDialerEventsHandler. +func (*httpsDialerCancelingContextStatsTracker) OnTLSVerifyError(tactic *httpsDialerTactic, err error) { + // nothing +} + +// OnSuccess implements httpsDialerEventsHandler. +func (st *httpsDialerCancelingContextStatsTracker) OnSuccess(tactic *httpsDialerTactic) { + if (st.flags & httpsDialerCancelingContextStatsTrackerOnSuccess) != 0 { + st.cancel() + } +} + +// QA using netem +func TestHTTPSDialerNetemQA(t *testing.T) { + // testcase is a test case implemented by this function + type testcase struct { + // name is the name of the test case + name string + + // short indicates whether this is a short test + short bool + + // stats is the stats tracker to use. + stats httpsDialerEventsHandler + + // endpoint is the endpoint to connect to consisting of a domain + // name or IP address followed by a TCP port + endpoint string + + // scenario is the netemx testing scenario to create + scenario []*netemx.ScenarioDomainAddresses + + // configureDPI configures DPI rules (just add an empty + // function if you don't need any) + configureDPI func(dpi *netem.DPIEngine) + + // expectErr is the error string we expect to see + expectErr string + } + + allTestCases := []testcase{ + + // This test case ensures that we handle the corner case of a missing port + { + name: "net.SplitHostPort failure", + short: true, + stats: &nullStatsManager{}, + endpoint: "www.example.com", // note: here the port is missing + scenario: netemx.InternetScenario, + configureDPI: func(dpi *netem.DPIEngine) { + // nothing + }, + expectErr: "address www.example.com: missing port in address", + }, + + // This test case ensures that we handle the case of a nonexistent domain + // where we get a dns_no_answer error. The original DNS error is lost in + // background goroutines and what we report to the caller is just that there + // is no available IP address and tactic to attempt using. + { + name: "hd.policy.LookupTactics failure", + short: true, + stats: &nullStatsManager{}, + endpoint: "www.example.nonexistent:443", // note: the domain does not exist + scenario: netemx.InternetScenario, + configureDPI: func(dpi *netem.DPIEngine) { + // nothing + }, + expectErr: "dns_no_answer", + }, + + // This test case is the common case: all is good with multiple addresses to dial (I am + // not testing the case of a single address because it's a subcase of this one) + { + name: "successful dial with multiple addresses", + short: true, + stats: &nullStatsManager{}, + endpoint: "www.example.com:443", + scenario: []*netemx.ScenarioDomainAddresses{{ + Domains: []string{ + "www.example.com", + }, + Addresses: []string{ + "93.184.216.34", + "93.184.216.35", + "93.184.216.36", + "93.184.216.37", + }, + Role: netemx.ScenarioRoleWebServer, + ServerNameMain: "www.example.com", + WebServerFactory: netemx.ExampleWebPageHandlerFactory(), + }}, + configureDPI: func(dpi *netem.DPIEngine) { + // nothing + }, + expectErr: "", + }, + + // Here we make sure that we're doing OK if the addresses are TCP-blocked + { + name: "with TCP connect errors", + short: true, + stats: &nullStatsManager{}, + endpoint: "www.example.com:443", + scenario: []*netemx.ScenarioDomainAddresses{{ + Domains: []string{ + "www.example.com", + }, + Addresses: []string{ + "93.184.216.34", + "93.184.216.35", + }, + Role: netemx.ScenarioRoleWebServer, + ServerNameMain: "www.example.com", + WebServerFactory: netemx.ExampleWebPageHandlerFactory(), + }}, + configureDPI: func(dpi *netem.DPIEngine) { + // we force closing the connection for all the known server endpoints + dpi.AddRule(&netem.DPICloseConnectionForServerEndpoint{ + Logger: log.Log, + ServerIPAddress: "93.184.216.34", + ServerPort: 443, + }) + dpi.AddRule(&netem.DPICloseConnectionForServerEndpoint{ + Logger: log.Log, + ServerIPAddress: "93.184.216.35", + ServerPort: 443, + }) + }, + expectErr: "connection_refused\nconnection_refused", + }, + + // Here we're making sure it's all WAI when there is TLS interference + { + name: "with TLS handshake errors", + short: true, + stats: &nullStatsManager{}, + endpoint: "www.example.com:443", + scenario: []*netemx.ScenarioDomainAddresses{{ + Domains: []string{ + "www.example.com", + }, + Addresses: []string{ + "93.184.216.34", + "93.184.216.35", + }, + Role: netemx.ScenarioRoleWebServer, + ServerNameMain: "www.example.com", + WebServerFactory: netemx.ExampleWebPageHandlerFactory(), + }}, + configureDPI: func(dpi *netem.DPIEngine) { + // we force resetting the connection for www.example.com + dpi.AddRule(&netem.DPIResetTrafficForTLSSNI{ + Logger: log.Log, + SNI: "www.example.com", + }) + }, + expectErr: "connection_reset\nconnection_reset", + }, + + // Note: this is where we test that TLS verification is WAI. The netemx scenario role + // constructs the equivalent of real world's badssl.com and we're checking whether + // we would accept a certificate valid for another hostname. The answer should be "NO!". + { + name: "with a TLS certificate valid for ANOTHER domain", + short: true, + stats: &nullStatsManager{}, + endpoint: "wrong.host.badssl.com:443", + scenario: []*netemx.ScenarioDomainAddresses{{ + Domains: []string{ + "wrong.host.badssl.com", + "untrusted-root.badssl.com", + "expired.badssl.com", + }, + Addresses: []string{ + "93.184.216.34", + "93.184.216.35", + }, + Role: netemx.ScenarioRoleBadSSL, + }}, + configureDPI: func(dpi *netem.DPIEngine) { + // nothing + }, + expectErr: "ssl_invalid_hostname\nssl_invalid_hostname", + }, + + // Note: this is another TLS related test case where we make sure that + // we can handle an untrusted root/self signed certificate + { + name: "with TLS certificate signed by an unknown authority", + short: true, + stats: &nullStatsManager{}, + endpoint: "untrusted-root.badssl.com:443", + scenario: []*netemx.ScenarioDomainAddresses{{ + Domains: []string{ + "wrong.host.badssl.com", + "untrusted-root.badssl.com", + "expired.badssl.com", + }, + Addresses: []string{ + "93.184.216.34", + "93.184.216.35", + }, + Role: netemx.ScenarioRoleBadSSL, + }}, + configureDPI: func(dpi *netem.DPIEngine) { + // nothing + }, + expectErr: "ssl_unknown_authority\nssl_unknown_authority", + }, + + // Note: this is another TLS related test case where we make sure that + // we can handle a certificate that has now expired. + { + name: "with expired TLS certificate", + short: true, + stats: &nullStatsManager{}, + endpoint: "expired.badssl.com:443", + scenario: []*netemx.ScenarioDomainAddresses{{ + Domains: []string{ + "wrong.host.badssl.com", + "untrusted-root.badssl.com", + "expired.badssl.com", + }, + Addresses: []string{ + "93.184.216.34", + "93.184.216.35", + }, + Role: netemx.ScenarioRoleBadSSL, + }}, + configureDPI: func(dpi *netem.DPIEngine) { + // nothing + }, + expectErr: "ssl_invalid_certificate\nssl_invalid_certificate", + }, + + // This is a corner case: what if the context is canceled after the DNS lookup + // but before we start dialing? Are we closing all goroutines and returning correctly? + { + name: "with context being canceled in OnStarting", + short: true, + stats: &httpsDialerCancelingContextStatsTracker{ + cancel: nil, + flags: httpsDialerCancelingContextStatsTrackerOnStarting, + }, + endpoint: "www.example.com:443", + scenario: []*netemx.ScenarioDomainAddresses{{ + Domains: []string{ + "www.example.com", + }, + Addresses: []string{ + "93.184.216.34", + "93.184.216.35", + }, + Role: netemx.ScenarioRoleWebServer, + ServerNameMain: "www.example.com", + WebServerFactory: netemx.ExampleWebPageHandlerFactory(), + }}, + configureDPI: func(dpi *netem.DPIEngine) { + // nothing + }, + expectErr: "interrupted\ninterrupted", + }, + + // This is another corner case: what happens if the context is canceled + // right after we eastablish a connection? Because of how the current code + // is written, the easiest thing to do is to just return the conn. + { + name: "with context being canceled in OnSuccess for the first success", + short: true, + stats: &httpsDialerCancelingContextStatsTracker{ + cancel: nil, + flags: httpsDialerCancelingContextStatsTrackerOnSuccess, + }, + endpoint: "www.example.com:443", + scenario: []*netemx.ScenarioDomainAddresses{{ + Domains: []string{ + "www.example.com", + }, + Addresses: []string{ + "93.184.216.34", + "93.184.216.35", + }, + Role: netemx.ScenarioRoleWebServer, + ServerNameMain: "www.example.com", + WebServerFactory: netemx.ExampleWebPageHandlerFactory(), + }}, + configureDPI: func(dpi *netem.DPIEngine) { + // nothing + }, + expectErr: "", + }} + + for _, tc := range allTestCases { + t.Run(tc.name, func(t *testing.T) { + // make sure we honor `go test -short` + if !tc.short && testing.Short() { + t.Skip("skip test in short mode") + } + + // track all the connections so we can check whether we close them all + cv := &testingx.CloseVerify{} + + func() { + // create the QA environment + env := netemx.MustNewScenario(tc.scenario) + defer env.Close() + + // possibly add specific DPI rules + tc.configureDPI(env.DPIEngine()) + + // create the proper underlying network and wrap it such that + // we track whether we close all the connections + unet := cv.WrapUnderlyingNetwork(&netxlite.NetemUnderlyingNetworkAdapter{UNet: env.ClientStack}) + + // create the network proper + netx := &netxlite.Netx{Underlying: unet} + + // create the getaddrinfo resolver + resolver := netx.NewStdlibResolver(log.Log) + + policy := &dnsPolicy{ + Logger: log.Log, + Resolver: resolver, + } + + // create the TLS dialer + dialer := newHTTPSDialer( + log.Log, + netx, + policy, + tc.stats, + ) + defer dialer.CloseIdleConnections() + + // configure cancellable context--some tests are going to use cancel + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Possibly tell the httpsDialerCancelingContextStatsTracker about the cancel func + // depending on which flags have been configured. + if p, ok := tc.stats.(*httpsDialerCancelingContextStatsTracker); ok { + p.cancel = cancel + } + + // dial the TLS connection + tlsConn, err := dialer.DialTLSContext(ctx, "tcp", tc.endpoint) + + t.Logf("%+v %+v", tlsConn, err) + + // make sure the error is the one we expected + switch { + case err != nil && tc.expectErr == "": + t.Fatal("expected", tc.expectErr, "got", err) + + case err == nil && tc.expectErr != "": + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr != "": + if diff := cmp.Diff(tc.expectErr, err.Error()); diff != "" { + t.Fatal(diff) + } + + case err == nil && tc.expectErr == "": + // all good + } + + // make sure we close the conn + if tlsConn != nil { + defer tlsConn.Close() + } + }() + + // now verify that we have closed all the connections + if err := cv.CheckForOpenConns(); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestHTTPSDialerTactic(t *testing.T) { + t.Run("String", func(t *testing.T) { + expected := `{"Address":"162.55.247.208","InitialDelay":150000000,"Port":"443","SNI":"www.example.com","VerifyHostname":"api.ooni.io"}` + ldt := &httpsDialerTactic{ + Address: "162.55.247.208", + InitialDelay: 150 * time.Millisecond, + Port: "443", + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + } + got := ldt.String() + if diff := cmp.Diff(expected, got); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("Clone", func(t *testing.T) { + ff := &testingx.FakeFiller{} + var expect httpsDialerTactic + ff.Fill(&expect) + got := expect.Clone() + if diff := cmp.Diff(expect.String(), got.String()); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("Summary", func(t *testing.T) { + expected := `162.55.247.208:443 sni=www.example.com verify=api.ooni.io` + ldt := &httpsDialerTactic{ + Address: "162.55.247.208", + InitialDelay: 150 * time.Millisecond, + Port: "443", + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + } + got := ldt.tacticSummaryKey() + if diff := cmp.Diff(expected, got); diff != "" { + t.Fatal(diff) + } + }) +} + +// QA using the host network +func TestHTTPSDialerHostNetworkQA(t *testing.T) { + t.Run("dnsPolicy allows connecting to https://127.0.0.1/ using a custom CA", func(t *testing.T) { + ca := netem.MustNewCA() + server := testingx.MustNewHTTPServerTLS( + testingx.HTTPHandlerBlockpage451(), + ca, + "server.local", + ) + defer server.Close() + + tproxy := &netxlite.DefaultTProxy{} + + // The resolver we're creating here reproduces the test case described by + // https://github.com/ooni/probe-cli/pull/1295#issuecomment-1731243994 + resolver := netxlite.MaybeWrapWithBogonResolver(true, netxlite.NewStdlibResolver(log.Log)) + + httpsDialer := newHTTPSDialer( + log.Log, + &netxlite.Netx{Underlying: &mocks.UnderlyingNetwork{ + MockDefaultCertPool: func() *x509.CertPool { + return ca.DefaultCertPool() // just override the CA + }, + MockDialTimeout: tproxy.DialTimeout, + MockDialContext: tproxy.DialContext, + MockListenTCP: tproxy.ListenTCP, + MockListenUDP: tproxy.ListenUDP, + MockGetaddrinfoLookupANY: tproxy.GetaddrinfoLookupANY, + MockGetaddrinfoResolverNetwork: tproxy.GetaddrinfoResolverNetwork, + }}, + &dnsPolicy{ + Logger: log.Log, + Resolver: resolver, + }, + &nullStatsManager{}, + ) + + URL := runtimex.Try1(url.Parse(server.URL)) + + ctx := context.Background() + tlsConn, err := httpsDialer.DialTLSContext(ctx, "tcp", URL.Host) + if err != nil { + t.Fatal(err) + } + tlsConn.Close() + }) +} + +func TestHTTPSDialerVerifyCertificateChain(t *testing.T) { + t.Run("without any peer certificate", func(t *testing.T) { + tlsConn := &mocks.TLSConn{ + MockConnectionState: func() tls.ConnectionState { + return tls.ConnectionState{} // empty! + }, + } + certPool := netxlite.NewMozillaCertPool() + err := httpsDialerVerifyCertificateChain("www.example.com", tlsConn, certPool) + if !errors.Is(err, errNoPeerCertificate) { + t.Fatal("unexpected error", err) + } + }) + + t.Run("with an empty hostname", func(t *testing.T) { + tlsConn := &mocks.TLSConn{ + MockConnectionState: func() tls.ConnectionState { + return tls.ConnectionState{} // empty but should not be an issue + }, + } + certPool := netxlite.NewMozillaCertPool() + err := httpsDialerVerifyCertificateChain("", tlsConn, certPool) + if !errors.Is(err, errEmptyVerifyHostname) { + t.Fatal("unexpected error", err) + } + }) +} + +func TestHTTPSDialerReduceResult(t *testing.T) { + t.Run("we return the first conn in a list of conns and close the other conns", func(t *testing.T) { + var closed int + expect := &mocks.TLSConn{} // empty + connv := []model.TLSConn{ + expect, + &mocks.TLSConn{ + Conn: mocks.Conn{ + MockClose: func() error { + closed++ + return nil + }, + }, + }, + &mocks.TLSConn{ + Conn: mocks.Conn{ + MockClose: func() error { + closed++ + return nil + }, + }, + }, + } + + conn, err := httpsDialerReduceResult(connv, nil) + if err != nil { + t.Fatal(err) + } + + if conn != expect { + t.Fatal("unexpected conn") + } + + if closed != 2 { + t.Fatal("did not call close") + } + }) + + t.Run("we join together a list of errors", func(t *testing.T) { + expectErr := "connection_refused\ninterrupted" + errorv := []error{errors.New("connection_refused"), errors.New("interrupted")} + + conn, err := httpsDialerReduceResult(nil, errorv) + if err == nil || err.Error() != expectErr { + t.Fatal("unexpected err", err) + } + + if conn != nil { + t.Fatal("expected nil conn") + } + }) + + t.Run("with a single error we return such an error", func(t *testing.T) { + expected := errors.New("connection_refused") + errorv := []error{expected} + + conn, err := httpsDialerReduceResult(nil, errorv) + if !errors.Is(err, expected) { + t.Fatal("unexpected err", err) + } + + if conn != nil { + t.Fatal("expected nil conn") + } + }) + + t.Run("we return errDNSNoAnswer if we don't have any conns or errors to return", func(t *testing.T) { + conn, err := httpsDialerReduceResult(nil, nil) + if !errors.Is(err, errDNSNoAnswer) { + t.Fatal("unexpected error", err) + } + + if conn != nil { + t.Fatal("expected nil conn") + } + }) +} diff --git a/pkg/enginenetx/network.go b/pkg/enginenetx/network.go new file mode 100644 index 000000000..a99a83ab1 --- /dev/null +++ b/pkg/enginenetx/network.go @@ -0,0 +1,173 @@ +package enginenetx + +// +// Network - the top-level object of this package, used by the +// OONI engine to communicate with several backends +// + +import ( + "net/http" + "net/http/cookiejar" + "net/url" + "time" + + "github.com/ooni/probe-engine/pkg/bytecounter" + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/netxlite" + "github.com/ooni/probe-engine/pkg/runtimex" + "golang.org/x/net/publicsuffix" +) + +// Network is the network abstraction used by the OONI engine. +// +// The zero value is invalid; construct using the [NewNetwork] func. +type Network struct { + reso model.Resolver + stats *statsManager + txp model.HTTPTransport +} + +// HTTPTransport returns the underlying [model.HTTPTransport]. +func (n *Network) HTTPTransport() model.HTTPTransport { + return n.txp +} + +// NewHTTPClient is a convenience function for building an [*http.Client] using +// the underlying [model.HTTPTransport] and the correct cookies configuration. +func (n *Network) NewHTTPClient() *http.Client { + // Note: cookiejar.New cannot fail, so we're using runtimex.Try1 here + return &http.Client{ + Transport: n.txp, + Jar: runtimex.Try1(cookiejar.New(&cookiejar.Options{ + PublicSuffixList: publicsuffix.List, + })), + } +} + +// Close ensures that we close idle connections and persist statistics. +func (n *Network) Close() error { + // TODO(bassosimone): do we want to introduce "once" semantics in this method? It + // does not seem necessary since there's no resource we can close just once. + + // make sure we close the transport's idle connections + n.txp.CloseIdleConnections() + + // same as above but for the resolver's connections + n.reso.CloseIdleConnections() + + // make sure we sync stats to disk and shutdown the background trimmer + return n.stats.Close() +} + +// NewNetwork creates a new [*Network] for the engine. This network MUST NOT be +// used for measuring because it implements engine-specific policies. +// +// You MUST call the Close method when done using the network. This method ensures +// that (i) we close idle connections and (ii) persist statistics. +// +// Arguments: +// +// - counter is the [*bytecounter.Counter] to use; +// +// - kvStore is a [model.KeyValueStore] for persisting stats; +// +// - logger is the [model.Logger] to use; +// +// - proxyURL is the OPTIONAL proxy URL; +// +// - resolver is the [model.Resolver] to use. +// +// The presence of the proxyURL MAY cause this function to possibly build a +// network with different behavior with respect to circumvention. If there is +// an upstream proxy we're going to trust it is doing circumvention for us. +func NewNetwork( + counter *bytecounter.Counter, + kvStore model.KeyValueStore, + logger model.Logger, + proxyURL *url.URL, + resolver model.Resolver, +) *Network { + // Create a dialer ONLY used for dialing unencrypted TCP connections. The common use + // case of this Network is to dial encrypted connections. For this reason, here it is + // reasonably fine to use the legacy sequential dialer implemented in netxlite. + dialer := netxlite.NewDialerWithResolver(logger, resolver) + + // Create manager for keeping track of statistics + const trimInterval = 30 * time.Second + stats := newStatsManager(kvStore, logger, trimInterval) + + // Create a TLS dialer ONLY used for dialing TLS connections. This dialer will use + // happy-eyeballs and possibly custom policies for dialing TLS connections. + httpsDialer := newHTTPSDialer( + logger, + &netxlite.Netx{Underlying: nil}, // nil means using netxlite's singleton + newHTTPSDialerPolicy(kvStore, logger, proxyURL, resolver, stats), + stats, + ) + + // Here we're creating a "new style" HTTPS transport, which has less + // restrictions compared to the "old style" one. + // + // Note that: + // + // - we're enabling compression, which is desiredable since this transport + // is not made for measuring and compression is good(TM); + // + // - if proxyURL is nil, the proxy option is equivalent to disabling + // the proxy, otherwise it means that we're using the ooni/oohttp library + // to dial for proxies, which has some restrictions. + // + // In particular, the returned transport uses dialer for dialing with + // cleartext proxies (e.g., socks5 and http) and httpsDialer for dialing + // with encrypted proxies (e.g., https). After this has happened, + // the code currently falls back to using the standard library's tls + // client code for establishing TLS connections over the proxy. The main + // implication here is that we're not using our custom mozilla CA for + // validating TLS certificates, rather we're using the system's cert store. + // + // Fixing this issue is TODO(https://github.com/ooni/probe/issues/2536). + txp := netxlite.NewHTTPTransportWithOptions( + logger, dialer, httpsDialer, + netxlite.HTTPTransportOptionDisableCompression(false), + netxlite.HTTPTransportOptionProxyURL(proxyURL), + ) + + // Make sure we count the bytes sent and received as part of the session + txp = bytecounter.WrapHTTPTransport(txp, counter) + + netx := &Network{ + reso: resolver, + stats: stats, + txp: txp, + } + return netx +} + +// newHTTPSDialerPolicy contains the logic to select the [HTTPSDialerPolicy] to use. +func newHTTPSDialerPolicy( + kvStore model.KeyValueStore, + logger model.Logger, + proxyURL *url.URL, // optional! + resolver model.Resolver, + stats *statsManager, +) httpsDialerPolicy { + // in case there's a proxy URL, we're going to trust the proxy to do the right thing and + // know what it's doing, hence we'll have a very simple DNS policy + if proxyURL != nil { + return &dnsPolicy{logger, resolver} + } + + // create a composed fallback TLS dialer policy + fallback := &statsPolicy{ + Fallback: &bridgesPolicy{Fallback: &dnsPolicy{logger, resolver}}, + Stats: stats, + } + + // make sure we honor a user-provided policy + policy, err := newUserPolicy(kvStore, fallback) + if err != nil { + return fallback + } + + return policy +} diff --git a/pkg/enginenetx/network_internal_test.go b/pkg/enginenetx/network_internal_test.go new file mode 100644 index 000000000..6184f793f --- /dev/null +++ b/pkg/enginenetx/network_internal_test.go @@ -0,0 +1,297 @@ +package enginenetx + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "sync" + "testing" + + "github.com/apex/log" + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-engine/pkg/bytecounter" + "github.com/ooni/probe-engine/pkg/kvstore" + "github.com/ooni/probe-engine/pkg/mocks" + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/netemx" + "github.com/ooni/probe-engine/pkg/netxlite" + "github.com/ooni/probe-engine/pkg/runtimex" +) + +func TestNetworkUnit(t *testing.T) { + t.Run("HTTPTransport returns the correct transport", func(t *testing.T) { + expected := &mocks.HTTPTransport{} + netx := &Network{txp: expected} + if netx.HTTPTransport() != expected { + t.Fatal("not the transport we expected") + } + }) + + t.Run("Close calls the transport's CloseIdleConnections method", func(t *testing.T) { + var called bool + expected := &mocks.HTTPTransport{ + MockCloseIdleConnections: func() { + called = true + }, + } + netx := &Network{ + reso: &mocks.Resolver{ + MockCloseIdleConnections: func() { + // nothing + }, + }, + stats: &statsManager{ + cancel: func() { /* nothing */ }, + closeOnce: sync.Once{}, + container: &statsContainer{}, + kvStore: &kvstore.Memory{}, + logger: model.DiscardLogger, + mu: sync.Mutex{}, + pruned: make(chan any), + wg: &sync.WaitGroup{}, + }, + txp: expected, + } + if err := netx.Close(); err != nil { + t.Fatal(err) + } + if !called { + t.Fatal("did not call the transport's CloseIdleConnections") + } + }) + + t.Run("Close calls the resolvers's CloseIdleConnections method", func(t *testing.T) { + var called bool + expected := &mocks.Resolver{ + MockCloseIdleConnections: func() { + called = true + }, + } + netx := &Network{ + reso: expected, + stats: &statsManager{ + cancel: func() { /* nothing */ }, + closeOnce: sync.Once{}, + container: &statsContainer{}, + kvStore: &kvstore.Memory{}, + logger: model.DiscardLogger, + mu: sync.Mutex{}, + pruned: make(chan any), + wg: &sync.WaitGroup{}, + }, + txp: &mocks.HTTPTransport{ + MockCloseIdleConnections: func() { + // nothing + }, + }, + } + if err := netx.Close(); err != nil { + t.Fatal(err) + } + if !called { + t.Fatal("did not call the resolver's CloseIdleConnections") + } + }) + + t.Run("Close calls the .cancel field of the statsManager as a side effect", func(t *testing.T) { + var called bool + netx := &Network{ + reso: &mocks.Resolver{ + MockCloseIdleConnections: func() { + // nothing + }, + }, + stats: &statsManager{ + cancel: func() { + called = true + }, + closeOnce: sync.Once{}, + container: &statsContainer{}, + kvStore: &kvstore.Memory{}, + logger: model.DiscardLogger, + mu: sync.Mutex{}, + pruned: make(chan any), + wg: &sync.WaitGroup{}, + }, + txp: &mocks.HTTPTransport{ + MockCloseIdleConnections: func() { + // nothing + }, + }, + } + if err := netx.Close(); err != nil { + t.Fatal(err) + } + if !called { + t.Fatal("did not call the .cancel field of the statsManager") + } + }) + + t.Run("NewNetwork uses the correct httpsDialerPolicy", func(t *testing.T) { + // testcase is a test case run by this func + type testcase struct { + name string + kvStore func() model.KeyValueStore + expectStatus int + expectBody []byte + } + + cases := []testcase{ + // Without a policy accessing www.example.com should lead to 200 as status + // code and the expected web page when we're using netem + { + name: "when there is no user-provided policy", + kvStore: func() model.KeyValueStore { + return &kvstore.Memory{} + }, + expectStatus: 200, + expectBody: []byte(netemx.ExampleWebPage), + }, + + // But we can create a policy that can land us on a different website (not the + // typical use case of the policy, but definitely demonstrating it works) + { + name: "when there's a user-provided policy", + kvStore: func() model.KeyValueStore { + policy := &userPolicyRoot{ + DomainEndpoints: map[string][]*httpsDialerTactic{ + "www.example.com:443": {{ + Address: netemx.AddressApiOONIIo, + InitialDelay: 0, + Port: "443", + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + }}, + }, + Version: userPolicyVersion, + } + rawPolicy := runtimex.Try1(json.Marshal(policy)) + kvStore := &kvstore.Memory{} + runtimex.Try0(kvStore.Set(userPolicyKey, rawPolicy)) + return kvStore + }, + expectStatus: 404, + expectBody: []byte{}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() + + env.Do(func() { + netx := NewNetwork( + bytecounter.New(), + tc.kvStore(), + log.Log, + nil, // proxy URL + netxlite.NewStdlibResolver(log.Log), + ) + defer netx.Close() + + client := netx.NewHTTPClient() + resp, err := client.Get("https://www.example.com/") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != tc.expectStatus { + t.Fatal("StatusCode: expected", tc.expectStatus, "got", resp.StatusCode) + } + data, err := netxlite.ReadAllContext(context.Background(), resp.Body) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(tc.expectBody, data); diff != "" { + t.Fatal(diff) + } + }) + }) + } + }) +} + +// Make sure we get the correct policy type depending on how we call newHTTPSDialerPolicy +func TestNewHTTPSDialerPolicy(t *testing.T) { + // testcase is a test case implemented by this function + type testcase struct { + // name is the name of the test case + name string + + // kvStore constructs the kvstore to use + kvStore func() model.KeyValueStore + + // proxyURL is the OPTIONAL proxy URL to use + proxyURL *url.URL + + // expectType is the string representation of the + // type constructed using these params + expectType string + } + + minimalUserPolicy := []byte(`{"Version":3}`) + + cases := []testcase{{ + name: "when there is a proxy URL and there is a user policy", + kvStore: func() model.KeyValueStore { + store := &kvstore.Memory{} + // this policy is mostly empty but it's enough to load + runtimex.Try0(store.Set(userPolicyKey, minimalUserPolicy)) + return store + }, + proxyURL: &url.URL{ + Scheme: "socks5", + Host: "127.0.0.1:9050", + Path: "/", + }, + expectType: "*enginenetx.dnsPolicy", + }, { + name: "when there is a proxy URL and there is no user policy", + kvStore: func() model.KeyValueStore { + return &kvstore.Memory{} + }, + proxyURL: &url.URL{ + Scheme: "socks5", + Host: "127.0.0.1:9050", + Path: "/", + }, + expectType: "*enginenetx.dnsPolicy", + }, { + name: "when there is no proxy URL and there is a user policy", + kvStore: func() model.KeyValueStore { + store := &kvstore.Memory{} + // this policy is mostly empty but it's enough to load + runtimex.Try0(store.Set(userPolicyKey, minimalUserPolicy)) + return store + }, + proxyURL: nil, + expectType: "*enginenetx.userPolicy", + }, { + name: "when there is no proxy URL and there is no user policy", + kvStore: func() model.KeyValueStore { + return &kvstore.Memory{} + }, + proxyURL: nil, + expectType: "*enginenetx.statsPolicy", + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + + p := newHTTPSDialerPolicy( + tc.kvStore(), + model.DiscardLogger, + tc.proxyURL, // possibly nil + &mocks.Resolver{}, // we are not using `out` so it does not matter + &statsManager{}, // ditto + ) + + got := fmt.Sprintf("%T", p) + if diff := cmp.Diff(tc.expectType, got); diff != "" { + t.Fatal(diff) + } + }) + } +} diff --git a/pkg/enginenetx/network_test.go b/pkg/enginenetx/network_test.go new file mode 100644 index 000000000..a47af92b8 --- /dev/null +++ b/pkg/enginenetx/network_test.go @@ -0,0 +1,272 @@ +package enginenetx_test + +import ( + "context" + "net" + "net/http" + "net/url" + "testing" + "time" + + "github.com/apex/log" + "github.com/ooni/probe-engine/pkg/bytecounter" + "github.com/ooni/probe-engine/pkg/enginenetx" + "github.com/ooni/probe-engine/pkg/kvstore" + "github.com/ooni/probe-engine/pkg/measurexlite" + "github.com/ooni/probe-engine/pkg/netemx" + "github.com/ooni/probe-engine/pkg/netxlite" + "github.com/ooni/probe-engine/pkg/testingsocks5" + "github.com/ooni/probe-engine/pkg/testingx" +) + +func TestNetworkQA(t *testing.T) { + t.Run("is WAI when not using any proxy", func(t *testing.T) { + env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() + + env.Do(func() { + txp := enginenetx.NewNetwork( + bytecounter.New(), + &kvstore.Memory{}, + log.Log, + nil, + netxlite.NewStdlibResolver(log.Log), + ) + client := txp.NewHTTPClient() + resp, err := client.Get("https://www.example.com/") + if err != nil { + t.Fatal(err) + } + t.Logf("%+v", resp) + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatal("unexpected status code") + } + }) + }) + + t.Run("is WAI when using a SOCKS5 proxy", func(t *testing.T) { + // create internet measurement scenario + env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() + + // create a proxy using the client's TCP/IP stack + proxy := testingsocks5.MustNewServer( + log.Log, + &netxlite.Netx{Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: env.ClientStack}}, + &net.TCPAddr{ + IP: net.ParseIP(env.ClientStack.IPAddress()), + Port: 9050, + }, + ) + defer proxy.Close() + + env.Do(func() { + txp := enginenetx.NewNetwork( + bytecounter.New(), + &kvstore.Memory{}, + log.Log, + &url.URL{ + Scheme: "socks5", + Host: net.JoinHostPort(env.ClientStack.IPAddress(), "9050"), + Path: "/", + }, + netxlite.NewStdlibResolver(log.Log), + ) + client := txp.NewHTTPClient() + + // To make sure we're connecting to the expected endpoint, we're going to use + // measurexlite and tracing to observe the destination endpoints + trace := measurexlite.NewTrace(0, time.Now()) + ctx := netxlite.ContextWithTrace(context.Background(), trace) + + // create request using the above context + // + // Implementation note: we cannot use HTTPS with netem here as explained + // by the https://github.com/ooni/probe/issues/2536 issue. + req, err := http.NewRequestWithContext(ctx, "GET", "http://www.example.com/", nil) + if err != nil { + t.Fatal(err) + } + + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + t.Logf("%+v", resp) + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatal("unexpected status code") + } + + // make sure that we only connected to the SOCKS5 proxy + tcpConnects := trace.TCPConnects() + if len(tcpConnects) <= 0 { + t.Fatal("expected at least one TCP connect") + } + for idx, entry := range tcpConnects { + t.Logf("%d: %+v", idx, entry) + if entry.IP != env.ClientStack.IPAddress() { + t.Fatal("unexpected IP address") + } + if entry.Port != 9050 { + t.Fatal("unexpected port") + } + } + }) + }) + + t.Run("is WAI when using an HTTP proxy", func(t *testing.T) { + // create internet measurement scenario + env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() + + // create a proxy using the client's TCP/IP stack + proxy := testingx.MustNewHTTPServerEx( + &net.TCPAddr{IP: net.ParseIP(env.ClientStack.IPAddress()), Port: 8080}, + env.ClientStack, + testingx.NewHTTPProxyHandler(log.Log, &netxlite.Netx{ + Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: env.ClientStack}}), + ) + defer proxy.Close() + + env.Do(func() { + txp := enginenetx.NewNetwork( + bytecounter.New(), + &kvstore.Memory{}, + log.Log, + &url.URL{ + Scheme: "http", + Host: net.JoinHostPort(env.ClientStack.IPAddress(), "8080"), + Path: "/", + }, + netxlite.NewStdlibResolver(log.Log), + ) + client := txp.NewHTTPClient() + + // To make sure we're connecting to the expected endpoint, we're going to use + // measurexlite and tracing to observe the destination endpoints + trace := measurexlite.NewTrace(0, time.Now()) + ctx := netxlite.ContextWithTrace(context.Background(), trace) + + // create request using the above context + // + // Implementation note: we cannot use HTTPS with netem here as explained + // by the https://github.com/ooni/probe/issues/2536 issue. + req, err := http.NewRequestWithContext(ctx, "GET", "http://www.example.com/", nil) + if err != nil { + t.Fatal(err) + } + + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + t.Logf("%+v", resp) + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatal("unexpected status code") + } + + // make sure that we only connected to the HTTP proxy + tcpConnects := trace.TCPConnects() + if len(tcpConnects) <= 0 { + t.Fatal("expected at least one TCP connect") + } + for idx, entry := range tcpConnects { + t.Logf("%d: %+v", idx, entry) + if entry.IP != env.ClientStack.IPAddress() { + t.Fatal("unexpected IP address") + } + if entry.Port != 8080 { + t.Fatal("unexpected port") + } + } + }) + }) + + t.Run("is WAI when using an HTTPS proxy", func(t *testing.T) { + // create internet measurement scenario + env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() + + // create a proxy using the client's TCP/IP stack + proxy := testingx.MustNewHTTPServerTLSEx( + &net.TCPAddr{IP: net.ParseIP(env.ClientStack.IPAddress()), Port: 4443}, + env.ClientStack, + testingx.NewHTTPProxyHandler(log.Log, &netxlite.Netx{ + Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: env.ClientStack}}), + env.ClientStack, + "proxy.local", + ) + defer proxy.Close() + + env.Do(func() { + txp := enginenetx.NewNetwork( + bytecounter.New(), + &kvstore.Memory{}, + log.Log, + &url.URL{ + Scheme: "https", + Host: net.JoinHostPort(env.ClientStack.IPAddress(), "4443"), + Path: "/", + }, + netxlite.NewStdlibResolver(log.Log), + ) + client := txp.NewHTTPClient() + + // To make sure we're connecting to the expected endpoint, we're going to use + // measurexlite and tracing to observe the destination endpoints + trace := measurexlite.NewTrace(0, time.Now()) + ctx := netxlite.ContextWithTrace(context.Background(), trace) + + // create request using the above context + // + // Implementation note: we cannot use HTTPS with netem here as explained + // by the https://github.com/ooni/probe/issues/2536 issue. + req, err := http.NewRequestWithContext(ctx, "GET", "http://www.example.com/", nil) + if err != nil { + t.Fatal(err) + } + + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + t.Logf("%+v", resp) + defer resp.Body.Close() + if resp.StatusCode != 200 { + t.Fatal("unexpected status code") + } + + // make sure that we only connected to the HTTPS proxy + tcpConnects := trace.TCPConnects() + if len(tcpConnects) <= 0 { + t.Fatal("expected at least one TCP connect") + } + for idx, entry := range tcpConnects { + t.Logf("%d: %+v", idx, entry) + if entry.IP != env.ClientStack.IPAddress() { + t.Fatal("unexpected IP address") + } + if entry.Port != 4443 { + t.Fatal("unexpected port") + } + } + }) + }) + + t.Run("NewHTTPClient returns a client with a cookie jar", func(t *testing.T) { + txp := enginenetx.NewNetwork( + bytecounter.New(), + &kvstore.Memory{}, + log.Log, + nil, + netxlite.NewStdlibResolver(log.Log), + ) + client := txp.NewHTTPClient() + if client.Jar == nil { + t.Fatal("expected non-nil cookie jar") + } + }) +} diff --git a/pkg/enginenetx/statsmanager.go b/pkg/enginenetx/statsmanager.go new file mode 100644 index 000000000..d1e9a802c --- /dev/null +++ b/pkg/enginenetx/statsmanager.go @@ -0,0 +1,671 @@ +package enginenetx + +// +// Code to keep statistics about the TLS dialing +// tactics that work and the ones that don't +// + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "sort" + "sync" + "time" + + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/runtimex" +) + +// nullStatsManager is the "null" [httpsDialerEventsHandler]. +type nullStatsManager struct{} + +var _ httpsDialerEventsHandler = &nullStatsManager{} + +// OnStarting implements httpsDialerEventsHandler. +func (*nullStatsManager) OnStarting(tactic *httpsDialerTactic) { + // nothing +} + +// OnSuccess implements httpsDialerEventsHandler. +func (*nullStatsManager) OnSuccess(tactic *httpsDialerTactic) { + // nothing +} + +// OnTCPConnectError implements httpsDialerEventsHandler. +func (*nullStatsManager) OnTCPConnectError(ctx context.Context, tactic *httpsDialerTactic, err error) { + // nothing +} + +// OnTLSHandshakeError implements httpsDialerEventsHandler. +func (*nullStatsManager) OnTLSHandshakeError(ctx context.Context, tactic *httpsDialerTactic, err error) { + // nothing +} + +// OnTLSVerifyError implements httpsDialerEventsHandler. +func (*nullStatsManager) OnTLSVerifyError(tactic *httpsDialerTactic, err error) { + // nothing +} + +// statsTactic keeps stats about an [*httpsDialerTactic]. +type statsTactic struct { + // CountStarted counts the number of operations we started. + CountStarted int64 + + // CountTCPConnectError counts the number of TCP connect errors. + CountTCPConnectError int64 + + // CountTCPConnectInterrupt counts the number of interrupted TCP connect attempts. + CountTCPConnectInterrupt int64 + + // CountTLSHandshakeError counts the number of TLS handshake errors. + CountTLSHandshakeError int64 + + // CountTLSHandshakeInterrupt counts the number of interrupted TLS handshakes. + CountTLSHandshakeInterrupt int64 + + // CountTLSVerificationError counts the number of TLS verification errors. + CountTLSVerificationError int64 + + // CountSuccess counts the number of successes. + CountSuccess int64 + + // HistoTCPConnectError contains an histogram of TCP connect errors. + HistoTCPConnectError map[string]int64 + + // HistoTLSHandshakeError contains an histogram of TLS handshake errors. + HistoTLSHandshakeError map[string]int64 + + // HistoTLSVerificationError contains an histogram of TLS verification errors. + HistoTLSVerificationError map[string]int64 + + // LastUpdated is the last time we updated this record. + LastUpdated time.Time + + // Tactic is the underlying tactic. + Tactic *httpsDialerTactic +} + +// statsNilSafeSuccessRate is a convenience function for computing the success rate +// which returns zero as the success rate if CountStarted is zero. +// +// For robustness, be paranoid about nils here because the stats are +// written on the disk and a user could potentially edit them. +func statsNilSafeSuccessRate(t *statsTactic) (rate float64) { + if t != nil && t.CountStarted > 0 { + rate = float64(t.CountSuccess) / float64(t.CountStarted) + } + return +} + +// statsNilSafeLastUpdated is a convenience function for getting the .LastUpdated +// field that takes into account the case where t is nil. +func statsNilSafeLastUpdated(t *statsTactic) (output time.Time) { + if t != nil { + output = t.LastUpdated + } + return +} + +// statsNilSafeCountSuccess is a convenience function for getting the .CountSuccess +// counter that takes into account the case where t is nil. +func statsNilSafeCountSuccess(t *statsTactic) (output int64) { + if t != nil { + output = t.CountSuccess + } + return +} + +// statsDefensivelySortTacticsByDescendingSuccessRateWithAcceptPredicate sorts the input list +// by success rate taking into account that several entries could be malformed, and then +// filters the sorted list using the given boolean predicate to accept elements. +// +// The sorting criteria takes into account: +// +// 1. the success rate; or +// +// 2. the last updated time; or +// +// 3. the number of successes. +// +// The predicate allows to further restrict the returned list. +// +// This function operates on a deep copy of the input list, so it does not create data races. +func statsDefensivelySortTacticsByDescendingSuccessRateWithAcceptPredicate( + input []*statsTactic, acceptfunc func(*statsTactic) bool) []*statsTactic { + // first let's create a working list such that we don't modify + // the input in place thus avoiding any data race + work := []*statsTactic{} + for _, t := range input { + if t != nil && t.Tactic != nil { + work = append(work, t.Clone()) // DEEP COPY! + } + } + + // now let's sort work in place + sort.SliceStable(work, func(i, j int) bool { + if statsNilSafeSuccessRate(work[i]) > statsNilSafeSuccessRate(work[j]) { + return true + } + if statsNilSafeCountSuccess(work[i]) > statsNilSafeCountSuccess(work[j]) { + return true + } + if statsNilSafeLastUpdated(work[i]).Sub(statsNilSafeLastUpdated(work[j])) > 0 { + return true + } + return false + }) + + // finally let's apply the predicate to produce output + output := []*statsTactic{} + for _, t := range work { + if acceptfunc(t) { + output = append(output, t) + } + } + return output +} + +func statsMaybeCloneMapStringInt64(input map[string]int64) (output map[string]int64) { + // distinguish and preserve nil versus empty + if input == nil { + return + } + output = make(map[string]int64) + for key, value := range input { + output[key] = value + } + return +} + +func statsMaybeCloneTactic(input *httpsDialerTactic) (output *httpsDialerTactic) { + if input != nil { + output = input.Clone() + } + return +} + +// Clone clones a given [*statsTactic] +func (st *statsTactic) Clone() *statsTactic { + // Implementation note: a time.Time consists of an uint16, an int64 and + // a pointer to a location which is typically immutable, so it's perfectly + // fine to copy the LastUpdate field by assignment. + // + // here we're using a bunch of robustness aware mechanisms to clone + // considering that the struct may be edited by the user + return &statsTactic{ + CountStarted: st.CountStarted, + CountTCPConnectError: st.CountTCPConnectError, + CountTCPConnectInterrupt: st.CountTCPConnectInterrupt, + CountTLSHandshakeError: st.CountTLSHandshakeError, + CountTLSHandshakeInterrupt: st.CountTLSHandshakeInterrupt, + CountTLSVerificationError: st.CountTLSVerificationError, + CountSuccess: st.CountSuccess, + HistoTCPConnectError: statsMaybeCloneMapStringInt64(st.HistoTCPConnectError), + HistoTLSHandshakeError: statsMaybeCloneMapStringInt64(st.HistoTLSHandshakeError), + HistoTLSVerificationError: statsMaybeCloneMapStringInt64(st.HistoTLSVerificationError), + LastUpdated: st.LastUpdated, + Tactic: statsMaybeCloneTactic(st.Tactic), + } +} + +// statsDomainEndpoint contains stats associated with a domain endpoint. +type statsDomainEndpoint struct { + Tactics map[string]*statsTactic +} + +// statsDomainEndpointPruneEntries returns a DEEP COPY of a [*statsDomainEndpoint] with old +// and excess entries removed, such that the overall size is not unbounded. +func statsDomainEndpointPruneEntries(input *statsDomainEndpoint) *statsDomainEndpoint { + tactics := []*statsTactic{} + now := time.Now() + + // if .Tactics is empty here we're just going to do nothing + for summary, tactic := range input.Tactics { + // we serialize stats to disk, so we cannot rule out the case where the user + // explicitly edits the stats to include a malformed entry + if summary == "" || tactic == nil || tactic.Tactic == nil { + continue + } + tactics = append(tactics, tactic) + } + + // oneWeek is a constant representing one week of data. + const oneWeek = 7 * 24 * time.Hour + + // maxEntriesPerDomainEndpoint is the maximum number of entries per + // domain endpoint that we would like to keep overall. + const maxEntriesPerDomainEndpoint = 10 + + // Sort by descending success rate and cut all the entries that are older than + // a given threshold. Note that we need to be defensive here because we are dealing + // with data stored on disk that might have been modified to crash us. + // + // Note that statsDefensivelySortTacticsByDescendingSuccessRateWithAcceptPredicate + // operates on and returns a DEEP COPY of the original list. + tactics = statsDefensivelySortTacticsByDescendingSuccessRateWithAcceptPredicate( + tactics, func(st *statsTactic) bool { + // When .LastUpdated is the zero time.Time value, the check is going to fail + // exactly like the time was 1 or 5 or 10 years ago instead. + // + // See https://go.dev/play/p/HGQT17ueIkq where we show that the zero time + // is handled exactly like any time in the past (it was kinda obvious, but + // sometimes it also make sense to double check assumptions!) + delta := now.Sub(statsNilSafeLastUpdated(st)) + return delta < oneWeek + }) + + // Cut excess entries, if needed + if len(tactics) > maxEntriesPerDomainEndpoint { + tactics = tactics[:maxEntriesPerDomainEndpoint] + } + + // return a new statsDomainEndpoint to the caller + output := &statsDomainEndpoint{ + Tactics: map[string]*statsTactic{}, + } + for _, t := range tactics { + output.Tactics[t.Tactic.tacticSummaryKey()] = t + } + return output +} + +// statsContainerVersion is the current version of [statsContainer]. +const statsContainerVersion = 5 + +// statsContainer is the root container for the stats. +// +// The zero value is invalid; construct using [newStatsContainer]. +type statsContainer struct { + // DomainEndpoints maps a domain endpoint to its tactics. + DomainEndpoints map[string]*statsDomainEndpoint + + // Version is the version of the container data format. + Version int +} + +// statsContainerPruneEntries returns a DEEP COPY of a [*statsContainer] with old entries removed. +func statsContainerPruneEntries(input *statsContainer) (output *statsContainer) { + output = newStatsContainer() + + // if .DomainEndpoints is nil here we're just going to do nothing + for domainEpnt, inputStats := range input.DomainEndpoints { + + // We serialize this data to disk, so we need to account for the case + // where a user has manually edited the JSON to add a nil value + if domainEpnt == "" || inputStats == nil || len(inputStats.Tactics) <= 0 { + continue + } + + prunedStats := statsDomainEndpointPruneEntries(inputStats) + + // We don't want to include an entry when it's empty because all the + // stats inside it have just been pruned + if len(prunedStats.Tactics) <= 0 { + continue + } + + output.DomainEndpoints[domainEpnt] = prunedStats + } + return +} + +// GetStatsTacticLocked returns the tactic record for the given [*statsTactic] instance. +// +// As the name implies, this function MUST be called while holding the [*statsManager] mutex. +func (c *statsContainer) GetStatsTacticLocked(tactic *httpsDialerTactic) (*statsTactic, bool) { + domainEpntRecord, found := c.DomainEndpoints[tactic.domainEndpointKey()] + if !found || domainEpntRecord == nil { + return nil, false + } + tacticRecord, found := domainEpntRecord.Tactics[tactic.tacticSummaryKey()] + return tacticRecord, found +} + +// SetStatsTacticLocked sets the tactic record for the given the given [*statsTactic] instance. +// +// As the name implies, this function MUST be called while holding the [*statsManager] mutex. +func (c *statsContainer) SetStatsTacticLocked(tactic *httpsDialerTactic, record *statsTactic) { + domainEpntRecord, found := c.DomainEndpoints[tactic.domainEndpointKey()] + if !found { + domainEpntRecord = &statsDomainEndpoint{ + Tactics: map[string]*statsTactic{}, + } + + // make sure the map is initialized -- not a void concern given that we're + // reading this structure from the disk + if len(c.DomainEndpoints) <= 0 { + c.DomainEndpoints = make(map[string]*statsDomainEndpoint) + } + + c.DomainEndpoints[tactic.domainEndpointKey()] = domainEpntRecord + // fallthrough + } + domainEpntRecord.Tactics[tactic.tacticSummaryKey()] = record +} + +// newStatsContainer creates a new empty [*statsContainer]. +func newStatsContainer() *statsContainer { + return &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{}, + Version: statsContainerVersion, + } +} + +// statsManager implements [httpsDialerEventsHandler] by storing the +// relevant statistics in a [model.KeyValueStore]. +// +// The zero value of this structure is not ready to use; please, use the +// [newStatsManager] factory to create a new instance. +type statsManager struct { + // cancel allows canceling the background stats pruner. + cancel context.CancelFunc + + // closeOnce gives .Close a "once" semantics + closeOnce sync.Once + + // container is the container container for stats + container *statsContainer + + // kvStore is the key-value store we're using + kvStore model.KeyValueStore + + // logger is the logger to use. + logger model.Logger + + // mu provides mutual exclusion when accessing the stats. + mu sync.Mutex + + // pruned is a channel pruned on a best effort basis + // by the background goroutine that prunes. + pruned chan any + + // wg tells us when the background goroutine joined. + wg *sync.WaitGroup +} + +// statsKey is the key used in the key-value store to access the state. +const statsKey = "httpsdialerstats.state" + +// errStatsContainerWrongVersion means that the stats container document has the wrong version number. +var errStatsContainerWrongVersion = errors.New("wrong stats container version") + +// loadStatsContainer loads a stats container from the given [model.KeyValueStore]. +func loadStatsContainer(kvStore model.KeyValueStore) (*statsContainer, error) { + // load data from the kvstore + data, err := kvStore.Get(statsKey) + if err != nil { + return nil, err + } + + // parse as JSON + var container statsContainer + if err := json.Unmarshal(data, &container); err != nil { + return nil, err + } + + // make sure the version is OK + if container.Version != statsContainerVersion { + err := fmt.Errorf( + "%s: %w: expected=%d got=%d", + statsKey, + errStatsContainerWrongVersion, + statsContainerVersion, + container.Version, + ) + return nil, err + } + + // make sure we prune the data structure + pruned := statsContainerPruneEntries(&container) + return pruned, nil +} + +// newStatsManager constructs a new instance of [*statsManager]. +func newStatsManager(kvStore model.KeyValueStore, logger model.Logger, trimInterval time.Duration) *statsManager { + runtimex.Assert(trimInterval > 0, "passed non-positive trimInterval") + + root, err := loadStatsContainer(kvStore) + if err != nil { + root = newStatsContainer() + } + + ctx, cancel := context.WithCancel(context.Background()) + + mt := &statsManager{ + cancel: cancel, + closeOnce: sync.Once{}, + container: root, + kvStore: kvStore, + logger: logger, + mu: sync.Mutex{}, + pruned: make(chan any), + wg: &sync.WaitGroup{}, + } + + // run a background goroutine that trims the stats by removing excessive + // entries until the programmer calls (*statsManager).Close + mt.wg.Add(1) + go func() { + defer mt.wg.Done() + mt.trim(ctx, trimInterval) + }() + + return mt +} + +var _ httpsDialerEventsHandler = &statsManager{} + +// OnStarting implements httpsDialerEventsHandler. +func (mt *statsManager) OnStarting(tactic *httpsDialerTactic) { + // get exclusive access + defer mt.mu.Unlock() + mt.mu.Lock() + + // get the record + record, found := mt.container.GetStatsTacticLocked(tactic) + if !found { + record = &statsTactic{ + CountStarted: 0, + CountTCPConnectError: 0, + CountTCPConnectInterrupt: 0, + CountTLSHandshakeError: 0, + CountTLSHandshakeInterrupt: 0, + CountTLSVerificationError: 0, + CountSuccess: 0, + HistoTCPConnectError: map[string]int64{}, + HistoTLSHandshakeError: map[string]int64{}, + HistoTLSVerificationError: map[string]int64{}, + LastUpdated: time.Time{}, + Tactic: tactic.Clone(), // avoid storing the original + } + mt.container.SetStatsTacticLocked(tactic, record) + } + + // update stats + record.CountStarted++ + record.LastUpdated = time.Now() +} + +func statsSafeIncrementMapStringInt64(input *map[string]int64, value string) { + runtimex.Assert(input != nil, "passed nil pointer to a map") + if *input == nil { + *input = make(map[string]int64) + } + (*input)[value]++ +} + +// OnTCPConnectError implements httpsDialerEventsHandler. +func (mt *statsManager) OnTCPConnectError(ctx context.Context, tactic *httpsDialerTactic, err error) { + // get exclusive access + defer mt.mu.Unlock() + mt.mu.Lock() + + // get the record + record, found := mt.container.GetStatsTacticLocked(tactic) + if !found { + mt.logger.Warnf("statsManager.OnTCPConnectError: not found: %+v", tactic) + return + } + + // update stats + record.LastUpdated = time.Now() + if ctx.Err() != nil { + record.CountTCPConnectInterrupt++ + return + } + + runtimex.Assert(err != nil, "OnTCPConnectError passed a nil error") + record.CountTCPConnectError++ + statsSafeIncrementMapStringInt64(&record.HistoTCPConnectError, err.Error()) +} + +// OnTLSHandshakeError implements httpsDialerEventsHandler. +func (mt *statsManager) OnTLSHandshakeError(ctx context.Context, tactic *httpsDialerTactic, err error) { + // get exclusive access + defer mt.mu.Unlock() + mt.mu.Lock() + + // get the record + record, found := mt.container.GetStatsTacticLocked(tactic) + if !found { + mt.logger.Warnf("statsManager.OnTLSHandshakeError: not found: %+v", tactic) + return + } + + // update stats + record.LastUpdated = time.Now() + if ctx.Err() != nil { + record.CountTLSHandshakeInterrupt++ + return + } + + runtimex.Assert(err != nil, "OnTLSHandshakeError passed a nil error") + record.CountTLSHandshakeError++ + statsSafeIncrementMapStringInt64(&record.HistoTLSHandshakeError, err.Error()) +} + +// OnTLSVerifyError implements httpsDialerEventsHandler. +func (mt *statsManager) OnTLSVerifyError(tactic *httpsDialerTactic, err error) { + // get exclusive access + defer mt.mu.Unlock() + mt.mu.Lock() + + // get the record + record, found := mt.container.GetStatsTacticLocked(tactic) + if !found { + mt.logger.Warnf("statsManager.OnTLSVerificationError: not found: %+v", tactic) + return + } + + // update stats + runtimex.Assert(err != nil, "OnTLSVerifyError passed a nil error") + record.CountTLSVerificationError++ + statsSafeIncrementMapStringInt64(&record.HistoTLSVerificationError, err.Error()) + record.LastUpdated = time.Now() +} + +// OnSuccess implements httpsDialerEventsHandler. +func (mt *statsManager) OnSuccess(tactic *httpsDialerTactic) { + // get exclusive access + defer mt.mu.Unlock() + mt.mu.Lock() + + // get the record + record, found := mt.container.GetStatsTacticLocked(tactic) + if !found { + mt.logger.Warnf("statsManager.OnSuccess: not found: %+v", tactic) + return + } + + // update stats + record.CountSuccess++ + record.LastUpdated = time.Now() +} + +// Close implements io.Closer +func (mt *statsManager) Close() (err error) { + mt.closeOnce.Do(func() { + // interrupt the background goroutine + mt.cancel() + + func() { + // get exclusive access + defer mt.mu.Unlock() + mt.mu.Lock() + + // make sure we remove the unneeded entries one last time before saving them + container := statsContainerPruneEntries(mt.container) + + // write updated stats into the underlying key-value store + err = mt.kvStore.Set(statsKey, runtimex.Try1(json.Marshal(container))) + }() + + // wait for background goroutine to join + mt.wg.Wait() + }) + return +} + +// trim runs in the background and trims the mt.container struct +func (mt *statsManager) trim(ctx context.Context, interval time.Duration) { + + // Note: we already manage mt.wg when we start this goroutine so there's NO NEED to do it here! + + t := time.NewTicker(interval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + + case <-t.C: + + // get exclusive access and edit the container + mt.mu.Lock() + mt.container = statsContainerPruneEntries(mt.container) + mt.mu.Unlock() + + // notify whoever's concerned that we pruned + // and do that best effort because it may be that nobody is concerned + select { + case mt.pruned <- true: + default: + } + + } + } +} + +// LookupTacticsStats returns stats about tactics for a given domain and port. The returned +// list is a clone of the one stored by [*statsManager] so, it can easily be modified. +func (mt *statsManager) LookupTactics(domain string, port string) ([]*statsTactic, bool) { + out := []*statsTactic{} + + // get exclusive access + defer mt.mu.Unlock() + mt.mu.Lock() + + // check whether we have information on this endpoint + // + // Note: in case mt.container.DomainEndpoints is nil, this access pattern + // will return to us a nil pointer and false + // + // we also protect against the case where a user has configured a nil + // domainEpnts value inside the serialized JSON to crash us + domainEpnts, good := mt.container.DomainEndpoints[net.JoinHostPort(domain, port)] + if !good || domainEpnts == nil { + return out, false + } + + // return a copy of each entry + // + // Note: if Tactics here is nil, we're just not going to have + // anything to include into the out list + for _, entry := range domainEpnts.Tactics { + out = append(out, entry.Clone()) + } + return out, len(out) > 0 +} diff --git a/pkg/enginenetx/statsmanager_test.go b/pkg/enginenetx/statsmanager_test.go new file mode 100644 index 000000000..a619530fb --- /dev/null +++ b/pkg/enginenetx/statsmanager_test.go @@ -0,0 +1,1635 @@ +package enginenetx + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math/rand" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/apex/log" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/ooni/netem" + "github.com/ooni/probe-engine/pkg/bytecounter" + "github.com/ooni/probe-engine/pkg/kvstore" + "github.com/ooni/probe-engine/pkg/mocks" + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/netemx" + "github.com/ooni/probe-engine/pkg/netxlite" + "github.com/ooni/probe-engine/pkg/runtimex" +) + +// This test ensures that a [*Network] created with [NewNetwork] collects stats. +func TestNetworkCollectsStats(t *testing.T) { + // testcase is a test case run by this function + type testcase struct { + // name is the test case name + name string + + // URL is the URL to GET + URL string + + // initialPolicy is the initial policy to configure into the key-value store + initialPolicy func() []byte + + // configureDPI is the function to configure DPI + configureDPI func(dpi *netem.DPIEngine) + + // expectErr is the expected error string + expectErr string + + // statsDomainEpnt is the domain endpoint to lookup inside the stats + statsDomainEpnt string + + // statsTacticsSummary is the summary to lookup inside the stats + // once we have used the statsDomain to get a record + statsTacticsSummary string + + // expectStats contains the expected record containing tactics stats + expectStats *statsTactic + } + + cases := []testcase{ + + { + name: "with TCP connect failure", + URL: "https://api.ooni.io/", + initialPolicy: func() []byte { + p0 := &userPolicyRoot{ + DomainEndpoints: map[string][]*httpsDialerTactic{ + // This policy has a different SNI and VerifyHostname, which gives + // us confidence that the stats are using the latter + "api.ooni.io:443": {{ + Address: netemx.AddressApiOONIIo, + InitialDelay: 0, + Port: "443", + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + }}, + }, + Version: userPolicyVersion, + } + return runtimex.Try1(json.Marshal(p0)) + }, + configureDPI: func(dpi *netem.DPIEngine) { + dpi.AddRule(&netem.DPICloseConnectionForServerEndpoint{ + Logger: log.Log, + ServerIPAddress: netemx.AddressApiOONIIo, + ServerPort: 443, + }) + }, + expectErr: `Get "https://api.ooni.io/": connection_refused`, + statsDomainEpnt: "api.ooni.io:443", + statsTacticsSummary: "162.55.247.208:443 sni=www.example.com verify=api.ooni.io", + expectStats: &statsTactic{ + CountStarted: 1, + CountTCPConnectError: 1, + CountTLSHandshakeError: 0, + CountTLSVerificationError: 0, + CountSuccess: 0, + HistoTCPConnectError: map[string]int64{ + "connection_refused": 1, + }, + HistoTLSHandshakeError: map[string]int64{}, + HistoTLSVerificationError: map[string]int64{}, + LastUpdated: time.Time{}, + Tactic: &httpsDialerTactic{ + Address: "162.55.247.208", + InitialDelay: 0, + Port: "443", + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + }, + }, + }, + + { + name: "with TLS handshake failure", + URL: "https://api.ooni.io/", + initialPolicy: func() []byte { + p0 := &userPolicyRoot{ + DomainEndpoints: map[string][]*httpsDialerTactic{ + // This policy has a different SNI and VerifyHostname, which gives + // us confidence that the stats are using the latter + "api.ooni.io:443": {{ + Address: netemx.AddressApiOONIIo, + InitialDelay: 0, + Port: "443", + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + }}, + }, + Version: userPolicyVersion, + } + return runtimex.Try1(json.Marshal(p0)) + }, + configureDPI: func(dpi *netem.DPIEngine) { + dpi.AddRule(&netem.DPIResetTrafficForTLSSNI{ + Logger: log.Log, + SNI: "www.example.com", + }) + }, + expectErr: `Get "https://api.ooni.io/": connection_reset`, + statsDomainEpnt: "api.ooni.io:443", + statsTacticsSummary: "162.55.247.208:443 sni=www.example.com verify=api.ooni.io", + expectStats: &statsTactic{ + CountStarted: 1, + CountTCPConnectError: 0, + CountTLSHandshakeError: 1, + CountTLSVerificationError: 0, + CountSuccess: 0, + HistoTCPConnectError: map[string]int64{}, + HistoTLSHandshakeError: map[string]int64{ + "connection_reset": 1, + }, + HistoTLSVerificationError: map[string]int64{}, + LastUpdated: time.Time{}, + Tactic: &httpsDialerTactic{ + Address: "162.55.247.208", + InitialDelay: 0, + Port: "443", + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + }, + }, + }, + + { + name: "with TLS verification failure", + URL: "https://api.ooni.io/", + initialPolicy: func() []byte { + p0 := &userPolicyRoot{ + DomainEndpoints: map[string][]*httpsDialerTactic{ + // This policy has a different SNI and VerifyHostname, which gives + // us confidence that the stats are using the latter + "api.ooni.io:443": {{ + Address: netemx.AddressBadSSLCom, + InitialDelay: 0, + Port: "443", + SNI: "untrusted-root.badssl.com", + VerifyHostname: "api.ooni.io", + }}, + }, + Version: userPolicyVersion, + } + return runtimex.Try1(json.Marshal(p0)) + }, + configureDPI: func(dpi *netem.DPIEngine) { + // nothing + }, + expectErr: `Get "https://api.ooni.io/": ssl_invalid_hostname`, + statsDomainEpnt: "api.ooni.io:443", + statsTacticsSummary: "104.154.89.105:443 sni=untrusted-root.badssl.com verify=api.ooni.io", + expectStats: &statsTactic{ + CountStarted: 1, + CountTCPConnectError: 0, + CountTLSHandshakeError: 0, + CountTLSVerificationError: 1, + CountSuccess: 0, + HistoTCPConnectError: map[string]int64{}, + HistoTLSHandshakeError: map[string]int64{}, + HistoTLSVerificationError: map[string]int64{ + "ssl_invalid_hostname": 1, + }, + LastUpdated: time.Time{}, + Tactic: &httpsDialerTactic{ + Address: "104.154.89.105", + InitialDelay: 0, + Port: "443", + SNI: "untrusted-root.badssl.com", + VerifyHostname: "api.ooni.io", + }, + }, + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + qa := netemx.MustNewScenario(netemx.InternetScenario) + defer qa.Close() + + // make sure we apply specific DPI rules + tc.configureDPI(qa.DPIEngine()) + + // create a memory key-value store where the engine will write stats that later we + // would be able to read to confirm we're collecting stats + kvStore := &kvstore.Memory{} + + initialPolicy := tc.initialPolicy() + t.Logf("initialPolicy: %s", string(initialPolicy)) + if err := kvStore.Set(userPolicyKey, initialPolicy); err != nil { + t.Fatal(err) + } + + qa.Do(func() { + byteCounter := bytecounter.New() + resolver := netxlite.NewStdlibResolver(log.Log) + + netx := NewNetwork(byteCounter, kvStore, log.Log, nil, resolver) + defer netx.Close() + + client := netx.NewHTTPClient() + + resp, err := client.Get(tc.URL) + + switch { + case err == nil && tc.expectErr == "": + // all good + + case err != nil && tc.expectErr == "": + t.Fatal("expected", tc.expectErr, "but got", err.Error()) + + case err == nil && tc.expectErr != "": + t.Fatal("expected", tc.expectErr, "but got", err) + + case err != nil && tc.expectErr != "": + if tc.expectErr != err.Error() { + t.Fatal("expected", tc.expectErr, "but got", err.Error()) + } + } + + if resp != nil { + defer resp.Body.Close() + } + }) + + // obtain the tactics container for the proper domain + rawStats, err := kvStore.Get(statsKey) + if err != nil { + t.Fatal(err) + } + var rootStats statsContainer + if err := json.Unmarshal(rawStats, &rootStats); err != nil { + t.Fatal(err) + } + tactics, good := rootStats.DomainEndpoints[tc.statsDomainEpnt] + if !good { + t.Fatalf("no such record for `%s`", tc.statsDomainEpnt) + } + t.Logf("%+v", tactics) + + // we expect to see a single record + if len(tactics.Tactics) != 1 { + t.Fatal("expected a single tactic") + } + tactic, good := tactics.Tactics[tc.statsTacticsSummary] + if !good { + t.Fatalf("no such record for: %s", tc.statsTacticsSummary) + } + + diffOptions := []cmp.Option{ + cmpopts.IgnoreFields(statsTactic{}, "LastUpdated"), + } + if diff := cmp.Diff(tc.expectStats, tactic, diffOptions...); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestLoadStatsContainer(t *testing.T) { + type testcase struct { + // name is the test case name + name string + + // input returns the bytes we should Set into the key-value store + input func() []byte + + // expectedErr is the expected error string or an empty string + expectErr string + + // expectRoot is the expected root container content + expectRoot *statsContainer + } + + fourtyFiveMinutesAgo := time.Now().Add(-45 * time.Minute) + + twoWeeksAgo := time.Now().Add(-14 * 24 * time.Hour) + + cases := []testcase{{ + name: "when the key-value store does not contain any data", + input: func() []byte { + // Note that returning nil causes the code to NOT set anything into the kvstore + return nil + }, + expectErr: "no such key", + expectRoot: nil, + }, { + name: "when we cannot parse the serialized JSON", + input: func() []byte { + return []byte(`{`) + }, + expectErr: "unexpected end of JSON input", + expectRoot: nil, + }, { + name: "with invalid version", + input: func() []byte { + return []byte(`{"Version":1}`) + }, + expectErr: "httpsdialerstats.state: wrong stats container version: expected=5 got=1", + expectRoot: nil, + }, { + name: "on success including correct entries pruning", + input: func() []byte { + root := &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{ + "api.ooni.io:443": { + Tactics: map[string]*statsTactic{ + "162.55.247.208:443 sni=www.example.com verify=api.ooni.io": { + CountStarted: 4, + CountTCPConnectError: 1, + CountTLSHandshakeError: 1, + CountTLSVerificationError: 1, + CountSuccess: 1, + HistoTCPConnectError: map[string]int64{ + "connection_refused": 1, + }, + HistoTLSHandshakeError: map[string]int64{ + "generic_timeout_error": 1, + }, + HistoTLSVerificationError: map[string]int64{ + "ssl_invalid_hostname": 1, + }, + LastUpdated: fourtyFiveMinutesAgo, + Tactic: &httpsDialerTactic{ + Address: "162.55.247.208", + InitialDelay: 0, + Port: "443", + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + }, + }, + "162.55.247.208:443 sni=www.example.org verify=api.ooni.io": { // should be skipped b/c it's old + CountStarted: 4, + CountTCPConnectError: 1, + CountTLSHandshakeError: 1, + CountTLSVerificationError: 1, + CountSuccess: 1, + HistoTCPConnectError: map[string]int64{ + "connection_refused": 1, + }, + HistoTLSHandshakeError: map[string]int64{ + "generic_timeout_error": 1, + }, + HistoTLSVerificationError: map[string]int64{ + "ssl_invalid_hostname": 1, + }, + LastUpdated: twoWeeksAgo, + Tactic: &httpsDialerTactic{ + Address: "162.55.247.208", + InitialDelay: 0, + Port: "443", + SNI: "www.example.org", + VerifyHostname: "api.ooni.io", + }, + }, + "162.55.247.208:443 sni=www.example.tk verify=api.ooni.io": { // should be skipped b/c time is zero + CountStarted: 4, + CountTCPConnectError: 1, + CountTLSHandshakeError: 1, + CountTLSVerificationError: 1, + CountSuccess: 1, + HistoTCPConnectError: map[string]int64{ + "connection_refused": 1, + }, + HistoTLSHandshakeError: map[string]int64{ + "generic_timeout_error": 1, + }, + HistoTLSVerificationError: map[string]int64{ + "ssl_invalid_hostname": 1, + }, + LastUpdated: time.Time{}, // zero value! + Tactic: &httpsDialerTactic{ + Address: "162.55.247.208", + InitialDelay: 0, + Port: "443", + SNI: "www.example.org", + VerifyHostname: "api.ooni.io", + }, + }, + + "162.55.247.208:443 sni=www.example.xyz verify=api.ooni.io": nil, // should be skipped because nil + "162.55.247.208:443 sni=www.example.it verify=api.ooni.io": { // should be skipped because nil tactic + CountStarted: 4, + CountTCPConnectError: 1, + CountTLSHandshakeError: 1, + CountTLSVerificationError: 1, + CountSuccess: 1, + HistoTCPConnectError: map[string]int64{ + "connection_refused": 1, + }, + HistoTLSHandshakeError: map[string]int64{ + "generic_timeout_error": 1, + }, + HistoTLSVerificationError: map[string]int64{ + "ssl_invalid_hostname": 1, + }, + LastUpdated: fourtyFiveMinutesAgo, + Tactic: nil, + }, + }, + }, + "www.kernel.org:443": { // this whole entry should be skipped because it's too old + Tactics: map[string]*statsTactic{ + "162.55.247.208:443 sni=www.example.com verify=www.kernel.org": { + CountStarted: 4, + CountTCPConnectError: 1, + CountTLSHandshakeError: 1, + CountTLSVerificationError: 1, + CountSuccess: 1, + HistoTCPConnectError: map[string]int64{ + "connection_refused": 1, + }, + HistoTLSHandshakeError: map[string]int64{ + "generic_timeout_error": 1, + }, + HistoTLSVerificationError: map[string]int64{ + "ssl_invalid_hostname": 1, + }, + LastUpdated: twoWeeksAgo, + Tactic: &httpsDialerTactic{ + Address: "162.55.247.208", + InitialDelay: 0, + Port: "443", + SNI: "www.example.com", + VerifyHostname: "www.kernel.org", + }, + }, + }, + }, + "www.kerneltrap.org:443": nil, // this whole entry should be skipped because it's nil + }, + Version: statsContainerVersion, + } + return runtimex.Try1(json.Marshal(root)) + }, + expectErr: "", + expectRoot: &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{ + "api.ooni.io:443": { + Tactics: map[string]*statsTactic{ + "162.55.247.208:443 sni=www.example.com verify=api.ooni.io": { + CountStarted: 4, + CountTCPConnectError: 1, + CountTLSHandshakeError: 1, + CountTLSVerificationError: 1, + CountSuccess: 1, + HistoTCPConnectError: map[string]int64{ + "connection_refused": 1, + }, + HistoTLSHandshakeError: map[string]int64{ + "generic_timeout_error": 1, + }, + HistoTLSVerificationError: map[string]int64{ + "ssl_invalid_hostname": 1, + }, + LastUpdated: fourtyFiveMinutesAgo, + Tactic: &httpsDialerTactic{ + Address: "162.55.247.208", + InitialDelay: 0, + Port: "443", + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + }, + }, + }, + }, + }, + Version: statsContainerVersion, + }, + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + kvStore := &kvstore.Memory{} + if input := tc.input(); len(input) > 0 { + if err := kvStore.Set(statsKey, input); err != nil { + t.Fatal(err) + } + } + + root, err := loadStatsContainer(kvStore) + + switch { + case err == nil && tc.expectErr == "": + // all good + + case err != nil && tc.expectErr == "": + t.Fatal("expected", tc.expectErr, "but got", err.Error()) + + case err == nil && tc.expectErr != "": + t.Fatal("expected", tc.expectErr, "but got", err) + + case err != nil && tc.expectErr != "": + if tc.expectErr != err.Error() { + t.Fatal("expected", tc.expectErr, "but got", err.Error()) + } + } + + if diff := cmp.Diff(tc.expectRoot, root); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestStatsManagerCallbacks(t *testing.T) { + type testcase struct { + name string + initialRoot *statsContainer + do func(stats *statsManager) + expectWarnf int + expectRoot *statsContainer + } + + fourtyFiveMinutesAgo := time.Now().Add(-45 * time.Minute) + + cases := []testcase{ + + // When TCP connect fails and the reason is a canceled context + { + name: "OnTCPConnectError with ctx.Error() != nil", + initialRoot: &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{ + "api.ooni.io:443": { + Tactics: map[string]*statsTactic{ + "162.55.247.208:443 sni=www.example.com verify=api.ooni.io": { + CountStarted: 1, + LastUpdated: fourtyFiveMinutesAgo, + Tactic: &httpsDialerTactic{ + Address: "162.55.247.208", + InitialDelay: 0, + Port: "443", + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + }, + }, + }, + }, + }, + Version: statsContainerVersion, + }, + do: func(stats *statsManager) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() // immediately! + + tactic := &httpsDialerTactic{ + Address: "162.55.247.208", + InitialDelay: 0, + Port: "443", + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + } + err := errors.New("generic_timeout_error") + + stats.OnTCPConnectError(ctx, tactic, err) + }, + expectWarnf: 0, + expectRoot: &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{ + "api.ooni.io:443": { + Tactics: map[string]*statsTactic{ + "162.55.247.208:443 sni=www.example.com verify=api.ooni.io": { + CountStarted: 1, + CountTCPConnectInterrupt: 1, + Tactic: &httpsDialerTactic{ + Address: "162.55.247.208", + InitialDelay: 0, + Port: "443", + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + }, + }, + }, + }, + }, + Version: statsContainerVersion, + }, + }, + + // When TCP connect fails and we don't already have a policy record + { + name: "OnTCPConnectError when we are missing the stats record for the domain", + initialRoot: &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{}, + Version: statsContainerVersion, + }, + do: func(stats *statsManager) { + ctx := context.Background() + + tactic := &httpsDialerTactic{ + Address: "162.55.247.208", + InitialDelay: 0, + Port: "443", + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + } + err := errors.New("generic_timeout_error") + + stats.OnTCPConnectError(ctx, tactic, err) + }, + expectWarnf: 1, + expectRoot: &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{}, + Version: statsContainerVersion, + }, + }, + + // When TLS handshake fails and the reason is a canceled context + { + name: "OnTLSHandshakeError with ctx.Error() != nil", + initialRoot: &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{ + "api.ooni.io:443": { + Tactics: map[string]*statsTactic{ + "162.55.247.208:443 sni=www.example.com verify=api.ooni.io": { + CountStarted: 1, + LastUpdated: fourtyFiveMinutesAgo, + Tactic: &httpsDialerTactic{ + Address: "162.55.247.208", + InitialDelay: 0, + Port: "443", + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + }, + }, + }, + }, + }, + Version: statsContainerVersion, + }, + do: func(stats *statsManager) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() // immediately! + + tactic := &httpsDialerTactic{ + Address: "162.55.247.208", + InitialDelay: 0, + Port: "443", + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + } + err := errors.New("generic_timeout_error") + + stats.OnTLSHandshakeError(ctx, tactic, err) + }, + expectWarnf: 0, + expectRoot: &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{ + "api.ooni.io:443": { + Tactics: map[string]*statsTactic{ + "162.55.247.208:443 sni=www.example.com verify=api.ooni.io": { + CountStarted: 1, + CountTLSHandshakeInterrupt: 1, + Tactic: &httpsDialerTactic{ + Address: "162.55.247.208", + InitialDelay: 0, + Port: "443", + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + }, + }, + }, + }, + }, + Version: statsContainerVersion, + }, + }, + + // When TLS handshake fails and we don't already have a policy record + { + name: "OnTLSHandshakeError when we are missing the stats record for the domain", + initialRoot: &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{}, + Version: statsContainerVersion, + }, + do: func(stats *statsManager) { + ctx := context.Background() + + tactic := &httpsDialerTactic{ + Address: "162.55.247.208", + InitialDelay: 0, + Port: "443", + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + } + err := errors.New("generic_timeout_error") + + stats.OnTLSHandshakeError(ctx, tactic, err) + }, + expectWarnf: 1, + expectRoot: &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{}, + Version: statsContainerVersion, + }, + }, + + // When TLS verification fails and we don't already have a policy record + { + name: "OnTLSVerifyError when we are missing the stats record for the domain", + initialRoot: &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{}, + Version: statsContainerVersion, + }, + do: func(stats *statsManager) { + tactic := &httpsDialerTactic{ + Address: "162.55.247.208", + InitialDelay: 0, + Port: "443", + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + } + err := errors.New("generic_timeout_error") + + stats.OnTLSVerifyError(tactic, err) + }, + expectWarnf: 1, + expectRoot: &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{}, + Version: statsContainerVersion, + }, + }, + + // With success when we don't already have a policy record + { + name: "OnSuccess when we are missing the stats record for the domain", + initialRoot: &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{}, + Version: statsContainerVersion, + }, + do: func(stats *statsManager) { + tactic := &httpsDialerTactic{ + Address: "162.55.247.208", + InitialDelay: 0, + Port: "443", + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + } + + stats.OnSuccess(tactic) + }, + expectWarnf: 1, + expectRoot: &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{}, + Version: statsContainerVersion, + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // configure the initial value of the stats + kvStore := &kvstore.Memory{} + if err := kvStore.Set(statsKey, runtimex.Try1(json.Marshal(tc.initialRoot))); err != nil { + t.Fatal(err) + } + + // create logger counting the number Warnf invocations + var warnfCount int + logger := &mocks.Logger{ + MockWarnf: func(format string, v ...any) { + warnfCount++ + }, + } + + // create the stats manager + const trimInterval = 30 * time.Second + stats := newStatsManager(kvStore, logger, trimInterval) + defer stats.Close() + + // invoke the proper stats callback + tc.do(stats) + + // close the stats to trigger a kvstore write + if err := stats.Close(); err != nil { + t.Fatal(err) + } + + // extract the possibly modified stats from the kvstore + var root *statsContainer + rawRoot, err := kvStore.Get(statsKey) + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(rawRoot, &root); err != nil { + t.Fatal(err) + } + + // make sure the stats are the ones we expect + diffOptions := []cmp.Option{ + cmpopts.IgnoreFields(statsTactic{}, "LastUpdated"), + } + if diff := cmp.Diff(tc.expectRoot, root, diffOptions...); diff != "" { + t.Fatal(diff) + } + + // make sure we logged if necessary + if tc.expectWarnf != warnfCount { + t.Fatal("expected", tc.expectWarnf, "got", warnfCount) + } + }) + } +} + +// Make sure that we can safely obtain statistics for a domain and a port. +func TestStatsManagerLookupTactics(t *testing.T) { + + // prepare the content of the stats + twentyMinutesAgo := time.Now().Add(-20 * time.Minute) + + expectTactics := []*statsTactic{{ + CountStarted: 5, + CountTCPConnectError: 0, + CountTCPConnectInterrupt: 0, + CountTLSHandshakeError: 0, + CountTLSHandshakeInterrupt: 0, + CountTLSVerificationError: 0, + CountSuccess: 5, + HistoTCPConnectError: map[string]int64{}, + HistoTLSHandshakeError: map[string]int64{}, + HistoTLSVerificationError: map[string]int64{}, + LastUpdated: twentyMinutesAgo, + Tactic: &httpsDialerTactic{ + Address: "162.55.247.208", + InitialDelay: 0, + Port: "443", + SNI: "www.repubblica.it", + VerifyHostname: "api.ooni.io", + }, + }, { + CountStarted: 1, + CountTCPConnectError: 0, + CountTCPConnectInterrupt: 0, + CountTLSHandshakeError: 0, + CountTLSHandshakeInterrupt: 0, + CountTLSVerificationError: 0, + CountSuccess: 1, + HistoTCPConnectError: map[string]int64{}, + HistoTLSHandshakeError: map[string]int64{}, + HistoTLSVerificationError: map[string]int64{}, + LastUpdated: twentyMinutesAgo, + Tactic: &httpsDialerTactic{ + Address: "162.55.247.208", + InitialDelay: 0, + Port: "443", + SNI: "www.kernel.org", + VerifyHostname: "api.ooni.io", + }, + }, { + CountStarted: 3, + CountTCPConnectError: 0, + CountTCPConnectInterrupt: 0, + CountTLSHandshakeError: 0, + CountTLSHandshakeInterrupt: 0, + CountTLSVerificationError: 0, + CountSuccess: 3, + HistoTCPConnectError: map[string]int64{}, + HistoTLSHandshakeError: map[string]int64{}, + HistoTLSVerificationError: map[string]int64{}, + LastUpdated: twentyMinutesAgo, + Tactic: &httpsDialerTactic{ + Address: "162.55.247.208", + InitialDelay: 0, + Port: "443", + SNI: "theconversation.com", + VerifyHostname: "api.ooni.io", + }, + }} + + expectContainer := &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{ + "api.ooni.io:443": { + Tactics: map[string]*statsTactic{}, + }, + }, + Version: statsContainerVersion, + } + + for _, tactic := range expectTactics { + expectContainer.DomainEndpoints["api.ooni.io:443"].Tactics[tactic.Tactic.tacticSummaryKey()] = tactic + } + + // configure the initial value of the stats + kvStore := &kvstore.Memory{} + if err := kvStore.Set(statsKey, runtimex.Try1(json.Marshal(expectContainer))); err != nil { + t.Fatal(err) + } + + // create the stats manager + const trimInterval = 30 * time.Second + stats := newStatsManager(kvStore, log.Log, trimInterval) + defer stats.Close() + + t.Run("when we're searching for a domain endpoint we know about", func(t *testing.T) { + // obtain tactics + tactics, good := stats.LookupTactics("api.ooni.io", "443") + if !good { + t.Fatal("expected good") + } + if len(tactics) != 3 { + t.Fatal("unexpected tactics length") + } + + // sort obtained tactics lexicographically + sort.SliceStable(tactics, func(i, j int) bool { + return tactics[i].Tactic.tacticSummaryKey() < tactics[j].Tactic.tacticSummaryKey() + }) + + // sort the initial tactics as well + sort.SliceStable(expectTactics, func(i, j int) bool { + return expectTactics[i].Tactic.tacticSummaryKey() < expectTactics[j].Tactic.tacticSummaryKey() + }) + + // compare once we have sorted + if diff := cmp.Diff(expectTactics, tactics); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("when we don't have information about a domain endpoint", func(t *testing.T) { + // obtain tactics + tactics, good := stats.LookupTactics("api.ooni.io", "444") // note: different port! + if good { + t.Fatal("expected !good") + } + if len(tactics) != 0 { + t.Fatal("unexpected tactics length") + } + }) + + t.Run("when the stats manager is manually configured to have an empty container", func(t *testing.T) { + stats := &statsManager{ + container: &statsContainer{ /* explicitly empty */ }, + kvStore: kvStore, + logger: model.DiscardLogger, + mu: sync.Mutex{}, + } + tactics, good := stats.LookupTactics("api.ooni.io", "443") + if good { + t.Fatal("expected !good") + } + if len(tactics) != 0 { + t.Fatal("unexpected tactics length") + } + }) + + t.Run("when the stats manager is manually configured to have nil tactics", func(t *testing.T) { + stats := &statsManager{ + container: &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{ + "api.ooni.io:443": nil, + }, + Version: 0, + }, + kvStore: kvStore, + logger: model.DiscardLogger, + mu: sync.Mutex{}, + } + tactics, good := stats.LookupTactics("api.ooni.io", "443") + if good { + t.Fatal("expected !good") + } + if len(tactics) != 0 { + t.Fatal("unexpected tactics length") + } + }) + + t.Run("when the stats manager is manually configured to have empty tactics", func(t *testing.T) { + stats := &statsManager{ + container: &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{ + "api.ooni.io:443": { /* explicitly left empty */ }, + }, + Version: 0, + }, + kvStore: kvStore, + logger: model.DiscardLogger, + mu: sync.Mutex{}, + } + tactics, good := stats.LookupTactics("api.ooni.io", "443") + if good { + t.Fatal("expected !good") + } + if len(tactics) != 0 { + t.Fatal("unexpected tactics length") + } + }) +} + +func TestStatsSafeIncrementMapStringInt64(t *testing.T) { + t.Run("with a nil map", func(t *testing.T) { + var m map[string]int64 + statsSafeIncrementMapStringInt64(&m, "foo") + if m["foo"] != 1 { + t.Fatal("unexpected result") + } + }) + + t.Run("with a non-nil map", func(t *testing.T) { + m := make(map[string]int64) + statsSafeIncrementMapStringInt64(&m, "foo") + if m["foo"] != 1 { + t.Fatal("unexpected result") + } + }) + + t.Run("with an already-initialized map", func(t *testing.T) { + m := make(map[string]int64) + m["foo"] = 16 + statsSafeIncrementMapStringInt64(&m, "foo") + if m["foo"] != 17 { + t.Fatal("unexpected result") + } + }) +} + +func TestStatsContainer(t *testing.T) { + t.Run("GetStatsTacticLocked", func(t *testing.T) { + t.Run("is robust with respect to c.DomainEndpoints containing a nil entry", func(t *testing.T) { + sc := &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{ + "api.ooni.io:443": nil, + }, + Version: statsContainerVersion, + } + tactic := &httpsDialerTactic{ + Address: "162.55.247.208", + InitialDelay: 0, + Port: "443", + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + } + record, good := sc.GetStatsTacticLocked(tactic) + if good { + t.Fatal("expected not good") + } + if record != nil { + t.Fatal("expected nil") + } + }) + }) +} + +func TestStatsNilSafeSuccessRate(t *testing.T) { + t.Run("with nil entry", func(t *testing.T) { + var st *statsTactic + if statsNilSafeSuccessRate(st) != 0 { + t.Fatal("unexpected result") + } + }) + + t.Run("with non-nil entry", func(t *testing.T) { + st := &statsTactic{ + CountStarted: 10, + CountSuccess: 5, + } + if statsNilSafeSuccessRate(st) != 0.5 { + t.Fatal("unexpected result") + } + }) +} + +func TestStatsNilSafeLastUpdated(t *testing.T) { + t.Run("with nil entry", func(t *testing.T) { + var st *statsTactic + if !statsNilSafeLastUpdated(st).IsZero() { + t.Fatal("unexpected result") + } + }) + + t.Run("with non-nil entry", func(t *testing.T) { + expect := time.Now() + st := &statsTactic{ + LastUpdated: expect, + } + if statsNilSafeLastUpdated(st) != expect { + t.Fatal("unexpected result") + } + }) +} + +func TestStatsNilSafeCountSuccess(t *testing.T) { + t.Run("with nil entry", func(t *testing.T) { + var st *statsTactic + if statsNilSafeCountSuccess(st) != 0 { + t.Fatal("unexpected result") + } + }) + + t.Run("with non-nil entry", func(t *testing.T) { + st := &statsTactic{ + CountSuccess: 11, + } + if statsNilSafeCountSuccess(st) != 11 { + t.Fatal("unexpected result") + } + }) +} + +func TestStatsDefensivelySortTacticsByDescendingSuccessRateWithAcceptPredicate(t *testing.T) { + now := time.Now() + + // expect shows what we expect to see in output + expect := []*statsTactic{ + + // this one should be first because it has 100% success rate + // and the highest number of successes + { + CountStarted: 5, + CountSuccess: 5, + LastUpdated: now.Add(-5 * time.Second), + Tactic: &httpsDialerTactic{ + Address: "130.192.91.211", + InitialDelay: 0, + Port: "443", + SNI: "www.repubblica.it", + VerifyHostname: "shelob.polito.it", + }, + }, + + // this one should be second because it has less successes + // than the first one albeit the same last updated + { + CountStarted: 4, + CountSuccess: 4, + LastUpdated: now.Add(-5 * time.Second), + Tactic: &httpsDialerTactic{ + Address: "130.192.91.211", + InitialDelay: 0, + Port: "443", + SNI: "www.ilfattoquotidiano.it", + VerifyHostname: "shelob.polito.it", + }, + }, + + // this one should be third because it is a bit older + // albeit it has the same number of successes + { + CountStarted: 4, + CountSuccess: 4, + LastUpdated: now.Add(-7 * time.Second), + Tactic: &httpsDialerTactic{ + Address: "130.192.91.211", + InitialDelay: 0, + Port: "443", + SNI: "www.ilpost.it", + VerifyHostname: "shelob.polito.it", + }, + }, + + // this one should come fourth because it has a lower success rate + { + CountStarted: 100, + CountSuccess: 95, + LastUpdated: now.Add(-2 * time.Second), + Tactic: &httpsDialerTactic{ + Address: "130.192.91.211", + InitialDelay: 0, + Port: "443", + SNI: "www.polito.it", + VerifyHostname: "shelob.polito.it", + }, + }, + } + + // input contains the input we provide, which should contain + // a mixture of the above entries together with a bunch of + // entries with very bad values + input := []*statsTactic{ + + // this is the one that should sort last in output + expect[3], + + // a nil entry is obviously a good test case + nil, + + // an entry with a nil Tactic is also quite annoying + { + CountStarted: 55, + CountSuccess: 55, + LastUpdated: now.Add(-3 * time.Second), + Tactic: nil, + }, + + expect[1], + expect[2], + + // another nil entry because why not + nil, + + // another entry with nil Tactic because why not + { + CountStarted: 101, + CountSuccess: 44, + LastUpdated: now.Add(-33 * time.Second), + Tactic: nil, + }, + + // a legitimate entry that is going to be filtered out + // by a custom filtering function + // + // otherwise, this one should be the first entry + { + CountStarted: 128, + CountSuccess: 128, + LastUpdated: now.Add(-130 * time.Millisecond), + Tactic: &httpsDialerTactic{ + Address: "130.192.91.211", + InitialDelay: 0, + Port: "443", + SNI: "kernel.org", + VerifyHostname: "shelob.polito.it", + }, + }, + + expect[0], + } + + got := statsDefensivelySortTacticsByDescendingSuccessRateWithAcceptPredicate( + input, func(st *statsTactic) bool { + return st != nil && st.Tactic != nil && strings.HasSuffix(st.Tactic.SNI, ".it") + }, + ) + + if diff := cmp.Diff(expect, got); diff != "" { + t.Fatal(diff) + } +} + +func TestStatsDomainEndpointPruneEntries(t *testing.T) { + t.Run("rejects tactics with empty summary, nil tactics and with nil .Tactics", func(t *testing.T) { + input := &statsDomainEndpoint{ + Tactics: map[string]*statsTactic{ + // empty summary + "": { + Tactic: &httpsDialerTactic{}, + }, + + // nil tactic + "antani": nil, + + // nil .Tactic + "foo": { + Tactic: nil, + }, + }, + } + + expect := &statsDomainEndpoint{ + Tactics: map[string]*statsTactic{}, + } + + got := statsDomainEndpointPruneEntries(input) + + if diff := cmp.Diff(expect, got); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("prunes entries older than one week", func(t *testing.T) { + now := time.Now() + + input := &statsDomainEndpoint{ + Tactics: map[string]*statsTactic{ + "130.192.91.211:443 sni=polito.it verify=shelob.polito.it": { + CountStarted: 10, + CountSuccess: 10, + LastUpdated: now.Add(-24 * time.Hour * 8), + Tactic: &httpsDialerTactic{ + Address: "130.192.91.211", + InitialDelay: 0, + Port: "443", + SNI: "polito.it", + VerifyHostname: "shelob.polito.it", + }, + }, + "130.192.91.211:443 sni=garr.it verify=shelob.polito.it": { + CountStarted: 10, + CountSuccess: 7, + LastUpdated: now.Add(-24 * time.Hour * 6), + Tactic: &httpsDialerTactic{ + Address: "130.192.91.211", + InitialDelay: 0, + Port: "443", + SNI: "garr.it", + VerifyHostname: "shelob.polito.it", + }, + }, + }, + } + + expect := &statsDomainEndpoint{ + Tactics: map[string]*statsTactic{ + "130.192.91.211:443 sni=garr.it verify=shelob.polito.it": { + CountStarted: 10, + CountSuccess: 7, + LastUpdated: now.Add(-24 * time.Hour * 6), + Tactic: &httpsDialerTactic{ + Address: "130.192.91.211", + InitialDelay: 0, + Port: "443", + SNI: "garr.it", + VerifyHostname: "shelob.polito.it", + }, + }, + }, + } + + got := statsDomainEndpointPruneEntries(input) + + if diff := cmp.Diff(expect, got); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("reduces the number of entries", func(t *testing.T) { + var ( + inputs []*statsTactic + ) + + expect := &statsDomainEndpoint{ + Tactics: map[string]*statsTactic{}, + } + now := time.Now() + + // create successful entries + for idx := int64(0); idx < 7; idx++ { + tactic := &statsTactic{ + CountStarted: 10, + CountTCPConnectError: idx, + CountSuccess: 10 - idx, + HistoTCPConnectError: map[string]int64{ + "generic_timeout_error": idx, + }, + LastUpdated: now.Add(-time.Duration(idx) * time.Second), + Tactic: &httpsDialerTactic{ + Address: "130.192.91.211", + InitialDelay: 0, + Port: "443", + SNI: fmt.Sprintf("host%d.garr.it", idx), + VerifyHostname: "shelob.polito.it", + }, + } + inputs = append(inputs, tactic) + + // note how we're making entries such that each entry is less + // good than the subsequent one in terms of the success rate + expect.Tactics[tactic.Tactic.tacticSummaryKey()] = tactic + } + + // create failed entries + for idx := int64(7); idx < 255; idx++ { + tactic := &statsTactic{ + CountStarted: idx, + CountTCPConnectError: idx, + HistoTCPConnectError: map[string]int64{ + "generic_timeout_error": idx, + }, + LastUpdated: now.Add(-time.Duration(idx) * time.Second), + Tactic: &httpsDialerTactic{ + Address: "130.192.91.211", + InitialDelay: 0, + Port: "443", + SNI: fmt.Sprintf("host%d.garr.it", idx), + VerifyHostname: "shelob.polito.it", + }, + } + inputs = append(inputs, tactic) + + // we need three extra failures in the expected results + // and they must sort after successful entries + if idx < 10 { + expect.Tactics[tactic.Tactic.tacticSummaryKey()] = tactic + } + } + + // shuffle the input order + r := rand.New(rand.NewSource(time.Now().UnixNano())) + r.Shuffle(len(inputs), func(i, j int) { + inputs[i], inputs[j] = inputs[j], inputs[i] + }) + + // fill the input struct + input := &statsDomainEndpoint{ + Tactics: map[string]*statsTactic{}, + } + for _, entry := range inputs { + input.Tactics[entry.Tactic.tacticSummaryKey()] = entry + } + + got := statsDomainEndpointPruneEntries(input) + + // log the results because it may be useful in case something is wrong + t.Log(string(runtimex.Try1(json.MarshalIndent(got, "", " ")))) + + if diff := cmp.Diff(expect, got); diff != "" { + t.Fatal(diff) + } + }) +} + +func TestStatsContainerPruneEntries(t *testing.T) { + t.Run("with a nil .DomainEndpoints field", func(t *testing.T) { + input := &statsContainer{ + DomainEndpoints: nil, // explicitly + Version: statsContainerVersion, + } + + output := statsContainerPruneEntries(input) + + expect := &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{}, + Version: statsContainerVersion, + } + + if diff := cmp.Diff(expect, output); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("we filter out empty summary, nil and nil/empty .Tactics", func(t *testing.T) { + input := &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{ + + // empty summary + "": {}, + + // nil entry + "antani": nil, + + // nil .Tactics + "foo": { + Tactics: nil, + }, + + // empty .Tactics + "bar": { + Tactics: map[string]*statsTactic{}, + }, + }, + Version: statsContainerVersion, + } + + output := statsContainerPruneEntries(input) + + expect := &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{}, + Version: statsContainerVersion, + } + + if diff := cmp.Diff(expect, output); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("we avoid including into the results expired entries", func(t *testing.T) { + input := &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{ + "shelob.polito.it:443": { + Tactics: map[string]*statsTactic{ + "130.192.91.211:443 sni=garr.it verify=shelob.polito.it": { + CountStarted: 10, + CountSuccess: 10, + LastUpdated: time.Time{}, // a long time ago! + Tactic: &httpsDialerTactic{ + Address: "130.192.91.211", + InitialDelay: 0, + Port: "443", + SNI: "garr.it", + VerifyHostname: "shelob.polito.it", + }, + }, + }, + }, + }, + Version: statsContainerVersion, + } + + output := statsContainerPruneEntries(input) + + expect := &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{}, + Version: statsContainerVersion, + } + + if diff := cmp.Diff(expect, output); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("on a successful case", func(t *testing.T) { + expectTactic := &statsTactic{ + CountStarted: 10, + CountSuccess: 10, + LastUpdated: time.Now().Add(-60 * time.Second), // recently + Tactic: &httpsDialerTactic{ + Address: "130.192.91.211", + InitialDelay: 0, + Port: "443", + SNI: "polito.it", + VerifyHostname: "shelob.polito.it", + }, + } + expectTacticSummary := expectTactic.Tactic.tacticSummaryKey() + + input := &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{ + "shelob.polito.it:443": { + Tactics: map[string]*statsTactic{ + "130.192.91.211:443 sni=garr.it verify=shelob.polito.it": { + CountStarted: 10, + CountSuccess: 10, + LastUpdated: time.Time{}, // a long time ago! + Tactic: &httpsDialerTactic{ + Address: "130.192.91.211", + InitialDelay: 0, + Port: "443", + SNI: "garr.it", + VerifyHostname: "shelob.polito.it", + }, + }, + expectTacticSummary: expectTactic, + }, + }, + }, + Version: statsContainerVersion, + } + + output := statsContainerPruneEntries(input) + + expect := &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{ + "shelob.polito.it:443": { + Tactics: map[string]*statsTactic{ + expectTacticSummary: expectTactic, + }, + }, + }, + Version: statsContainerVersion, + } + + if diff := cmp.Diff(expect, output); diff != "" { + t.Fatal(diff) + } + }) +} + +func TestStatsManagerTrimEntriesConcurrently(t *testing.T) { + // start stats manager that trims very frequently + store := &kvstore.Memory{} + sm := newStatsManager(store, model.DiscardLogger, 1*time.Second) + + // obtain exclusive access + sm.mu.Lock() + + // insert some data that needs pruning + sm.container = &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{ + "shelob.polito.it:443": { + Tactics: map[string]*statsTactic{ + "130.192.91.211:443 sni=garr.it verify=shelob.polito.it": { + CountStarted: 10, + CountSuccess: 10, + LastUpdated: time.Time{}, // a long time ago! + Tactic: &httpsDialerTactic{ + Address: "130.192.91.211", + InitialDelay: 0, + Port: "443", + SNI: "garr.it", + VerifyHostname: "shelob.polito.it", + }, + }, + }, + }, + }, + Version: statsContainerVersion, + } + + // let the worker continue to run + sm.mu.Unlock() + + // wait for pruning to happen + <-sm.pruned + + // order the background goroutine to shutdown + // and wait for the shutdown to complete + if err := sm.Close(); err != nil { + t.Fatal(err) + } + + // now check what actually ended up being written; note that we expect + // to see empty domain endpoints because we added a too old entry + expectedData := []byte(`{"DomainEndpoints":{},"Version":5}`) + data, err := store.Get(statsKey) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(expectedData, data); diff != "" { + t.Fatal(diff) + } +} diff --git a/pkg/enginenetx/statspolicy.go b/pkg/enginenetx/statspolicy.go new file mode 100644 index 000000000..68a708f6e --- /dev/null +++ b/pkg/enginenetx/statspolicy.go @@ -0,0 +1,96 @@ +package enginenetx + +// +// Scheduling policy based on stats that fallbacks to +// another policy after it has produced all the working +// tactics we can produce given the current stats. +// + +import ( + "context" + + "github.com/ooni/probe-engine/pkg/runtimex" +) + +// statsPolicy is a policy that schedules tactics already known +// to work based on statistics and defers to a fallback policy +// once it has generated all the tactics known to work. +// +// The zero value of this struct is invalid; please, make sure you +// fill all the fields marked as MANDATORY. +type statsPolicy struct { + // Fallback is the MANDATORY fallback policy. + Fallback httpsDialerPolicy + + // Stats is the MANDATORY stats manager. + Stats *statsManager +} + +var _ httpsDialerPolicy = &statsPolicy{} + +// LookupTactics implements HTTPSDialerPolicy. +func (p *statsPolicy) LookupTactics(ctx context.Context, domain string, port string) <-chan *httpsDialerTactic { + out := make(chan *httpsDialerTactic) + + go func() { + defer close(out) // make sure the parent knows when we're done + index := 0 + + // useful to make sure we don't emit two equal policy in a single run + uniq := make(map[string]int) + + // function that emits a given tactic unless we already emitted it + maybeEmitTactic := func(t *httpsDialerTactic) { + // as a safety mechanism let's gracefully handle the + // case in which the tactic is nil + if t == nil { + return + } + + // handle the case in which we already emitted a policy + key := t.tacticSummaryKey() + if uniq[key] > 0 { + return + } + uniq[key]++ + + // 🚀!!! + t.InitialDelay = happyEyeballsDelay(index) + index += 1 + out <- t + } + + // give priority to what we know from stats + for _, t := range statsPolicyPostProcessTactics(p.Stats.LookupTactics(domain, port)) { + maybeEmitTactic(t) + } + + // fallback to the secondary policy + for t := range p.Fallback.LookupTactics(ctx, domain, port) { + maybeEmitTactic(t) + } + }() + + return out +} + +func statsPolicyPostProcessTactics(tactics []*statsTactic, good bool) (out []*httpsDialerTactic) { + // when good is false, it means p.Stats.LookupTactics failed + if !good { + return + } + + // only keep well-formed successful entries + onlySuccesses := statsDefensivelySortTacticsByDescendingSuccessRateWithAcceptPredicate( + tactics, func(st *statsTactic) bool { + return st != nil && st.Tactic != nil && st.CountSuccess > 0 + }, + ) + + // convert the statsTactic list into a list of tactics + for _, t := range onlySuccesses { + runtimex.Assert(t != nil && t.Tactic != nil && t.CountSuccess > 0, "expected well-formed *statsTactic") + out = append(out, t.Tactic) + } + return +} diff --git a/pkg/enginenetx/statspolicy_test.go b/pkg/enginenetx/statspolicy_test.go new file mode 100644 index 000000000..959152536 --- /dev/null +++ b/pkg/enginenetx/statspolicy_test.go @@ -0,0 +1,403 @@ +package enginenetx + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/apex/log" + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-engine/pkg/kvstore" + "github.com/ooni/probe-engine/pkg/mocks" + "github.com/ooni/probe-engine/pkg/netemx" + "github.com/ooni/probe-engine/pkg/netxlite" + "github.com/ooni/probe-engine/pkg/runtimex" +) + +func TestStatsPolicyWorkingAsIntended(t *testing.T) { + // prepare the content of the stats + twentyMinutesAgo := time.Now().Add(-20 * time.Minute) + + const bridgeAddress = netemx.AddressApiOONIIo + + expectTacticsStats := []*statsTactic{{ + CountStarted: 5, + CountTCPConnectError: 0, + CountTCPConnectInterrupt: 0, + CountTLSHandshakeError: 0, + CountTLSHandshakeInterrupt: 0, + CountTLSVerificationError: 0, + CountSuccess: 5, // this one always succeeds, so it should be there + HistoTCPConnectError: map[string]int64{}, + HistoTLSHandshakeError: map[string]int64{}, + HistoTLSVerificationError: map[string]int64{}, + LastUpdated: twentyMinutesAgo, + Tactic: &httpsDialerTactic{ + Address: bridgeAddress, + InitialDelay: 0, + Port: "443", + SNI: "www.repubblica.it", + VerifyHostname: "api.ooni.io", + }, + }, { + CountStarted: 3, + CountTCPConnectError: 0, + CountTCPConnectInterrupt: 0, + CountTLSHandshakeError: 1, + CountTLSHandshakeInterrupt: 0, + CountTLSVerificationError: 0, + CountSuccess: 2, // this one sometimes succeded so it should be added + HistoTCPConnectError: map[string]int64{}, + HistoTLSHandshakeError: map[string]int64{}, + HistoTLSVerificationError: map[string]int64{}, + LastUpdated: twentyMinutesAgo, + Tactic: &httpsDialerTactic{ + Address: bridgeAddress, + InitialDelay: 0, + Port: "443", + SNI: "www.kernel.org", + VerifyHostname: "api.ooni.io", + }, + }, { + CountStarted: 3, + CountTCPConnectError: 0, + CountTCPConnectInterrupt: 0, + CountTLSHandshakeError: 3, // this one always failed, so should not be added + CountTLSHandshakeInterrupt: 0, + CountTLSVerificationError: 0, + CountSuccess: 0, + HistoTCPConnectError: map[string]int64{}, + HistoTLSHandshakeError: map[string]int64{}, + HistoTLSVerificationError: map[string]int64{}, + LastUpdated: twentyMinutesAgo, + Tactic: &httpsDialerTactic{ + Address: bridgeAddress, + InitialDelay: 0, + Port: "443", + SNI: "theconversation.com", + VerifyHostname: "api.ooni.io", + }, + }, { + CountStarted: 4, + CountTCPConnectError: 0, + CountTCPConnectInterrupt: 0, + CountTLSHandshakeError: 0, + CountTLSHandshakeInterrupt: 0, + CountTLSVerificationError: 0, + CountSuccess: 4, + HistoTCPConnectError: map[string]int64{}, + HistoTLSHandshakeError: map[string]int64{}, + HistoTLSVerificationError: map[string]int64{}, + LastUpdated: twentyMinutesAgo, + Tactic: nil, // the nil policy here should cause this entry to be filtered out + }, { + CountStarted: 0, + CountTCPConnectError: 0, + CountTCPConnectInterrupt: 0, + CountTLSHandshakeError: 0, + CountTLSHandshakeInterrupt: 0, + CountTLSVerificationError: 0, + CountSuccess: 0, + HistoTCPConnectError: map[string]int64{}, + HistoTLSHandshakeError: map[string]int64{}, + HistoTLSVerificationError: map[string]int64{}, + LastUpdated: time.Time{}, // the zero time should exclude this one + Tactic: &httpsDialerTactic{ + Address: bridgeAddress, + InitialDelay: 0, + Port: "443", + SNI: "ilpost.it", + VerifyHostname: "api.ooni.io", + }, + }} + + // createStatsManager creates a stats manager given some baseline stats + createStatsManager := func(domainEndpoint string, tactics ...*statsTactic) *statsManager { + container := &statsContainer{ + DomainEndpoints: map[string]*statsDomainEndpoint{ + domainEndpoint: { + Tactics: map[string]*statsTactic{}, + }, + }, + Version: statsContainerVersion, + } + + for _, tx := range tactics { + if tx.Tactic != nil { + container.DomainEndpoints[domainEndpoint].Tactics[tx.Tactic.tacticSummaryKey()] = tx + } + } + + kvStore := &kvstore.Memory{} + if err := kvStore.Set(statsKey, runtimex.Try1(json.Marshal(container))); err != nil { + t.Fatal(err) + } + + const trimInterval = 30 * time.Second + return newStatsManager(kvStore, log.Log, trimInterval) + } + + t.Run("when we have unique statistics", func(t *testing.T) { + // create stats manager + stats := createStatsManager("api.ooni.io:443", expectTacticsStats...) + defer stats.Close() + + // create the composed policy + policy := &statsPolicy{ + Fallback: &dnsPolicy{ + Logger: log.Log, + Resolver: &mocks.Resolver{ + MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { + switch domain { + case "api.ooni.io": + return []string{bridgeAddress}, nil + default: + return nil, netxlite.ErrOODNSNoSuchHost + } + }, + }, + }, + Stats: stats, + } + + // obtain the tactics from the saved stats + var tactics []*httpsDialerTactic + for entry := range policy.LookupTactics(context.Background(), "api.ooni.io", "443") { + tactics = append(tactics, entry) + } + + // compute the list of results we expect to see from the stats data + var expect []*httpsDialerTactic + idx := 0 + for _, entry := range expectTacticsStats { + if entry.CountSuccess <= 0 || entry.Tactic == nil { + continue // we SHOULD NOT include entries that systematically failed + } + t := entry.Tactic.Clone() + t.InitialDelay = happyEyeballsDelay(idx) + expect = append(expect, t) + idx++ + } + + // extend the expected list to include DNS results + expect = append(expect, &httpsDialerTactic{ + Address: bridgeAddress, + InitialDelay: 2 * time.Second, + Port: "443", + SNI: "api.ooni.io", + VerifyHostname: "api.ooni.io", + }) + + // perform the actual comparison + if diff := cmp.Diff(expect, tactics); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("when we have duplicates", func(t *testing.T) { + // add each entry twice to create obvious duplicates + statsWithDupes := []*statsTactic{} + for _, entry := range expectTacticsStats { + statsWithDupes = append(statsWithDupes, entry.Clone()) + statsWithDupes = append(statsWithDupes, entry.Clone()) + } + + // create stats manager + stats := createStatsManager("api.ooni.io:443", statsWithDupes...) + defer stats.Close() + + // create the composed policy + policy := &statsPolicy{ + Fallback: &dnsPolicy{ + Logger: log.Log, + Resolver: &mocks.Resolver{ + MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { + switch domain { + case "api.ooni.io": + // Twice so we try to cause duplicate entries also with the DNS policy + return []string{bridgeAddress, bridgeAddress}, nil + default: + return nil, netxlite.ErrOODNSNoSuchHost + } + }, + }, + }, + Stats: stats, + } + + // obtain the tactics from the saved stats + var tactics []*httpsDialerTactic + for entry := range policy.LookupTactics(context.Background(), "api.ooni.io", "443") { + tactics = append(tactics, entry) + } + + // compute the list of results we expect to see from the stats data + var expect []*httpsDialerTactic + idx := 0 + for _, entry := range expectTacticsStats { + if entry.CountSuccess <= 0 || entry.Tactic == nil { + continue // we SHOULD NOT include entries that systematically failed + } + t := entry.Tactic.Clone() + t.InitialDelay = happyEyeballsDelay(idx) + expect = append(expect, t) + idx++ + } + + // extend the expected list to include DNS results + expect = append(expect, &httpsDialerTactic{ + Address: bridgeAddress, + InitialDelay: 2 * time.Second, + Port: "443", + SNI: "api.ooni.io", + VerifyHostname: "api.ooni.io", + }) + + // perform the actual comparison + if diff := cmp.Diff(expect, tactics); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("we avoid manipulating nil tactics", func(t *testing.T) { + // create stats manager + stats := createStatsManager("api.ooni.io:443", expectTacticsStats...) + defer stats.Close() + + // create the composed policy + policy := &statsPolicy{ + Fallback: &mocksPolicy{ + MockLookupTactics: func(ctx context.Context, domain, port string) <-chan *httpsDialerTactic { + out := make(chan *httpsDialerTactic) + go func() { + defer close(out) + + // explicitly send nil on the channel + out <- nil + }() + return out + }, + }, + Stats: stats, + } + + // obtain the tactics from the saved stats + var tactics []*httpsDialerTactic + for entry := range policy.LookupTactics(context.Background(), "api.ooni.io", "443") { + tactics = append(tactics, entry) + } + + // compute the list of results we expect to see from the stats data + var expect []*httpsDialerTactic + idx := 0 + for _, entry := range expectTacticsStats { + if entry.CountSuccess <= 0 || entry.Tactic == nil { + continue // we SHOULD NOT include entries that systematically failed + } + t := entry.Tactic.Clone() + t.InitialDelay = happyEyeballsDelay(idx) + expect = append(expect, t) + idx++ + } + + // perform the actual comparison + if diff := cmp.Diff(expect, tactics); diff != "" { + t.Fatal(diff) + } + }) +} + +type mocksPolicy struct { + MockLookupTactics func(ctx context.Context, domain string, port string) <-chan *httpsDialerTactic +} + +var _ httpsDialerPolicy = &mocksPolicy{} + +// LookupTactics implements httpsDialerPolicy. +func (p *mocksPolicy) LookupTactics(ctx context.Context, domain string, port string) <-chan *httpsDialerTactic { + return p.MockLookupTactics(ctx, domain, port) +} + +func TestStatsPolicyPostProcessTactics(t *testing.T) { + t.Run("we do nothing when good is false", func(t *testing.T) { + tactics := statsPolicyPostProcessTactics(nil, false) + if len(tactics) != 0 { + t.Fatal("expected zero-lenght return value") + } + }) + + t.Run("we filter out cases in which t or t.Tactic are nil or entry has no successes", func(t *testing.T) { + expected := &statsTactic{ + CountStarted: 7, + CountTCPConnectError: 3, + CountSuccess: 4, + HistoTCPConnectError: map[string]int64{ + "generic_timeout_error": 3, + }, + LastUpdated: time.Now().Add(-11 * time.Second), + Tactic: &httpsDialerTactic{ + Address: "130.192.91.211", + InitialDelay: 0, + Port: "443", + SNI: "garr.it", + VerifyHostname: "shelob.polito.it", + }, + } + + input := []*statsTactic{ + // nil entry + nil, + + // entry with nil tactic + { + CountStarted: 0, + CountTCPConnectError: 0, + CountTCPConnectInterrupt: 0, + CountTLSHandshakeError: 0, + CountTLSHandshakeInterrupt: 0, + CountTLSVerificationError: 0, + CountSuccess: 0, + HistoTCPConnectError: map[string]int64{}, + HistoTLSHandshakeError: map[string]int64{}, + HistoTLSVerificationError: map[string]int64{}, + LastUpdated: time.Time{}, + Tactic: nil, + }, + + // another nil entry + nil, + + // an entry that should be OK + expected, + + // entry that is OK except that it does not contain any + // success so we don't expect to see it + { + CountStarted: 10, + CountTLSHandshakeError: 10, + HistoTLSHandshakeError: map[string]int64{ + "generic_timeout_error": 10, + }, + LastUpdated: time.Now().Add(-4 * time.Second), + Tactic: &httpsDialerTactic{ + Address: "130.192.91.211", + InitialDelay: 0, + Port: "443", + SNI: "polito.it", + VerifyHostname: "shelob.polito.it", + }, + }, + } + + got := statsPolicyPostProcessTactics(input, true) + + if len(got) != 1 { + t.Fatal("expected just one element") + } + + if diff := cmp.Diff(expected.Tactic, got[0]); diff != "" { + t.Fatal(diff) + } + }) +} diff --git a/pkg/enginenetx/userpolicy.go b/pkg/enginenetx/userpolicy.go new file mode 100644 index 000000000..50ce9b1e8 --- /dev/null +++ b/pkg/enginenetx/userpolicy.go @@ -0,0 +1,125 @@ +package enginenetx + +// +// user policy - the possibility of loading a user policy from a JSON +// document named `bridges.conf` in $OONI_HOME/engine that contains +// a specific policy for TLS dialing for specific endpoints. +// +// This policy helps a lot with exploration and experimentation. +// + +import ( + "context" + "errors" + "fmt" + "net" + + "github.com/ooni/probe-engine/pkg/hujsonx" + "github.com/ooni/probe-engine/pkg/model" +) + +// userPolicy is an [httpsDialerPolicy] incorporating verbatim +// a user policy loaded from the engine's key-value store. +// +// This policy is very useful for exploration and experimentation. +type userPolicy struct { + // Fallback is the fallback policy in case the user one does not + // contain a rule for a specific domain. + Fallback httpsDialerPolicy + + // Root is the root of the user policy loaded from disk. + Root *userPolicyRoot +} + +// userPolicyKey is the kvstore key used to retrieve the user policy. +const userPolicyKey = "bridges.conf" + +// errUserPolicyWrongVersion means that the user policy document has the wrong version number. +var errUserPolicyWrongVersion = errors.New("wrong user policy version") + +// newUserPolicy attempts to constructs a user policy using a given fallback +// policy and either returns a good policy or an error. The typical error case is the one +// in which there's no httpsDialerUserPolicyKey in the key-value store. +func newUserPolicy( + kvStore model.KeyValueStore, fallback httpsDialerPolicy) (*userPolicy, error) { + // attempt to read the user policy bytes from the kvstore + data, err := kvStore.Get(userPolicyKey) + if err != nil { + return nil, err + } + + // attempt to parse the user policy using human-readable JSON + var root userPolicyRoot + if err := hujsonx.Unmarshal(data, &root); err != nil { + return nil, err + } + + // make sure the version is OK + if root.Version != userPolicyVersion { + err := fmt.Errorf( + "%s: %w: expected=%d got=%d", + userPolicyKey, + errUserPolicyWrongVersion, + userPolicyVersion, + root.Version, + ) + return nil, err + } + + out := &userPolicy{ + Fallback: fallback, + Root: &root, + } + return out, nil +} + +// userPolicyVersion is the current version of the user policy file. +const userPolicyVersion = 3 + +// userPolicyRoot is the root of the user policy. +type userPolicyRoot struct { + // DomainEndpoints maps each domain endpoint to its policies. + DomainEndpoints map[string][]*httpsDialerTactic + + // Version is the data structure version. + Version int +} + +var _ httpsDialerPolicy = &userPolicy{} + +// LookupTactics implements httpsDialerPolicy. +func (ldp *userPolicy) LookupTactics( + ctx context.Context, domain string, port string) <-chan *httpsDialerTactic { + // check whether an entry exists in the user-provided map, which MAY be nil + // if/when the user has chosen their policy to be as such + tactics, found := ldp.Root.DomainEndpoints[net.JoinHostPort(domain, port)] + if !found { + return ldp.Fallback.LookupTactics(ctx, domain, port) + } + + // note that we also need to fallback when the tactics contains an empty list + // or a list that only contains nil entries + tactics = userPolicyRemoveNilEntries(tactics) + if len(tactics) <= 0 { + return ldp.Fallback.LookupTactics(ctx, domain, port) + } + + // emit the resuults, which may possibly be empty + out := make(chan *httpsDialerTactic) + go func() { + defer close(out) // let the caller know we're done + for _, tactic := range tactics { + out <- tactic + } + }() + return out +} + +func userPolicyRemoveNilEntries(input []*httpsDialerTactic) (output []*httpsDialerTactic) { + for _, entry := range input { + if entry != nil { + output = append(output, entry) + } + } + return +} diff --git a/pkg/enginenetx/userpolicy_test.go b/pkg/enginenetx/userpolicy_test.go new file mode 100644 index 000000000..0774efecc --- /dev/null +++ b/pkg/enginenetx/userpolicy_test.go @@ -0,0 +1,314 @@ +package enginenetx + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/apex/log" + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-engine/pkg/kvstore" + "github.com/ooni/probe-engine/pkg/mocks" + "github.com/ooni/probe-engine/pkg/runtimex" +) + +func TestUserPolicy(t *testing.T) { + t.Run("newUserPolicy", func(t *testing.T) { + // testcase is a test case implemented by this function + type testcase struct { + // name is the test case name + name string + + // key is the key to use for settings the input inside the kvstore + key string + + // input contains the serialized input bytes + input []byte + + // expectErr contains the expected error string or the empty string on success + expectErr string + + // expectRoot contains the expected policy we loaded or nil + expectedPolicy *userPolicy + } + + fallback := &dnsPolicy{} + + cases := []testcase{{ + name: "when there is no key in the kvstore", + key: "", + input: []byte{}, + expectErr: "no such key", + expectedPolicy: nil, + }, { + name: "with nil input", + key: userPolicyKey, + input: nil, + expectErr: "hujson: line 1, column 1: parsing value: unexpected EOF", + expectedPolicy: nil, + }, { + name: "with invalid serialized JSON", + key: userPolicyKey, + input: []byte(`{`), + expectErr: "hujson: line 1, column 2: parsing value: unexpected EOF", + expectedPolicy: nil, + }, { + name: "with empty JSON", + key: userPolicyKey, + input: []byte(`{}`), + expectErr: "bridges.conf: wrong user policy version: expected=3 got=0", + expectedPolicy: nil, + }, { + name: "with real serialized policy", + key: userPolicyKey, + input: (func() []byte { + return runtimex.Try1(json.Marshal(&userPolicyRoot{ + DomainEndpoints: map[string][]*httpsDialerTactic{ + + // Please, note how the input includes explicitly nil entries + // with the purpose of making sure the code can handle them + "api.ooni.io:443": {{ + Address: "162.55.247.208", + InitialDelay: 0, + Port: "443", + SNI: "api.ooni.io", + VerifyHostname: "api.ooni.io", + }, nil, { + Address: "46.101.82.151", + InitialDelay: 300 * time.Millisecond, + Port: "443", + SNI: "api.ooni.io", + VerifyHostname: "api.ooni.io", + }, { + Address: "2a03:b0c0:1:d0::ec4:9001", + InitialDelay: 600 * time.Millisecond, + Port: "443", + SNI: "api.ooni.io", + VerifyHostname: "api.ooni.io", + }, nil, { + Address: "46.101.82.151", + InitialDelay: 3000 * time.Millisecond, + Port: "443", + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + }, { + Address: "2a03:b0c0:1:d0::ec4:9001", + InitialDelay: 3300 * time.Millisecond, + Port: "443", + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + }, nil}, + // + + }, + Version: userPolicyVersion, + })) + })(), + expectErr: "", + expectedPolicy: &userPolicy{ + Fallback: fallback, + Root: &userPolicyRoot{ + DomainEndpoints: map[string][]*httpsDialerTactic{ + "api.ooni.io:443": {{ + Address: "162.55.247.208", + InitialDelay: 0, + Port: "443", + SNI: "api.ooni.io", + VerifyHostname: "api.ooni.io", + }, nil, { + Address: "46.101.82.151", + InitialDelay: 300 * time.Millisecond, + Port: "443", + SNI: "api.ooni.io", + VerifyHostname: "api.ooni.io", + }, { + Address: "2a03:b0c0:1:d0::ec4:9001", + InitialDelay: 600 * time.Millisecond, + Port: "443", + SNI: "api.ooni.io", + VerifyHostname: "api.ooni.io", + }, nil, { + Address: "46.101.82.151", + InitialDelay: 3000 * time.Millisecond, + Port: "443", + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + }, { + Address: "2a03:b0c0:1:d0::ec4:9001", + InitialDelay: 3300 * time.Millisecond, + Port: "443", + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + }, nil}, + }, + Version: userPolicyVersion, + }, + }, + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + kvStore := &kvstore.Memory{} + runtimex.Try0(kvStore.Set(tc.key, tc.input)) + + policy, err := newUserPolicy(kvStore, fallback) + + switch { + case err != nil && tc.expectErr == "": + t.Fatal("expected", tc.expectErr, "got", err) + + case err == nil && tc.expectErr != "": + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr != "": + if diff := cmp.Diff(tc.expectErr, err.Error()); diff != "" { + t.Fatal(diff) + } + + case err == nil && tc.expectErr == "": + // all good + } + + if diff := cmp.Diff(tc.expectedPolicy, policy); diff != "" { + t.Fatal(diff) + } + }) + } + }) + + t.Run("LookupTactics", func(t *testing.T) { + expectedTactic := &httpsDialerTactic{ + Address: "162.55.247.208", + InitialDelay: 0, + Port: "443", + SNI: "www.example.com", + VerifyHostname: "api.ooni.io", + } + userPolicyRoot := &userPolicyRoot{ + DomainEndpoints: map[string][]*httpsDialerTactic{ + // Note that here we're adding explicitly nil entries + // to make sure that the code correctly handles 'em + "api.ooni.io:443": { + nil, + expectedTactic, + nil, + }, + + // We add additional entries to make sure that in those + // cases we are going to fallback as they're basically empty + // and so non-actionable for us. + "api.ooni.xyz:443": nil, + "api.ooni.org:443": {}, + "api.ooni.com:443": {nil, nil, nil}, + }, + Version: userPolicyVersion, + } + kvStore := &kvstore.Memory{} + rawUserPolicyRoot := runtimex.Try1(json.Marshal(userPolicyRoot)) + if err := kvStore.Set(userPolicyKey, rawUserPolicyRoot); err != nil { + t.Fatal(err) + } + + t.Run("with user policy", func(t *testing.T) { + ctx := context.Background() + + policy, err := newUserPolicy(kvStore, nil /* explictly to crash if used */) + if err != nil { + t.Fatal(err) + } + + tactics := policy.LookupTactics(ctx, "api.ooni.io", "443") + got := []*httpsDialerTactic{} + for tactic := range tactics { + t.Logf("%+v", tactic) + got = append(got, tactic) + } + + expect := []*httpsDialerTactic{expectedTactic} + + if diff := cmp.Diff(expect, got); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("we fallback if there is no entry in the user policy", func(t *testing.T) { + ctx := context.Background() + + fallback := &dnsPolicy{ + Logger: log.Log, + Resolver: &mocks.Resolver{ + MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { + return []string{"93.184.216.34"}, nil + }, + }, + } + + policy, err := newUserPolicy(kvStore, fallback) + if err != nil { + t.Fatal(err) + } + + tactics := policy.LookupTactics(ctx, "www.example.com", "443") + got := []*httpsDialerTactic{} + for tactic := range tactics { + t.Logf("%+v", tactic) + got = append(got, tactic) + } + + expect := []*httpsDialerTactic{{ + Address: "93.184.216.34", + InitialDelay: 0, + Port: "443", + SNI: "www.example.com", + VerifyHostname: "www.example.com", + }} + + if diff := cmp.Diff(expect, got); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("we fallback if the entry in the user policy is ~empty", func(t *testing.T) { + ctx := context.Background() + + fallback := &dnsPolicy{ + Logger: log.Log, + Resolver: &mocks.Resolver{ + MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { + return []string{"93.184.216.34"}, nil + }, + }, + } + + policy, err := newUserPolicy(kvStore, fallback) + if err != nil { + t.Fatal(err) + } + + // these cases are specially constructed to be empty/invalid user policies + for _, domain := range []string{"api.ooni.xyz", "api.ooni.org", "api.ooni.com"} { + t.Run(domain, func(t *testing.T) { + tactics := policy.LookupTactics(ctx, domain, "443") + got := []*httpsDialerTactic{} + for tactic := range tactics { + t.Logf("%+v", tactic) + got = append(got, tactic) + } + + expect := []*httpsDialerTactic{{ + Address: "93.184.216.34", + InitialDelay: 0, + Port: "443", + SNI: domain, + VerifyHostname: domain, + }} + + if diff := cmp.Diff(expect, got); diff != "" { + t.Fatal(diff) + } + }) + } + }) + }) +} diff --git a/pkg/sessionresolver/doc.go b/pkg/engineresolver/doc.go similarity index 79% rename from pkg/sessionresolver/doc.go rename to pkg/engineresolver/doc.go index 266312bb2..c0cdb2d99 100644 --- a/pkg/sessionresolver/doc.go +++ b/pkg/engineresolver/doc.go @@ -1,4 +1,4 @@ -// Package sessionresolver contains the resolver used by the session. This +// Package engineresolver contains the resolver used by the OONI engine. This // resolver will try to figure out which is the best service for running // domain name resolutions and will consistently use it. // @@ -13,12 +13,12 @@ // have any preferential ordering. The initial resolutions may be slower // if there are many issues with resolvers. // -// The system resolver is given the lowest priority at the beginning -// but it will of course be the most popular resolver if anything else +// The system resolver is given intermediate priority at the beginning (i.e., +// 0.5) but it will of course be the most popular resolver if anything else // is failing us. (We will still occasionally probe for other working // resolvers and increase their score on success.) // // We also support a socks5 proxy. When such a proxy is configured, // the code WILL skip http3 resolvers AS WELL AS the system // resolver, in an attempt to avoid leaking your queries. -package sessionresolver +package engineresolver diff --git a/pkg/sessionresolver/errwrapper.go b/pkg/engineresolver/errwrapper.go similarity index 96% rename from pkg/sessionresolver/errwrapper.go rename to pkg/engineresolver/errwrapper.go index 8f43a6640..f5e2f24bd 100644 --- a/pkg/sessionresolver/errwrapper.go +++ b/pkg/engineresolver/errwrapper.go @@ -1,4 +1,4 @@ -package sessionresolver +package engineresolver // // Error wrapping diff --git a/pkg/sessionresolver/errwrapper_test.go b/pkg/engineresolver/errwrapper_test.go similarity index 94% rename from pkg/sessionresolver/errwrapper_test.go rename to pkg/engineresolver/errwrapper_test.go index 1db791e5d..3b99f7084 100644 --- a/pkg/sessionresolver/errwrapper_test.go +++ b/pkg/engineresolver/errwrapper_test.go @@ -1,4 +1,4 @@ -package sessionresolver +package engineresolver import ( "errors" diff --git a/pkg/sessionresolver/factory.go b/pkg/engineresolver/factory.go similarity index 88% rename from pkg/sessionresolver/factory.go rename to pkg/engineresolver/factory.go index 20ab47a4c..71f244999 100644 --- a/pkg/sessionresolver/factory.go +++ b/pkg/engineresolver/factory.go @@ -1,4 +1,4 @@ -package sessionresolver +package engineresolver import ( "errors" @@ -80,13 +80,14 @@ func newChildResolverHTTPS( var txp model.HTTPTransport switch http3Enabled { case false: - dialer := netxlite.MaybeWrapWithProxyDialer( - netxlite.NewDialerWithStdlibResolver(logger), - proxyURL, // handles correctly the case where proxyURL is nil - ) + dialer := netxlite.NewDialerWithStdlibResolver(logger) thx := netxlite.NewTLSHandshakerStdlib(logger) tlsDialer := netxlite.NewTLSDialer(dialer, thx) - txp = netxlite.NewHTTPTransport(logger, dialer, tlsDialer) + txp = netxlite.NewHTTPTransportWithOptions( + logger, dialer, tlsDialer, + netxlite.HTTPTransportOptionDisableCompression(false), // defaults to true but compression is fine here + netxlite.HTTPTransportOptionProxyURL(proxyURL), // nil here disables using the proxy + ) case true: txp = netxlite.NewHTTP3TransportStdlib(logger) } diff --git a/pkg/sessionresolver/factory_test.go b/pkg/engineresolver/factory_test.go similarity index 99% rename from pkg/sessionresolver/factory_test.go rename to pkg/engineresolver/factory_test.go index fd44c108f..f5cf51dd8 100644 --- a/pkg/sessionresolver/factory_test.go +++ b/pkg/engineresolver/factory_test.go @@ -1,4 +1,4 @@ -package sessionresolver +package engineresolver import ( "context" diff --git a/pkg/sessionresolver/integration_test.go b/pkg/engineresolver/integration_test.go similarity index 82% rename from pkg/sessionresolver/integration_test.go rename to pkg/engineresolver/integration_test.go index a1c2fd9d6..25124a295 100644 --- a/pkg/sessionresolver/integration_test.go +++ b/pkg/engineresolver/integration_test.go @@ -1,18 +1,18 @@ -package sessionresolver_test +package engineresolver_test import ( "context" "testing" + "github.com/ooni/probe-engine/pkg/engineresolver" "github.com/ooni/probe-engine/pkg/kvstore" - "github.com/ooni/probe-engine/pkg/sessionresolver" ) func TestSessionResolverGood(t *testing.T) { if testing.Short() { t.Skip("skip test in short mode") } - reso := &sessionresolver.Resolver{ + reso := &engineresolver.Resolver{ KVStore: &kvstore.Memory{}, } defer reso.CloseIdleConnections() diff --git a/pkg/sessionresolver/jsoncodec.go b/pkg/engineresolver/jsoncodec.go similarity index 97% rename from pkg/sessionresolver/jsoncodec.go rename to pkg/engineresolver/jsoncodec.go index d88420496..f61846474 100644 --- a/pkg/sessionresolver/jsoncodec.go +++ b/pkg/engineresolver/jsoncodec.go @@ -1,4 +1,4 @@ -package sessionresolver +package engineresolver // // JSON codec diff --git a/pkg/sessionresolver/jsoncodec_test.go b/pkg/engineresolver/jsoncodec_test.go similarity index 97% rename from pkg/sessionresolver/jsoncodec_test.go rename to pkg/engineresolver/jsoncodec_test.go index 77d90c08c..f1436671d 100644 --- a/pkg/sessionresolver/jsoncodec_test.go +++ b/pkg/engineresolver/jsoncodec_test.go @@ -1,4 +1,4 @@ -package sessionresolver +package engineresolver import ( "testing" diff --git a/pkg/sessionresolver/lookup.go b/pkg/engineresolver/lookup.go similarity index 98% rename from pkg/sessionresolver/lookup.go rename to pkg/engineresolver/lookup.go index 10c2dd19b..6ea76c55a 100644 --- a/pkg/sessionresolver/lookup.go +++ b/pkg/engineresolver/lookup.go @@ -1,4 +1,4 @@ -package sessionresolver +package engineresolver // // Actual lookup code diff --git a/pkg/sessionresolver/lookup_test.go b/pkg/engineresolver/lookup_test.go similarity index 97% rename from pkg/sessionresolver/lookup_test.go rename to pkg/engineresolver/lookup_test.go index 8fc68c11b..bf6e99777 100644 --- a/pkg/sessionresolver/lookup_test.go +++ b/pkg/engineresolver/lookup_test.go @@ -1,4 +1,4 @@ -package sessionresolver +package engineresolver import ( "context" diff --git a/pkg/sessionresolver/resolver.go b/pkg/engineresolver/resolver.go similarity index 93% rename from pkg/sessionresolver/resolver.go rename to pkg/engineresolver/resolver.go index 602366129..60f912bd6 100644 --- a/pkg/sessionresolver/resolver.go +++ b/pkg/engineresolver/resolver.go @@ -1,4 +1,4 @@ -package sessionresolver +package engineresolver // // Implementation of model.Resolver @@ -14,7 +14,7 @@ import ( "time" "github.com/ooni/probe-engine/pkg/bytecounter" - "github.com/ooni/probe-engine/pkg/measurexlite" + "github.com/ooni/probe-engine/pkg/logx" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/multierror" ) @@ -109,6 +109,16 @@ func (r *Resolver) LookupHost(ctx context.Context, hostname string) ([]string, e r.logger().Infof("sessionresolver: skipping with proxy: %+v", e) continue // we cannot proxy this URL so ignore it } + + // Hotfix: if the context has been canceled from the outside avoid + // doing a dnslookup, which would mark the resolver as not WAI. + // + // See https://github.com/ooni/probe/issues/2544 + if err := ctx.Err(); err != nil { + me.Add(newErrWrapper(err, e.URL)) + continue + } + addrs, err := r.lookupHost(ctx, e, hostname) if err == nil { return addrs, nil @@ -139,7 +149,7 @@ func (r *Resolver) lookupHost(ctx context.Context, ri *resolverinfo, hostname st ri.Score = 0 // this is a hard error return nil, err } - op := measurexlite.NewOperationLogger( + op := logx.NewOperationLogger( r.logger(), "sessionresolver: lookup %s using %s", hostname, ri.URL) addrs, err := timeLimitedLookup(ctx, re, hostname) op.Stop(err) diff --git a/pkg/sessionresolver/resolver_test.go b/pkg/engineresolver/resolver_test.go similarity index 94% rename from pkg/sessionresolver/resolver_test.go rename to pkg/engineresolver/resolver_test.go index c0d7b6913..8678ff8b5 100644 --- a/pkg/sessionresolver/resolver_test.go +++ b/pkg/engineresolver/resolver_test.go @@ -1,4 +1,4 @@ -package sessionresolver +package engineresolver import ( "context" @@ -34,7 +34,7 @@ func TestAddressWorks(t *testing.T) { } } -func TestTypicalUsageWithFailure(t *testing.T) { +func TestResolverLookupHostUsingACanceledContext(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() // fail immediately reso := &Resolver{KVStore: &kvstore.Memory{}} @@ -47,6 +47,7 @@ func TestTypicalUsageWithFailure(t *testing.T) { t.Fatal("cannot convert error") } for _, child := range me.Children { + // net.DNSError does not include the underlying error // but just a string representing the error. This // means that we need to go down hunting what's the @@ -54,7 +55,7 @@ func TestTypicalUsageWithFailure(t *testing.T) { { var ew *errWrapper if !errors.As(child, &ew) { - t.Fatal("not an instance of errwrapper") + t.Fatalf("not an instance of errwrapper: '%v' [%T]", child, child) } var de *net.DNSError if errors.As(ew, &de) { @@ -64,6 +65,7 @@ func TestTypicalUsageWithFailure(t *testing.T) { continue } } + // otherwise just unwrap and check whether it's // a real context.Canceled error. if !errors.Is(child, context.Canceled) { @@ -73,9 +75,22 @@ func TestTypicalUsageWithFailure(t *testing.T) { if addrs != nil { t.Fatal("expected nil here") } - if len(reso.res) < 1 { - t.Fatal("expected to see some resolvers here") + + // Since https://github.com/ooni/probe-cli/pull/1351 we avoid + // constructing any resolver if we start running with a canceled context + // because of https://github.com/ooni/probe/issues/2544. + // + // In other words, as long as we see zero here we can be confident + // that we're not creating a resolver if the context has been canceled, + // which should imply we're not changing its score. + // + // Heavier refactoring of this package should probably more aggressively + // ensure that we're not changing the score, but for now this test + // is sufficient given that we are committing an hotfix. + if len(reso.res) != 0 { + t.Fatal("expected to see no resolvers here") } + reso.CloseIdleConnections() if len(reso.res) != 0 { t.Fatal("expected to see no resolvers after CloseIdleConnections") diff --git a/pkg/sessionresolver/resolvermaker.go b/pkg/engineresolver/resolvermaker.go similarity index 83% rename from pkg/sessionresolver/resolvermaker.go rename to pkg/engineresolver/resolvermaker.go index 35279297b..f1fe05a9c 100644 --- a/pkg/sessionresolver/resolvermaker.go +++ b/pkg/engineresolver/resolvermaker.go @@ -1,4 +1,4 @@ -package sessionresolver +package engineresolver // // High-level code for creating a new child resolver @@ -43,21 +43,29 @@ var allmakers = []*resolvermaker{{ }} // allbyurl contains all the resolvermakers by URL -var allbyurl map[string]*resolvermaker +var allbyurl = resolverMakeInitialState() -// init fills allbyname and gives a nonzero initial score +// resolverMakeInitialState initializes the initial +// state by giving a nonzero initial score // to all resolvers except for the system resolver. We set -// the system resolver score to zero, so that it's less +// the system resolver score to be 0.5, so that it's less // likely than other resolvers in this list. -func init() { - allbyurl = make(map[string]*resolvermaker) +// +// We used to set this value to 0, but this proved to +// create issues when it was the only available resolver, +// see https://github.com/ooni/probe/issues/2544. +func resolverMakeInitialState() map[string]*resolvermaker { + output := make(map[string]*resolvermaker) rng := rand.New(rand.NewSource(time.Now().UnixNano())) for _, e := range allmakers { - allbyurl[e.url] = e + output[e.url] = e if e.url != systemResolverURL { e.score = rng.Float64() + } else { + e.score = 0.5 } } + return output } // logger returns the configured logger or a default diff --git a/pkg/sessionresolver/resolvermaker_test.go b/pkg/engineresolver/resolvermaker_test.go similarity index 83% rename from pkg/sessionresolver/resolvermaker_test.go rename to pkg/engineresolver/resolvermaker_test.go index 32bfcf449..48c2d5701 100644 --- a/pkg/sessionresolver/resolvermaker_test.go +++ b/pkg/engineresolver/resolvermaker_test.go @@ -1,4 +1,4 @@ -package sessionresolver +package engineresolver import ( "strings" @@ -121,3 +121,19 @@ func TestGetResolverHTTP3(t *testing.T) { t.Fatal("expected true") } } + +func TestResolverMakeInitialState(t *testing.T) { + t.Run("the system resolver has a default score of 0.5", func(t *testing.T) { + state := resolverMakeInitialState() + var okay bool + for URL, entry := range state { + t.Logf("entry: %v %v", URL, *entry) + if URL == systemResolverURL && entry.score == 0.5 { + okay = true + } + } + if !okay { + t.Fatal("expected to see the system resolver with 0.5 as its score") + } + }) +} diff --git a/pkg/sessionresolver/state.go b/pkg/engineresolver/state.go similarity index 94% rename from pkg/sessionresolver/state.go rename to pkg/engineresolver/state.go index 513c320ad..70524d228 100644 --- a/pkg/sessionresolver/state.go +++ b/pkg/engineresolver/state.go @@ -1,4 +1,4 @@ -package sessionresolver +package engineresolver // // Persistent on-disk state @@ -9,6 +9,9 @@ import ( "sort" ) +// TODO(bassosimone): we may want to change the key and rename or +// remove the old file inside the statedir + // storekey is the key used by the key value store to store // the state required by this package. const storekey = "sessionresolver.state" diff --git a/pkg/sessionresolver/state_test.go b/pkg/engineresolver/state_test.go similarity index 99% rename from pkg/sessionresolver/state_test.go rename to pkg/engineresolver/state_test.go index 854927d41..75e0ac851 100644 --- a/pkg/sessionresolver/state_test.go +++ b/pkg/engineresolver/state_test.go @@ -1,4 +1,4 @@ -package sessionresolver +package engineresolver import ( "errors" diff --git a/pkg/experiment/dash/measurer.go b/pkg/experiment/dash/measurer.go index 2f8cfb4f9..01990a88b 100644 --- a/pkg/experiment/dash/measurer.go +++ b/pkg/experiment/dash/measurer.go @@ -10,9 +10,9 @@ import ( "net/http" "github.com/ooni/probe-engine/pkg/legacy/netx" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" - "github.com/ooni/probe-engine/pkg/tracex" ) // Config contains the experiment config. diff --git a/pkg/experiment/dash/runner.go b/pkg/experiment/dash/runner.go index 82bea8804..3a2ed8087 100644 --- a/pkg/experiment/dash/runner.go +++ b/pkg/experiment/dash/runner.go @@ -14,9 +14,9 @@ import ( "github.com/montanaflynn/stats" "github.com/ooni/probe-engine/pkg/humanize" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/runtimex" - "github.com/ooni/probe-engine/pkg/tracex" ) // runnerConfig contains settings for running the dash experiment. This struct diff --git a/pkg/experiment/dash/runner_test.go b/pkg/experiment/dash/runner_test.go index 88666fb9e..626dcbf9f 100644 --- a/pkg/experiment/dash/runner_test.go +++ b/pkg/experiment/dash/runner_test.go @@ -10,9 +10,9 @@ import ( "time" "github.com/apex/log" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/mocks" "github.com/ooni/probe-engine/pkg/model" - "github.com/ooni/probe-engine/pkg/tracex" ) func TestRunnerRunAllPhasesLocateFailure(t *testing.T) { diff --git a/pkg/experiment/dnscheck/dnscheck.go b/pkg/experiment/dnscheck/dnscheck.go index a413aa3cd..49d772b17 100644 --- a/pkg/experiment/dnscheck/dnscheck.go +++ b/pkg/experiment/dnscheck/dnscheck.go @@ -16,9 +16,9 @@ import ( "github.com/ooni/probe-engine/pkg/experiment/urlgetter" "github.com/ooni/probe-engine/pkg/legacy/netx" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/runtimex" - "github.com/ooni/probe-engine/pkg/tracex" ) const ( diff --git a/pkg/experiment/dnscheck/dnscheck_test.go b/pkg/experiment/dnscheck/dnscheck_test.go index aac8fcd29..8fa4bef9a 100644 --- a/pkg/experiment/dnscheck/dnscheck_test.go +++ b/pkg/experiment/dnscheck/dnscheck_test.go @@ -143,6 +143,10 @@ func TestMakeResolverURL(t *testing.T) { } func TestDNSCheckValid(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + measurer := NewExperimentMeasurer(Config{ DefaultAddrs: "1.1.1.1 1.0.0.1", }) @@ -189,6 +193,10 @@ func TestSummaryKeysGeneric(t *testing.T) { } func TestDNSCheckWait(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + endpoints := &Endpoints{ WaitTime: 1 * time.Second, } diff --git a/pkg/experiment/dnsping/dnsping.go b/pkg/experiment/dnsping/dnsping.go index d6dfab5fc..fe8a2db70 100644 --- a/pkg/experiment/dnsping/dnsping.go +++ b/pkg/experiment/dnsping/dnsping.go @@ -12,6 +12,7 @@ import ( "sync" "time" + "github.com/ooni/probe-engine/pkg/logx" "github.com/ooni/probe-engine/pkg/measurexlite" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" @@ -136,7 +137,7 @@ func (m *Measurer) dnsRoundTrip(ctx context.Context, index int64, zeroTime time. defer wg.Done() pings := []*SinglePing{} trace := measurexlite.NewTrace(index, zeroTime) - ol := measurexlite.NewOperationLogger(logger, "DNSPing #%d %s %s", index, address, domain) + ol := logx.NewOperationLogger(logger, "DNSPing #%d %s %s", index, address, domain) // TODO(bassosimone, DecFox): what should we do if the user passes us a resolver with a // domain name in terms of saving its results? Shall we save also the system resolver's lookups? // Shall we, otherwise, pre-resolve the domain name to IP addresses once and for all? In such diff --git a/pkg/experiment/echcheck/handshake.go b/pkg/experiment/echcheck/handshake.go index 279f364f2..c50a0e0ea 100644 --- a/pkg/experiment/echcheck/handshake.go +++ b/pkg/experiment/echcheck/handshake.go @@ -41,9 +41,10 @@ func handshakeWithExtension(ctx context.Context, conn net.Conn, zeroTime time.Ti tracedHandshaker := handshakerConstructor(log.Log, &utls.HelloFirefox_Auto) start := time.Now() - _, connState, err := tracedHandshaker.Handshake(ctx, conn, tlsConfig) + maybeTLSConn, err := tracedHandshaker.Handshake(ctx, conn, tlsConfig) finish := time.Now() + connState := netxlite.MaybeTLSConnectionState(maybeTLSConn) return measurexlite.NewArchivalTLSOrQUICHandshakeResult(0, start.Sub(zeroTime), "tcp", address, tlsConfig, connState, err, finish.Sub(zeroTime)) } diff --git a/pkg/experiment/echcheck/measure_test.go b/pkg/experiment/echcheck/measure_test.go index d20c96b6d..529624413 100644 --- a/pkg/experiment/echcheck/measure_test.go +++ b/pkg/experiment/echcheck/measure_test.go @@ -67,6 +67,10 @@ func TestMeasurerMeasureWithInvalidInput2(t *testing.T) { } func TestMeasurementSuccess(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + sess := &mockable.Session{MockableLogger: log.Log} callbacks := model.NewPrinterCallbacks(sess.Logger()) measurer := NewExperimentMeasurer(Config{}) @@ -84,12 +88,10 @@ func TestMeasurementSuccess(t *testing.T) { } summary, err := measurer.GetSummaryKeys(&model.Measurement{}) - + if err != nil { + t.Fatal(err) + } if summary.(SummaryKeys).IsAnomaly != false { t.Fatal("expected false") } } - -func newsession() model.ExperimentSession { - return &mockable.Session{MockableLogger: log.Log} -} diff --git a/pkg/experiment/echcheck/utls.go b/pkg/experiment/echcheck/utls.go index 3f98ffce1..49b50bed3 100644 --- a/pkg/experiment/echcheck/utls.go +++ b/pkg/experiment/echcheck/utls.go @@ -12,7 +12,6 @@ import ( ) type tlsHandshakerWithExtensions struct { - conn *netxlite.UTLSConn extensions []utls.TLSExtension dl model.DebugLogger id *utls.ClientHelloID @@ -32,18 +31,19 @@ func newHandshakerWithExtensions(extensions []utls.TLSExtension) func(dl model.D } } -func (t *tlsHandshakerWithExtensions) Handshake(ctx context.Context, conn net.Conn, tlsConfig *tls.Config) ( - net.Conn, tls.ConnectionState, error) { - var err error - t.conn, err = netxlite.NewUTLSConn(conn, tlsConfig, t.id) +func (t *tlsHandshakerWithExtensions) Handshake( + ctx context.Context, tcpConn net.Conn, tlsConfig *tls.Config) (model.TLSConn, error) { + tlsConn, err := netxlite.NewUTLSConn(tcpConn, tlsConfig, t.id) runtimex.Assert(err == nil, "unexpected error when creating UTLSConn") if t.extensions != nil && len(t.extensions) != 0 { - t.conn.BuildHandshakeState() - t.conn.Extensions = append(t.conn.Extensions, t.extensions...) + tlsConn.BuildHandshakeState() + tlsConn.Extensions = append(tlsConn.Extensions, t.extensions...) } - err = t.conn.Handshake() + if err := tlsConn.Handshake(); err != nil { + return nil, err + } - return t.conn.NetConn(), t.conn.ConnectionState(), err + return tlsConn, nil } diff --git a/pkg/experiment/echcheck/utls_test.go b/pkg/experiment/echcheck/utls_test.go new file mode 100644 index 000000000..1fd5cfe0e --- /dev/null +++ b/pkg/experiment/echcheck/utls_test.go @@ -0,0 +1,41 @@ +package echcheck + +import ( + "context" + "crypto/tls" + "errors" + "testing" + + "github.com/ooni/probe-engine/pkg/mocks" + "github.com/ooni/probe-engine/pkg/model" + utls "gitlab.com/yawning/utls.git" +) + +func TestTLSHandshakerWithExtension(t *testing.T) { + t.Run("when the TLS handshake fails", func(t *testing.T) { + thx := &tlsHandshakerWithExtensions{ + extensions: []utls.TLSExtension{}, + dl: model.DiscardLogger, + id: &utls.HelloChrome_70, + } + + expected := errors.New("mocked error") + tcpConn := &mocks.Conn{ + MockWrite: func(b []byte) (int, error) { + return 0, expected + }, + } + + tlsConfig := &tls.Config{ + InsecureSkipVerify: true, + } + + tlsConn, err := thx.Handshake(context.Background(), tcpConn, tlsConfig) + if !errors.Is(err, expected) { + t.Fatal(err) + } + if tlsConn != nil { + t.Fatal("expected nil tls conn") + } + }) +} diff --git a/pkg/experiment/fbmessenger/fbmessenger_test.go b/pkg/experiment/fbmessenger/fbmessenger_test.go index 12d9298d6..b7e08e8b0 100644 --- a/pkg/experiment/fbmessenger/fbmessenger_test.go +++ b/pkg/experiment/fbmessenger/fbmessenger_test.go @@ -11,12 +11,12 @@ import ( "github.com/ooni/netem" "github.com/ooni/probe-engine/pkg/experiment/fbmessenger" "github.com/ooni/probe-engine/pkg/experiment/urlgetter" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/mocks" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netemx" "github.com/ooni/probe-engine/pkg/netxlite" "github.com/ooni/probe-engine/pkg/runtimex" - "github.com/ooni/probe-engine/pkg/tracex" ) // servicesAddr is the IP address implementing al fbmessenger services in netem-based tests @@ -74,7 +74,21 @@ func TestMeasurerRun(t *testing.T) { } // create a new test environment - env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer(servicesAddr, netemx.ExampleWebPageHandlerFactory())) + env := netemx.MustNewQAEnv(netemx.QAEnvOptionNetStack( + servicesAddr, + &netemx.HTTPSecureServerFactory{ + Factory: netemx.ExampleWebPageHandlerFactory(), + Ports: []int{443}, + ServerNameMain: "b-api.facebook.com", + ServerNameExtras: []string{ + "b-graph.facebook.com", + "edge-mqtt.facebook.com", + "external.xx.fbcdn.net", + "scontent.xx.fbcdn.net", + "star.c10r.facebook.com", + }, + }, + )) defer env.Close() // configure the DNS for all resolvers @@ -125,7 +139,21 @@ func TestMeasurerRun(t *testing.T) { } // create a new test environment - env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer(servicesAddr, netemx.ExampleWebPageHandlerFactory())) + env := netemx.MustNewQAEnv(netemx.QAEnvOptionNetStack( + servicesAddr, + &netemx.HTTPSecureServerFactory{ + Factory: netemx.ExampleWebPageHandlerFactory(), + Ports: []int{443}, + ServerNameMain: "b-api.facebook.com", + ServerNameExtras: []string{ + "b-graph.facebook.com", + "edge-mqtt.facebook.com", + "external.xx.fbcdn.net", + "scontent.xx.fbcdn.net", + "star.c10r.facebook.com", + }, + }, + )) defer env.Close() // configure the DNS for all resolvers @@ -185,7 +213,21 @@ func TestMeasurerRun(t *testing.T) { }() // create a new test environment - env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer(servicesAddr, netemx.ExampleWebPageHandlerFactory())) + env := netemx.MustNewQAEnv(netemx.QAEnvOptionNetStack( + servicesAddr, + &netemx.HTTPSecureServerFactory{ + Factory: netemx.ExampleWebPageHandlerFactory(), + Ports: []int{443}, + ServerNameMain: "b-api.facebook.com", + ServerNameExtras: []string{ + "b-graph.facebook.com", + "edge-mqtt.facebook.com", + "external.xx.fbcdn.net", + "scontent.xx.fbcdn.net", + "star.c10r.facebook.com", + }, + }, + )) defer env.Close() // configure the DNS for all resolvers @@ -245,7 +287,21 @@ func TestMeasurerRun(t *testing.T) { } // create a new test environment - env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer(servicesAddr, netemx.ExampleWebPageHandlerFactory())) + env := netemx.MustNewQAEnv(netemx.QAEnvOptionNetStack( + servicesAddr, + &netemx.HTTPSecureServerFactory{ + Factory: netemx.ExampleWebPageHandlerFactory(), + Ports: []int{443}, + ServerNameMain: "b-api.facebook.com", + ServerNameExtras: []string{ + "b-graph.facebook.com", + "edge-mqtt.facebook.com", + "external.xx.fbcdn.net", + "scontent.xx.fbcdn.net", + "star.c10r.facebook.com", + }, + }, + )) defer env.Close() // configure all DNS servers but the ISP's one diff --git a/pkg/experiment/hhfm/hhfm.go b/pkg/experiment/hhfm/hhfm.go index 78f90a009..220fd91b0 100644 --- a/pkg/experiment/hhfm/hhfm.go +++ b/pkg/experiment/hhfm/hhfm.go @@ -11,14 +11,13 @@ import ( "fmt" "net" "net/http" - "sort" "time" "github.com/ooni/probe-engine/pkg/experiment/urlgetter" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" "github.com/ooni/probe-engine/pkg/randx" - "github.com/ooni/probe-engine/pkg/tracex" ) const ( @@ -246,27 +245,32 @@ func (tk *TestKeys) FillTampering( } } +// newHeadersFromMap converts the definition of headers used when sending the +// request to something that looks like http.Header. QUIRK: because we need to +// preserve the original random casing of headers (which is what we are in +// fact testing with this experiment), the implementation of this func should +// stay clear of using ordinary http.Header and specifically its .Set func. +func newHeadersFromMap(input map[string]string) map[string][]string { + out := map[string][]string{} + for key, value := range input { + out[key] = []string{value} + } + return out +} + // NewRequestEntryList creates a new []tracex.RequestEntry given a // specific *http.Request and headers with random case. func NewRequestEntryList(req *http.Request, headers map[string]string) (out []tracex.RequestEntry) { + // Note: using the random capitalization headers here + realHeaders := newHeadersFromMap(headers) out = []tracex.RequestEntry{{ Request: tracex.HTTPRequest{ - Headers: make(map[string]tracex.MaybeBinaryValue), - HeadersList: []tracex.HTTPHeader{}, + Headers: model.ArchivalNewHTTPHeadersMap(realHeaders), + HeadersList: model.ArchivalNewHTTPHeadersList(realHeaders), Method: req.Method, URL: req.URL.String(), }, }} - for key, value := range headers { - // Using the random capitalization headers here - mbv := tracex.MaybeBinaryValue{Value: value} - out[0].Request.Headers[key] = mbv - out[0].Request.HeadersList = append(out[0].Request.HeadersList, - tracex.HTTPHeader{Key: key, Value: mbv}) - } - sort.Slice(out[0].Request.HeadersList, func(i, j int) bool { - return out[0].Request.HeadersList[i].Key < out[0].Request.HeadersList[j].Key - }) return } @@ -274,19 +278,11 @@ func NewRequestEntryList(req *http.Request, headers map[string]string) (out []tr // specific *http.Response instance and its body. func NewHTTPResponse(resp *http.Response, data []byte) (out tracex.HTTPResponse) { out = tracex.HTTPResponse{ - Body: tracex.HTTPBody{Value: string(data)}, + Body: model.ArchivalScrubbedMaybeBinaryString(data), Code: int64(resp.StatusCode), - Headers: make(map[string]tracex.MaybeBinaryValue), - HeadersList: []tracex.HTTPHeader{}, - } - for key := range resp.Header { - mbv := tracex.MaybeBinaryValue{Value: resp.Header.Get(key)} - out.Headers[key] = mbv - out.HeadersList = append(out.HeadersList, tracex.HTTPHeader{Key: key, Value: mbv}) + Headers: model.ArchivalNewHTTPHeadersMap(resp.Header), + HeadersList: model.ArchivalNewHTTPHeadersList(resp.Header), } - sort.Slice(out.HeadersList, func(i, j int) bool { - return out.HeadersList[i].Key < out.HeadersList[j].Key - }) return } diff --git a/pkg/experiment/hhfm/hhfm_internal_test.go b/pkg/experiment/hhfm/hhfm_internal_test.go new file mode 100644 index 000000000..0ea764e1e --- /dev/null +++ b/pkg/experiment/hhfm/hhfm_internal_test.go @@ -0,0 +1,49 @@ +package hhfm + +import ( + "net/http" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestNewHeadersFromMap(t *testing.T) { + + // testcase is a test case run by this func + type testcase struct { + name string + input map[string]string + expect map[string][]string + } + + cases := []testcase{{ + name: "with nil input", + input: nil, + expect: http.Header{}, + }, { + name: "with empty input", + input: map[string]string{}, + expect: http.Header{}, + }, { + name: "common case: headers with mixed casing should be preserved", + input: map[string]string{ + "ConTent-TyPe": "text/html; charset=utf-8", + "ViA": "a", + "User-AgeNt": "miniooni/0.1.0", + }, + expect: map[string][]string{ + "ConTent-TyPe": {"text/html; charset=utf-8"}, + "ViA": {"a"}, + "User-AgeNt": {"miniooni/0.1.0"}, + }, + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := newHeadersFromMap(tc.input) + if diff := cmp.Diff(tc.expect, got); diff != "" { + t.Fatal(diff) + } + }) + } +} diff --git a/pkg/experiment/hhfm/hhfm_test.go b/pkg/experiment/hhfm/hhfm_test.go index 357c7bafc..4c68efe4a 100644 --- a/pkg/experiment/hhfm/hhfm_test.go +++ b/pkg/experiment/hhfm/hhfm_test.go @@ -16,9 +16,9 @@ import ( "github.com/ooni/probe-engine/pkg/experiment/hhfm" "github.com/ooni/probe-engine/pkg/experiment/urlgetter" "github.com/ooni/probe-engine/pkg/legacy/mockable" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" - "github.com/ooni/probe-engine/pkg/tracex" ) func TestNewExperimentMeasurer(t *testing.T) { @@ -32,6 +32,10 @@ func TestNewExperimentMeasurer(t *testing.T) { } func TestSuccess(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + measurer := hhfm.NewExperimentMeasurer(hhfm.Config{}) ctx := context.Background() sess := &mockable.Session{ @@ -68,7 +72,7 @@ func TestSuccess(t *testing.T) { if request.Failure != nil { t.Fatal("invalid Requests[0].Failure") } - if request.Request.Body.Value != "" { + if request.Request.Body != "" { t.Fatal("invalid Requests[0].Request.Body.Value") } if request.Request.BodyIsTruncated != false { @@ -99,7 +103,7 @@ func TestSuccess(t *testing.T) { if request.Request.URL != ths[0].Address { t.Fatal("invalid Requests[0].Request.URL") } - if len(request.Response.Body.Value) < 1 { + if len(request.Response.Body) < 1 { t.Fatal("invalid Requests[0].Response.Body.Value length") } if request.Response.BodyIsTruncated != false { @@ -181,7 +185,7 @@ func TestCancelledContext(t *testing.T) { if *request.Failure != netxlite.FailureInterrupted { t.Fatal("invalid Requests[0].Failure") } - if request.Request.Body.Value != "" { + if request.Request.Body != "" { t.Fatal("invalid Requests[0].Request.Body.Value") } if request.Request.BodyIsTruncated != false { @@ -212,7 +216,7 @@ func TestCancelledContext(t *testing.T) { if request.Request.URL != ths[0].Address { t.Fatal("invalid Requests[0].Request.URL") } - if len(request.Response.Body.Value) != 0 { + if len(request.Response.Body) != 0 { t.Fatal("invalid Requests[0].Response.Body.Value length") } if request.Response.BodyIsTruncated != false { @@ -743,16 +747,16 @@ func TestNewRequestEntryList(t *testing.T) { }, wantOut: []tracex.RequestEntry{{ Request: tracex.HTTPRequest{ - HeadersList: []tracex.HTTPHeader{{ - Key: "ContENt-tYPE", - Value: tracex.MaybeBinaryValue{Value: "text/plain"}, + HeadersList: []model.ArchivalHTTPHeader{{ + model.ArchivalScrubbedMaybeBinaryString("ContENt-tYPE"), + model.ArchivalScrubbedMaybeBinaryString("text/plain"), }, { - Key: "User-aGENT", - Value: tracex.MaybeBinaryValue{Value: "foo/1.0"}, + model.ArchivalScrubbedMaybeBinaryString("User-aGENT"), + model.ArchivalScrubbedMaybeBinaryString("foo/1.0"), }}, - Headers: map[string]tracex.MaybeBinaryValue{ - "ContENt-tYPE": {Value: "text/plain"}, - "User-aGENT": {Value: "foo/1.0"}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "ContENt-tYPE": "text/plain", + "User-aGENT": "foo/1.0", }, Method: "GeT", URL: "http://10.0.0.1/", @@ -773,8 +777,8 @@ func TestNewRequestEntryList(t *testing.T) { wantOut: []tracex.RequestEntry{{ Request: tracex.HTTPRequest{ Method: "GeT", - Headers: make(map[string]tracex.MaybeBinaryValue), - HeadersList: []tracex.HTTPHeader{}, + Headers: make(map[string]model.ArchivalScrubbedMaybeBinaryString), + HeadersList: []model.ArchivalHTTPHeader{}, URL: "http://10.0.0.1/", }, }}, @@ -811,18 +815,18 @@ func TestNewHTTPResponse(t *testing.T) { data: []byte("deadbeef"), }, wantOut: tracex.HTTPResponse{ - Body: tracex.MaybeBinaryValue{Value: "deadbeef"}, + Body: model.ArchivalScrubbedMaybeBinaryString("deadbeef"), Code: 200, - HeadersList: []tracex.HTTPHeader{{ - Key: "Content-Type", - Value: tracex.MaybeBinaryValue{Value: "text/plain"}, + HeadersList: []model.ArchivalHTTPHeader{{ + model.ArchivalScrubbedMaybeBinaryString("Content-Type"), + model.ArchivalScrubbedMaybeBinaryString("text/plain"), }, { - Key: "User-Agent", - Value: tracex.MaybeBinaryValue{Value: "foo/1.0"}, + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("foo/1.0"), }}, - Headers: map[string]tracex.MaybeBinaryValue{ - "Content-Type": {Value: "text/plain"}, - "User-Agent": {Value: "foo/1.0"}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "Content-Type": "text/plain", + "User-Agent": "foo/1.0", }, }, }, { @@ -831,10 +835,10 @@ func TestNewHTTPResponse(t *testing.T) { resp: &http.Response{StatusCode: 200}, }, wantOut: tracex.HTTPResponse{ - Body: tracex.MaybeBinaryValue{Value: ""}, + Body: model.ArchivalScrubbedMaybeBinaryString(""), Code: 200, - HeadersList: []tracex.HTTPHeader{}, - Headers: map[string]tracex.MaybeBinaryValue{}, + HeadersList: []model.ArchivalHTTPHeader{}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{}, }, }} for _, tt := range tests { diff --git a/pkg/experiment/hirl/hirl.go b/pkg/experiment/hirl/hirl.go index daeb23fea..feb665e76 100644 --- a/pkg/experiment/hirl/hirl.go +++ b/pkg/experiment/hirl/hirl.go @@ -11,11 +11,12 @@ import ( "strings" "time" + "github.com/ooni/probe-engine/pkg/legacy/legacymodel" "github.com/ooni/probe-engine/pkg/legacy/netx" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" "github.com/ooni/probe-engine/pkg/randx" - "github.com/ooni/probe-engine/pkg/tracex" ) const ( @@ -29,11 +30,11 @@ type Config struct{} // TestKeys contains the experiment test keys. type TestKeys struct { - FailureList []*string `json:"failure_list"` - Received []tracex.MaybeBinaryValue `json:"received"` - Sent []string `json:"sent"` - TamperingList []bool `json:"tampering_list"` - Tampering bool `json:"tampering"` + FailureList []*string `json:"failure_list"` + Received []legacymodel.ArchivalMaybeBinaryData `json:"received"` + Sent []string `json:"sent"` + TamperingList []bool `json:"tampering_list"` + Tampering bool `json:"tampering"` } // NewExperimentMeasurer creates a new ExperimentMeasurer. @@ -151,7 +152,7 @@ type MethodConfig struct { type MethodResult struct { Err error Name string - Received tracex.MaybeBinaryValue + Received legacymodel.ArchivalMaybeBinaryData Sent string Tampering bool } diff --git a/pkg/experiment/hirl/hirl_test.go b/pkg/experiment/hirl/hirl_test.go index 37fa9600b..58668274a 100644 --- a/pkg/experiment/hirl/hirl_test.go +++ b/pkg/experiment/hirl/hirl_test.go @@ -8,11 +8,11 @@ import ( "github.com/apex/log" "github.com/ooni/probe-engine/pkg/experiment/hirl" + "github.com/ooni/probe-engine/pkg/legacy/legacymodel" "github.com/ooni/probe-engine/pkg/legacy/mockable" "github.com/ooni/probe-engine/pkg/legacy/netx" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" - "github.com/ooni/probe-engine/pkg/tracex" ) func TestNewExperimentMeasurer(t *testing.T) { @@ -159,7 +159,7 @@ func (FakeMethodSuccessful) Name() string { func (meth FakeMethodSuccessful) Run(ctx context.Context, config hirl.MethodConfig) { config.Out <- hirl.MethodResult{ Name: meth.Name(), - Received: tracex.MaybeBinaryValue{Value: "antani"}, + Received: legacymodel.ArchivalMaybeBinaryData{Value: "antani"}, Sent: "antani", Tampering: false, } @@ -174,7 +174,7 @@ func (FakeMethodFailure) Name() string { func (meth FakeMethodFailure) Run(ctx context.Context, config hirl.MethodConfig) { config.Out <- hirl.MethodResult{ Name: meth.Name(), - Received: tracex.MaybeBinaryValue{Value: "antani"}, + Received: legacymodel.ArchivalMaybeBinaryData{Value: "antani"}, Sent: "melandri", Tampering: true, } diff --git a/pkg/experiment/ndt7/dial_test.go b/pkg/experiment/ndt7/dial_test.go index 223c0ad7c..92551192a 100644 --- a/pkg/experiment/ndt7/dial_test.go +++ b/pkg/experiment/ndt7/dial_test.go @@ -14,6 +14,10 @@ import ( ) func TestDialDownloadWithCancelledContext(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + ctx, cancel := context.WithCancel(context.Background()) cancel() // immediately halt mgr := newDialManager("wss://hostname.fake", log.Log, "miniooni/0.1.0-dev") @@ -27,6 +31,10 @@ func TestDialDownloadWithCancelledContext(t *testing.T) { } func TestDialUploadWithCancelledContext(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + ctx, cancel := context.WithCancel(context.Background()) cancel() // immediately halt mgr := newDialManager("wss://hostname.fake", log.Log, "miniooni/0.1.0-dev") diff --git a/pkg/experiment/ndt7/ndt7.go b/pkg/experiment/ndt7/ndt7.go index c92f9ee84..98b1cd42e 100644 --- a/pkg/experiment/ndt7/ndt7.go +++ b/pkg/experiment/ndt7/ndt7.go @@ -77,6 +77,10 @@ type Measurer struct { func (m *Measurer) discover( ctx context.Context, sess model.ExperimentSession) (*mlablocatev2.NDT7Result, error) { + // Implementation note: here we cannot use the session's HTTP client because it MAY be proxied + // and instead we need to connect directly to M-Lab's locate service. + // + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPClientStdlib has QUIRKS and maybe here it doesn't matter? httpClient := netxlite.NewHTTPClientStdlib(sess.Logger()) defer httpClient.CloseIdleConnections() client := mlablocatev2.NewClient(httpClient, sess.Logger(), sess.UserAgent()) diff --git a/pkg/experiment/ndt7/ndt7_test.go b/pkg/experiment/ndt7/ndt7_test.go index 3e59d8d4b..ab7ad9251 100644 --- a/pkg/experiment/ndt7/ndt7_test.go +++ b/pkg/experiment/ndt7/ndt7_test.go @@ -131,6 +131,10 @@ func TestGood(t *testing.T) { } func TestFailDownload(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + ctx, cancel := context.WithCancel(context.Background()) defer cancel() measurer := NewExperimentMeasurer(Config{}).(*Measurer) @@ -162,6 +166,10 @@ func TestFailDownload(t *testing.T) { } func TestFailUpload(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + ctx, cancel := context.WithCancel(context.Background()) defer cancel() measurer := NewExperimentMeasurer(Config{noDownload: true}).(*Measurer) diff --git a/pkg/experiment/portfiltering/tcpconnect.go b/pkg/experiment/portfiltering/tcpconnect.go index 744897124..28cc4b1cd 100644 --- a/pkg/experiment/portfiltering/tcpconnect.go +++ b/pkg/experiment/portfiltering/tcpconnect.go @@ -10,6 +10,7 @@ import ( "net" "time" + "github.com/ooni/probe-engine/pkg/logx" "github.com/ooni/probe-engine/pkg/measurexlite" "github.com/ooni/probe-engine/pkg/model" ) @@ -39,7 +40,7 @@ func (m *Measurer) tcpConnectAsync(ctx context.Context, index int64, func (m *Measurer) tcpConnect(ctx context.Context, index int64, zeroTime time.Time, logger model.Logger, address string) *model.ArchivalTCPConnectResult { trace := measurexlite.NewTrace(index, zeroTime) - ol := measurexlite.NewOperationLogger(logger, "TCPConnect #%d %s", index, address) + ol := logx.NewOperationLogger(logger, "TCPConnect #%d %s", index, address) dialer := trace.NewDialerWithoutResolver(logger) conn, err := dialer.DialContext(ctx, "tcp", address) ol.Stop(err) diff --git a/pkg/experiment/quicping/quicping.go b/pkg/experiment/quicping/quicping.go index 0cbd6bb61..8c9ab09c0 100644 --- a/pkg/experiment/quicping/quicping.go +++ b/pkg/experiment/quicping/quicping.go @@ -17,8 +17,9 @@ import ( _ "crypto/sha256" + "github.com/ooni/probe-engine/pkg/legacy/legacymodel" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/model" - "github.com/ooni/probe-engine/pkg/tracex" ) // A connectionID in QUIC @@ -32,7 +33,7 @@ const ( const ( testName = "quicping" - testVersion = "0.1.0" + testVersion = "0.1.1" ) // Config contains the experiment configuration. @@ -78,26 +79,26 @@ type TestKeys struct { // SinglePing is a result of a single ping operation. type SinglePing struct { - ConnIdDst string `json:"conn_id_dst"` - ConnIdSrc string `json:"conn_id_src"` - Failure *string `json:"failure"` - Request *model.ArchivalMaybeBinaryData `json:"request"` - T float64 `json:"t"` - Responses []*SinglePingResponse `json:"responses"` + ConnIdDst string `json:"conn_id_dst"` + ConnIdSrc string `json:"conn_id_src"` + Failure *string `json:"failure"` + Request *legacymodel.ArchivalMaybeBinaryData `json:"request"` + T float64 `json:"t"` + Responses []*SinglePingResponse `json:"responses"` } type SinglePingResponse struct { - Data *model.ArchivalMaybeBinaryData `json:"response_data"` - Failure *string `json:"failure"` - T float64 `json:"t"` - SupportedVersions []uint32 `json:"supported_versions"` + Data *legacymodel.ArchivalMaybeBinaryData `json:"response_data"` + Failure *string `json:"failure"` + T float64 `json:"t"` + SupportedVersions []uint32 `json:"supported_versions"` } // makeResponse is a utility function to create a SinglePingResponse func makeResponse(resp *responseInfo) *SinglePingResponse { - var data *model.ArchivalMaybeBinaryData + var data *legacymodel.ArchivalMaybeBinaryData if resp.raw != nil { - data = &model.ArchivalMaybeBinaryData{Value: string(resp.raw)} + data = &legacymodel.ArchivalMaybeBinaryData{Value: string(resp.raw)} } return &SinglePingResponse{ Data: data, @@ -227,12 +228,18 @@ func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { sess := args.Session host := string(measurement.Input) + var port = "" // allow URL input if u, err := url.ParseRequestURI(host); err == nil { - host = u.Host + host = u.Hostname() + port = u.Port() } - service := net.JoinHostPort(host, m.config.port()) - udpAddr, err := net.ResolveUDPAddr("udp4", service) + var service string + if port == "" { + port = m.config.port() + } + service = net.JoinHostPort(host, port) + udpAddr, err := net.ResolveUDPAddr("udp", service) if err != nil { return err } @@ -272,7 +279,7 @@ L: ConnIdDst: req.dstID, ConnIdSrc: req.srcID, Failure: tracex.NewFailure(req.err), - Request: &model.ArchivalMaybeBinaryData{Value: string(req.raw)}, + Request: &legacymodel.ArchivalMaybeBinaryData{Value: string(req.raw)}, T: req.t, }) continue @@ -312,7 +319,7 @@ L: ConnIdDst: ping.request.dstID, ConnIdSrc: ping.request.srcID, Failure: tracex.NewFailure(timeoutErr), - Request: &model.ArchivalMaybeBinaryData{Value: string(ping.request.raw)}, + Request: &legacymodel.ArchivalMaybeBinaryData{Value: string(ping.request.raw)}, T: ping.request.t, }) continue @@ -325,7 +332,7 @@ L: ConnIdDst: ping.request.dstID, ConnIdSrc: ping.request.srcID, Failure: nil, - Request: &model.ArchivalMaybeBinaryData{Value: string(ping.request.raw)}, + Request: &legacymodel.ArchivalMaybeBinaryData{Value: string(ping.request.raw)}, T: ping.request.t, Responses: responses, }) diff --git a/pkg/experiment/quicping/quicping_test.go b/pkg/experiment/quicping/quicping_test.go index 675c1b702..2a1f1874c 100644 --- a/pkg/experiment/quicping/quicping_test.go +++ b/pkg/experiment/quicping/quicping_test.go @@ -20,7 +20,7 @@ func TestNewExperimentMeasurer(t *testing.T) { if measurer.ExperimentName() != "quicping" { t.Fatal("unexpected name") } - if measurer.ExperimentVersion() != "0.1.0" { + if measurer.ExperimentVersion() != "0.1.1" { t.Fatal("unexpected version") } } @@ -123,6 +123,10 @@ func TestSuccess(t *testing.T) { } func TestWithCancelledContext(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + measurer := NewExperimentMeasurer(Config{}) measurement := new(model.Measurement) measurement.Input = model.MeasurementTarget("google.com") @@ -145,6 +149,10 @@ func TestWithCancelledContext(t *testing.T) { } func TestListenFails(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + expected := errors.New("expected") measurer := NewExperimentMeasurer(Config{ netListenUDP: func(network string, laddr *net.UDPAddr) (model.UDPLikeConn, error) { diff --git a/pkg/experiment/riseupvpn/riseupvpn.go b/pkg/experiment/riseupvpn/riseupvpn.go index bdf045cc4..48d57739c 100644 --- a/pkg/experiment/riseupvpn/riseupvpn.go +++ b/pkg/experiment/riseupvpn/riseupvpn.go @@ -10,9 +10,9 @@ import ( "time" "github.com/ooni/probe-engine/pkg/experiment/urlgetter" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" - "github.com/ooni/probe-engine/pkg/tracex" ) const ( @@ -304,7 +304,7 @@ func parseGateways(testKeys *TestKeys) []GatewayV3 { // TODO(bassosimone,cyberta): is it reasonable that we discard // the error when the JSON we fetched cannot be parsed? // See https://github.com/ooni/probe/issues/1432 - eipService, err := DecodeEIP3(requestEntry.Response.Body.Value) + eipService, err := DecodeEIP3(string(requestEntry.Response.Body)) if err == nil { return eipService.Gateways } diff --git a/pkg/experiment/riseupvpn/riseupvpn_test.go b/pkg/experiment/riseupvpn/riseupvpn_test.go index c53c6d200..df6ae8c35 100644 --- a/pkg/experiment/riseupvpn/riseupvpn_test.go +++ b/pkg/experiment/riseupvpn/riseupvpn_test.go @@ -15,9 +15,9 @@ import ( "github.com/ooni/probe-engine/pkg/experiment/riseupvpn" "github.com/ooni/probe-engine/pkg/experiment/urlgetter" "github.com/ooni/probe-engine/pkg/legacy/mockable" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" - "github.com/ooni/probe-engine/pkg/tracex" ) const ( @@ -773,13 +773,13 @@ func generateMockGetter(requestResponse map[string]string, responseStatus map[st Failure: failure, Request: tracex.HTTPRequest{ URL: url, - Body: tracex.MaybeBinaryValue{}, + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, }, Response: tracex.HTTPResponse{ - Body: tracex.HTTPBody{ - Value: responseBody, - }, + Body: model.ArchivalScrubbedMaybeBinaryString( + responseBody, + ), BodyIsTruncated: false, }}, }, diff --git a/pkg/experiment/signal/signal_test.go b/pkg/experiment/signal/signal_test.go index 0af8a4335..c826400b2 100644 --- a/pkg/experiment/signal/signal_test.go +++ b/pkg/experiment/signal/signal_test.go @@ -23,6 +23,10 @@ func TestNewExperimentMeasurer(t *testing.T) { } func TestGood(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + measurer := signal.NewExperimentMeasurer(signal.Config{}) measurement := new(model.Measurement) args := &model.ExperimentArgs{ diff --git a/pkg/experiment/simplequicping/simplequicping.go b/pkg/experiment/simplequicping/simplequicping.go index 1cb97aa95..3f9278075 100644 --- a/pkg/experiment/simplequicping/simplequicping.go +++ b/pkg/experiment/simplequicping/simplequicping.go @@ -13,6 +13,7 @@ import ( "strings" "time" + "github.com/ooni/probe-engine/pkg/logx" "github.com/ooni/probe-engine/pkg/measurexlite" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" @@ -170,8 +171,8 @@ func (m *Measurer) quicHandshake(ctx context.Context, index int64, sni := m.config.sni(address) alpn := strings.Split(m.config.alpn(), " ") trace := measurexlite.NewTrace(index, zeroTime) - ol := measurexlite.NewOperationLogger(logger, "SimpleQUICPing #%d %s %s %v", index, address, sni, alpn) - listener := netxlite.NewQUICListener() + ol := logx.NewOperationLogger(logger, "SimpleQUICPing #%d %s %s %v", index, address, sni, alpn) + listener := netxlite.NewUDPListener() dialer := trace.NewQUICDialerWithoutResolver(listener, logger) // See https://github.com/ooni/probe/issues/2413 to understand // why we're using nil to force netxlite to use the cached diff --git a/pkg/experiment/simplequicping/simplequicping_test.go b/pkg/experiment/simplequicping/simplequicping_test.go index 6505fa1f6..b6f69afaf 100644 --- a/pkg/experiment/simplequicping/simplequicping_test.go +++ b/pkg/experiment/simplequicping/simplequicping_test.go @@ -104,7 +104,15 @@ func TestMeasurerRun(t *testing.T) { t.Run("with netem: without DPI: expect success", func(t *testing.T) { // create a new test environment - env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.ExampleWebPageHandlerFactory())) + env := netemx.MustNewQAEnv(netemx.QAEnvOptionNetStack( + "8.8.8.8", + &netemx.HTTP3ServerFactory{ + Factory: netemx.ExampleWebPageHandlerFactory(), + Ports: []int{443}, + ServerNameMain: SNI, + ServerNameExtras: []string{}, + }, + )) defer env.Close() env.Do(func() { @@ -140,7 +148,15 @@ func TestMeasurerRun(t *testing.T) { t.Run("with netem: with DPI that drops UDP datagrams to 8.8.8.8:443: expect failure", func(t *testing.T) { // create a new test environment - env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.ExampleWebPageHandlerFactory())) + env := netemx.MustNewQAEnv(netemx.QAEnvOptionNetStack( + "8.8.8.8", + &netemx.HTTP3ServerFactory{ + Factory: netemx.ExampleWebPageHandlerFactory(), + Ports: []int{443}, + ServerNameMain: SNI, + ServerNameExtras: []string{}, + }, + )) defer env.Close() // add DPI engine to emulate the censorship condition diff --git a/pkg/experiment/sniblocking/sniblocking_test.go b/pkg/experiment/sniblocking/sniblocking_test.go index 3b7965b6d..0ad7661d1 100644 --- a/pkg/experiment/sniblocking/sniblocking_test.go +++ b/pkg/experiment/sniblocking/sniblocking_test.go @@ -214,7 +214,7 @@ func configureDNSWithDefaults(config *netem.DNSConfig) { func TestMeasurerWithInvalidInput(t *testing.T) { t.Run("with no measurement input: expect input error", func(t *testing.T) { // create a new test environment - env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.ExampleWebPageHandlerFactory())) + env := netemx.MustNewQAEnv() defer env.Close() // we use the same valid DNS config for client and servers here @@ -237,7 +237,7 @@ func TestMeasurerWithInvalidInput(t *testing.T) { t.Run("with invalid MeasurementInput: expect parsing error", func(t *testing.T) { // create a new test environment - env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.ExampleWebPageHandlerFactory())) + env := netemx.MustNewQAEnv() defer env.Close() // we use the same valid DNS config for client and servers here @@ -269,7 +269,15 @@ func TestMeasurerWithInvalidInput(t *testing.T) { func TestMeasurerRun(t *testing.T) { t.Run("without DPI: expect success", func(t *testing.T) { // create a new test environment - env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.ExampleWebPageHandlerFactory())) + env := netemx.MustNewQAEnv(netemx.QAEnvOptionNetStack( + exampleOrgAddr, + &netemx.HTTPSecureServerFactory{ + Factory: netemx.ExampleWebPageHandlerFactory(), + Ports: []int{443}, + ServerNameMain: "example.org", + ServerNameExtras: []string{}, + }, + )) defer env.Close() // we use the same valid DNS config for client and servers here @@ -300,9 +308,13 @@ func TestMeasurerRun(t *testing.T) { t.Fatalf("Unexpected Control Failure %s", *tk.Control.Failure) } target := tk.Target - if target.Failure != nil { + + // note: the target IS EXPECTED TO FAIL here because we are connecting to the + // control host which does not have the target certificate configured + if target.Failure == nil || *target.Failure != "ssl_invalid_hostname" { t.Fatalf("Unexpected Target Failure %s", *tk.Target.Failure) } + if target.Agent != "redirect" { t.Fatal("not the expected Agent") } @@ -312,12 +324,9 @@ func TestMeasurerRun(t *testing.T) { if target.DNSCache != nil { t.Fatal("not the expected DNSCache") } - if target.FailedOperation != nil { + if target.FailedOperation == nil || *target.FailedOperation != "tls_handshake" { t.Fatal("unexpected FailedOperation") } - if target.Failure != nil { - t.Fatal("unexpected failure") - } if len(target.NetworkEvents) < 1 { t.Fatal("not the expected NetworkEvents") } @@ -424,7 +433,15 @@ func TestMeasurerRun(t *testing.T) { t.Run("with cache: expect to see cached entry", func(t *testing.T) { // create a new test environment - env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.ExampleWebPageHandlerFactory())) + env := netemx.MustNewQAEnv(netemx.QAEnvOptionNetStack( + exampleOrgAddr, + &netemx.HTTPSecureServerFactory{ + Factory: netemx.ExampleWebPageHandlerFactory(), + Ports: []int{443}, + ServerNameMain: "example.org", + ServerNameExtras: []string{}, + }, + )) defer env.Close() // we use the same valid DNS config for client and servers here @@ -480,7 +497,15 @@ func TestMeasurerRun(t *testing.T) { t.Run("with DPI that blocks target SNI", func(t *testing.T) { // create a new test environment - env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.ExampleWebPageHandlerFactory())) + env := netemx.MustNewQAEnv(netemx.QAEnvOptionNetStack( + exampleOrgAddr, + &netemx.HTTPSecureServerFactory{ + Factory: netemx.ExampleWebPageHandlerFactory(), + Ports: []int{443}, + ServerNameMain: "example.org", + ServerNameExtras: []string{}, + }, + )) defer env.Close() // we use the same valid DNS config for client and servers here @@ -529,7 +554,15 @@ func TestMeasurerRun(t *testing.T) { func TestMeasureonewithcacheWorks(t *testing.T) { // create a new test environment - env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer(exampleOrgAddr, netemx.ExampleWebPageHandlerFactory())) + env := netemx.MustNewQAEnv(netemx.QAEnvOptionNetStack( + exampleOrgAddr, + &netemx.HTTPSecureServerFactory{ + Factory: netemx.ExampleWebPageHandlerFactory(), + Ports: []int{443}, + ServerNameMain: "example.org", + ServerNameExtras: []string{}, + }, + )) defer env.Close() // we use the same valid DNS config for client and servers here @@ -554,9 +587,13 @@ func TestMeasureonewithcacheWorks(t *testing.T) { if result.Cached != expected { t.Fatal("unexpected cached") } - if result.Failure != nil { - t.Fatal("unexpected failure") + + // note: the target IS EXPECTED TO FAIL here because we are connecting to the + // control host which does not have the target certificate configured + if result.Failure == nil || *result.Failure != "ssl_invalid_hostname" { + t.Fatal("unexpected failure", *result.Failure) } + if result.SNI != "kernel.org" { t.Fatal("unexpected SNI") } diff --git a/pkg/experiment/stunreachability/stunreachability.go b/pkg/experiment/stunreachability/stunreachability.go index e802d9c3f..26f5e0641 100644 --- a/pkg/experiment/stunreachability/stunreachability.go +++ b/pkg/experiment/stunreachability/stunreachability.go @@ -11,9 +11,9 @@ import ( "time" "github.com/ooni/probe-engine/pkg/legacy/netx" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" - "github.com/ooni/probe-engine/pkg/tracex" "github.com/pion/stun" ) diff --git a/pkg/experiment/stunreachability/stunreachability_test.go b/pkg/experiment/stunreachability/stunreachability_test.go index f817fad99..3250f275e 100644 --- a/pkg/experiment/stunreachability/stunreachability_test.go +++ b/pkg/experiment/stunreachability/stunreachability_test.go @@ -89,6 +89,10 @@ func TestRunWithUnsupportedURLScheme(t *testing.T) { } func TestRunWithInput(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + measurer := NewExperimentMeasurer(Config{}) measurement := new(model.Measurement) measurement.Input = model.MeasurementTarget(defaultInput) @@ -158,6 +162,10 @@ func TestCancelledContext(t *testing.T) { } func TestNewClientFailure(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + config := &Config{} expected := errors.New("mocked error") config.newClient = func(conn stun.Connection, options ...stun.ClientOption) (*stun.Client, error) { diff --git a/pkg/experiment/tcpping/tcpping.go b/pkg/experiment/tcpping/tcpping.go index a2deb804d..3cabf7e9a 100644 --- a/pkg/experiment/tcpping/tcpping.go +++ b/pkg/experiment/tcpping/tcpping.go @@ -10,6 +10,7 @@ import ( "net/url" "time" + "github.com/ooni/probe-engine/pkg/logx" "github.com/ooni/probe-engine/pkg/measurexlite" "github.com/ooni/probe-engine/pkg/model" ) @@ -134,7 +135,7 @@ func (m *Measurer) tcpConnect(ctx context.Context, index int64, defer cancel() trace := measurexlite.NewTrace(index, zeroTime) dialer := trace.NewDialerWithoutResolver(logger) - ol := measurexlite.NewOperationLogger(logger, "TCPPing #%d %s", index, address) + ol := logx.NewOperationLogger(logger, "TCPPing #%d %s", index, address) conn, err := dialer.DialContext(ctx, "tcp", address) ol.Stop(err) measurexlite.MaybeClose(conn) diff --git a/pkg/experiment/tcpping/tcpping_test.go b/pkg/experiment/tcpping/tcpping_test.go index 5d18d7622..0dcf1b2f2 100644 --- a/pkg/experiment/tcpping/tcpping_test.go +++ b/pkg/experiment/tcpping/tcpping_test.go @@ -90,7 +90,15 @@ func TestMeasurer_run(t *testing.T) { }) t.Run("with netem: without DPI: expect success", func(t *testing.T) { - env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.ExampleWebPageHandlerFactory())) + env := netemx.MustNewQAEnv(netemx.QAEnvOptionNetStack( + "8.8.8.8", + &netemx.HTTPSecureServerFactory{ + Factory: netemx.ExampleWebPageHandlerFactory(), + Ports: []int{443}, + ServerNameMain: "dns.google", + ServerNameExtras: []string{}, + }, + )) defer env.Close() env.Do(func() { @@ -132,7 +140,15 @@ func TestMeasurer_run(t *testing.T) { t.Run("with netem: with DPI that drops TCP segments to 8.8.8.8:443: expect failure", func(t *testing.T) { // create a new test environment - env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.ExampleWebPageHandlerFactory())) + env := netemx.MustNewQAEnv(netemx.QAEnvOptionNetStack( + "8.8.8.8", + &netemx.HTTPSecureServerFactory{ + Factory: netemx.ExampleWebPageHandlerFactory(), + Ports: []int{443}, + ServerNameMain: "dns.google", + ServerNameExtras: []string{}, + }, + )) defer env.Close() // add DPI engine to emulate the censorship condition diff --git a/pkg/experiment/telegram/telegram_test.go b/pkg/experiment/telegram/telegram_test.go index 936e2353f..95eaba40b 100644 --- a/pkg/experiment/telegram/telegram_test.go +++ b/pkg/experiment/telegram/telegram_test.go @@ -299,7 +299,15 @@ func newQAEnvironment(ipaddrs ...string) *netemx.QAEnv { // add handler for telegram web (we're using a different-from-reality HTTP handler // but we're not testing for the returned webpage, so we should be fine) - options = append(options, netemx.QAEnvOptionHTTPServer(telegramWebAddr, netemx.ExampleWebPageHandlerFactory())) + options = append(options, netemx.QAEnvOptionNetStack( + telegramWebAddr, + &netemx.HTTPSecureServerFactory{ + Factory: netemx.ExampleWebPageHandlerFactory(), + Ports: []int{443}, + ServerNameMain: "web.telegram.org", + ServerNameExtras: []string{}, + }, + )) // create the environment proper with all the options env := netemx.MustNewQAEnv(options...) diff --git a/pkg/experiment/tlsmiddlebox/connect.go b/pkg/experiment/tlsmiddlebox/connect.go index 224a5b5d9..f505b14bb 100644 --- a/pkg/experiment/tlsmiddlebox/connect.go +++ b/pkg/experiment/tlsmiddlebox/connect.go @@ -8,6 +8,7 @@ import ( "context" "time" + "github.com/ooni/probe-engine/pkg/logx" "github.com/ooni/probe-engine/pkg/measurexlite" "github.com/ooni/probe-engine/pkg/model" ) @@ -17,7 +18,7 @@ func (m *Measurer) TCPConnect(ctx context.Context, index int64, zeroTime time.Ti logger model.Logger, address string, tk *TestKeys) error { trace := measurexlite.NewTrace(index, zeroTime) dialer := trace.NewDialerWithoutResolver(logger) - ol := measurexlite.NewOperationLogger(logger, "TCPConnect #%d %s", index, address) + ol := logx.NewOperationLogger(logger, "TCPConnect #%d %s", index, address) conn, err := dialer.DialContext(ctx, "tcp", address) ol.Stop(err) measurexlite.MaybeClose(conn) diff --git a/pkg/experiment/tlsmiddlebox/dns.go b/pkg/experiment/tlsmiddlebox/dns.go index 9a55693cb..782880be2 100644 --- a/pkg/experiment/tlsmiddlebox/dns.go +++ b/pkg/experiment/tlsmiddlebox/dns.go @@ -8,6 +8,7 @@ import ( "context" "time" + "github.com/ooni/probe-engine/pkg/logx" "github.com/ooni/probe-engine/pkg/measurexlite" "github.com/ooni/probe-engine/pkg/model" ) @@ -17,7 +18,7 @@ func (m *Measurer) DNSLookup(ctx context.Context, index int64, zeroTime time.Tim logger model.Logger, domain string, tk *TestKeys) ([]string, error) { url := m.config.resolverURL() trace := measurexlite.NewTrace(index, zeroTime) - ol := measurexlite.NewOperationLogger(logger, "DNSLookup #%d, %s, %s", index, url, domain) + ol := logx.NewOperationLogger(logger, "DNSLookup #%d, %s, %s", index, url, domain) // TODO(DecFox, bassosimone): We are currently using the DoH resolver, we will // switch to the TRR2 resolver once we have it in measurexlite // Issue: https://github.com/ooni/probe/issues/2185 diff --git a/pkg/experiment/tlsmiddlebox/trace.go b/pkg/experiment/tlsmiddlebox/trace.go index 303dcae3a..ab722b425 100644 --- a/pkg/experiment/tlsmiddlebox/trace.go +++ b/pkg/experiment/tlsmiddlebox/trace.go @@ -3,8 +3,8 @@ package tlsmiddlebox import ( "sync" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/model" - "github.com/ooni/probe-engine/pkg/tracex" ) // CompleteTrace records the result of the network trace diff --git a/pkg/experiment/tlsmiddlebox/tracing.go b/pkg/experiment/tlsmiddlebox/tracing.go index 4838c980f..093ba14c1 100644 --- a/pkg/experiment/tlsmiddlebox/tracing.go +++ b/pkg/experiment/tlsmiddlebox/tracing.go @@ -14,6 +14,7 @@ import ( "syscall" "time" + "github.com/ooni/probe-engine/pkg/logx" "github.com/ooni/probe-engine/pkg/measurexlite" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" @@ -72,7 +73,7 @@ func (m *Measurer) handshakeWithTTL(ctx context.Context, index int64, zeroTime t // 1. Connect to the target IP // TODO(DecFox, bassosimone): Do we need a trace for this TCP connect? d := NewDialerTTLWrapper() - ol := measurexlite.NewOperationLogger(logger, "Handshake Trace #%d TTL %d %s %s", index, ttl, address, sni) + ol := logx.NewOperationLogger(logger, "Handshake Trace #%d TTL %d %s %s", index, ttl, address, sni) conn, err := d.DialContext(ctx, "tcp", address) if err != nil { iteration := newIterationFromHandshake(ttl, err, nil, nil) @@ -96,7 +97,7 @@ func (m *Measurer) handshakeWithTTL(ctx context.Context, index int64, zeroTime t if clientId > 0 { thx = trace.NewTLSHandshakerUTLS(logger, ClientIDs[clientId]) } - _, _, err = thx.Handshake(ctx, conn, genTLSConfig(sni)) + _, err = thx.Handshake(ctx, conn, genTLSConfig(sni)) ol.Stop(err) soErr := extractSoError(conn) // 4. reset the TTL value to ensure that conn closes successfully diff --git a/pkg/experiment/tlsping/tlsping.go b/pkg/experiment/tlsping/tlsping.go index d6c434dc6..b78f4c299 100644 --- a/pkg/experiment/tlsping/tlsping.go +++ b/pkg/experiment/tlsping/tlsping.go @@ -13,6 +13,7 @@ import ( "strings" "time" + "github.com/ooni/probe-engine/pkg/logx" "github.com/ooni/probe-engine/pkg/measurexlite" "github.com/ooni/probe-engine/pkg/model" ) @@ -171,7 +172,7 @@ func (m *Measurer) tlsConnectAndHandshake(ctx context.Context, index int64, dialer := trace.NewDialerWithoutResolver(logger) alpn := strings.Split(m.config.alpn(), " ") sni := m.config.sni(address) - ol := measurexlite.NewOperationLogger(logger, "TLSPing #%d %s %s %v", index, address, sni, alpn) + ol := logx.NewOperationLogger(logger, "TLSPing #%d %s %s %v", index, address, sni, alpn) conn, err := dialer.DialContext(ctx, "tcp", address) sp.TCPConnect = trace.FirstTCPConnectOrNil() // record the first connect from the buffer if err != nil { @@ -188,7 +189,7 @@ func (m *Measurer) tlsConnectAndHandshake(ctx context.Context, index int64, RootCAs: nil, ServerName: sni, } - _, _, err = thx.Handshake(ctx, conn, config) + _, err = thx.Handshake(ctx, conn, config) ol.Stop(err) sp.TLSHandshake = trace.FirstTLSHandshakeOrNil() // record the first handshake from the buffer sp.NetworkEvents = trace.NetworkEvents() diff --git a/pkg/experiment/tlsping/tlsping_test.go b/pkg/experiment/tlsping/tlsping_test.go index 5f3c01d10..627bc86fb 100644 --- a/pkg/experiment/tlsping/tlsping_test.go +++ b/pkg/experiment/tlsping/tlsping_test.go @@ -105,7 +105,15 @@ func TestMeasurerRun(t *testing.T) { t.Run("with netem: without DPI: expect success", func(t *testing.T) { // create a new test environment - env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.ExampleWebPageHandlerFactory())) + env := netemx.MustNewQAEnv(netemx.QAEnvOptionNetStack( + "8.8.8.8", + &netemx.HTTPSecureServerFactory{ + Factory: netemx.ExampleWebPageHandlerFactory(), + Ports: []int{443}, + ServerNameMain: SNI, + ServerNameExtras: []string{}, + }, + )) defer env.Close() env.Do(func() { @@ -147,7 +155,15 @@ func TestMeasurerRun(t *testing.T) { t.Run("with netem: with DPI that drops TCP segments to 8.8.8.8:443: expect failure", func(t *testing.T) { // create a new test environment - env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.ExampleWebPageHandlerFactory())) + env := netemx.MustNewQAEnv(netemx.QAEnvOptionNetStack( + "8.8.8.8", + &netemx.HTTPSecureServerFactory{ + Factory: netemx.ExampleWebPageHandlerFactory(), + Ports: []int{443}, + ServerNameMain: SNI, + ServerNameExtras: []string{}, + }, + )) defer env.Close() // add DPI engine to emulate the censorship condition @@ -205,7 +221,15 @@ func TestMeasurerRun(t *testing.T) { t.Run("with netem: with DPI that resets TLS to SNI blocked.com: expect failure", func(t *testing.T) { // create a new test environment - env := netemx.MustNewQAEnv(netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.ExampleWebPageHandlerFactory())) + env := netemx.MustNewQAEnv(netemx.QAEnvOptionNetStack( + "8.8.8.8", + &netemx.HTTPSecureServerFactory{ + Factory: netemx.ExampleWebPageHandlerFactory(), + Ports: []int{443}, + ServerNameMain: SNI, + ServerNameExtras: []string{}, + }, + )) defer env.Close() // add DPI engine to emulate the censorship condition diff --git a/pkg/experiment/tlstool/internal/internal_test.go b/pkg/experiment/tlstool/internal/internal_test.go index e887b5e1e..8127aad8d 100644 --- a/pkg/experiment/tlstool/internal/internal_test.go +++ b/pkg/experiment/tlstool/internal/internal_test.go @@ -25,17 +25,33 @@ func dial(t *testing.T, d model.Dialer) { } func TestNewSNISplitterDialer(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + dial(t, internal.NewSNISplitterDialer(config)) } func TestNewThriceSplitterDialer(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + dial(t, internal.NewThriceSplitterDialer(config)) } func TestNewRandomSplitterDialer(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + dial(t, internal.NewRandomSplitterDialer(config)) } func TestNewVanillaDialer(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + dial(t, internal.NewVanillaDialer(config)) } diff --git a/pkg/experiment/tlstool/tlstool.go b/pkg/experiment/tlstool/tlstool.go index f1ff8a0b7..53374f505 100644 --- a/pkg/experiment/tlstool/tlstool.go +++ b/pkg/experiment/tlstool/tlstool.go @@ -17,9 +17,9 @@ import ( "github.com/ooni/probe-engine/pkg/experiment/tlstool/internal" "github.com/ooni/probe-engine/pkg/legacy/netx" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/runtimex" - "github.com/ooni/probe-engine/pkg/tracex" ) const ( diff --git a/pkg/experiment/tor/tor.go b/pkg/experiment/tor/tor.go index f641f573a..5d4868f0f 100644 --- a/pkg/experiment/tor/tor.go +++ b/pkg/experiment/tor/tor.go @@ -13,12 +13,13 @@ import ( "sync/atomic" "time" - "github.com/ooni/probe-engine/pkg/measurex" + "github.com/ooni/probe-engine/pkg/legacy/measurex" + "github.com/ooni/probe-engine/pkg/legacy/tracex" + "github.com/ooni/probe-engine/pkg/logx" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" "github.com/ooni/probe-engine/pkg/runtimex" "github.com/ooni/probe-engine/pkg/scrubber" - "github.com/ooni/probe-engine/pkg/tracex" ) const ( @@ -275,7 +276,7 @@ func maybeSanitize(input TargetResults, kt keytarget) TargetResults { // Implementation note: here we are using a strict scrubbing policy where // we remove all IP _endpoints_, mainly for convenience, because we already // have a well tested implementation that does that. - data = []byte(scrubber.Scrub(string(data))) + data = []byte(scrubber.ScrubString(string(data))) var out TargetResults err = json.Unmarshal(data, &out) runtimex.PanicOnError(err, "json.Unmarshal should not fail here") @@ -320,7 +321,7 @@ func maybeScrubbingLogger(input model.Logger, kt keytarget) model.Logger { if !kt.private() { return input } - return &scrubber.Logger{Logger: input} + return &logx.ScrubberLogger{Logger: input} } // defaultFlexibleConnect is the default implementation of the diff --git a/pkg/experiment/tor/tor_test.go b/pkg/experiment/tor/tor_test.go index b1b3ba06a..78b1cbbd7 100644 --- a/pkg/experiment/tor/tor_test.go +++ b/pkg/experiment/tor/tor_test.go @@ -13,11 +13,11 @@ import ( "github.com/apex/log" "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-engine/pkg/legacy/measurex" "github.com/ooni/probe-engine/pkg/legacy/mockable" - "github.com/ooni/probe-engine/pkg/measurex" + "github.com/ooni/probe-engine/pkg/logx" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" - "github.com/ooni/probe-engine/pkg/scrubber" ) func TestNewExperimentMeasurer(t *testing.T) { @@ -717,7 +717,7 @@ func TestMaybeScrubbingLogger(t *testing.T) { if out != input { t.Fatal("not the output we expected") } - if _, ok := out.(*scrubber.Logger); ok { + if _, ok := out.(*logx.ScrubberLogger); ok { t.Fatal("not the output type we expected") } }) @@ -730,7 +730,7 @@ func TestMaybeScrubbingLogger(t *testing.T) { if out == input { t.Fatal("not the output value we expected") } - if _, ok := out.(*scrubber.Logger); !ok { + if _, ok := out.(*logx.ScrubberLogger); !ok { t.Fatal("not the output type we expected") } }) diff --git a/pkg/experiment/torsf/torsf.go b/pkg/experiment/torsf/torsf.go index 319f3bf5a..620d980df 100644 --- a/pkg/experiment/torsf/torsf.go +++ b/pkg/experiment/torsf/torsf.go @@ -11,11 +11,11 @@ import ( "time" "github.com/ooni/probe-engine/pkg/bytecounter" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/ptx" "github.com/ooni/probe-engine/pkg/runtimex" "github.com/ooni/probe-engine/pkg/torlogs" - "github.com/ooni/probe-engine/pkg/tracex" "github.com/ooni/probe-engine/pkg/tunnel" ) diff --git a/pkg/experiment/urlgetter/configurer.go b/pkg/experiment/urlgetter/configurer.go index ded71c29d..c41fd3644 100644 --- a/pkg/experiment/urlgetter/configurer.go +++ b/pkg/experiment/urlgetter/configurer.go @@ -9,9 +9,9 @@ import ( "strings" "github.com/ooni/probe-engine/pkg/legacy/netx" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" - "github.com/ooni/probe-engine/pkg/tracex" ) // The Configurer job is to construct a Configuration that can diff --git a/pkg/experiment/urlgetter/configurer_test.go b/pkg/experiment/urlgetter/configurer_test.go index d607398ab..0fec8ddca 100644 --- a/pkg/experiment/urlgetter/configurer_test.go +++ b/pkg/experiment/urlgetter/configurer_test.go @@ -9,8 +9,8 @@ import ( "github.com/apex/log" "github.com/ooni/probe-engine/pkg/experiment/urlgetter" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/netxlite" - "github.com/ooni/probe-engine/pkg/tracex" ) func TestConfigurerNewConfigurationVanilla(t *testing.T) { diff --git a/pkg/experiment/urlgetter/getter.go b/pkg/experiment/urlgetter/getter.go index 6adf2e885..fb57fb952 100644 --- a/pkg/experiment/urlgetter/getter.go +++ b/pkg/experiment/urlgetter/getter.go @@ -6,9 +6,9 @@ import ( "net/url" "time" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" - "github.com/ooni/probe-engine/pkg/tracex" "github.com/ooni/probe-engine/pkg/tunnel" ) @@ -69,7 +69,7 @@ func (g Getter) Get(ctx context.Context) (TestKeys, error) { if len(tk.Requests) > 0 { // OONI's convention is that the last request appears first tk.HTTPResponseStatus = tk.Requests[0].Response.Code - tk.HTTPResponseBody = tk.Requests[0].Response.Body.Value + tk.HTTPResponseBody = string(tk.Requests[0].Response.Body) tk.HTTPResponseLocations = tk.Requests[0].Response.Locations } tk.TCPConnect = append( diff --git a/pkg/experiment/urlgetter/getter_integration_test.go b/pkg/experiment/urlgetter/getter_integration_test.go index 0764e9437..6ad87f6e9 100644 --- a/pkg/experiment/urlgetter/getter_integration_test.go +++ b/pkg/experiment/urlgetter/getter_integration_test.go @@ -386,6 +386,10 @@ func TestGetterWithCancelledContextUnknownResolverURL(t *testing.T) { } func TestGetterIntegrationHTTPS(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + ctx := context.Background() g := urlgetter.Getter{ Config: urlgetter.Config{ @@ -510,6 +514,10 @@ func TestGetterIntegrationRedirect(t *testing.T) { } func TestGetterIntegrationTLSHandshake(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + ctx := context.Background() g := urlgetter.Getter{ Config: urlgetter.Config{ @@ -611,6 +619,10 @@ func TestGetterIntegrationTLSHandshake(t *testing.T) { } func TestGetterHTTPSWithTunnel(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + // quick enough (0.4s) to run with every run ctx := context.Background() g := urlgetter.Getter{ diff --git a/pkg/experiment/urlgetter/multi_test.go b/pkg/experiment/urlgetter/multi_test.go index a4e813862..690aae01d 100644 --- a/pkg/experiment/urlgetter/multi_test.go +++ b/pkg/experiment/urlgetter/multi_test.go @@ -17,6 +17,10 @@ import ( ) func TestMultiIntegration(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + multi := urlgetter.Multi{Session: &mockable.Session{}} inputs := []urlgetter.MultiInput{{ Config: urlgetter.Config{Method: "HEAD", NoFollowRedirects: true}, diff --git a/pkg/experiment/urlgetter/urlgetter.go b/pkg/experiment/urlgetter/urlgetter.go index 7a7bc9cdd..280945461 100644 --- a/pkg/experiment/urlgetter/urlgetter.go +++ b/pkg/experiment/urlgetter/urlgetter.go @@ -13,8 +13,8 @@ import ( "crypto/x509" "time" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/model" - "github.com/ooni/probe-engine/pkg/tracex" ) const ( diff --git a/pkg/experiment/vanillator/vanillator.go b/pkg/experiment/vanillator/vanillator.go index a0cc948aa..d86c319a1 100644 --- a/pkg/experiment/vanillator/vanillator.go +++ b/pkg/experiment/vanillator/vanillator.go @@ -10,10 +10,10 @@ import ( "path" "time" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/runtimex" "github.com/ooni/probe-engine/pkg/torlogs" - "github.com/ooni/probe-engine/pkg/tracex" "github.com/ooni/probe-engine/pkg/tunnel" ) diff --git a/pkg/experiment/webconnectivity/dnsanalysis.go b/pkg/experiment/webconnectivity/dnsanalysis.go index a4c714e86..0c9367fa3 100644 --- a/pkg/experiment/webconnectivity/dnsanalysis.go +++ b/pkg/experiment/webconnectivity/dnsanalysis.go @@ -34,12 +34,14 @@ func DNSAnalysis(URL *url.URL, measurement DNSLookupResult, control ControlResponse) (out DNSAnalysisResult) { // 0. start assuming it's not consistent out.DNSConsistency = &DNSInconsistent + // 1. flip to consistent if we're targeting an IP address because the // control will actually return dns_name_error in this case. if net.ParseIP(URL.Hostname()) != nil { out.DNSConsistency = &DNSConsistent return } + // 2. flip to consistent if the failures are compatible if measurement.Failure != nil && control.DNS.Failure != nil { switch *control.DNS.Failure { @@ -57,6 +59,7 @@ func DNSAnalysis(URL *url.URL, measurement DNSLookupResult, } return } + // 3. flip to consistent if measurement and control returned IP addresses // that belong to the same Autonomous System(s). // @@ -85,6 +88,7 @@ func DNSAnalysis(URL *url.URL, measurement DNSLookupResult, return } } + // 4. when ASN lookup failed (unlikely), check whether // there is overlap in the returned IP addresses ipmap := make(map[string]int) @@ -101,6 +105,7 @@ func DNSAnalysis(URL *url.URL, measurement DNSLookupResult, return } } + // 5. conclude that measurement and control are inconsistent return } diff --git a/pkg/experiment/webconnectivity/httpanalysis.go b/pkg/experiment/webconnectivity/httpanalysis.go index 63ead3838..3cc0233e3 100644 --- a/pkg/experiment/webconnectivity/httpanalysis.go +++ b/pkg/experiment/webconnectivity/httpanalysis.go @@ -56,7 +56,7 @@ func HTTPBodyLengthChecks( if response.BodyIsTruncated { return } - measurement := int64(len(response.Body.Value)) + measurement := int64(len(response.Body)) if measurement <= 0 { return } @@ -201,7 +201,7 @@ func HTTPTitleMatch(tk urlgetter.TestKeys, ctrl ControlResponse) (out *bool) { return } control := ctrl.HTTPRequest.Title - measurementBody := response.Body.Value + measurementBody := string(response.Body) measurement := measurexlite.WebGetTitle(measurementBody) if measurement == "" { return diff --git a/pkg/experiment/webconnectivity/httpanalysis_test.go b/pkg/experiment/webconnectivity/httpanalysis_test.go index 7d55c4d46..c9d8303d0 100644 --- a/pkg/experiment/webconnectivity/httpanalysis_test.go +++ b/pkg/experiment/webconnectivity/httpanalysis_test.go @@ -6,8 +6,9 @@ import ( "github.com/google/go-cmp/cmp" "github.com/ooni/probe-engine/pkg/experiment/urlgetter" "github.com/ooni/probe-engine/pkg/experiment/webconnectivity" + "github.com/ooni/probe-engine/pkg/legacy/tracex" + "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/randx" - "github.com/ooni/probe-engine/pkg/tracex" ) func TestHTTPBodyLengthChecks(t *testing.T) { @@ -76,9 +77,9 @@ func TestHTTPBodyLengthChecks(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Body: tracex.MaybeBinaryValue{ - Value: randx.Letters(768), - }, + Body: model.ArchivalScrubbedMaybeBinaryString( + randx.Letters(768), + ), }, }}, }, @@ -95,9 +96,9 @@ func TestHTTPBodyLengthChecks(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Body: tracex.MaybeBinaryValue{ - Value: randx.Letters(768), - }, + Body: model.ArchivalScrubbedMaybeBinaryString( + randx.Letters(768), + ), }, }}, }, @@ -115,9 +116,9 @@ func TestHTTPBodyLengthChecks(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Body: tracex.MaybeBinaryValue{ - Value: randx.Letters(1024), - }, + Body: model.ArchivalScrubbedMaybeBinaryString( + randx.Letters(1024), + ), }, }}, }, @@ -135,9 +136,9 @@ func TestHTTPBodyLengthChecks(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Body: tracex.MaybeBinaryValue{ - Value: randx.Letters(8), - }, + Body: model.ArchivalScrubbedMaybeBinaryString( + randx.Letters(8), + ), }, }}, }, @@ -155,9 +156,9 @@ func TestHTTPBodyLengthChecks(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Body: tracex.MaybeBinaryValue{ - Value: randx.Letters(16), - }, + Body: model.ArchivalScrubbedMaybeBinaryString( + randx.Letters(16), + ), }, }}, }, @@ -365,8 +366,8 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]tracex.MaybeBinaryValue{ - "Date": {Value: "Mon Jul 13 21:10:08 CEST 2020"}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "Date": "Mon Jul 13 21:10:08 CEST 2020", }, Code: 200, }, @@ -381,8 +382,8 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]tracex.MaybeBinaryValue{ - "Date": {Value: "Mon Jul 13 21:10:08 CEST 2020"}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "Date": "Mon Jul 13 21:10:08 CEST 2020", }, Code: 200, }, @@ -401,8 +402,8 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]tracex.MaybeBinaryValue{ - "Date": {Value: "Mon Jul 13 21:10:08 CEST 2020"}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "Date": "Mon Jul 13 21:10:08 CEST 2020", }, Code: 200, }, @@ -424,9 +425,9 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]tracex.MaybeBinaryValue{ - "Date": {Value: "Mon Jul 13 21:10:08 CEST 2020"}, - "Antani": {Value: "MASCETTI"}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "Date": "Mon Jul 13 21:10:08 CEST 2020", + "Antani": "MASCETTI", }, Code: 200, }, @@ -449,9 +450,9 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]tracex.MaybeBinaryValue{ - "Date": {Value: "Mon Jul 13 21:10:08 CEST 2020"}, - "Antani": {Value: "MASCETTI"}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "Date": "Mon Jul 13 21:10:08 CEST 2020", + "Antani": "MASCETTI", }, Code: 200, }, @@ -474,19 +475,19 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]tracex.MaybeBinaryValue{ - "Accept-Ranges": {Value: "bytes"}, - "Age": {Value: "404727"}, - "Cache-Control": {Value: "max-age=604800"}, - "Content-Length": {Value: "1256"}, - "Content-Type": {Value: "text/html; charset=UTF-8"}, - "Date": {Value: "Tue, 14 Jul 2020 22:26:09 GMT"}, - "Etag": {Value: "\"3147526947\""}, - "Expires": {Value: "Tue, 21 Jul 2020 22:26:09 GMT"}, - "Last-Modified": {Value: "Thu, 17 Oct 2019 07:18:26 GMT"}, - "Server": {Value: "ECS (dcb/7F3C)"}, - "Vary": {Value: "Accept-Encoding"}, - "X-Cache": {Value: "HIT"}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "Accept-Ranges": "bytes", + "Age": "404727", + "Cache-Control": "max-age=604800", + "Content-Length": "1256", + "Content-Type": "text/html; charset=UTF-8", + "Date": "Tue, 14 Jul 2020 22:26:09 GMT", + "Etag": "\"3147526947\"", + "Expires": "Tue, 21 Jul 2020 22:26:09 GMT", + "Last-Modified": "Thu, 17 Oct 2019 07:18:26 GMT", + "Server": "ECS (dcb/7F3C)", + "Vary": "Accept-Encoding", + "X-Cache": "HIT", }, Code: 200, }, @@ -521,18 +522,18 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]tracex.MaybeBinaryValue{ - "Accept-Ranges": {Value: "bytes"}, - "Age": {Value: "404727"}, - "Cache-Control": {Value: "max-age=604800"}, - "Content-Length": {Value: "1256"}, - "Content-Type": {Value: "text/html; charset=UTF-8"}, - "Date": {Value: "Tue, 14 Jul 2020 22:26:09 GMT"}, - "Etag": {Value: "\"3147526947\""}, - "Expires": {Value: "Tue, 21 Jul 2020 22:26:09 GMT"}, - "Last-Modified": {Value: "Thu, 17 Oct 2019 07:18:26 GMT"}, - "Server": {Value: "ECS (dcb/7F3C)"}, - "Vary": {Value: "Accept-Encoding"}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "Accept-Ranges": "bytes", + "Age": "404727", + "Cache-Control": "max-age=604800", + "Content-Length": "1256", + "Content-Type": "text/html; charset=UTF-8", + "Date": "Tue, 14 Jul 2020 22:26:09 GMT", + "Etag": "\"3147526947\"", + "Expires": "Tue, 21 Jul 2020 22:26:09 GMT", + "Last-Modified": "Thu, 17 Oct 2019 07:18:26 GMT", + "Server": "ECS (dcb/7F3C)", + "Vary": "Accept-Encoding", }, Code: 200, }, @@ -566,17 +567,17 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]tracex.MaybeBinaryValue{ - "Accept-Ranges": {Value: "bytes"}, - "Age": {Value: "404727"}, - "Cache-Control": {Value: "max-age=604800"}, - "Content-Type": {Value: "text/html; charset=UTF-8"}, - "Date": {Value: "Tue, 14 Jul 2020 22:26:09 GMT"}, - "Etag": {Value: "\"3147526947\""}, - "Expires": {Value: "Tue, 21 Jul 2020 22:26:09 GMT"}, - "Last-Modified": {Value: "Thu, 17 Oct 2019 07:18:26 GMT"}, - "Server": {Value: "ECS (dcb/7F3C)"}, - "Vary": {Value: "Accept-Encoding"}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "Accept-Ranges": "bytes", + "Age": "404727", + "Cache-Control": "max-age=604800", + "Content-Type": "text/html; charset=UTF-8", + "Date": "Tue, 14 Jul 2020 22:26:09 GMT", + "Etag": "\"3147526947\"", + "Expires": "Tue, 21 Jul 2020 22:26:09 GMT", + "Last-Modified": "Thu, 17 Oct 2019 07:18:26 GMT", + "Server": "ECS (dcb/7F3C)", + "Vary": "Accept-Encoding", }, Code: 200, }, @@ -607,17 +608,17 @@ func TestHeadersMatch(t *testing.T) { tk: urlgetter.TestKeys{ Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ - Headers: map[string]tracex.MaybeBinaryValue{ - "accept-ranges": {Value: "bytes"}, - "AGE": {Value: "404727"}, - "cache-Control": {Value: "max-age=604800"}, - "Content-TyPe": {Value: "text/html; charset=UTF-8"}, - "DatE": {Value: "Tue, 14 Jul 2020 22:26:09 GMT"}, - "etag": {Value: "\"3147526947\""}, - "expires": {Value: "Tue, 21 Jul 2020 22:26:09 GMT"}, - "Last-Modified": {Value: "Thu, 17 Oct 2019 07:18:26 GMT"}, - "SerVer": {Value: "ECS (dcb/7F3C)"}, - "Vary": {Value: "Accept-Encoding"}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "accept-ranges": "bytes", + "AGE": "404727", + "cache-Control": "max-age=604800", + "Content-TyPe": "text/html; charset=UTF-8", + "DatE": "Tue, 14 Jul 2020 22:26:09 GMT", + "etag": "\"3147526947\"", + "expires": "Tue, 21 Jul 2020 22:26:09 GMT", + "Last-Modified": "Thu, 17 Oct 2019 07:18:26 GMT", + "SerVer": "ECS (dcb/7F3C)", + "Vary": "Accept-Encoding", }, Code: 200, }, @@ -698,7 +699,7 @@ func TestTitleMatch(t *testing.T) { Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ Code: 200, - Body: tracex.MaybeBinaryValue{Value: ""}, + Body: model.ArchivalScrubbedMaybeBinaryString(""), }, }}, }, @@ -711,7 +712,7 @@ func TestTitleMatch(t *testing.T) { Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ Code: 200, - Body: tracex.MaybeBinaryValue{Value: ""}, + Body: model.ArchivalScrubbedMaybeBinaryString(""), }, }}, }, @@ -730,8 +731,8 @@ func TestTitleMatch(t *testing.T) { Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ Code: 200, - Body: tracex.MaybeBinaryValue{ - Value: "La community di MSN"}, + Body: model.ArchivalScrubbedMaybeBinaryString( + "La community di MSN"), }, }}, }, @@ -750,8 +751,8 @@ func TestTitleMatch(t *testing.T) { Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ Code: 200, - Body: tracex.MaybeBinaryValue{ - Value: "La communità di MSN"}, + Body: model.ArchivalScrubbedMaybeBinaryString( + "La communità di MSN"), }, }}, }, @@ -770,8 +771,8 @@ func TestTitleMatch(t *testing.T) { Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ Code: 200, - Body: tracex.MaybeBinaryValue{ - Value: "" + randx.Letters(1024) + ""}, + Body: model.ArchivalScrubbedMaybeBinaryString( + "" + randx.Letters(1024) + ""), }, }}, }, @@ -790,8 +791,8 @@ func TestTitleMatch(t *testing.T) { Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ Code: 200, - Body: tracex.MaybeBinaryValue{ - Value: "La commUNity di MSN"}, + Body: model.ArchivalScrubbedMaybeBinaryString( + "La commUNity di MSN"), }, }}, }, @@ -810,8 +811,8 @@ func TestTitleMatch(t *testing.T) { Requests: []tracex.RequestEntry{{ Response: tracex.HTTPResponse{ Code: 200, - Body: tracex.MaybeBinaryValue{ - Value: "La commUNity di MSN"}, + Body: model.ArchivalScrubbedMaybeBinaryString( + "La commUNity di MSN"), }, }}, }, diff --git a/pkg/experiment/webconnectivity/summary.go b/pkg/experiment/webconnectivity/summary.go index 8db3053c6..b3154b9b8 100644 --- a/pkg/experiment/webconnectivity/summary.go +++ b/pkg/experiment/webconnectivity/summary.go @@ -100,6 +100,7 @@ func Summarize(tk *TestKeys) (out Summary) { defer func() { out.Blocking = DetermineBlocking(out) }() + var ( accessible = true inaccessible = false @@ -108,6 +109,7 @@ func Summarize(tk *TestKeys) (out Summary) { httpFailure = "http-failure" tcpIP = "tcp_ip" ) + // If the measurement was for an HTTPS website and the HTTP experiment // succeeded, then either there is a compromised CA in our pool (which is // certifi-go), or there is transparent proxying, or we are actually @@ -119,11 +121,13 @@ func Summarize(tk *TestKeys) (out Summary) { out.Status |= StatusSuccessSecure return } + // If we couldn't contact the control, we cannot do much more here. if tk.ControlFailure != nil { out.Status |= StatusAnomalyControlUnreachable return } + // If DNS failed with NXDOMAIN and the control DNS is consistent, then it // means this website does not exist anymore. We need to include the weird // cache failure on Android into this analysis because that failure means @@ -142,15 +146,33 @@ func Summarize(tk *TestKeys) (out Summary) { out.Status |= StatusSuccessNXDOMAIN | StatusExperimentDNS return } - // Otherwise, if DNS failed with NXDOMAIN, it's DNS based blocking. - // TODO(bassosimone): do we wanna include other errors here? Like timeout? - if tk.DNSExperimentFailure != nil && - *tk.DNSExperimentFailure == netxlite.FailureDNSNXDOMAINError { + + // Web Connectivity's analysis algorithm up until v0.4.2 gave priority to checking for http-diff + // over saying that there's "dns" blocking when the DNS is inconsistent. + // + // In v0.4.3, we want to address https://github.com/ooni/probe/issues/2499 while still + // trying to preserve the original spirit of the v0.4.2 analysis algorithm. + // + // To this end, we _only_ flag anomaly if the following happens: + // + // 1. the DNS is inconsistent; and + // + // 2. the probe's DNS lookup has failed with dns_nxdomain_error or android_dns_cache_no_data. + // + // By using this algorithm, we narrow the scope and impact of this change but we are, at + // the same time, able to catch cases such as the one mentioned by the issue above. + // + // A more aggressive approach would flag as "dns" blocking any inconsistent result but + // that would depart quite a lot from the behavior of v0.4.2. + if tk.DNSConsistency != nil && *tk.DNSConsistency == DNSInconsistent && + tk.DNSExperimentFailure != nil && (*tk.DNSExperimentFailure == netxlite.FailureDNSNXDOMAINError || + *tk.DNSExperimentFailure == netxlite.FailureAndroidDNSCacheNoData) { out.Accessible = &inaccessible out.BlockingReason = &dns out.Status |= StatusAnomalyDNS | StatusExperimentDNS return } + // If we tried to connect more than once and never succedeed and we were // able to measure DNS consistency, then we can conclude something. if tk.TCPConnectAttempts > 0 && tk.TCPConnectSuccesses <= 0 && tk.DNSConsistency != nil { diff --git a/pkg/experiment/webconnectivity/summary_test.go b/pkg/experiment/webconnectivity/summary_test.go index 530c7e386..c868c9054 100644 --- a/pkg/experiment/webconnectivity/summary_test.go +++ b/pkg/experiment/webconnectivity/summary_test.go @@ -6,8 +6,8 @@ import ( "github.com/google/go-cmp/cmp" "github.com/ooni/probe-engine/pkg/experiment/webconnectivity" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/netxlite" - "github.com/ooni/probe-engine/pkg/tracex" ) func TestSummarize(t *testing.T) { diff --git a/pkg/experiment/webconnectivity/webconnectivity.go b/pkg/experiment/webconnectivity/webconnectivity.go index 86b57ecfe..9ef3186c5 100644 --- a/pkg/experiment/webconnectivity/webconnectivity.go +++ b/pkg/experiment/webconnectivity/webconnectivity.go @@ -9,13 +9,13 @@ import ( "time" "github.com/ooni/probe-engine/pkg/experiment/webconnectivity/internal" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/model" - "github.com/ooni/probe-engine/pkg/tracex" ) const ( testName = "web_connectivity" - testVersion = "0.4.2" + testVersion = "0.4.3" ) // Config contains the experiment config. diff --git a/pkg/experiment/webconnectivity/webconnectivity_test.go b/pkg/experiment/webconnectivity/webconnectivity_test.go index 6ee5fbfa3..751643845 100644 --- a/pkg/experiment/webconnectivity/webconnectivity_test.go +++ b/pkg/experiment/webconnectivity/webconnectivity_test.go @@ -11,9 +11,9 @@ import ( "github.com/google/go-cmp/cmp" "github.com/ooni/probe-engine/pkg/engine" "github.com/ooni/probe-engine/pkg/experiment/webconnectivity" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" - "github.com/ooni/probe-engine/pkg/tracex" ) func TestNewExperimentMeasurer(t *testing.T) { @@ -21,7 +21,7 @@ func TestNewExperimentMeasurer(t *testing.T) { if measurer.ExperimentName() != "web_connectivity" { t.Fatal("unexpected name") } - if measurer.ExperimentVersion() != "0.4.2" { + if measurer.ExperimentVersion() != "0.4.3" { t.Fatal("unexpected version") } } diff --git a/pkg/experiment/webconnectivitylte/analysishttpdiff.go b/pkg/experiment/webconnectivitylte/analysishttpdiff.go index 66af13b36..d7dfc41ee 100644 --- a/pkg/experiment/webconnectivitylte/analysishttpdiff.go +++ b/pkg/experiment/webconnectivitylte/analysishttpdiff.go @@ -91,7 +91,7 @@ func (tk *TestKeys) httpDiffBodyLengthChecks( if response.BodyIsTruncated { return // cannot trust body length in this case } - measurement := int64(len(response.Body.Value)) + measurement := int64(len(response.Body)) if measurement <= 0 { return // no actual length } @@ -230,7 +230,7 @@ func (tk *TestKeys) httpDiffTitleMatch( return } control := ctrl.Title - measurementBody := response.Body.Value + measurementBody := string(response.Body) measurement := measurexlite.WebGetTitle(measurementBody) if control == "" || measurement == "" { return diff --git a/pkg/experiment/webconnectivitylte/cleartextflow.go b/pkg/experiment/webconnectivitylte/cleartextflow.go index 62b687479..bdfa78a5a 100644 --- a/pkg/experiment/webconnectivitylte/cleartextflow.go +++ b/pkg/experiment/webconnectivitylte/cleartextflow.go @@ -16,6 +16,7 @@ import ( "sync/atomic" "time" + "github.com/ooni/probe-engine/pkg/logx" "github.com/ooni/probe-engine/pkg/measurexlite" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" @@ -107,7 +108,7 @@ func (t *CleartextFlow) Run(parentCtx context.Context, index int64) error { }() // start the operation logger - ol := measurexlite.NewOperationLogger( + ol := logx.NewOperationLogger( t.Logger, "[#%d] GET http://%s using %s", index, t.HostHeader, t.Address, ) @@ -136,6 +137,9 @@ func (t *CleartextFlow) Run(parentCtx context.Context, index int64) error { } // create HTTP transport + // TODO(https://github.com/ooni/probe/issues/2534): here we're using the QUIRKY netxlite.NewHTTPTransport + // function, but we can probably avoid using it, given that this code is + // not using tracing and does not care about those quirks. httpTransport := netxlite.NewHTTPTransport( t.Logger, netxlite.NewSingleUseDialer(tcpConn), diff --git a/pkg/experiment/webconnectivitylte/control.go b/pkg/experiment/webconnectivitylte/control.go index e4a4a7901..c9351c5d8 100644 --- a/pkg/experiment/webconnectivitylte/control.go +++ b/pkg/experiment/webconnectivitylte/control.go @@ -9,7 +9,7 @@ import ( "github.com/ooni/probe-engine/pkg/experiment/webconnectivity" "github.com/ooni/probe-engine/pkg/httpapi" - "github.com/ooni/probe-engine/pkg/measurexlite" + "github.com/ooni/probe-engine/pkg/logx" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" "github.com/ooni/probe-engine/pkg/ooapi" @@ -102,7 +102,7 @@ func (c *Control) Run(parentCtx context.Context) { c.TestKeys.SetControlRequest(creq) // create logger for this operation - ol := measurexlite.NewOperationLogger( + ol := logx.NewOperationLogger( c.Logger, "control for %s using %+v", creq.HTTPRequest, diff --git a/pkg/experiment/webconnectivitylte/dnsresolvers.go b/pkg/experiment/webconnectivitylte/dnsresolvers.go index 6dd382cd1..2660fabd7 100644 --- a/pkg/experiment/webconnectivitylte/dnsresolvers.go +++ b/pkg/experiment/webconnectivitylte/dnsresolvers.go @@ -16,6 +16,7 @@ import ( "sync/atomic" "time" + "github.com/ooni/probe-engine/pkg/logx" "github.com/ooni/probe-engine/pkg/measurexlite" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" @@ -212,7 +213,7 @@ func (t *DNSResolvers) lookupHostSystem(parentCtx context.Context, out chan<- [] trace := measurexlite.NewTrace(index, t.ZeroTime) // start the operation logger - ol := measurexlite.NewOperationLogger( + ol := logx.NewOperationLogger( t.Logger, "[#%d] lookup %s using system", index, t.Domain, ) @@ -239,7 +240,7 @@ func (t *DNSResolvers) lookupHostUDP(parentCtx context.Context, udpAddress strin trace := measurexlite.NewTrace(index, t.ZeroTime) // start the operation logger - ol := measurexlite.NewOperationLogger( + ol := logx.NewOperationLogger( t.Logger, "[#%d] lookup %s using %s", index, t.Domain, udpAddress, ) @@ -377,7 +378,7 @@ func (t *DNSResolvers) lookupHostDNSOverHTTPS(parentCtx context.Context, out cha trace := measurexlite.NewTrace(index, t.ZeroTime) // start the operation logger - ol := measurexlite.NewOperationLogger( + ol := logx.NewOperationLogger( t.Logger, "[#%d] lookup %s using %s", index, t.Domain, URL, ) diff --git a/pkg/experiment/webconnectivitylte/secureflow.go b/pkg/experiment/webconnectivitylte/secureflow.go index 21ee19414..311abea1b 100644 --- a/pkg/experiment/webconnectivitylte/secureflow.go +++ b/pkg/experiment/webconnectivitylte/secureflow.go @@ -17,6 +17,7 @@ import ( "sync/atomic" "time" + "github.com/ooni/probe-engine/pkg/logx" "github.com/ooni/probe-engine/pkg/measurexlite" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" @@ -114,7 +115,7 @@ func (t *SecureFlow) Run(parentCtx context.Context, index int64) error { }() // start the operation logger - ol := measurexlite.NewOperationLogger( + ol := logx.NewOperationLogger( t.Logger, "[#%d] GET https://%s using %s", index, t.HostHeader, t.Address, ) @@ -153,7 +154,7 @@ func (t *SecureFlow) Run(parentCtx context.Context, index int64) error { const tlsTimeout = 10 * time.Second tlsCtx, tlsCancel := context.WithTimeout(parentCtx, tlsTimeout) defer tlsCancel() - tlsConn, tlsConnState, err := tlsHandshaker.Handshake(tlsCtx, tcpConn, tlsConfig) + tlsConn, err := tlsHandshaker.Handshake(tlsCtx, tcpConn, tlsConfig) t.TestKeys.AppendTLSHandshakes(trace.TLSHandshakes()...) if err != nil { ol.Stop(err) @@ -161,6 +162,7 @@ func (t *SecureFlow) Run(parentCtx context.Context, index int64) error { } defer tlsConn.Close() + tlsConnState := netxlite.MaybeTLSConnectionState(tlsConn) alpn := tlsConnState.NegotiatedProtocol // Determine whether we're allowed to fetch the webpage @@ -170,11 +172,13 @@ func (t *SecureFlow) Run(parentCtx context.Context, index int64) error { } // create HTTP transport + // TODO(https://github.com/ooni/probe/issues/2534): here we're using the QUIRKY netxlite.NewHTTPTransport + // function, but we can probably avoid using it, given that this code is + // not using tracing and does not care about those quirks. httpTransport := netxlite.NewHTTPTransport( t.Logger, netxlite.NewNullDialer(), - // note: netxlite guarantees that here tlsConn is a netxlite.TLSConn - netxlite.NewSingleUseTLSDialer(tlsConn.(netxlite.TLSConn)), + netxlite.NewSingleUseTLSDialer(tlsConn), ) // create HTTP request diff --git a/pkg/experiment/webconnectivitylte/testkeys.go b/pkg/experiment/webconnectivitylte/testkeys.go index d84e8db23..67278e221 100644 --- a/pkg/experiment/webconnectivitylte/testkeys.go +++ b/pkg/experiment/webconnectivitylte/testkeys.go @@ -12,8 +12,8 @@ import ( "sync" "github.com/ooni/probe-engine/pkg/experiment/webconnectivity" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/model" - "github.com/ooni/probe-engine/pkg/tracex" ) // TestKeys contains the results produced by web_connectivity. diff --git a/pkg/experiment/webconnectivityqa/badssl_test.go b/pkg/experiment/webconnectivityqa/badssl_test.go index 04fb863c9..56eeb0df8 100644 --- a/pkg/experiment/webconnectivityqa/badssl_test.go +++ b/pkg/experiment/webconnectivityqa/badssl_test.go @@ -33,9 +33,12 @@ func TestBadSSLConditions(t *testing.T) { for _, tc := range testcases { t.Run(tc.testCase.Name, func(t *testing.T) { env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() + tc.testCase.Configure(env) env.Do(func() { + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPClientStdlib has QUIRKS but they're not needed here client := netxlite.NewHTTPClientStdlib(log.Log) req := runtimex.Try1(http.NewRequest("GET", tc.testCase.Input, nil)) resp, err := client.Do(req) diff --git a/pkg/experiment/webconnectivityqa/control_test.go b/pkg/experiment/webconnectivityqa/control_test.go index c0cf29298..d348a4bcd 100644 --- a/pkg/experiment/webconnectivityqa/control_test.go +++ b/pkg/experiment/webconnectivityqa/control_test.go @@ -12,6 +12,8 @@ import ( func TestControlFailureWithSuccessfulHTTPWebsite(t *testing.T) { env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() + tc := controlFailureWithSuccessfulHTTPWebsite() tc.Configure(env) @@ -33,6 +35,8 @@ func TestControlFailureWithSuccessfulHTTPWebsite(t *testing.T) { func TestControlFailureWithSuccessfulHTTPSWebsite(t *testing.T) { env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() + tc := controlFailureWithSuccessfulHTTPSWebsite() tc.Configure(env) diff --git a/pkg/experiment/webconnectivityqa/dnsblocking.go b/pkg/experiment/webconnectivityqa/dnsblocking.go index 34c6ed57a..1a22dd368 100644 --- a/pkg/experiment/webconnectivityqa/dnsblocking.go +++ b/pkg/experiment/webconnectivityqa/dnsblocking.go @@ -9,7 +9,7 @@ import ( func dnsBlockingAndroidDNSCacheNoData() *TestCase { return &TestCase{ Name: "dnsBlockingAndroidDNSCacheNoData", - Flags: TestCaseFlagNoV04, // see https://github.com/ooni/probe-cli/pull/1211 + Flags: 0, Input: "https://www.example.com/", Configure: func(env *netemx.QAEnv) { // make sure the env knows we want to emulate our getaddrinfo wrapper behavior @@ -23,8 +23,9 @@ func dnsBlockingAndroidDNSCacheNoData() *TestCase { ExpectTestKeys: &testKeys{ DNSExperimentFailure: "android_dns_cache_no_data", DNSConsistency: "inconsistent", - XDNSFlags: 2, // AnalysisDNSUnexpectedFailure - XBlockingFlags: 33, // analysisFlagDNSBlocking | analysisFlagSuccess + XStatus: 2080, // StatusExperimentDNS | StatusAnomalyDNS + XDNSFlags: 2, // AnalysisDNSUnexpectedFailure + XBlockingFlags: 33, // analysisFlagDNSBlocking | analysisFlagSuccess Accessible: false, Blocking: "dns", }, diff --git a/pkg/experiment/webconnectivityqa/dnsblocking_test.go b/pkg/experiment/webconnectivityqa/dnsblocking_test.go index 2f4ed62c0..ceafaf1f5 100644 --- a/pkg/experiment/webconnectivityqa/dnsblocking_test.go +++ b/pkg/experiment/webconnectivityqa/dnsblocking_test.go @@ -12,6 +12,8 @@ import ( func TestDNSBlockingAndroidDNSCacheNoData(t *testing.T) { env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() + tc := dnsBlockingAndroidDNSCacheNoData() tc.Configure(env) @@ -29,6 +31,8 @@ func TestDNSBlockingAndroidDNSCacheNoData(t *testing.T) { func TestDNSBlockingNXDOMAIN(t *testing.T) { env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() + tc := dnsBlockingNXDOMAIN() tc.Configure(env) diff --git a/pkg/experiment/webconnectivityqa/dnshijacking_test.go b/pkg/experiment/webconnectivityqa/dnshijacking_test.go index 954d03cce..65197f314 100644 --- a/pkg/experiment/webconnectivityqa/dnshijacking_test.go +++ b/pkg/experiment/webconnectivityqa/dnshijacking_test.go @@ -19,6 +19,8 @@ func TestDNSHijackingTestCases(t *testing.T) { for _, tc := range testcases { t.Run(tc.Name, func(t *testing.T) { env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() + tc.Configure(env) env.Do(func() { diff --git a/pkg/experiment/webconnectivityqa/httpdiff_test.go b/pkg/experiment/webconnectivityqa/httpdiff_test.go index 3b82bf645..160d6a578 100644 --- a/pkg/experiment/webconnectivityqa/httpdiff_test.go +++ b/pkg/experiment/webconnectivityqa/httpdiff_test.go @@ -20,9 +20,12 @@ func TestHTTPDiffWithConsistentDNS(t *testing.T) { for _, tc := range testcases { t.Run(tc.Name, func(t *testing.T) { env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() + tc.Configure(env) env.Do(func() { + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPClientStdlib has QUIRKS but they're not needed here client := netxlite.NewHTTPClientStdlib(log.Log) req := runtimex.Try1(http.NewRequest("GET", "http://www.example.com/", nil)) resp, err := client.Do(req) @@ -50,10 +53,13 @@ func TestHTTPDiffWithInconsistentDNS(t *testing.T) { for _, tc := range testcases { t.Run(tc.Name, func(t *testing.T) { env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() + tc.Configure(env) env.Do(func() { t.Run("there is blockpage spoofing", func(t *testing.T) { + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPClientStdlib has QUIRKS but they're not needed here client := netxlite.NewHTTPClientStdlib(log.Log) req := runtimex.Try1(http.NewRequest("GET", "http://www.example.com/", nil)) resp, err := client.Do(req) diff --git a/pkg/experiment/webconnectivityqa/redirect_test.go b/pkg/experiment/webconnectivityqa/redirect_test.go index efaee7cb1..696aad2bc 100644 --- a/pkg/experiment/webconnectivityqa/redirect_test.go +++ b/pkg/experiment/webconnectivityqa/redirect_test.go @@ -23,6 +23,8 @@ func TestRedirectWithConsistentDNSAndThenConnectionRefused(t *testing.T) { for _, tc := range testcases { t.Run(tc.Name, func(t *testing.T) { env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() + tc.Configure(env) env.Do(func() { @@ -55,6 +57,8 @@ func TestRedirectWithConsistentDNSAndThenConnectionReset(t *testing.T) { for _, tc := range testcases { t.Run(tc.Name, func(t *testing.T) { env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() + tc.Configure(env) env.Do(func() { @@ -62,6 +66,7 @@ func TestRedirectWithConsistentDNSAndThenConnectionReset(t *testing.T) { for _, URL := range urls { t.Run(fmt.Sprintf("for URL %s", URL), func(t *testing.T) { + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPClientStdlib has QUIRKS but they're not needed here client := netxlite.NewHTTPClientStdlib(log.Log) req := runtimex.Try1(http.NewRequest("GET", URL, nil)) resp, err := client.Do(req) @@ -86,6 +91,8 @@ func TestRedirectWithConsistentDNSAndThenNXDOMAIN(t *testing.T) { for _, tc := range testcases { t.Run(tc.Name, func(t *testing.T) { env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() + tc.Configure(env) env.Do(func() { @@ -125,6 +132,8 @@ func TestRedirectWithConsistentDNSAndThenEOF(t *testing.T) { for _, tc := range testcases { t.Run(tc.Name, func(t *testing.T) { env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() + tc.Configure(env) env.Do(func() { @@ -132,6 +141,7 @@ func TestRedirectWithConsistentDNSAndThenEOF(t *testing.T) { for _, URL := range urls { t.Run(fmt.Sprintf("for URL %s", URL), func(t *testing.T) { + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPClientStdlib has QUIRKS but they're not needed here client := netxlite.NewHTTPClientStdlib(log.Log) req := runtimex.Try1(http.NewRequest("GET", URL, nil)) resp, err := client.Do(req) @@ -157,6 +167,8 @@ func TestRedirectWithConsistentDNSAndThenTimeout(t *testing.T) { for _, tc := range testcases { t.Run(tc.Name, func(t *testing.T) { env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() + tc.Configure(env) env.Do(func() { @@ -166,6 +178,7 @@ func TestRedirectWithConsistentDNSAndThenTimeout(t *testing.T) { t.Run(fmt.Sprintf("for URL %s", URL), func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPClientStdlib has QUIRKS but they're not needed here client := netxlite.NewHTTPClientStdlib(log.Log) req := runtimex.Try1(http.NewRequestWithContext(ctx, "GET", URL, nil)) resp, err := client.Do(req) diff --git a/pkg/experiment/webconnectivityqa/run.go b/pkg/experiment/webconnectivityqa/run.go index 79f38aa70..c213c5cf2 100644 --- a/pkg/experiment/webconnectivityqa/run.go +++ b/pkg/experiment/webconnectivityqa/run.go @@ -34,6 +34,7 @@ func RunTestCase(measurer model.ExperimentMeasurer, tc *TestCase) error { var err error env.Do(func() { // create an HTTP client inside the env.Do function so we're using netem + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPClientStdlib has QUIRKS but they're not needed here httpClient := netxlite.NewHTTPClientStdlib(prefixLogger) arguments := &model.ExperimentArgs{ Callbacks: model.NewPrinterCallbacks(prefixLogger), diff --git a/pkg/experiment/webconnectivityqa/run_test.go b/pkg/experiment/webconnectivityqa/run_test.go index ebd9ccb20..59b2e4f93 100644 --- a/pkg/experiment/webconnectivityqa/run_test.go +++ b/pkg/experiment/webconnectivityqa/run_test.go @@ -78,7 +78,7 @@ func TestRunTestCase(t *testing.T) { return "web_connectivity" }, MockExperimentVersion: func() string { - return "0.4.2" + return "0.4.3" }, MockRun: func(ctx context.Context, args *model.ExperimentArgs) error { args.Measurement.TestKeys = &testKeys{} @@ -107,7 +107,7 @@ func TestRunTestCase(t *testing.T) { return "web_connectivity" }, MockExperimentVersion: func() string { - return "0.4.2" + return "0.4.3" }, MockRun: func(ctx context.Context, args *model.ExperimentArgs) error { args.Measurement.TestKeys = &testKeys{ @@ -139,7 +139,7 @@ func TestRunTestCase(t *testing.T) { return "web_connectivity" }, MockExperimentVersion: func() string { - return "0.4.2" + return "0.4.3" }, MockRun: func(ctx context.Context, args *model.ExperimentArgs) error { args.Measurement.TestKeys = &testKeys{ @@ -173,7 +173,7 @@ func TestRunTestCase(t *testing.T) { return "web_connectivity" }, MockExperimentVersion: func() string { - return "0.4.2" + return "0.4.3" }, MockRun: func(ctx context.Context, args *model.ExperimentArgs) error { args.Measurement.TestKeys = &testKeys{ diff --git a/pkg/experiment/webconnectivityqa/tcpblocking_test.go b/pkg/experiment/webconnectivityqa/tcpblocking_test.go index 46b3526aa..765cf6167 100644 --- a/pkg/experiment/webconnectivityqa/tcpblocking_test.go +++ b/pkg/experiment/webconnectivityqa/tcpblocking_test.go @@ -12,6 +12,8 @@ import ( func TestTCPBlockingConnectTimeout(t *testing.T) { env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() + tc := tcpBlockingConnectTimeout() tc.Configure(env) @@ -30,6 +32,8 @@ func TestTCPBlockingConnectTimeout(t *testing.T) { func TestTCPBlockingConnectionRefusedWithInconsistentDNS(t *testing.T) { env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() + tc := tcpBlockingConnectionRefusedWithInconsistentDNS() tc.Configure(env) diff --git a/pkg/experiment/webconnectivityqa/testkeys.go b/pkg/experiment/webconnectivityqa/testkeys.go index 67170bce7..4ac9b69d2 100644 --- a/pkg/experiment/webconnectivityqa/testkeys.go +++ b/pkg/experiment/webconnectivityqa/testkeys.go @@ -69,7 +69,7 @@ func compareTestKeys(expected, got *testKeys) error { } switch got.XExperimentVersion { - case "0.4.2": + case "0.4.3": // ignore the fields that are specific to LTE options = append(options, cmpopts.IgnoreFields(testKeys{}, "XDNSFlags", "XBlockingFlags", "XNullNullFlags")) diff --git a/pkg/experiment/webconnectivityqa/tlsblocking_test.go b/pkg/experiment/webconnectivityqa/tlsblocking_test.go index 43c67c44c..bea81a867 100644 --- a/pkg/experiment/webconnectivityqa/tlsblocking_test.go +++ b/pkg/experiment/webconnectivityqa/tlsblocking_test.go @@ -14,6 +14,8 @@ import ( func TestBlockingTLSConnectionResetWithConsistentDNS(t *testing.T) { env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() + tc := tlsBlockingConnectionResetWithConsistentDNS() tc.Configure(env) @@ -21,6 +23,7 @@ func TestBlockingTLSConnectionResetWithConsistentDNS(t *testing.T) { urls := []string{"https://www.example.com/", "https://www.example.com/"} for _, URL := range urls { t.Run(fmt.Sprintf("for %s", URL), func(t *testing.T) { + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPClientStdlib has QUIRKS but they're not needed here client := netxlite.NewHTTPClientStdlib(log.Log) req, err := http.NewRequest("GET", URL, nil) if err != nil { @@ -40,6 +43,8 @@ func TestBlockingTLSConnectionResetWithConsistentDNS(t *testing.T) { func TestBlockingTLSConnectionResetWithInconsistentDNS(t *testing.T) { env := netemx.MustNewScenario(netemx.InternetScenario) + defer env.Close() + tc := tlsBlockingConnectionResetWithInconsistentDNS() tc.Configure(env) @@ -47,6 +52,7 @@ func TestBlockingTLSConnectionResetWithInconsistentDNS(t *testing.T) { urls := []string{"https://www.example.com/", "https://www.example.com/"} for _, URL := range urls { t.Run(fmt.Sprintf("for %s", URL), func(t *testing.T) { + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPClientStdlib has QUIRKS but they're not needed here client := netxlite.NewHTTPClientStdlib(log.Log) req, err := http.NewRequest("GET", URL, nil) if err != nil { diff --git a/pkg/experiment/whatsapp/whatsapp_test.go b/pkg/experiment/whatsapp/whatsapp_test.go index 507ed915b..26665ee81 100644 --- a/pkg/experiment/whatsapp/whatsapp_test.go +++ b/pkg/experiment/whatsapp/whatsapp_test.go @@ -73,7 +73,17 @@ func newQAEnvironment() *netemx.QAEnv { // - TCP listeners for endpoints on 443 and 5222 env := netemx.MustNewQAEnv( netemx.QAEnvOptionLogger(log.Log), - netemx.QAEnvOptionHTTPServer(whatsappWebAddr, netemx.ExampleWebPageHandlerFactory()), + netemx.QAEnvOptionNetStack( + whatsappWebAddr, + &netemx.HTTPSecureServerFactory{ + Factory: netemx.ExampleWebPageHandlerFactory(), + Ports: []int{443}, + ServerNameMain: "web.whatsapp.com", + ServerNameExtras: []string{ + "v.whatsapp.net", + }, + }, + ), netemx.QAEnvOptionNetStack(whatsappEndpointAddr, endpointsNetStack), ) diff --git a/pkg/hujsonx/hujsonx.go b/pkg/hujsonx/hujsonx.go new file mode 100644 index 000000000..ac44ebf45 --- /dev/null +++ b/pkg/hujsonx/hujsonx.go @@ -0,0 +1,18 @@ +// Package hujsonx contains github.com/tailscale/hujson extensions. +package hujsonx + +import ( + "encoding/json" + + "github.com/tailscale/hujson" +) + +// Unmarshal is like [json.Unmarshal] except that it first removes comments and +// extra commas using the [hujson.Standardize] function. +func Unmarshal(data []byte, v any) error { + data, err := hujson.Standardize(data) + if err != nil { + return err + } + return json.Unmarshal(data, v) +} diff --git a/pkg/hujsonx/hujsonx_test.go b/pkg/hujsonx/hujsonx_test.go new file mode 100644 index 000000000..c4e613cb8 --- /dev/null +++ b/pkg/hujsonx/hujsonx_test.go @@ -0,0 +1,33 @@ +package hujsonx + +import ( + "errors" + "io" + "testing" +) + +type user struct { + Name string + Age int +} + +func TestHuJSONXWorkingAsIntended(t *testing.T) { + t.Run("for invalid input", func(t *testing.T) { + input := []byte("{") + var v user + err := Unmarshal(input, &v) + if !errors.Is(err, io.ErrUnexpectedEOF) { + t.Fatal("unexpected error", err) + } + }) + + t.Run("for valid JSON we cannot map to a real struct", func(t *testing.T) { + input := []byte(`{"Name": {}, "Age": []}`) + var v user + err := Unmarshal(input, &v) + expected := "json: cannot unmarshal object into Go struct field user.Name of type string" + if err == nil || err.Error() != expected { + t.Fatal("unexpected error", err) + } + }) +} diff --git a/pkg/kvstore/memory.go b/pkg/kvstore/memory.go index e441c060f..30f28acfd 100644 --- a/pkg/kvstore/memory.go +++ b/pkg/kvstore/memory.go @@ -11,6 +11,8 @@ import ( var ErrNoSuchKey = errors.New("no such key") // Memory is an in-memory key-value store. +// +// The zero value is ready to use. type Memory struct { // m is the underlying map. m map[string][]byte diff --git a/pkg/legacy/legacymodel/archival.go b/pkg/legacy/legacymodel/archival.go new file mode 100644 index 000000000..e11203e5a --- /dev/null +++ b/pkg/legacy/legacymodel/archival.go @@ -0,0 +1,60 @@ +package legacymodel + +import ( + "encoding/base64" + "encoding/json" + "errors" + "unicode/utf8" +) + +// ArchivalMaybeBinaryData is a possibly binary string. We use this helper class +// to define a custom JSON encoder that allows us to choose the proper +// representation depending on whether the Value field is valid UTF-8 or not. +// +// See https://github.com/ooni/spec/blob/master/data-formats/df-001-httpt.md#maybebinarydata +// +// Deprecated: do not use this type in new code. +// +// Removing this struct is TODO(https://github.com/ooni/probe/issues/2543). +type ArchivalMaybeBinaryData struct { + Value string +} + +// MarshalJSON marshals a string-like to JSON following the OONI spec that +// says that UTF-8 content is represented as string and non-UTF-8 content is +// instead represented using `{"format":"base64","data":"..."}`. +func (hb ArchivalMaybeBinaryData) MarshalJSON() ([]byte, error) { + // if we can serialize as UTF-8 string, do that + if utf8.ValidString(hb.Value) { + return json.Marshal(hb.Value) + } + + // otherwise fallback to the ooni/spec representation for binary data + er := make(map[string]string) + er["format"] = "base64" + er["data"] = base64.StdEncoding.EncodeToString([]byte(hb.Value)) + return json.Marshal(er) +} + +// UnmarshalJSON is the opposite of MarshalJSON. +func (hb *ArchivalMaybeBinaryData) UnmarshalJSON(d []byte) error { + if err := json.Unmarshal(d, &hb.Value); err == nil { + return nil + } + er := make(map[string]string) + if err := json.Unmarshal(d, &er); err != nil { + return err + } + if v, ok := er["format"]; !ok || v != "base64" { + return errors.New("missing or invalid format field") + } + if _, ok := er["data"]; !ok { + return errors.New("missing data field") + } + b64, err := base64.StdEncoding.DecodeString(er["data"]) + if err != nil { + return err + } + hb.Value = string(b64) + return nil +} diff --git a/pkg/legacy/legacymodel/archival_test.go b/pkg/legacy/legacymodel/archival_test.go new file mode 100644 index 000000000..5f9462cbc --- /dev/null +++ b/pkg/legacy/legacymodel/archival_test.go @@ -0,0 +1,111 @@ +package legacymodel_test + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-engine/pkg/legacy/legacymodel" +) + +// we use this value below to test we can handle binary data +var archivalBinaryInput = []uint8{ + 0x57, 0xe5, 0x79, 0xfb, 0xa6, 0xbb, 0x0d, 0xbc, 0xce, 0xbd, 0xa7, 0xa0, + 0xba, 0xa4, 0x78, 0x78, 0x12, 0x59, 0xee, 0x68, 0x39, 0xa4, 0x07, 0x98, + 0xc5, 0x3e, 0xbc, 0x55, 0xcb, 0xfe, 0x34, 0x3c, 0x7e, 0x1b, 0x5a, 0xb3, + 0x22, 0x9d, 0xc1, 0x2d, 0x6e, 0xca, 0x5b, 0xf1, 0x10, 0x25, 0x47, 0x1e, + 0x44, 0xe2, 0x2d, 0x60, 0x08, 0xea, 0xb0, 0x0a, 0xcc, 0x05, 0x48, 0xa0, + 0xf5, 0x78, 0x38, 0xf0, 0xdb, 0x3f, 0x9d, 0x9f, 0x25, 0x6f, 0x89, 0x00, + 0x96, 0x93, 0xaf, 0x43, 0xac, 0x4d, 0xc9, 0xac, 0x13, 0xdb, 0x22, 0xbe, + 0x7a, 0x7d, 0xd9, 0x24, 0xa2, 0x52, 0x69, 0xd8, 0x89, 0xc1, 0xd1, 0x57, + 0xaa, 0x04, 0x2b, 0xa2, 0xd8, 0xb1, 0x19, 0xf6, 0xd5, 0x11, 0x39, 0xbb, + 0x80, 0xcf, 0x86, 0xf9, 0x5f, 0x9d, 0x8c, 0xab, 0xf5, 0xc5, 0x74, 0x24, + 0x3a, 0xa2, 0xd4, 0x40, 0x4e, 0xd7, 0x10, 0x1f, +} + +// we use this value below to test we can handle binary data +var archivalEncodedBinaryInput = []byte(`{"data":"V+V5+6a7DbzOvaeguqR4eBJZ7mg5pAeYxT68Vcv+NDx+G1qzIp3BLW7KW/EQJUceROItYAjqsArMBUig9Xg48Ns/nZ8lb4kAlpOvQ6xNyawT2yK+en3ZJKJSadiJwdFXqgQrotixGfbVETm7gM+G+V+djKv1xXQkOqLUQE7XEB8=","format":"base64"}`) + +func TestArchivalMaybeBinaryData(t *testing.T) { + t.Run("MarshalJSON", func(t *testing.T) { + tests := []struct { + name string // test name + input string // value to marshal + want []byte // expected result + wantErr bool // whether we expect an error + }{{ + name: "with string input", + input: "antani", + want: []byte(`"antani"`), + wantErr: false, + }, { + name: "with binary input", + input: string(archivalBinaryInput), + want: archivalEncodedBinaryInput, + wantErr: false, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hb := legacymodel.ArchivalMaybeBinaryData{ + Value: tt.input, + } + got, err := hb.MarshalJSON() + if (err != nil) != tt.wantErr { + t.Fatalf("ArchivalMaybeBinaryData.MarshalJSON() error = %v, wantErr %v", err, tt.wantErr) + } + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Fatal(diff) + } + }) + } + }) + + t.Run("UnmarshalJSON", func(t *testing.T) { + tests := []struct { + name string // test name + input []byte // value to unmarshal + want string // expected result + wantErr bool // whether we want an error + }{{ + name: "with string input", + input: []byte(`"xo"`), + want: "xo", + wantErr: false, + }, { + name: "with nil input", + input: nil, + want: "", + wantErr: true, + }, { + name: "with missing/invalid format", + input: []byte(`{"format": "foo"}`), + want: "", + wantErr: true, + }, { + name: "with missing data", + input: []byte(`{"format": "base64"}`), + want: "", + wantErr: true, + }, { + name: "with invalid base64 data", + input: []byte(`{"format": "base64", "data": "x"}`), + want: "", + wantErr: true, + }, { + name: "with valid base64 data", + input: archivalEncodedBinaryInput, + want: string(archivalBinaryInput), + wantErr: false, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hb := &legacymodel.ArchivalMaybeBinaryData{} + if err := hb.UnmarshalJSON(tt.input); (err != nil) != tt.wantErr { + t.Fatalf("ArchivalMaybeBinaryData.UnmarshalJSON() error = %v, wantErr %v", err, tt.wantErr) + } + if d := cmp.Diff(tt.want, hb.Value); d != "" { + t.Fatal(d) + } + }) + } + }) +} diff --git a/pkg/legacy/legacymodel/doc.go b/pkg/legacy/legacymodel/doc.go new file mode 100644 index 000000000..de378bc7d --- /dev/null +++ b/pkg/legacy/legacymodel/doc.go @@ -0,0 +1,2 @@ +// Package legacymodel contains legacy content that used to be in internal/model +package legacymodel diff --git a/pkg/measurex/archival.go b/pkg/legacy/measurex/archival.go similarity index 100% rename from pkg/measurex/archival.go rename to pkg/legacy/measurex/archival.go index 055d25038..ad9c30da9 100644 --- a/pkg/measurex/archival.go +++ b/pkg/legacy/measurex/archival.go @@ -1,5 +1,11 @@ package measurex +// +// Archival +// +// This file defines helpers to serialize to the OONI data format. +// + import ( "net" "net/http" @@ -8,12 +14,6 @@ import ( "time" ) -// -// Archival -// -// This file defines helpers to serialize to the OONI data format. -// - // // BinaryData // diff --git a/pkg/measurex/db.go b/pkg/legacy/measurex/db.go similarity index 100% rename from pkg/measurex/db.go rename to pkg/legacy/measurex/db.go diff --git a/pkg/measurex/dialer.go b/pkg/legacy/measurex/dialer.go similarity index 100% rename from pkg/measurex/dialer.go rename to pkg/legacy/measurex/dialer.go diff --git a/pkg/measurex/dnsx.go b/pkg/legacy/measurex/dnsx.go similarity index 97% rename from pkg/measurex/dnsx.go rename to pkg/legacy/measurex/dnsx.go index 02a741f6d..5ec4e4b85 100644 --- a/pkg/measurex/dnsx.go +++ b/pkg/legacy/measurex/dnsx.go @@ -10,8 +10,8 @@ import ( "context" "time" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/model" - "github.com/ooni/probe-engine/pkg/tracex" ) // WrapDNSXRoundTripper creates a new DNSXRoundTripper that diff --git a/pkg/measurex/dnsx_test.go b/pkg/legacy/measurex/dnsx_test.go similarity index 100% rename from pkg/measurex/dnsx_test.go rename to pkg/legacy/measurex/dnsx_test.go diff --git a/pkg/measurex/doc.go b/pkg/legacy/measurex/doc.go similarity index 100% rename from pkg/measurex/doc.go rename to pkg/legacy/measurex/doc.go diff --git a/pkg/measurex/easy.go b/pkg/legacy/measurex/easy.go similarity index 100% rename from pkg/measurex/easy.go rename to pkg/legacy/measurex/easy.go diff --git a/pkg/measurex/endpoint.go b/pkg/legacy/measurex/endpoint.go similarity index 100% rename from pkg/measurex/endpoint.go rename to pkg/legacy/measurex/endpoint.go diff --git a/pkg/measurex/failure.go b/pkg/legacy/measurex/failure.go similarity index 100% rename from pkg/measurex/failure.go rename to pkg/legacy/measurex/failure.go diff --git a/pkg/measurex/http.go b/pkg/legacy/measurex/http.go similarity index 100% rename from pkg/measurex/http.go rename to pkg/legacy/measurex/http.go diff --git a/pkg/measurex/logger.go b/pkg/legacy/measurex/logger.go similarity index 53% rename from pkg/measurex/logger.go rename to pkg/legacy/measurex/logger.go index fa4b26221..172b5c9ba 100644 --- a/pkg/measurex/logger.go +++ b/pkg/legacy/measurex/logger.go @@ -1,6 +1,6 @@ package measurex -import "github.com/ooni/probe-engine/pkg/measurexlite" +import "github.com/ooni/probe-engine/pkg/logx" // // Logger @@ -9,7 +9,7 @@ import "github.com/ooni/probe-engine/pkg/measurexlite" // // NewOperationLogger is an alias for measurex.NewOperationLogger. -var NewOperationLogger = measurexlite.NewOperationLogger +var NewOperationLogger = logx.NewOperationLogger // OperationLogger is an alias for measurex.OperationLogger. -type OperationLogger = measurexlite.OperationLogger +type OperationLogger = logx.OperationLogger diff --git a/pkg/measurex/measurement.go b/pkg/legacy/measurex/measurement.go similarity index 100% rename from pkg/measurex/measurement.go rename to pkg/legacy/measurex/measurement.go diff --git a/pkg/measurex/measurer.go b/pkg/legacy/measurex/measurer.go similarity index 99% rename from pkg/measurex/measurer.go rename to pkg/legacy/measurex/measurer.go index 6aafc00ef..b1ecdd182 100644 --- a/pkg/measurex/measurer.go +++ b/pkg/legacy/measurex/measurer.go @@ -355,13 +355,12 @@ func (mx *Measurer) TLSConnectAndHandshakeWithDB(ctx context.Context, ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() th := mx.WrapTLSHandshaker(db, mx.TLSHandshaker) - tlsConn, _, err := th.Handshake(ctx, conn, config) + tlsConn, err := th.Handshake(ctx, conn, config) ol.Stop(err) if err != nil { return nil, err } - // cast safe according to the docs of netxlite's handshaker - return tlsConn.(netxlite.TLSConn), nil + return tlsConn, nil } // QUICHandshake connects and TLS handshakes with a QUIC endpoint. diff --git a/pkg/measurex/oddity.go b/pkg/legacy/measurex/oddity.go similarity index 100% rename from pkg/measurex/oddity.go rename to pkg/legacy/measurex/oddity.go diff --git a/pkg/measurex/quic.go b/pkg/legacy/measurex/quic.go similarity index 90% rename from pkg/measurex/quic.go rename to pkg/legacy/measurex/quic.go index 6a7196ef8..cb4036880 100644 --- a/pkg/measurex/quic.go +++ b/pkg/legacy/measurex/quic.go @@ -17,14 +17,14 @@ import ( "github.com/quic-go/quic-go" ) -type quicListenerDB struct { - model.QUICListener +type udpListenerDB struct { + model.UDPListener begin time.Time db WritableDB } -func (ql *quicListenerDB) Listen(addr *net.UDPAddr) (model.UDPLikeConn, error) { - pconn, err := ql.QUICListener.Listen(addr) +func (ql *udpListenerDB) Listen(addr *net.UDPAddr) (model.UDPLikeConn, error) { + pconn, err := ql.UDPListener.Listen(addr) if err != nil { return nil, err } @@ -109,17 +109,17 @@ func (qh *quicDialerDB) DialContext(ctx context.Context, address string, tlsConfig *tls.Config, quicConfig *quic.Config) (quic.EarlyConnection, error) { started := time.Since(qh.begin).Seconds() var state tls.ConnectionState - listener := &quicListenerDB{ - QUICListener: netxlite.NewQUICListener(), - begin: qh.begin, - db: qh.db, + listener := &udpListenerDB{ + UDPListener: netxlite.NewUDPListener(), + begin: qh.begin, + db: qh.db, } dialer := netxlite.NewQUICDialerWithoutResolver(listener, qh.logger) defer dialer.CloseIdleConnections() sess, err := dialer.DialContext(ctx, address, tlsConfig, quicConfig) if err == nil { - <-sess.HandshakeComplete().Done() // robustness (the dialer already does that) - state = sess.ConnectionState().TLS.ConnectionState + <-sess.HandshakeComplete() // robustness (the dialer already does that) + state = sess.ConnectionState().TLS } finished := time.Since(qh.begin).Seconds() qh.db.InsertIntoQUICHandshake(&QUICTLSHandshakeEvent{ diff --git a/pkg/measurex/resolver.go b/pkg/legacy/measurex/resolver.go similarity index 99% rename from pkg/measurex/resolver.go rename to pkg/legacy/measurex/resolver.go index 7800f7a33..8e62e4bc3 100644 --- a/pkg/measurex/resolver.go +++ b/pkg/legacy/measurex/resolver.go @@ -11,9 +11,9 @@ import ( "strings" "time" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" - "github.com/ooni/probe-engine/pkg/tracex" ) // WrapResolver creates a new Resolver that saves events into the WritableDB. diff --git a/pkg/measurex/resolver_test.go b/pkg/legacy/measurex/resolver_test.go similarity index 100% rename from pkg/measurex/resolver_test.go rename to pkg/legacy/measurex/resolver_test.go diff --git a/pkg/measurex/tls.go b/pkg/legacy/measurex/tls.go similarity index 87% rename from pkg/measurex/tls.go rename to pkg/legacy/measurex/tls.go index 00327ba77..c8a2d6038 100644 --- a/pkg/measurex/tls.go +++ b/pkg/legacy/measurex/tls.go @@ -11,7 +11,6 @@ import ( "crypto/tls" "crypto/x509" "errors" - "net" "time" "github.com/ooni/probe-engine/pkg/model" @@ -53,13 +52,13 @@ type QUICTLSHandshakeEvent struct { Started float64 } -func (thx *tlsHandshakerDB) Handshake(ctx context.Context, - conn Conn, config *tls.Config) (net.Conn, tls.ConnectionState, error) { +func (thx *tlsHandshakerDB) Handshake(ctx context.Context, conn Conn, config *tls.Config) (model.TLSConn, error) { network := conn.RemoteAddr().Network() remoteAddr := conn.RemoteAddr().String() started := time.Since(thx.begin).Seconds() - tconn, state, err := thx.TLSHandshaker.Handshake(ctx, conn, config) + tconn, err := thx.TLSHandshaker.Handshake(ctx, conn, config) finished := time.Since(thx.begin).Seconds() + tstate := netxlite.MaybeTLSConnectionState(tconn) thx.db.InsertIntoTLSHandshake(&QUICTLSHandshakeEvent{ Network: network, RemoteAddr: remoteAddr, @@ -70,12 +69,12 @@ func (thx *tlsHandshakerDB) Handshake(ctx context.Context, Finished: finished, Failure: NewFailure(err), Oddity: thx.computeOddity(err), - TLSVersion: netxlite.TLSVersionString(state.Version), - CipherSuite: netxlite.TLSCipherSuiteString(state.CipherSuite), - NegotiatedProto: state.NegotiatedProtocol, - PeerCerts: peerCerts(err, &state), + TLSVersion: netxlite.TLSVersionString(tstate.Version), + CipherSuite: netxlite.TLSCipherSuiteString(tstate.CipherSuite), + NegotiatedProto: tstate.NegotiatedProtocol, + PeerCerts: peerCerts(err, &tstate), }) - return tconn, state, err + return tconn, err } func (thx *tlsHandshakerDB) computeOddity(err error) Oddity { diff --git a/pkg/measurex/tracing.go b/pkg/legacy/measurex/tracing.go similarity index 100% rename from pkg/measurex/tracing.go rename to pkg/legacy/measurex/tracing.go diff --git a/pkg/measurex/utils.go b/pkg/legacy/measurex/utils.go similarity index 100% rename from pkg/measurex/utils.go rename to pkg/legacy/measurex/utils.go diff --git a/pkg/legacy/netx/config.go b/pkg/legacy/netx/config.go index 911f63b50..eb86adde6 100644 --- a/pkg/legacy/netx/config.go +++ b/pkg/legacy/netx/config.go @@ -9,8 +9,8 @@ import ( "net/url" "github.com/ooni/probe-engine/pkg/bytecounter" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/model" - "github.com/ooni/probe-engine/pkg/tracex" ) // Config contains configuration for creating new transports, dialers, etc. When diff --git a/pkg/legacy/netx/dnstransport_test.go b/pkg/legacy/netx/dnstransport_test.go index 87a9550c1..cd63832ac 100644 --- a/pkg/legacy/netx/dnstransport_test.go +++ b/pkg/legacy/netx/dnstransport_test.go @@ -5,8 +5,8 @@ import ( "strings" "testing" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/netxlite" - "github.com/ooni/probe-engine/pkg/tracex" ) func TestNewDNSClientInvalidURL(t *testing.T) { diff --git a/pkg/legacy/netx/http_test.go b/pkg/legacy/netx/http_test.go index d04c86083..23cf1e6c8 100644 --- a/pkg/legacy/netx/http_test.go +++ b/pkg/legacy/netx/http_test.go @@ -7,8 +7,8 @@ import ( "net/http" "testing" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/mocks" - "github.com/ooni/probe-engine/pkg/tracex" ) func TestNewHTTPTransportWithDialer(t *testing.T) { diff --git a/pkg/legacy/netx/integration_test.go b/pkg/legacy/netx/integration_test.go index 3413b7995..937c03072 100644 --- a/pkg/legacy/netx/integration_test.go +++ b/pkg/legacy/netx/integration_test.go @@ -7,15 +7,14 @@ import ( "github.com/apex/log" "github.com/ooni/probe-engine/pkg/bytecounter" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/netxlite" - "github.com/ooni/probe-engine/pkg/tracex" ) func TestHTTPTransportWorkingAsIntended(t *testing.T) { if testing.Short() { t.Skip("skip test in short mode") } - log.SetLevel(log.DebugLevel) counter := bytecounter.New() config := Config{ BogonIsError: true, diff --git a/pkg/legacy/netx/quic.go b/pkg/legacy/netx/quic.go index 8298324ad..3df035e20 100644 --- a/pkg/legacy/netx/quic.go +++ b/pkg/legacy/netx/quic.go @@ -16,7 +16,7 @@ func NewQUICDialer(config Config) model.QUICDialer { } // TODO(https://github.com/ooni/probe/issues/2121#issuecomment-1147424810): we // should count the bytes consumed by this QUIC dialer - ql := config.ReadWriteSaver.WrapQUICListener(netxlite.NewQUICListener()) + ql := config.ReadWriteSaver.WrapUDPListener(netxlite.NewUDPListener()) logger := model.ValidLoggerOrDefault(config.Logger) return netxlite.NewQUICDialerWithResolver(ql, logger, config.FullResolver, config.Saver) } diff --git a/pkg/legacy/netx/resolver_test.go b/pkg/legacy/netx/resolver_test.go index 367065f6e..17c70e644 100644 --- a/pkg/legacy/netx/resolver_test.go +++ b/pkg/legacy/netx/resolver_test.go @@ -6,8 +6,8 @@ import ( "testing" "github.com/apex/log" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/netxlite" - "github.com/ooni/probe-engine/pkg/tracex" ) func TestNewResolverBogonResolutionNotBroken(t *testing.T) { diff --git a/pkg/legacy/netx/tls_test.go b/pkg/legacy/netx/tls_test.go index 8e523d36f..95cba7ef5 100644 --- a/pkg/legacy/netx/tls_test.go +++ b/pkg/legacy/netx/tls_test.go @@ -5,10 +5,10 @@ import ( "crypto/tls" "testing" + "github.com/ooni/netem" + "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/netxlite" - "github.com/ooni/probe-engine/pkg/runtimex" "github.com/ooni/probe-engine/pkg/testingx" - "github.com/ooni/probe-engine/pkg/tracex" ) func TestNewTLSDialer(t *testing.T) { @@ -45,8 +45,9 @@ func TestNewTLSDialer(t *testing.T) { }) t.Run("we can skip TLS verification", func(t *testing.T) { - mitm := testingx.MustNewTLSMITMProviderNetem() - server := testingx.MustNewTLSServer(testingx.TLSHandlerHandshakeAndWriteText(mitm, testingx.HTTPBlockpage451)) + ca := netem.MustNewCA() + cert := ca.MustNewTLSCertificate("www.example.com") + server := testingx.MustNewTLSServer(testingx.TLSHandlerHandshakeAndWriteText(cert, testingx.HTTPBlockpage451)) defer server.Close() tdx := NewTLSDialer(Config{TLSConfig: &tls.Config{ InsecureSkipVerify: true, @@ -59,12 +60,13 @@ func TestNewTLSDialer(t *testing.T) { }) t.Run("we can set the cert pool", func(t *testing.T) { - mitm := testingx.MustNewTLSMITMProviderNetem() - server := testingx.MustNewTLSServer(testingx.TLSHandlerHandshakeAndWriteText(mitm, testingx.HTTPBlockpage451)) + ca := netem.MustNewCA() + cert := ca.MustNewTLSCertificate("dns.google") + server := testingx.MustNewTLSServer(testingx.TLSHandlerHandshakeAndWriteText(cert, testingx.HTTPBlockpage451)) defer server.Close() tdx := NewTLSDialer(Config{ TLSConfig: &tls.Config{ - RootCAs: runtimex.Try1(mitm.DefaultCertPool()), + RootCAs: ca.DefaultCertPool(), ServerName: "dns.google", }, }) diff --git a/pkg/tracex/archival.go b/pkg/legacy/tracex/archival.go similarity index 85% rename from pkg/tracex/archival.go rename to pkg/legacy/tracex/archival.go index 4e8c250ae..1348c65e6 100644 --- a/pkg/tracex/archival.go +++ b/pkg/legacy/tracex/archival.go @@ -7,8 +7,6 @@ package tracex import ( "errors" "net" - "net/http" - "sort" "strconv" "strings" "time" @@ -23,12 +21,9 @@ type ( ExtSpec = model.ArchivalExtSpec TCPConnectEntry = model.ArchivalTCPConnectResult TCPConnectStatus = model.ArchivalTCPConnectStatus - MaybeBinaryValue = model.ArchivalMaybeBinaryData DNSQueryEntry = model.ArchivalDNSLookupResult DNSAnswerEntry = model.ArchivalDNSAnswer TLSHandshake = model.ArchivalTLSOrQUICHandshakeResult - HTTPBody = model.ArchivalHTTPBody - HTTPHeader = model.ArchivalHTTPHeader RequestEntry = model.ArchivalHTTPRequestResult HTTPRequest = model.ArchivalHTTPRequest HTTPResponse = model.ArchivalHTTPResponse @@ -96,31 +91,6 @@ func NewFailedOperation(err error) *string { return &s } -// httpAddHeaders adds the headers inside source into destList and destMap. -func httpAddHeaders(source http.Header, destList *[]HTTPHeader, - destMap *map[string]MaybeBinaryValue) { - *destList = []HTTPHeader{} - *destMap = make(map[string]model.ArchivalMaybeBinaryData) - for key, values := range source { - for index, value := range values { - value := MaybeBinaryValue{Value: value} - // With the map representation we can only represent a single - // value for every key. Hence the list representation. - if index == 0 { - (*destMap)[key] = value - } - *destList = append(*destList, HTTPHeader{ - Key: key, - Value: value, - }) - } - } - // Sorting helps with unit testing (map keys are unordered) - sort.Slice(*destList, func(i, j int) bool { - return (*destList)[i].Key < (*destList)[j].Key - }) -} - // NewRequestList returns the list for "requests" func NewRequestList(begin time.Time, events []Event) (out []RequestEntry) { // OONI wants the last request to appear first @@ -138,16 +108,16 @@ func newRequestList(begin time.Time, events []Event) (out []RequestEntry) { case *EventHTTPTransactionDone: entry := RequestEntry{} entry.T = ev.Time.Sub(begin).Seconds() - httpAddHeaders( - ev.HTTPRequestHeaders, &entry.Request.HeadersList, &entry.Request.Headers) + entry.Request.Headers = model.ArchivalNewHTTPHeadersMap(ev.HTTPRequestHeaders) + entry.Request.HeadersList = model.ArchivalNewHTTPHeadersList(ev.HTTPRequestHeaders) entry.Request.Method = ev.HTTPMethod entry.Request.URL = ev.HTTPURL entry.Request.Transport = ev.Transport - httpAddHeaders( - ev.HTTPResponseHeaders, &entry.Response.HeadersList, &entry.Response.Headers) + entry.Response.Headers = model.ArchivalNewHTTPHeadersMap(ev.HTTPResponseHeaders) + entry.Response.HeadersList = model.ArchivalNewHTTPHeadersList(ev.HTTPResponseHeaders) entry.Response.Code = int64(ev.HTTPStatusCode) entry.Response.Locations = ev.HTTPResponseHeaders.Values("Location") - entry.Response.Body.Value = string(ev.HTTPResponseBody) + entry.Response.Body = model.ArchivalScrubbedMaybeBinaryString(ev.HTTPResponseBody) entry.Response.BodyIsTruncated = ev.HTTPResponseBodyIsTruncated entry.Failure = ev.Err.ToFailure() out = append(out, entry) @@ -305,9 +275,9 @@ func NewTLSHandshakesList(begin time.Time, events []Event) (out []TLSHandshake) return } -func tlsMakePeerCerts(in [][]byte) (out []MaybeBinaryValue) { +func tlsMakePeerCerts(in [][]byte) (out []model.ArchivalBinaryData) { for _, entry := range in { - out = append(out, MaybeBinaryValue{Value: string(entry)}) + out = append(out, model.ArchivalBinaryData(entry)) } return } diff --git a/pkg/tracex/archival_test.go b/pkg/legacy/tracex/archival_test.go similarity index 88% rename from pkg/tracex/archival_test.go rename to pkg/legacy/tracex/archival_test.go index d5fbd72e3..d2e441d44 100644 --- a/pkg/tracex/archival_test.go +++ b/pkg/legacy/tracex/archival_test.go @@ -10,6 +10,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/gorilla/websocket" + "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" ) @@ -171,53 +172,43 @@ func TestNewRequestList(t *testing.T) { want: []RequestEntry{{ Failure: NewFailure(io.EOF), Request: HTTPRequest{ - HeadersList: []HTTPHeader{{ - Key: "User-Agent", - Value: MaybeBinaryValue{ - Value: "miniooni/0.1.0-dev", - }, + HeadersList: []model.ArchivalHTTPHeader{{ + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0-dev"), }}, - Headers: map[string]MaybeBinaryValue{ - "User-Agent": {Value: "miniooni/0.1.0-dev"}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "User-Agent": "miniooni/0.1.0-dev", }, Method: "GET", URL: "https://www.example.com/result", }, Response: HTTPResponse{ - HeadersList: []HTTPHeader{}, - Headers: make(map[string]MaybeBinaryValue), + HeadersList: []model.ArchivalHTTPHeader{}, + Headers: make(map[string]model.ArchivalScrubbedMaybeBinaryString), }, T: 0.02, }, { Request: HTTPRequest{ - Body: MaybeBinaryValue{ - Value: "", - }, - HeadersList: []HTTPHeader{{ - Key: "User-Agent", - Value: MaybeBinaryValue{ - Value: "miniooni/0.1.0-dev", - }, + Body: model.ArchivalScrubbedMaybeBinaryString(""), + HeadersList: []model.ArchivalHTTPHeader{{ + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0-dev"), }}, - Headers: map[string]MaybeBinaryValue{ - "User-Agent": {Value: "miniooni/0.1.0-dev"}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "User-Agent": "miniooni/0.1.0-dev", }, Method: "POST", URL: "https://www.example.com/submit", }, Response: HTTPResponse{ - Body: MaybeBinaryValue{ - Value: "{}", - }, + Body: model.ArchivalScrubbedMaybeBinaryString("{}"), Code: 200, - HeadersList: []HTTPHeader{{ - Key: "Server", - Value: MaybeBinaryValue{ - Value: "miniooni/0.1.0-dev", - }, + HeadersList: []model.ArchivalHTTPHeader{{ + model.ArchivalScrubbedMaybeBinaryString("Server"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0-dev"), }}, - Headers: map[string]MaybeBinaryValue{ - "Server": {Value: "miniooni/0.1.0-dev"}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "Server": "miniooni/0.1.0-dev", }, Locations: nil, }, @@ -245,39 +236,31 @@ func TestNewRequestList(t *testing.T) { }, want: []RequestEntry{{ Request: HTTPRequest{ - HeadersList: []HTTPHeader{{ - Key: "User-Agent", - Value: MaybeBinaryValue{ - Value: "miniooni/0.1.0-dev", - }, + HeadersList: []model.ArchivalHTTPHeader{{ + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0-dev"), }}, - Headers: map[string]MaybeBinaryValue{ - "User-Agent": {Value: "miniooni/0.1.0-dev"}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "User-Agent": "miniooni/0.1.0-dev", }, Method: "GET", URL: "https://www.example.com/", }, Response: HTTPResponse{ Code: 302, - HeadersList: []HTTPHeader{{ - Key: "Location", - Value: MaybeBinaryValue{ - Value: "https://x.example.com", - }, + HeadersList: []model.ArchivalHTTPHeader{{ + model.ArchivalScrubbedMaybeBinaryString("Location"), + model.ArchivalScrubbedMaybeBinaryString("https://x.example.com"), }, { - Key: "Location", - Value: MaybeBinaryValue{ - Value: "https://y.example.com", - }, + model.ArchivalScrubbedMaybeBinaryString("Location"), + model.ArchivalScrubbedMaybeBinaryString("https://y.example.com"), }, { - Key: "Server", - Value: MaybeBinaryValue{ - Value: "miniooni/0.1.0-dev", - }, + model.ArchivalScrubbedMaybeBinaryString("Server"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0-dev"), }}, - Headers: map[string]MaybeBinaryValue{ - "Server": {Value: "miniooni/0.1.0-dev"}, - "Location": {Value: "https://x.example.com"}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "Server": "miniooni/0.1.0-dev", + "Location": "https://x.example.com", }, Locations: []string{ "https://x.example.com", "https://y.example.com", @@ -547,11 +530,10 @@ func TestNewTLSHandshakesList(t *testing.T) { Failure: NewFailure(io.EOF), NegotiatedProtocol: "h2", NoTLSVerify: false, - PeerCertificates: []MaybeBinaryValue{{ - Value: "deadbeef", - }, { - Value: "abad1dea", - }}, + PeerCertificates: []model.ArchivalBinaryData{ + model.ArchivalBinaryData("deadbeef"), + model.ArchivalBinaryData("abad1dea"), + }, ServerName: "x.org", T: 0.055, TLSVersion: "TLSv1.3", @@ -582,11 +564,10 @@ func TestNewTLSHandshakesList(t *testing.T) { Failure: NewFailure(io.EOF), NegotiatedProtocol: "h3", NoTLSVerify: false, - PeerCertificates: []MaybeBinaryValue{{ - Value: "deadbeef", - }, { - Value: "abad1dea", - }}, + PeerCertificates: []model.ArchivalBinaryData{ + model.ArchivalBinaryData("deadbeef"), + model.ArchivalBinaryData("abad1dea"), + }, ServerName: "x.org", T: 0.055, TLSVersion: "TLSv1.3", diff --git a/pkg/tracex/dialer.go b/pkg/legacy/tracex/dialer.go similarity index 100% rename from pkg/tracex/dialer.go rename to pkg/legacy/tracex/dialer.go diff --git a/pkg/tracex/dialer_test.go b/pkg/legacy/tracex/dialer_test.go similarity index 100% rename from pkg/tracex/dialer_test.go rename to pkg/legacy/tracex/dialer_test.go diff --git a/pkg/tracex/doc.go b/pkg/legacy/tracex/doc.go similarity index 100% rename from pkg/tracex/doc.go rename to pkg/legacy/tracex/doc.go diff --git a/pkg/tracex/event.go b/pkg/legacy/tracex/event.go similarity index 100% rename from pkg/tracex/event.go rename to pkg/legacy/tracex/event.go diff --git a/pkg/tracex/event_test.go b/pkg/legacy/tracex/event_test.go similarity index 100% rename from pkg/tracex/event_test.go rename to pkg/legacy/tracex/event_test.go diff --git a/pkg/tracex/http.go b/pkg/legacy/tracex/http.go similarity index 100% rename from pkg/tracex/http.go rename to pkg/legacy/tracex/http.go diff --git a/pkg/tracex/http_test.go b/pkg/legacy/tracex/http_test.go similarity index 100% rename from pkg/tracex/http_test.go rename to pkg/legacy/tracex/http_test.go diff --git a/pkg/tracex/quic.go b/pkg/legacy/tracex/quic.go similarity index 85% rename from pkg/tracex/quic.go rename to pkg/legacy/tracex/quic.go index 73419fc86..bd5fe03cc 100644 --- a/pkg/tracex/quic.go +++ b/pkg/legacy/tracex/quic.go @@ -93,36 +93,36 @@ func (h *QUICDialerSaver) CloseIdleConnections() { // quicConnectionState returns the ConnectionState of a QUIC Session. func quicConnectionState(sess quic.EarlyConnection) tls.ConnectionState { - return sess.ConnectionState().TLS.ConnectionState + return sess.ConnectionState().TLS } -// QUICListenerSaver is a QUICListener that also implements saving events. -type QUICListenerSaver struct { - // QUICListener is the underlying QUICListener. - QUICListener model.QUICListener +// UDPListenerSaver is a UDPListener that also implements saving events. +type UDPListenerSaver struct { + // UDPListener is the underlying UDPListener. + UDPListener model.UDPListener // Saver is the underlying Saver. Saver *Saver } -// WrapQUICListener wraps a model.QUICDialer with a QUICListenerSaver that will +// WrapUDPListener wraps a model.UDPDialer with a UDPListenerSaver that will // save the QUIC I/O packet conn events into this Saver. // // When this function is invoked on a nil Saver, it will directly return -// the original QUICListener without any wrapping. -func (s *Saver) WrapQUICListener(ql model.QUICListener) model.QUICListener { +// the original UDPListener without any wrapping. +func (s *Saver) WrapUDPListener(ql model.UDPListener) model.UDPListener { if s == nil { return ql } - return &QUICListenerSaver{ - QUICListener: ql, - Saver: s, + return &UDPListenerSaver{ + UDPListener: ql, + Saver: s, } } -// Listen implements QUICListener.Listen. -func (qls *QUICListenerSaver) Listen(addr *net.UDPAddr) (model.UDPLikeConn, error) { - pconn, err := qls.QUICListener.Listen(addr) +// Listen implements UDPListener.Listen. +func (qls *UDPListenerSaver) Listen(addr *net.UDPAddr) (model.UDPLikeConn, error) { + pconn, err := qls.UDPListener.Listen(addr) if err != nil { return nil, err } @@ -184,5 +184,5 @@ func (c *quicPacketConnWrapper) safeAddrString(addr net.Addr) (out string) { } var _ model.QUICDialer = &QUICDialerSaver{} -var _ model.QUICListener = &QUICListenerSaver{} +var _ model.UDPListener = &UDPListenerSaver{} var _ model.UDPLikeConn = &quicPacketConnWrapper{} diff --git a/pkg/tracex/quic_test.go b/pkg/legacy/tracex/quic_test.go similarity index 97% rename from pkg/tracex/quic_test.go rename to pkg/legacy/tracex/quic_test.go index 46570042b..ee1af3530 100644 --- a/pkg/tracex/quic_test.go +++ b/pkg/legacy/tracex/quic_test.go @@ -81,7 +81,7 @@ func TestQUICDialerSaver(t *testing.T) { returnedConn := &mocks.QUICEarlyConnection{ MockConnectionState: func() quic.ConnectionState { cs := quic.ConnectionState{} - cs.TLS.ConnectionState.CipherSuite = tls.TLS_RSA_WITH_RC4_128_SHA + cs.TLS.CipherSuite = tls.TLS_RSA_WITH_RC4_128_SHA cs.TLS.NegotiatedProtocol = "h3" cs.TLS.PeerCertificates = []*x509.Certificate{{ Raw: []byte{1, 2, 3, 4}, @@ -177,19 +177,19 @@ func TestQUICDialerSaver(t *testing.T) { }) } -func TestWrapQUICListener(t *testing.T) { +func TestWrapUDPListener(t *testing.T) { var saver *Saver - ql := &mocks.QUICListener{} - if saver.WrapQUICListener(ql) != ql { + ql := &mocks.UDPListener{} + if saver.WrapUDPListener(ql) != ql { t.Fatal("unexpected result") } } -func TestQUICListenerSaver(t *testing.T) { +func TestUDPListenerSaver(t *testing.T) { t.Run("on failure", func(t *testing.T) { expected := errors.New("mocked error") saver := &Saver{} - qls := saver.WrapQUICListener(&mocks.QUICListener{ + qls := saver.WrapUDPListener(&mocks.UDPListener{ MockListen: func(addr *net.UDPAddr) (model.UDPLikeConn, error) { return nil, expected }, @@ -210,7 +210,7 @@ func TestQUICListenerSaver(t *testing.T) { t.Run("on success", func(t *testing.T) { saver := &Saver{} returnedConn := &mocks.UDPLikeConn{} - qls := saver.WrapQUICListener(&mocks.QUICListener{ + qls := saver.WrapUDPListener(&mocks.UDPListener{ MockListen: func(addr *net.UDPAddr) (model.UDPLikeConn, error) { return returnedConn, nil }, diff --git a/pkg/tracex/resolver.go b/pkg/legacy/tracex/resolver.go similarity index 100% rename from pkg/tracex/resolver.go rename to pkg/legacy/tracex/resolver.go diff --git a/pkg/tracex/resolver_test.go b/pkg/legacy/tracex/resolver_test.go similarity index 100% rename from pkg/tracex/resolver_test.go rename to pkg/legacy/tracex/resolver_test.go diff --git a/pkg/tracex/saver.go b/pkg/legacy/tracex/saver.go similarity index 100% rename from pkg/tracex/saver.go rename to pkg/legacy/tracex/saver.go diff --git a/pkg/tracex/saver_test.go b/pkg/legacy/tracex/saver_test.go similarity index 100% rename from pkg/tracex/saver_test.go rename to pkg/legacy/tracex/saver_test.go diff --git a/pkg/tracex/tls.go b/pkg/legacy/tracex/tls.go similarity index 85% rename from pkg/tracex/tls.go rename to pkg/legacy/tracex/tls.go index 5af3476b7..c393fe6b6 100644 --- a/pkg/tracex/tls.go +++ b/pkg/legacy/tracex/tls.go @@ -42,7 +42,7 @@ func (s *Saver) WrapTLSHandshaker(thx model.TLSHandshaker) model.TLSHandshaker { // Handshake implements model.TLSHandshaker.Handshake func (h *TLSHandshakerSaver) Handshake( - ctx context.Context, conn net.Conn, config *tls.Config) (net.Conn, tls.ConnectionState, error) { + ctx context.Context, conn net.Conn, config *tls.Config) (model.TLSConn, error) { proto := conn.RemoteAddr().Network() remoteAddr := conn.RemoteAddr().String() start := time.Now() @@ -54,23 +54,24 @@ func (h *TLSHandshakerSaver) Handshake( TLSServerName: config.ServerName, Time: start, }}) - tlsconn, state, err := h.TLSHandshaker.Handshake(ctx, conn, config) + tlsconn, err := h.TLSHandshaker.Handshake(ctx, conn, config) stop := time.Now() + tstate := netxlite.MaybeTLSConnectionState(tlsconn) h.Saver.Write(&EventTLSHandshakeDone{&EventValue{ Address: remoteAddr, Duration: stop.Sub(start), Err: NewFailureStr(err), NoTLSVerify: config.InsecureSkipVerify, Proto: proto, - TLSCipherSuite: netxlite.TLSCipherSuiteString(state.CipherSuite), - TLSNegotiatedProto: state.NegotiatedProtocol, + TLSCipherSuite: netxlite.TLSCipherSuiteString(tstate.CipherSuite), + TLSNegotiatedProto: tstate.NegotiatedProtocol, TLSNextProtos: config.NextProtos, - TLSPeerCerts: tlsPeerCerts(state, err), + TLSPeerCerts: tlsPeerCerts(tstate, err), TLSServerName: config.ServerName, - TLSVersion: netxlite.TLSVersionString(state.Version), + TLSVersion: netxlite.TLSVersionString(tstate.Version), Time: stop, }}) - return tlsconn, state, err + return tlsconn, err } var _ model.TLSHandshaker = &TLSHandshakerSaver{} diff --git a/pkg/tracex/tls_test.go b/pkg/legacy/tracex/tls_test.go similarity index 92% rename from pkg/tracex/tls_test.go rename to pkg/legacy/tracex/tls_test.go index 95d9ac3a3..eb8180ae3 100644 --- a/pkg/tracex/tls_test.go +++ b/pkg/legacy/tracex/tls_test.go @@ -10,6 +10,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/ooni/probe-engine/pkg/mocks" + "github.com/ooni/probe-engine/pkg/model" ) func TestWrapTLSHandshaker(t *testing.T) { @@ -98,9 +99,8 @@ func TestTLSHandshakerSaver(t *testing.T) { }, } thx := saver.WrapTLSHandshaker(&mocks.TLSHandshaker{ - MockHandshake: func(ctx context.Context, conn net.Conn, - config *tls.Config) (net.Conn, tls.ConnectionState, error) { - return returnedConn, returnedConnState, nil + MockHandshake: func(ctx context.Context, conn net.Conn, config *tls.Config) (model.TLSConn, error) { + return returnedConn, nil }, }) ctx := context.Background() @@ -121,7 +121,7 @@ func TestTLSHandshakerSaver(t *testing.T) { } }, } - conn, _, err := thx.Handshake(ctx, tcpConn, tlsConfig) + conn, err := thx.Handshake(ctx, tcpConn, tlsConfig) if err != nil { t.Fatal(err) } @@ -161,9 +161,8 @@ func TestTLSHandshakerSaver(t *testing.T) { expected := errors.New("mocked error") saver := &Saver{} thx := saver.WrapTLSHandshaker(&mocks.TLSHandshaker{ - MockHandshake: func(ctx context.Context, conn net.Conn, - config *tls.Config) (net.Conn, tls.ConnectionState, error) { - return nil, tls.ConnectionState{}, expected + MockHandshake: func(ctx context.Context, conn net.Conn, config *tls.Config) (model.TLSConn, error) { + return nil, expected }, }) ctx := context.Background() @@ -184,7 +183,7 @@ func TestTLSHandshakerSaver(t *testing.T) { } }, } - conn, _, err := thx.Handshake(ctx, tcpConn, tlsConfig) + conn, err := thx.Handshake(ctx, tcpConn, tlsConfig) if !errors.Is(err, expected) { t.Fatal("unexpected err", err) } diff --git a/pkg/measurexlite/logger.go b/pkg/logx/operation.go similarity index 94% rename from pkg/measurexlite/logger.go rename to pkg/logx/operation.go index ad0d89c88..07338d76f 100644 --- a/pkg/measurexlite/logger.go +++ b/pkg/logx/operation.go @@ -1,8 +1,4 @@ -package measurexlite - -// -// Logging support -// +package logx import ( "fmt" @@ -12,8 +8,6 @@ import ( "github.com/ooni/probe-engine/pkg/model" ) -// TODO(bassosimone): consider moving inside the logx package? - // NewOperationLogger creates a new logger that logs // about an in-progress operation. If it takes too much // time to emit the result of the operation, the code @@ -33,6 +27,7 @@ func newOperationLogger(maxwait time.Duration, logger model.Logger, format strin wg: &sync.WaitGroup{}, } ol.wg.Add(1) + ol.logger.Infof("%s... started", ol.message) go ol.maybeEmitProgress() return ol } diff --git a/pkg/measurexlite/logger_test.go b/pkg/logx/operation_test.go similarity index 80% rename from pkg/measurexlite/logger_test.go rename to pkg/logx/operation_test.go index b66c89834..dd64ced79 100644 --- a/pkg/measurexlite/logger_test.go +++ b/pkg/logx/operation_test.go @@ -1,4 +1,4 @@ -package measurexlite +package logx import ( "fmt" @@ -26,10 +26,13 @@ func TestNewOperationLogger(t *testing.T) { } ol := NewOperationLogger(logger, "antani%d", 0) ol.Stop(nil) - if len(lines) != 1 { + if len(lines) != 2 { t.Fatal("unexpected number of lines") } - if lines[0] != "antani0... ok" { + if lines[0] != "antani0... started" { + t.Fatal("unexpected first line", lines[0]) + } + if lines[1] != "antani0... ok" { t.Fatal("unexpected first line", lines[0]) } }) @@ -49,10 +52,13 @@ func TestNewOperationLogger(t *testing.T) { } ol := NewOperationLogger(logger, "antani%d", 0) ol.Stop(io.EOF) - if len(lines) != 1 { + if len(lines) != 2 { t.Fatal("unexpected number of lines") } - if lines[0] != "antani0... EOF" { + if lines[0] != "antani0... started" { + t.Fatal("unexpected first line", lines[0]) + } + if lines[1] != "antani0... EOF" { t.Fatal("unexpected first line", lines[0]) } }) @@ -74,13 +80,16 @@ func TestNewOperationLogger(t *testing.T) { ol := newOperationLogger(maxwait, logger, "antani%d", 0) ol.wg.Wait() // wait for the message to be emitted ol.Stop(nil) - if len(lines) != 2 { + if len(lines) != 3 { t.Fatal("unexpected number of lines") } - if lines[0] != "antani0... in progress" { + if lines[0] != "antani0... started" { t.Fatal("unexpected first line", lines[0]) } - if lines[1] != "antani0... ok" { + if lines[1] != "antani0... in progress" { + t.Fatal("unexpected first line", lines[0]) + } + if lines[2] != "antani0... ok" { t.Fatal("unexpected first line", lines[0]) } }) @@ -102,13 +111,16 @@ func TestNewOperationLogger(t *testing.T) { ol := newOperationLogger(maxwait, logger, "antani%d", 0) ol.wg.Wait() // wait for the message to be emitted ol.Stop(io.EOF) - if len(lines) != 2 { + if len(lines) != 3 { t.Fatal("unexpected number of lines") } - if lines[0] != "antani0... in progress" { + if lines[0] != "antani0... started" { t.Fatal("unexpected first line", lines[0]) } - if lines[1] != "antani0... EOF" { + if lines[1] != "antani0... in progress" { + t.Fatal("unexpected first line", lines[0]) + } + if lines[2] != "antani0... EOF" { t.Fatal("unexpected first line", lines[0]) } }) diff --git a/pkg/logx/scrubber.go b/pkg/logx/scrubber.go new file mode 100644 index 000000000..0295ea78d --- /dev/null +++ b/pkg/logx/scrubber.go @@ -0,0 +1,47 @@ +package logx + +import ( + "fmt" + + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/scrubber" +) + +// ScrubberLogger is a [model.Logger] with scrubbing. All messages are scrubbed including the ones +// that won't be emitted. As such, this logger is less efficient than a logger without scrubbing. +// +// The zero value is invalid; please init all MANDATORY fields. +type ScrubberLogger struct { + // Logger is the MANDATORY underlying logger to use. + Logger model.Logger +} + +// Debug scrubs and emits a debug message. +func (sl *ScrubberLogger) Debug(message string) { + sl.Logger.Debug(scrubber.ScrubString(message)) +} + +// Debugf scrubs, formats, and emits a debug message. +func (sl *ScrubberLogger) Debugf(format string, v ...interface{}) { + sl.Debug(fmt.Sprintf(format, v...)) +} + +// Info scrubs and emits an informational message. +func (sl *ScrubberLogger) Info(message string) { + sl.Logger.Info(scrubber.ScrubString(message)) +} + +// Infof scrubs, formats, and emits an informational message. +func (sl *ScrubberLogger) Infof(format string, v ...interface{}) { + sl.Info(fmt.Sprintf(format, v...)) +} + +// Warn scrubs and emits a warning message. +func (sl *ScrubberLogger) Warn(message string) { + sl.Logger.Warn(scrubber.ScrubString(message)) +} + +// Warnf scrubs, formats, and emits a warning message. +func (sl *ScrubberLogger) Warnf(format string, v ...interface{}) { + sl.Warn(fmt.Sprintf(format, v...)) +} diff --git a/pkg/scrubber/logger_test.go b/pkg/logx/scrubber_test.go similarity index 67% rename from pkg/scrubber/logger_test.go rename to pkg/logx/scrubber_test.go index 1ffad86b8..2e16b455a 100644 --- a/pkg/scrubber/logger_test.go +++ b/pkg/logx/scrubber_test.go @@ -1,47 +1,48 @@ -package scrubber +package logx import ( "fmt" "testing" ) -type savingLogger struct { +// scrubberSavingLogger helps writing tests for [ScrubberLogger]. +type scrubberSavingLogger struct { debug []string info []string warn []string } -func (sl *savingLogger) Debug(message string) { +func (sl *scrubberSavingLogger) Debug(message string) { sl.debug = append(sl.debug, message) } -func (sl *savingLogger) Debugf(format string, v ...interface{}) { +func (sl *scrubberSavingLogger) Debugf(format string, v ...interface{}) { sl.Debug(fmt.Sprintf(format, v...)) } -func (sl *savingLogger) Info(message string) { +func (sl *scrubberSavingLogger) Info(message string) { sl.info = append(sl.info, message) } -func (sl *savingLogger) Infof(format string, v ...interface{}) { +func (sl *scrubberSavingLogger) Infof(format string, v ...interface{}) { sl.Info(fmt.Sprintf(format, v...)) } -func (sl *savingLogger) Warn(message string) { +func (sl *scrubberSavingLogger) Warn(message string) { sl.warn = append(sl.warn, message) } -func (sl *savingLogger) Warnf(format string, v ...interface{}) { +func (sl *scrubberSavingLogger) Warnf(format string, v ...interface{}) { sl.Warn(fmt.Sprintf(format, v...)) } -func TestScrubLogger(t *testing.T) { +func TestScrubberLogger(t *testing.T) { input := "failure: 130.192.91.211:443: no route the host" expect := "failure: [scrubbed]: no route the host" t.Run("for debug", func(t *testing.T) { - logger := new(savingLogger) - scrubber := &Logger{Logger: logger} + logger := new(scrubberSavingLogger) + scrubber := &ScrubberLogger{Logger: logger} scrubber.Debug(input) if len(logger.debug) != 1 && len(logger.info) != 0 && len(logger.warn) != 0 { t.Fatal("unexpected number of log lines written") @@ -52,8 +53,8 @@ func TestScrubLogger(t *testing.T) { }) t.Run("for debugf", func(t *testing.T) { - logger := new(savingLogger) - scrubber := &Logger{Logger: logger} + logger := new(scrubberSavingLogger) + scrubber := &ScrubberLogger{Logger: logger} scrubber.Debugf("%s", input) if len(logger.debug) != 1 && len(logger.info) != 0 && len(logger.warn) != 0 { t.Fatal("unexpected number of log lines written") @@ -64,8 +65,8 @@ func TestScrubLogger(t *testing.T) { }) t.Run("for info", func(t *testing.T) { - logger := new(savingLogger) - scrubber := &Logger{Logger: logger} + logger := new(scrubberSavingLogger) + scrubber := &ScrubberLogger{Logger: logger} scrubber.Info(input) if len(logger.debug) != 0 && len(logger.info) != 1 && len(logger.warn) != 0 { t.Fatal("unexpected number of log lines written") @@ -76,8 +77,8 @@ func TestScrubLogger(t *testing.T) { }) t.Run("for infof", func(t *testing.T) { - logger := new(savingLogger) - scrubber := &Logger{Logger: logger} + logger := new(scrubberSavingLogger) + scrubber := &ScrubberLogger{Logger: logger} scrubber.Infof("%s", input) if len(logger.debug) != 0 && len(logger.info) != 1 && len(logger.warn) != 0 { t.Fatal("unexpected number of log lines written") @@ -88,8 +89,8 @@ func TestScrubLogger(t *testing.T) { }) t.Run("for warn", func(t *testing.T) { - logger := new(savingLogger) - scrubber := &Logger{Logger: logger} + logger := new(scrubberSavingLogger) + scrubber := &ScrubberLogger{Logger: logger} scrubber.Warn(input) if len(logger.debug) != 0 && len(logger.info) != 0 && len(logger.warn) != 1 { t.Fatal("unexpected number of log lines written") @@ -100,8 +101,8 @@ func TestScrubLogger(t *testing.T) { }) t.Run("for warnf", func(t *testing.T) { - logger := new(savingLogger) - scrubber := &Logger{Logger: logger} + logger := new(scrubberSavingLogger) + scrubber := &ScrubberLogger{Logger: logger} scrubber.Warnf("%s", input) if len(logger.debug) != 0 && len(logger.info) != 0 && len(logger.warn) != 1 { t.Fatal("unexpected number of log lines written") diff --git a/pkg/measurexlite/dialer.go b/pkg/measurexlite/dialer.go index 6cbf9c2ef..d97c163e5 100644 --- a/pkg/measurexlite/dialer.go +++ b/pkg/measurexlite/dialer.go @@ -18,9 +18,12 @@ import ( // NewDialerWithoutResolver is equivalent to netxlite.NewDialerWithoutResolver // except that it returns a model.Dialer that uses this trace. -func (tx *Trace) NewDialerWithoutResolver(dl model.DebugLogger) model.Dialer { +// +// Caveat: the dialer wrappers are there to implement the [model.MeasuringNetwork] +// interface, but they're not used by this function. +func (tx *Trace) NewDialerWithoutResolver(dl model.DebugLogger, wrappers ...model.DialerWrapper) model.Dialer { return &dialerTrace{ - d: tx.newDialerWithoutResolver(dl), + d: tx.Netx.NewDialerWithoutResolver(dl), tx: tx, } } diff --git a/pkg/measurexlite/dialer_test.go b/pkg/measurexlite/dialer_test.go index b6a56d29f..447250f23 100644 --- a/pkg/measurexlite/dialer_test.go +++ b/pkg/measurexlite/dialer_test.go @@ -21,8 +21,10 @@ func TestNewDialerWithoutResolver(t *testing.T) { underlying := &mocks.Dialer{} zeroTime := time.Now() trace := NewTrace(0, zeroTime) - trace.newDialerWithoutResolverFn = func(dl model.DebugLogger) model.Dialer { - return underlying + trace.Netx = &mocks.MeasuringNetwork{ + MockNewDialerWithoutResolver: func(dl model.DebugLogger, w ...model.DialerWrapper) model.Dialer { + return underlying + }, } dialer := trace.NewDialerWithoutResolver(model.DiscardLogger) dt := dialer.(*dialerTrace) @@ -46,8 +48,10 @@ func TestNewDialerWithoutResolver(t *testing.T) { return nil, expectedErr }, } - trace.newDialerWithoutResolverFn = func(dl model.DebugLogger) model.Dialer { - return underlying + trace.Netx = &mocks.MeasuringNetwork{ + MockNewDialerWithoutResolver: func(dl model.DebugLogger, w ...model.DialerWrapper) model.Dialer { + return underlying + }, } dialer := trace.NewDialerWithoutResolver(model.DiscardLogger) ctx := context.Background() @@ -72,8 +76,10 @@ func TestNewDialerWithoutResolver(t *testing.T) { called = true }, } - trace.newDialerWithoutResolverFn = func(dl model.DebugLogger) model.Dialer { - return underlying + trace.Netx = &mocks.MeasuringNetwork{ + MockNewDialerWithoutResolver: func(dl model.DebugLogger, w ...model.DialerWrapper) model.Dialer { + return underlying + }, } dialer := trace.NewDialerWithoutResolver(model.DiscardLogger) dialer.CloseIdleConnections() diff --git a/pkg/measurexlite/dns.go b/pkg/measurexlite/dns.go index 10aca62e7..3694591b9 100644 --- a/pkg/measurexlite/dns.go +++ b/pkg/measurexlite/dns.go @@ -92,18 +92,18 @@ func (r *resolverTrace) LookupNS(ctx context.Context, domain string) ([]*net.NS, } // NewStdlibResolver returns a trace-ware system resolver -func (tx *Trace) NewStdlibResolver(logger model.Logger) model.Resolver { - return tx.wrapResolver(tx.newStdlibResolver(logger)) +func (tx *Trace) NewStdlibResolver(logger model.DebugLogger) model.Resolver { + return tx.wrapResolver(tx.Netx.NewStdlibResolver(logger)) } // NewParallelUDPResolver returns a trace-ware parallel UDP resolver -func (tx *Trace) NewParallelUDPResolver(logger model.Logger, dialer model.Dialer, address string) model.Resolver { - return tx.wrapResolver(tx.newParallelUDPResolver(logger, dialer, address)) +func (tx *Trace) NewParallelUDPResolver(logger model.DebugLogger, dialer model.Dialer, address string) model.Resolver { + return tx.wrapResolver(tx.Netx.NewParallelUDPResolver(logger, dialer, address)) } // NewParallelDNSOverHTTPSResolver returns a trace-aware parallel DoH resolver -func (tx *Trace) NewParallelDNSOverHTTPSResolver(logger model.Logger, URL string) model.Resolver { - return tx.wrapResolver(tx.newParallelDNSOverHTTPSResolver(logger, URL)) +func (tx *Trace) NewParallelDNSOverHTTPSResolver(logger model.DebugLogger, URL string) model.Resolver { + return tx.wrapResolver(tx.Netx.NewParallelDNSOverHTTPSResolver(logger, URL)) } // OnDNSRoundTripForLookupHost implements model.Trace.OnDNSRoundTripForLookupHost diff --git a/pkg/measurexlite/http.go b/pkg/measurexlite/http.go index 1bc1ba905..52a94a1e7 100644 --- a/pkg/measurexlite/http.go +++ b/pkg/measurexlite/http.go @@ -6,7 +6,6 @@ package measurexlite import ( "net/http" - "sort" "time" "github.com/ooni/probe-engine/pkg/model" @@ -52,7 +51,7 @@ func NewArchivalHTTPRequestResult(index int64, started time.Duration, network, a ALPN: alpn, Failure: NewFailure(err), Request: model.ArchivalHTTPRequest{ - Body: model.ArchivalMaybeBinaryData{}, + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: newHTTPRequestHeaderList(req), Headers: newHTTPRequestHeaderMap(req), @@ -62,7 +61,7 @@ func NewArchivalHTTPRequestResult(index int64, started time.Duration, network, a URL: httpRequestURL(req), }, Response: model.ArchivalHTTPResponse{ - Body: httpResponseBody(body), + Body: model.ArchivalScrubbedMaybeBinaryString(body), BodyIsTruncated: httpResponseBodyIsTruncated(body, maxRespBodySize), Code: httpResponseStatusCode(resp), HeadersList: newHTTPResponseHeaderList(resp), @@ -91,17 +90,17 @@ func newHTTPRequestHeaderList(req *http.Request) []model.ArchivalHTTPHeader { if req != nil { m = req.Header } - return newHTTPHeaderList(m) + return model.ArchivalNewHTTPHeadersList(m) } // newHTTPRequestHeaderMap calls newHTTPHeaderMap with the request headers or // return an empty map in case the request is nil. -func newHTTPRequestHeaderMap(req *http.Request) map[string]model.ArchivalMaybeBinaryData { +func newHTTPRequestHeaderMap(req *http.Request) map[string]model.ArchivalScrubbedMaybeBinaryString { m := http.Header{} if req != nil { m = req.Header } - return newHTTPHeaderMap(m) + return model.ArchivalNewHTTPHeadersMap(m) } // httpRequestURL returns the req.URL.String() or an empty string. @@ -112,14 +111,6 @@ func httpRequestURL(req *http.Request) (out string) { return } -// httpResponseBody returns the response body, if possible, or an empty body. -func httpResponseBody(body []byte) (out model.ArchivalMaybeBinaryData) { - if body != nil { - out.Value = string(body) - } - return -} - // httpResponseBodyIsTruncated determines whether the body is truncated (if possible) func httpResponseBodyIsTruncated(body []byte, maxSnapSize int64) (out bool) { if len(body) > 0 && maxSnapSize > 0 { @@ -143,17 +134,17 @@ func newHTTPResponseHeaderList(resp *http.Response) (out []model.ArchivalHTTPHea if resp != nil { m = resp.Header } - return newHTTPHeaderList(m) + return model.ArchivalNewHTTPHeadersList(m) } // newHTTPResponseHeaderMap calls newHTTPHeaderMap with the request headers or // return an empty map in case the request is nil. -func newHTTPResponseHeaderMap(resp *http.Response) (out map[string]model.ArchivalMaybeBinaryData) { +func newHTTPResponseHeaderMap(resp *http.Response) (out map[string]model.ArchivalScrubbedMaybeBinaryString) { m := http.Header{} if resp != nil { m = resp.Header } - return newHTTPHeaderMap(m) + return model.ArchivalNewHTTPHeadersMap(m) } // httpResponseLocations returns the locations inside the response (if possible) @@ -167,43 +158,3 @@ func httpResponseLocations(resp *http.Response) []string { } return []string{loc.String()} } - -// newHTTPHeaderList creates a list representation of HTTP headers -func newHTTPHeaderList(header http.Header) (out []model.ArchivalHTTPHeader) { - out = []model.ArchivalHTTPHeader{} - keys := []string{} - for key := range header { - keys = append(keys, key) - } - - // ensure the output is consistent, which helps with testing; - // for an example of why we need to sort headers, see - // https://github.com/ooni/probe-engine/pull/751/checks?check_run_id=853562310 - sort.Strings(keys) - - for _, key := range keys { - for _, value := range header[key] { - out = append(out, model.ArchivalHTTPHeader{ - Key: key, - Value: model.ArchivalMaybeBinaryData{ - Value: value, - }, - }) - } - } - return -} - -// newHTTPHeaderMap creates a map representation of HTTP headers -func newHTTPHeaderMap(header http.Header) (out map[string]model.ArchivalMaybeBinaryData) { - out = make(map[string]model.ArchivalMaybeBinaryData) - for key, values := range header { - for _, value := range values { - out[key] = model.ArchivalMaybeBinaryData{ - Value: value, - } - break - } - } - return -} diff --git a/pkg/measurexlite/http_test.go b/pkg/measurexlite/http_test.go index b0da466e6..b6892590e 100644 --- a/pkg/measurexlite/http_test.go +++ b/pkg/measurexlite/http_test.go @@ -58,21 +58,21 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { ALPN: "", Failure: nil, Request: model.ArchivalHTTPRequest{ - Body: model.ArchivalMaybeBinaryData{}, + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: []model.ArchivalHTTPHeader{}, - Headers: map[string]model.ArchivalMaybeBinaryData{}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{}, Method: "", Tor: model.ArchivalHTTPTor{}, Transport: "", URL: "", }, Response: model.ArchivalHTTPResponse{ - Body: model.ArchivalMaybeBinaryData{}, + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, Code: 0, HeadersList: []model.ArchivalHTTPHeader{}, - Headers: map[string]model.ArchivalMaybeBinaryData{}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{}, Locations: []string{}, }, T0: 0, @@ -117,22 +117,18 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { return &s }(), Request: model.ArchivalHTTPRequest{ - Body: model.ArchivalMaybeBinaryData{}, + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: []model.ArchivalHTTPHeader{{ - Key: "Accept", - Value: model.ArchivalMaybeBinaryData{ - Value: "*/*", - }, + model.ArchivalScrubbedMaybeBinaryString("Accept"), + model.ArchivalScrubbedMaybeBinaryString("*/*"), }, { - Key: "User-Agent", - Value: model.ArchivalMaybeBinaryData{ - Value: "miniooni/0.1.0-dev", - }, + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0-dev"), }}, - Headers: map[string]model.ArchivalMaybeBinaryData{ - "Accept": {Value: "*/*"}, - "User-Agent": {Value: "miniooni/0.1.0-dev"}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "Accept": "*/*", + "User-Agent": "miniooni/0.1.0-dev", }, Method: "GET", Tor: model.ArchivalHTTPTor{}, @@ -140,11 +136,11 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { URL: "http://dns.google/", }, Response: model.ArchivalHTTPResponse{ - Body: model.ArchivalMaybeBinaryData{}, + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, Code: 0, HeadersList: []model.ArchivalHTTPHeader{}, - Headers: map[string]model.ArchivalMaybeBinaryData{}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{}, Locations: []string{}, }, T0: 0.25, @@ -192,22 +188,18 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { ALPN: "h3", Failure: nil, Request: model.ArchivalHTTPRequest{ - Body: model.ArchivalMaybeBinaryData{}, + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: []model.ArchivalHTTPHeader{{ - Key: "Accept", - Value: model.ArchivalMaybeBinaryData{ - Value: "*/*", - }, + model.ArchivalScrubbedMaybeBinaryString("Accept"), + model.ArchivalScrubbedMaybeBinaryString("*/*"), }, { - Key: "User-Agent", - Value: model.ArchivalMaybeBinaryData{ - Value: "miniooni/0.1.0-dev", - }, + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0-dev"), }}, - Headers: map[string]model.ArchivalMaybeBinaryData{ - "Accept": {Value: "*/*"}, - "User-Agent": {Value: "miniooni/0.1.0-dev"}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "Accept": "*/*", + "User-Agent": "miniooni/0.1.0-dev", }, Method: "GET", Tor: model.ArchivalHTTPTor{}, @@ -215,25 +207,21 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { URL: "https://dns.google/", }, Response: model.ArchivalHTTPResponse{ - Body: model.ArchivalMaybeBinaryData{ - Value: string(testingx.HTTPBlockpage451), - }, + Body: model.ArchivalScrubbedMaybeBinaryString( + testingx.HTTPBlockpage451, + ), BodyIsTruncated: false, Code: 200, HeadersList: []model.ArchivalHTTPHeader{{ - Key: "Content-Type", - Value: model.ArchivalMaybeBinaryData{ - Value: "text/html; charset=iso-8859-1", - }, + model.ArchivalScrubbedMaybeBinaryString("Content-Type"), + model.ArchivalScrubbedMaybeBinaryString("text/html; charset=iso-8859-1"), }, { - Key: "Server", - Value: model.ArchivalMaybeBinaryData{ - Value: "Apache", - }, + model.ArchivalScrubbedMaybeBinaryString("Server"), + model.ArchivalScrubbedMaybeBinaryString("Apache"), }}, - Headers: map[string]model.ArchivalMaybeBinaryData{ - "Content-Type": {Value: "text/html; charset=iso-8859-1"}, - "Server": {Value: "Apache"}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "Content-Type": "text/html; charset=iso-8859-1", + "Server": "Apache", }, Locations: []string{}, }, @@ -290,22 +278,18 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { ALPN: "h3", Failure: nil, Request: model.ArchivalHTTPRequest{ - Body: model.ArchivalMaybeBinaryData{}, + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, HeadersList: []model.ArchivalHTTPHeader{{ - Key: "Accept", - Value: model.ArchivalMaybeBinaryData{ - Value: "*/*", - }, + model.ArchivalScrubbedMaybeBinaryString("Accept"), + model.ArchivalScrubbedMaybeBinaryString("*/*"), }, { - Key: "User-Agent", - Value: model.ArchivalMaybeBinaryData{ - Value: "miniooni/0.1.0-dev", - }, + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0-dev"), }}, - Headers: map[string]model.ArchivalMaybeBinaryData{ - "Accept": {Value: "*/*"}, - "User-Agent": {Value: "miniooni/0.1.0-dev"}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "Accept": "*/*", + "User-Agent": "miniooni/0.1.0-dev", }, Method: "GET", Tor: model.ArchivalHTTPTor{}, @@ -313,31 +297,23 @@ func TestNewArchivalHTTPRequestResult(t *testing.T) { URL: "https://dns.google/", }, Response: model.ArchivalHTTPResponse{ - Body: model.ArchivalMaybeBinaryData{ - Value: "", - }, + Body: model.ArchivalScrubbedMaybeBinaryString(""), BodyIsTruncated: false, Code: 302, HeadersList: []model.ArchivalHTTPHeader{{ - Key: "Content-Type", - Value: model.ArchivalMaybeBinaryData{ - Value: "text/html; charset=iso-8859-1", - }, + model.ArchivalScrubbedMaybeBinaryString("Content-Type"), + model.ArchivalScrubbedMaybeBinaryString("text/html; charset=iso-8859-1"), }, { - Key: "Location", - Value: model.ArchivalMaybeBinaryData{ - Value: "/v2/index.html", - }, + model.ArchivalScrubbedMaybeBinaryString("Location"), + model.ArchivalScrubbedMaybeBinaryString("/v2/index.html"), }, { - Key: "Server", - Value: model.ArchivalMaybeBinaryData{ - Value: "Apache", - }, + model.ArchivalScrubbedMaybeBinaryString("Server"), + model.ArchivalScrubbedMaybeBinaryString("Apache"), }}, - Headers: map[string]model.ArchivalMaybeBinaryData{ - "Content-Type": {Value: "text/html; charset=iso-8859-1"}, - "Location": {Value: "/v2/index.html"}, - "Server": {Value: "Apache"}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "Content-Type": "text/html; charset=iso-8859-1", + "Location": "/v2/index.html", + "Server": "Apache", }, Locations: []string{ "https://dns.google/v2/index.html", diff --git a/pkg/measurexlite/quic.go b/pkg/measurexlite/quic.go index a71defb76..a7155cb40 100644 --- a/pkg/measurexlite/quic.go +++ b/pkg/measurexlite/quic.go @@ -16,9 +16,13 @@ import ( // NewQUICDialerWithoutResolver is equivalent to netxlite.NewQUICDialerWithoutResolver // except that it returns a model.QUICDialer that uses this trace. -func (tx *Trace) NewQUICDialerWithoutResolver(listener model.QUICListener, dl model.DebugLogger) model.QUICDialer { +// +// Caveat: the dialer wrappers are there to implement the [model.MeasuringNetwork] +// interface, but they're not used by this function. +func (tx *Trace) NewQUICDialerWithoutResolver( + listener model.UDPListener, dl model.DebugLogger, wrappers ...model.QUICDialerWrapper) model.QUICDialer { return &quicDialerTrace{ - qd: tx.newQUICDialerWithoutResolver(listener, dl), + qd: tx.Netx.NewQUICDialerWithoutResolver(listener, dl), tx: tx, } } @@ -60,7 +64,7 @@ func (tx *Trace) OnQUICHandshakeDone(started time.Time, remoteAddr string, qconn state := tls.ConnectionState{} if qconn != nil { - state = qconn.ConnectionState().TLS.ConnectionState + state = qconn.ConnectionState().TLS } select { diff --git a/pkg/measurexlite/quic_test.go b/pkg/measurexlite/quic_test.go index e27183dae..85d4bcde9 100644 --- a/pkg/measurexlite/quic_test.go +++ b/pkg/measurexlite/quic_test.go @@ -13,7 +13,7 @@ import ( "github.com/ooni/probe-engine/pkg/mocks" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" - "github.com/ooni/probe-engine/pkg/netxlite/quictesting" + "github.com/ooni/probe-engine/pkg/testingquic" "github.com/ooni/probe-engine/pkg/testingx" "github.com/quic-go/quic-go" ) @@ -23,10 +23,12 @@ func TestNewQUICDialerWithoutResolver(t *testing.T) { underlying := &mocks.QUICDialer{} zeroTime := time.Now() trace := NewTrace(0, zeroTime) - trace.newQUICDialerWithoutResolverFn = func(listener model.QUICListener, dl model.DebugLogger) model.QUICDialer { - return underlying + trace.Netx = &mocks.MeasuringNetwork{ + MockNewQUICDialerWithoutResolver: func(listener model.UDPListener, logger model.DebugLogger, w ...model.QUICDialerWrapper) model.QUICDialer { + return underlying + }, } - listener := &mocks.QUICListener{} + listener := &mocks.UDPListener{} dialer := trace.NewQUICDialerWithoutResolver(listener, model.DiscardLogger) dt := dialer.(*quicDialerTrace) if dt.qd != underlying { @@ -50,10 +52,12 @@ func TestNewQUICDialerWithoutResolver(t *testing.T) { return nil, expectedErr }, } - trace.newQUICDialerWithoutResolverFn = func(listener model.QUICListener, dl model.DebugLogger) model.QUICDialer { - return underlying + trace.Netx = &mocks.MeasuringNetwork{ + MockNewQUICDialerWithoutResolver: func(listener model.UDPListener, logger model.DebugLogger, w ...model.QUICDialerWrapper) model.QUICDialer { + return underlying + }, } - listener := &mocks.QUICListener{} + listener := &mocks.UDPListener{} dialer := trace.NewQUICDialerWithoutResolver(listener, model.DiscardLogger) ctx := context.Background() conn, err := dialer.DialContext(ctx, "1.1.1.1:443", &tls.Config{}, &quic.Config{}) @@ -77,10 +81,12 @@ func TestNewQUICDialerWithoutResolver(t *testing.T) { called = true }, } - trace.newQUICDialerWithoutResolverFn = func(listener model.QUICListener, dl model.DebugLogger) model.QUICDialer { - return underlying + trace.Netx = &mocks.MeasuringNetwork{ + MockNewQUICDialerWithoutResolver: func(listener model.UDPListener, logger model.DebugLogger, w ...model.QUICDialerWrapper) model.QUICDialer { + return underlying + }, } - listener := &mocks.QUICListener{} + listener := &mocks.UDPListener{} dialer := trace.NewQUICDialerWithoutResolver(listener, model.DiscardLogger) dialer.CloseIdleConnections() if !called { @@ -97,6 +103,9 @@ func TestNewQUICDialerWithoutResolver(t *testing.T) { pconn := &mocks.UDPLikeConn{ MockLocalAddr: func() net.Addr { return &net.UDPAddr{ + // quic-go does not allow the use of the same net.PacketConn for multiple "Dial" + // calls (unless a quic.Transport is used), so we have to make sure to mock local + // addresses with different ports, as tests run in parallel. Port: 0, } }, @@ -111,8 +120,11 @@ func TestNewQUICDialerWithoutResolver(t *testing.T) { MockClose: func() error { return nil }, + MockSetReadBuffer: func(n int) error { + return nil + }, } - listener := &mocks.QUICListener{ + listener := &mocks.UDPListener{ MockListen: func(addr *net.UDPAddr) (model.UDPLikeConn, error) { return pconn, nil }, @@ -144,7 +156,7 @@ func TestNewQUICDialerWithoutResolver(t *testing.T) { Failure: &expectedFailure, NegotiatedProtocol: "", NoTLSVerify: true, - PeerCertificates: []model.ArchivalMaybeBinaryData{}, + PeerCertificates: []model.ArchivalBinaryData{}, ServerName: "dns.cloudflare.com", T: time.Second.Seconds(), Tags: []string{"antani"}, @@ -207,7 +219,10 @@ func TestNewQUICDialerWithoutResolver(t *testing.T) { pconn := &mocks.UDPLikeConn{ MockLocalAddr: func() net.Addr { return &net.UDPAddr{ - Port: 0, + // quic-go does not allow the use of the same net.PacketConn for multiple "Dial" + // calls (unless a quic.Transport is used), so we have to make sure to mock local + // addresses with different ports, as tests run in parallel. + Port: 1, } }, MockRemoteAddr: func() net.Addr { @@ -221,8 +236,11 @@ func TestNewQUICDialerWithoutResolver(t *testing.T) { MockClose: func() error { return nil }, + MockSetReadBuffer: func(n int) error { + return nil + }, } - listener := &mocks.QUICListener{ + listener := &mocks.UDPListener{ MockListen: func(addr *net.UDPAddr) (model.UDPLikeConn, error) { return pconn, nil }, @@ -258,17 +276,21 @@ func TestNewQUICDialerWithoutResolver(t *testing.T) { } func TestOnQUICHandshakeDoneExtractsTheConnectionState(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + // create a trace trace := NewTrace(0, time.Now()) // create a QUIC dialer - quicListener := netxlite.NewQUICListener() - quicDialer := trace.NewQUICDialerWithoutResolver(quicListener, model.DiscardLogger) + udpListener := netxlite.NewUDPListener() + quicDialer := trace.NewQUICDialerWithoutResolver(udpListener, model.DiscardLogger) // dial with the endpoint we use for testing quicConn, err := quicDialer.DialContext( context.Background(), - quictesting.Endpoint("443"), + testingquic.MustEndpoint("443"), &tls.Config{ InsecureSkipVerify: true, }, @@ -318,7 +340,7 @@ func TestFirstQUICHandshake(t *testing.T) { Failure: nil, NegotiatedProtocol: "", NoTLSVerify: true, - PeerCertificates: []model.ArchivalMaybeBinaryData{}, + PeerCertificates: []model.ArchivalBinaryData{}, ServerName: "dns.cloudflare.com", T: time.Second.Seconds(), Tags: []string{}, @@ -330,7 +352,7 @@ func TestFirstQUICHandshake(t *testing.T) { Failure: nil, NegotiatedProtocol: "", NoTLSVerify: true, - PeerCertificates: []model.ArchivalMaybeBinaryData{}, + PeerCertificates: []model.ArchivalBinaryData{}, ServerName: "dns.google.com", T: time.Second.Seconds(), Tags: []string{}, diff --git a/pkg/measurexlite/tls.go b/pkg/measurexlite/tls.go index 3602bd323..e75aa4267 100644 --- a/pkg/measurexlite/tls.go +++ b/pkg/measurexlite/tls.go @@ -20,7 +20,7 @@ import ( // except that it returns a model.TLSHandshaker that uses this trace. func (tx *Trace) NewTLSHandshakerStdlib(dl model.DebugLogger) model.TLSHandshaker { return &tlsHandshakerTrace{ - thx: tx.newTLSHandshakerStdlib(dl), + thx: tx.Netx.NewTLSHandshakerStdlib(dl), tx: tx, } } @@ -35,7 +35,7 @@ var _ model.TLSHandshaker = &tlsHandshakerTrace{} // Handshake implements model.TLSHandshaker.Handshake. func (thx *tlsHandshakerTrace) Handshake( - ctx context.Context, conn net.Conn, tlsConfig *tls.Config) (net.Conn, tls.ConnectionState, error) { + ctx context.Context, conn net.Conn, tlsConfig *tls.Config) (model.TLSConn, error) { return thx.thx.Handshake(netxlite.ContextWithTrace(ctx, thx.tx), conn, tlsConfig) } @@ -99,27 +99,16 @@ func NewArchivalTLSOrQUICHandshakeResult( } } -// newArchivalBinaryData is a factory that adapts binary data to the -// model.ArchivalMaybeBinaryData format. -func newArchivalBinaryData(data []byte) model.ArchivalMaybeBinaryData { - // TODO(https://github.com/ooni/probe/issues/2165): we should actually extend the - // model's archival data format to have a pure-binary-data type for the cases in which - // we know in advance we're dealing with binary data. - return model.ArchivalMaybeBinaryData{ - Value: string(data), - } -} - // TLSPeerCerts extracts the certificates either from the list of certificates // in the connection state or from the error that occurred. func TLSPeerCerts( - state tls.ConnectionState, err error) (out []model.ArchivalMaybeBinaryData) { - out = []model.ArchivalMaybeBinaryData{} + state tls.ConnectionState, err error) (out []model.ArchivalBinaryData) { + out = []model.ArchivalBinaryData{} var x509HostnameError x509.HostnameError if errors.As(err, &x509HostnameError) { // Test case: https://wrong.host.badssl.com/ - out = append(out, newArchivalBinaryData(x509HostnameError.Certificate.Raw)) + out = append(out, model.ArchivalBinaryData(x509HostnameError.Certificate.Raw)) return } @@ -127,19 +116,19 @@ func TLSPeerCerts( if errors.As(err, &x509UnknownAuthorityError) { // Test case: https://self-signed.badssl.com/. This error has // never been among the ones returned by MK. - out = append(out, newArchivalBinaryData(x509UnknownAuthorityError.Cert.Raw)) + out = append(out, model.ArchivalBinaryData(x509UnknownAuthorityError.Cert.Raw)) return } var x509CertificateInvalidError x509.CertificateInvalidError if errors.As(err, &x509CertificateInvalidError) { // Test case: https://expired.badssl.com/ - out = append(out, newArchivalBinaryData(x509CertificateInvalidError.Cert.Raw)) + out = append(out, model.ArchivalBinaryData(x509CertificateInvalidError.Cert.Raw)) return } for _, cert := range state.PeerCertificates { - out = append(out, newArchivalBinaryData(cert.Raw)) + out = append(out, model.ArchivalBinaryData(cert.Raw)) } return } diff --git a/pkg/measurexlite/tls_test.go b/pkg/measurexlite/tls_test.go index d3f7cdee9..462e60cd4 100644 --- a/pkg/measurexlite/tls_test.go +++ b/pkg/measurexlite/tls_test.go @@ -7,15 +7,14 @@ import ( "crypto/x509" "errors" "net" - "reflect" "testing" "time" "github.com/google/go-cmp/cmp" + "github.com/ooni/netem" "github.com/ooni/probe-engine/pkg/mocks" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" - "github.com/ooni/probe-engine/pkg/runtimex" "github.com/ooni/probe-engine/pkg/testingx" ) @@ -24,8 +23,10 @@ func TestNewTLSHandshakerStdlib(t *testing.T) { underlying := &mocks.TLSHandshaker{} zeroTime := time.Now() trace := NewTrace(0, zeroTime) - trace.newTLSHandshakerStdlibFn = func(dl model.DebugLogger) model.TLSHandshaker { - return underlying + trace.Netx = &mocks.MeasuringNetwork{ + MockNewTLSHandshakerStdlib: func(logger model.DebugLogger) model.TLSHandshaker { + return underlying + }, } thx := trace.NewTLSHandshakerStdlib(model.DiscardLogger) thxt := thx.(*tlsHandshakerTrace) @@ -43,24 +44,23 @@ func TestNewTLSHandshakerStdlib(t *testing.T) { trace := NewTrace(0, zeroTime) var hasCorrectTrace bool underlying := &mocks.TLSHandshaker{ - MockHandshake: func(ctx context.Context, conn net.Conn, config *tls.Config) (net.Conn, tls.ConnectionState, error) { + MockHandshake: func(ctx context.Context, conn net.Conn, config *tls.Config) (model.TLSConn, error) { gotTrace := netxlite.ContextTraceOrDefault(ctx) hasCorrectTrace = (gotTrace == trace) - return nil, tls.ConnectionState{}, expectedErr + return nil, expectedErr }, } - trace.newTLSHandshakerStdlibFn = func(dl model.DebugLogger) model.TLSHandshaker { - return underlying + trace.Netx = &mocks.MeasuringNetwork{ + MockNewTLSHandshakerStdlib: func(logger model.DebugLogger) model.TLSHandshaker { + return underlying + }, } thx := trace.NewTLSHandshakerStdlib(model.DiscardLogger) ctx := context.Background() - conn, state, err := thx.Handshake(ctx, &mocks.Conn{}, &tls.Config{}) + conn, err := thx.Handshake(ctx, &mocks.Conn{}, &tls.Config{}) if !errors.Is(err, expectedErr) { t.Fatal("unexpected err", err) } - if !reflect.ValueOf(state).IsZero() { - t.Fatal("expected zero-value state") - } if conn != nil { t.Fatal("expected nil conn") } @@ -102,13 +102,10 @@ func TestNewTLSHandshakerStdlib(t *testing.T) { InsecureSkipVerify: true, ServerName: "dns.cloudflare.com", } - conn, state, err := thx.Handshake(ctx, tcpConn, tlsConfig) + conn, err := thx.Handshake(ctx, tcpConn, tlsConfig) if !errors.Is(err, mockedErr) { t.Fatal("unexpected err", err) } - if !reflect.ValueOf(state).IsZero() { - t.Fatal("expected zero-value state") - } if conn != nil { t.Fatal("expected nil conn") } @@ -126,7 +123,7 @@ func TestNewTLSHandshakerStdlib(t *testing.T) { Failure: &expectedFailure, NegotiatedProtocol: "", NoTLSVerify: true, - PeerCertificates: []model.ArchivalMaybeBinaryData{}, + PeerCertificates: []model.ArchivalBinaryData{}, ServerName: "dns.cloudflare.com", T: time.Second.Seconds(), Tags: []string{"antani"}, @@ -212,13 +209,10 @@ func TestNewTLSHandshakerStdlib(t *testing.T) { InsecureSkipVerify: true, ServerName: "dns.cloudflare.com", } - conn, state, err := thx.Handshake(ctx, tcpConn, tlsConfig) + conn, err := thx.Handshake(ctx, tcpConn, tlsConfig) if !errors.Is(err, mockedErr) { t.Fatal("unexpected err", err) } - if !reflect.ValueOf(state).IsZero() { - t.Fatal("expected zero-value state") - } if conn != nil { t.Fatal("expected nil conn") } @@ -239,8 +233,9 @@ func TestNewTLSHandshakerStdlib(t *testing.T) { }) t.Run("we collect the desired data with a local TLS server", func(t *testing.T) { - mitm := testingx.MustNewTLSMITMProviderNetem() - server := testingx.MustNewTLSServer(testingx.TLSHandlerHandshakeAndWriteText(mitm, testingx.HTTPBlockpage451)) + ca := netem.MustNewCA() + cert := ca.MustNewTLSCertificate("dns.google") + server := testingx.MustNewTLSServer(testingx.TLSHandlerHandshakeAndWriteText(cert, testingx.HTTPBlockpage451)) dialer := netxlite.NewDialerWithoutResolver(model.DiscardLogger) ctx := context.Background() conn, err := dialer.DialContext(ctx, "tcp", server.Endpoint()) @@ -254,10 +249,10 @@ func TestNewTLSHandshakerStdlib(t *testing.T) { trace.timeNowFn = dt.Now // deterministic timing thx := trace.NewTLSHandshakerStdlib(model.DiscardLogger) tlsConfig := &tls.Config{ - RootCAs: runtimex.Try1(mitm.DefaultCertPool()), + RootCAs: ca.DefaultCertPool(), ServerName: "dns.google", } - tlsConn, connState, err := thx.Handshake(ctx, conn, tlsConfig) + tlsConn, err := thx.Handshake(ctx, conn, tlsConfig) if err != nil { t.Fatal(err) } @@ -270,6 +265,8 @@ func TestNewTLSHandshakerStdlib(t *testing.T) { t.Fatal("bytes should match") } + connState := netxlite.MaybeTLSConnectionState(tlsConn) + t.Run("TLSHandshake events", func(t *testing.T) { events := trace.TLSHandshakes() if len(events) != 1 { @@ -282,7 +279,7 @@ func TestNewTLSHandshakerStdlib(t *testing.T) { Failure: nil, NegotiatedProtocol: "", NoTLSVerify: false, - PeerCertificates: []model.ArchivalMaybeBinaryData{}, + PeerCertificates: []model.ArchivalBinaryData{}, ServerName: "dns.google", T: time.Second.Seconds(), Tags: []string{}, @@ -296,7 +293,7 @@ func TestNewTLSHandshakerStdlib(t *testing.T) { if len(got.PeerCertificates) != 2 { t.Fatal("expected to see two certificates") } - got.PeerCertificates = []model.ArchivalMaybeBinaryData{} // see above + got.PeerCertificates = []model.ArchivalBinaryData{} // see above if diff := cmp.Diff(expected, got); diff != "" { t.Fatal(diff) } @@ -369,7 +366,7 @@ func TestFirstTLSHandshake(t *testing.T) { Failure: nil, NegotiatedProtocol: "", NoTLSVerify: true, - PeerCertificates: []model.ArchivalMaybeBinaryData{}, + PeerCertificates: []model.ArchivalBinaryData{}, ServerName: "dns.cloudflare.com", T: time.Second.Seconds(), Tags: []string{}, @@ -381,7 +378,7 @@ func TestFirstTLSHandshake(t *testing.T) { Failure: nil, NegotiatedProtocol: "", NoTLSVerify: true, - PeerCertificates: []model.ArchivalMaybeBinaryData{}, + PeerCertificates: []model.ArchivalBinaryData{}, ServerName: "dns.google.com", T: time.Second.Seconds(), Tags: []string{}, @@ -403,7 +400,7 @@ func TestTLSPeerCerts(t *testing.T) { tests := []struct { name string args args - wantOut []model.ArchivalMaybeBinaryData + wantOut []model.ArchivalBinaryData }{{ name: "x509.HostnameError", args: args{ @@ -414,9 +411,9 @@ func TestTLSPeerCerts(t *testing.T) { }, }, }, - wantOut: []model.ArchivalMaybeBinaryData{{ - Value: "deadbeef", - }}, + wantOut: []model.ArchivalBinaryData{ + model.ArchivalBinaryData("deadbeef"), + }, }, { name: "x509.UnknownAuthorityError", args: args{ @@ -427,9 +424,9 @@ func TestTLSPeerCerts(t *testing.T) { }, }, }, - wantOut: []model.ArchivalMaybeBinaryData{{ - Value: "deadbeef", - }}, + wantOut: []model.ArchivalBinaryData{ + model.ArchivalBinaryData("deadbeef"), + }, }, { name: "x509.CertificateInvalidError", args: args{ @@ -440,9 +437,9 @@ func TestTLSPeerCerts(t *testing.T) { }, }, }, - wantOut: []model.ArchivalMaybeBinaryData{{ - Value: "deadbeef", - }}, + wantOut: []model.ArchivalBinaryData{ + model.ArchivalBinaryData("deadbeef"), + }, }, { name: "successful case", args: args{ @@ -455,11 +452,10 @@ func TestTLSPeerCerts(t *testing.T) { }, err: nil, }, - wantOut: []model.ArchivalMaybeBinaryData{{ - Value: "deadbeef", - }, { - Value: "abad1dea", - }}, + wantOut: []model.ArchivalBinaryData{ + model.ArchivalBinaryData("deadbeef"), + model.ArchivalBinaryData("abad1dea"), + }, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/measurexlite/trace.go b/pkg/measurexlite/trace.go index 7a89cd8a3..bc85d145e 100644 --- a/pkg/measurexlite/trace.go +++ b/pkg/measurexlite/trace.go @@ -10,7 +10,6 @@ import ( "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" - utls "gitlab.com/yawning/utls.git" ) // Trace implements [model.Trace]. We use a [context.Context] to register ourselves @@ -31,6 +30,11 @@ type Trace struct { // once you have constructed a trace MAY lead to data races. Index int64 + // Netx is the network to use for measuring. The constructor inits this + // field using a [*netxlite.Netx]. You MAY override this field for testing. Make + // sure you do that before you start measuring to avoid data races. + Netx model.MeasuringNetwork + // bytesReceivedMap maps a remote host with the bytes we received // from such a remote host. Accessing this map requires one to // additionally hold the bytesReceivedMu mutex. @@ -40,43 +44,15 @@ type Trace struct { // access from multiple goroutines. bytesReceivedMu *sync.Mutex - // networkEvent is MANDATORY and buffers network events. - networkEvent chan *model.ArchivalNetworkEvent - - // newStdlibResolverFn is OPTIONAL and can be used to overide - // calls to the netxlite.NewStdlibResolver factory. - newStdlibResolverFn func(logger model.Logger) model.Resolver - - // newParallelUDPResolverFn is OPTIONAL and can be used to overide - // calls to the netxlite.NewParallelUDPResolver factory. - newParallelUDPResolverFn func(logger model.Logger, dialer model.Dialer, address string) model.Resolver - - // newParallelDNSOverHTTPSResolverFn is OPTIONAL and can be used to overide - // calls to the netxlite.NewParallelDNSOverHTTPSUDPResolver factory. - newParallelDNSOverHTTPSResolverFn func(logger model.Logger, URL string) model.Resolver - - // newDialerWithoutResolverFn is OPTIONAL and can be used to override - // calls to the netxlite.NewDialerWithoutResolver factory. - newDialerWithoutResolverFn func(dl model.DebugLogger) model.Dialer - - // newTLSHandshakerStdlibFn is OPTIONAL and can be used to overide - // calls to the netxlite.NewTLSHandshakerStdlib factory. - newTLSHandshakerStdlibFn func(dl model.DebugLogger) model.TLSHandshaker - - // newTLSHandshakerUTLSFn is OPTIONAL and can be used to overide - // calls to the netxlite.NewTLSHandshakerUTLS factory. - newTLSHandshakerUTLSFn func(dl model.DebugLogger, id *utls.ClientHelloID) model.TLSHandshaker - - // NewDialerWithoutResolverFn is OPTIONAL and can be used to override - // calls to the netxlite.NewQUICDialerWithoutResolver factory. - newQUICDialerWithoutResolverFn func(listener model.QUICListener, dl model.DebugLogger) model.QUICDialer - // dnsLookup is MANDATORY and buffers DNS Lookup observations. dnsLookup chan *model.ArchivalDNSLookupResult // delayedDNSResponse is MANDATORY and buffers delayed DNS responses. delayedDNSResponse chan *model.ArchivalDNSLookupResult + // networkEvent is MANDATORY and buffers network events. + networkEvent chan *model.ArchivalNetworkEvent + // tcpConnect is MANDATORY and buffers TCP connect observations. tcpConnect chan *model.ArchivalTCPConnectResult @@ -99,6 +75,8 @@ type Trace struct { ZeroTime time.Time } +var _ model.MeasuringNetwork = &Trace{} + // NetworkEventBufferSize is the [*Trace] buffer size for network I/O events. const NetworkEventBufferSize = 64 @@ -134,19 +112,9 @@ const QUICHandshakeBufferSize = 8 func NewTrace(index int64, zeroTime time.Time, tags ...string) *Trace { return &Trace{ Index: index, + Netx: &netxlite.Netx{Underlying: nil}, // use the host network bytesReceivedMap: make(map[string]int64), bytesReceivedMu: &sync.Mutex{}, - networkEvent: make( - chan *model.ArchivalNetworkEvent, - NetworkEventBufferSize, - ), - newStdlibResolverFn: nil, // use default - newParallelUDPResolverFn: nil, // use default - newParallelDNSOverHTTPSResolverFn: nil, // use default - newDialerWithoutResolverFn: nil, // use default - newTLSHandshakerStdlibFn: nil, // use default - newTLSHandshakerUTLSFn: nil, // use default - newQUICDialerWithoutResolverFn: nil, // use default dnsLookup: make( chan *model.ArchivalDNSLookupResult, DNSLookupBufferSize, @@ -155,6 +123,10 @@ func NewTrace(index int64, zeroTime time.Time, tags ...string) *Trace { chan *model.ArchivalDNSLookupResult, DelayedDNSResponseBufferSize, ), + networkEvent: make( + chan *model.ArchivalNetworkEvent, + NetworkEventBufferSize, + ), tcpConnect: make( chan *model.ArchivalTCPConnectResult, TCPConnectBufferSize, @@ -173,69 +145,6 @@ func NewTrace(index int64, zeroTime time.Time, tags ...string) *Trace { } } -// newStdlibResolver indirectly calls the passed netxlite.NewStdlibResolver -// thus allowing us to mock this function for testing -func (tx *Trace) newStdlibResolver(logger model.Logger) model.Resolver { - if tx.newStdlibResolverFn != nil { - return tx.newStdlibResolverFn(logger) - } - return netxlite.NewStdlibResolver(logger) -} - -// newParallelUDPResolver indirectly calls the passed netxlite.NewParallerUDPResolver -// thus allowing us to mock this function for testing -func (tx *Trace) newParallelUDPResolver(logger model.Logger, dialer model.Dialer, address string) model.Resolver { - if tx.newParallelUDPResolverFn != nil { - return tx.newParallelUDPResolverFn(logger, dialer, address) - } - return netxlite.NewParallelUDPResolver(logger, dialer, address) -} - -// newParallelDNSOverHTTPSResolver indirectly calls the passed netxlite.NewParallerDNSOverHTTPSResolver -// thus allowing us to mock this function for testing -func (tx *Trace) newParallelDNSOverHTTPSResolver(logger model.Logger, URL string) model.Resolver { - if tx.newParallelDNSOverHTTPSResolverFn != nil { - return tx.newParallelDNSOverHTTPSResolverFn(logger, URL) - } - return netxlite.NewParallelDNSOverHTTPSResolver(logger, URL) -} - -// newDialerWithoutResolver indirectly calls netxlite.NewDialerWithoutResolver -// thus allowing us to mock this func for testing. -func (tx *Trace) newDialerWithoutResolver(dl model.DebugLogger) model.Dialer { - if tx.newDialerWithoutResolverFn != nil { - return tx.newDialerWithoutResolverFn(dl) - } - return netxlite.NewDialerWithoutResolver(dl) -} - -// newTLSHandshakerStdlib indirectly calls netxlite.NewTLSHandshakerStdlib -// thus allowing us to mock this func for testing. -func (tx *Trace) newTLSHandshakerStdlib(dl model.DebugLogger) model.TLSHandshaker { - if tx.newTLSHandshakerStdlibFn != nil { - return tx.newTLSHandshakerStdlibFn(dl) - } - return netxlite.NewTLSHandshakerStdlib(dl) -} - -// newTLSHandshakerUTLS indirectly calls netxlite.NewTLSHandshakerUTLS -// thus allowing us to mock this func for testing. -func (tx *Trace) newTLSHandshakerUTLS(dl model.DebugLogger, id *utls.ClientHelloID) model.TLSHandshaker { - if tx.newTLSHandshakerUTLSFn != nil { - return tx.newTLSHandshakerUTLSFn(dl, id) - } - return netxlite.NewTLSHandshakerUTLS(dl, id) -} - -// newQUICDialerWithoutResolver indirectly calls netxlite.NewQUICDialerWithoutResolver -// thus allowing us to mock this func for testing. -func (tx *Trace) newQUICDialerWithoutResolver(listener model.QUICListener, dl model.DebugLogger) model.QUICDialer { - if tx.newQUICDialerWithoutResolverFn != nil { - return tx.newQUICDialerWithoutResolverFn(listener, dl) - } - return netxlite.NewQUICDialerWithoutResolver(listener, dl) -} - // TimeNow implements model.Trace.TimeNow. func (tx *Trace) TimeNow() time.Time { if tx.timeNowFn != nil { diff --git a/pkg/measurexlite/trace_test.go b/pkg/measurexlite/trace_test.go index 21860ffc6..742682404 100644 --- a/pkg/measurexlite/trace_test.go +++ b/pkg/measurexlite/trace_test.go @@ -5,7 +5,6 @@ import ( "crypto/tls" "errors" "net" - "reflect" "syscall" "testing" "time" @@ -50,45 +49,16 @@ func TestNewTrace(t *testing.T) { } }) - t.Run("NewStdlibResolverFn is nil", func(t *testing.T) { - if trace.newStdlibResolverFn != nil { - t.Fatal("expected nil NewStdlibResolverFn") + t.Run("Netx is an instance of *netxlite.Netx with a nil .Underlying", func(t *testing.T) { + if trace.Netx == nil { + t.Fatal("expected non-nil .Netx") } - }) - - t.Run("NewParallelUDPResolverFn is nil", func(t *testing.T) { - if trace.newParallelUDPResolverFn != nil { - t.Fatal("expected nil NewParallelUDPResolverFn") + netx, good := trace.Netx.(*netxlite.Netx) + if !good { + t.Fatal("not a *netxlite.Netx") } - }) - - t.Run("NewParallelDNSOverHTTPSResolverFn is nil", func(t *testing.T) { - if trace.newParallelDNSOverHTTPSResolverFn != nil { - t.Fatal("expected nil NewParallelDNSOverHTTPSResolverFn") - } - }) - - t.Run("NewDialerWithoutResolverFn is nil", func(t *testing.T) { - if trace.newDialerWithoutResolverFn != nil { - t.Fatal("expected nil NewDialerWithoutResolverFn") - } - }) - - t.Run("NewTLSHandshakerStdlibFn is nil", func(t *testing.T) { - if trace.newTLSHandshakerStdlibFn != nil { - t.Fatal("expected nil NewTLSHandshakerStdlibFn") - } - }) - - t.Run("newTLShandshakerUTLSFn is nil", func(t *testing.T) { - if trace.newTLSHandshakerUTLSFn != nil { - t.Fatal("expected nil NewTLSHandshakerUTLSfn") - } - }) - - t.Run("NewQUICDialerWithoutResolverFn is nil", func(t *testing.T) { - if trace.newQUICDialerWithoutResolverFn != nil { - t.Fatal("expected nil NewQUICDialerQithoutResolverFn") + if netx.Underlying != nil { + t.Fatal(".Underlying is not nil") } }) @@ -202,34 +172,10 @@ func TestNewTrace(t *testing.T) { } func TestTrace(t *testing.T) { - t.Run("NewStdlibResolverFn works as intended", func(t *testing.T) { - t.Run("when not nil", func(t *testing.T) { - mockedErr := errors.New("mocked") - tx := &Trace{ - newStdlibResolverFn: func(logger model.Logger) model.Resolver { - return &mocks.Resolver{ - MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { - return []string{}, mockedErr - }, - } - }, - } - resolver := tx.newStdlibResolver(model.DiscardLogger) - ctx := context.Background() - addrs, err := resolver.LookupHost(ctx, "example.com") - if !errors.Is(err, mockedErr) { - t.Fatal("unexpected err", err) - } - if len(addrs) != 0 { - t.Fatal("expected array of size 0") - } - }) - + t.Run("NewStdlibResolver works as intended", func(t *testing.T) { t.Run("when nil", func(t *testing.T) { - tx := &Trace{ - newParallelUDPResolverFn: nil, - } - resolver := tx.newStdlibResolver(model.DiscardLogger) + tx := NewTrace(0, time.Now()) + resolver := tx.NewStdlibResolver(model.DiscardLogger) ctx, cancel := context.WithCancel(context.Background()) cancel() addrs, err := resolver.LookupHost(ctx, "example.com") @@ -242,333 +188,169 @@ func TestTrace(t *testing.T) { }) }) - t.Run("NewParallelUDPResolverFn works as intended", func(t *testing.T) { - t.Run("when not nil", func(t *testing.T) { - mockedErr := errors.New("mocked") - tx := &Trace{ - newParallelUDPResolverFn: func(logger model.Logger, dialer model.Dialer, address string) model.Resolver { - return &mocks.Resolver{ - MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { - return []string{}, mockedErr - }, - } - }, - } - dialer := &mocks.Dialer{} - resolver := tx.newParallelUDPResolver(model.DiscardLogger, dialer, "1.1.1.1:53") - ctx := context.Background() - addrs, err := resolver.LookupHost(ctx, "example.com") - if !errors.Is(err, mockedErr) { - t.Fatal("unexpected err", err) - } - if len(addrs) != 0 { - t.Fatal("expected array of size 0") - } - }) - - t.Run("when nil", func(t *testing.T) { - tx := &Trace{ - newParallelUDPResolverFn: nil, - } - dialer := netxlite.NewDialerWithoutResolver(model.DiscardLogger) - resolver := tx.newParallelUDPResolver(model.DiscardLogger, dialer, "1.1.1.1:53") - ctx, cancel := context.WithCancel(context.Background()) - cancel() - addrs, err := resolver.LookupHost(ctx, "example.com") - if err == nil || err.Error() != netxlite.FailureInterrupted { - t.Fatal("unexpected err", err) - } - if len(addrs) != 0 { - t.Fatal("expected array of size 0") - } - }) + t.Run("NewParallelUDPResolver works as intended", func(t *testing.T) { + tx := NewTrace(0, time.Now()) + dialer := netxlite.NewDialerWithoutResolver(model.DiscardLogger) + resolver := tx.NewParallelUDPResolver(model.DiscardLogger, dialer, "1.1.1.1:53") + ctx, cancel := context.WithCancel(context.Background()) + cancel() + addrs, err := resolver.LookupHost(ctx, "example.com") + if err == nil || err.Error() != netxlite.FailureInterrupted { + t.Fatal("unexpected err", err) + } + if len(addrs) != 0 { + t.Fatal("expected array of size 0") + } }) - t.Run("NewParallelDNSOverHTTPSResolverFn works as intended", func(t *testing.T) { - t.Run("when not nil", func(t *testing.T) { - mockedErr := errors.New("mocked") - tx := &Trace{ - newParallelDNSOverHTTPSResolverFn: func(logger model.Logger, URL string) model.Resolver { - return &mocks.Resolver{ - MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { - return []string{}, mockedErr - }, - } - }, - } - resolver := tx.newParallelDNSOverHTTPSResolver(model.DiscardLogger, "https://dns.google.com") - ctx := context.Background() - addrs, err := resolver.LookupHost(ctx, "example.com") - if !errors.Is(err, mockedErr) { - t.Fatal("unexpected err", err) - } - if len(addrs) != 0 { - t.Fatal("expected array of size 0") - } - }) - - t.Run("when nil", func(t *testing.T) { - tx := &Trace{ - newParallelDNSOverHTTPSResolverFn: nil, - } - resolver := tx.newParallelDNSOverHTTPSResolver(model.DiscardLogger, "https://dns.google.com") - ctx, cancel := context.WithCancel(context.Background()) - cancel() - addrs, err := resolver.LookupHost(ctx, "example.com") - if err == nil || err.Error() != netxlite.FailureInterrupted { - t.Fatal("unexpected err", err) - } - if len(addrs) != 0 { - t.Fatal("expected array of size 0") - } - }) + t.Run("NewParallelDNSOverHTTPSResolver works as intended", func(t *testing.T) { + tx := NewTrace(0, time.Now()) + resolver := tx.NewParallelDNSOverHTTPSResolver(model.DiscardLogger, "https://dns.google.com") + ctx, cancel := context.WithCancel(context.Background()) + cancel() + addrs, err := resolver.LookupHost(ctx, "example.com") + if err == nil || err.Error() != netxlite.FailureInterrupted { + t.Fatal("unexpected err", err) + } + if len(addrs) != 0 { + t.Fatal("expected array of size 0") + } }) - t.Run("NewDialerWithoutResolverFn works as intended", func(t *testing.T) { - t.Run("when not nil", func(t *testing.T) { - mockedErr := errors.New("mocked") - tx := &Trace{ - newDialerWithoutResolverFn: func(dl model.DebugLogger) model.Dialer { - return &mocks.Dialer{ - MockDialContext: func(ctx context.Context, network, address string) (net.Conn, error) { - return nil, mockedErr - }, - } - }, - } - dialer := tx.NewDialerWithoutResolver(model.DiscardLogger) - ctx := context.Background() - conn, err := dialer.DialContext(ctx, "tcp", "1.1.1.1:443") - if !errors.Is(err, mockedErr) { - t.Fatal("unexpected err", err) - } - if conn != nil { - t.Fatal("expected nil conn") - } - }) - - t.Run("when nil", func(t *testing.T) { - tx := &Trace{ - newDialerWithoutResolverFn: nil, - } - dialer := tx.NewDialerWithoutResolver(model.DiscardLogger) - ctx, cancel := context.WithCancel(context.Background()) - cancel() // fail immediately - conn, err := dialer.DialContext(ctx, "tcp", "1.1.1.1:443") - if err == nil || err.Error() != netxlite.FailureInterrupted { - t.Fatal("unexpected err", err) - } - if conn != nil { - t.Fatal("expected nil conn") - } - }) + t.Run("NewDialerWithoutResolver works as intended", func(t *testing.T) { + tx := NewTrace(0, time.Now()) + dialer := tx.NewDialerWithoutResolver(model.DiscardLogger) + ctx, cancel := context.WithCancel(context.Background()) + cancel() // fail immediately + conn, err := dialer.DialContext(ctx, "tcp", "1.1.1.1:443") + if err == nil || err.Error() != netxlite.FailureInterrupted { + t.Fatal("unexpected err", err) + } + if conn != nil { + t.Fatal("expected nil conn") + } }) - t.Run("NewTLSHandshakerStdlibFn works as intended", func(t *testing.T) { - t.Run("when not nil", func(t *testing.T) { - mockedErr := errors.New("mocked") - tx := &Trace{ - newTLSHandshakerStdlibFn: func(dl model.DebugLogger) model.TLSHandshaker { - return &mocks.TLSHandshaker{ - MockHandshake: func(ctx context.Context, conn net.Conn, config *tls.Config) (net.Conn, tls.ConnectionState, error) { - return nil, tls.ConnectionState{}, mockedErr - }, - } - }, - } - thx := tx.NewTLSHandshakerStdlib(model.DiscardLogger) - ctx := context.Background() - conn, state, err := thx.Handshake(ctx, &mocks.Conn{}, &tls.Config{}) - if !errors.Is(err, mockedErr) { - t.Fatal("unexpected err", err) - } - if !reflect.ValueOf(state).IsZero() { - t.Fatal("state is not a zero value") - } - if conn != nil { - t.Fatal("expected nil conn") - } - }) - - t.Run("when nil", func(t *testing.T) { - mockedErr := errors.New("mocked") - tx := &Trace{ - newTLSHandshakerStdlibFn: nil, - } - thx := tx.NewTLSHandshakerStdlib(model.DiscardLogger) - tcpConn := &mocks.Conn{ - MockSetDeadline: func(t time.Time) error { - return nil - }, - MockRemoteAddr: func() net.Addr { - return &mocks.Addr{ - MockNetwork: func() string { - return "tcp" - }, - MockString: func() string { - return "1.1.1.1:443" - }, - } - }, - MockWrite: func(b []byte) (int, error) { - return 0, mockedErr - }, - MockClose: func() error { - return nil - }, - } - tlsConfig := &tls.Config{ - InsecureSkipVerify: true, - } - ctx := context.Background() - conn, state, err := thx.Handshake(ctx, tcpConn, tlsConfig) - if !errors.Is(err, mockedErr) { - t.Fatal("unexpected err", err) - } - if !reflect.ValueOf(state).IsZero() { - t.Fatal("state is not a zero value") - } - if conn != nil { - t.Fatal("expected nil conn") - } - }) + t.Run("NewTLSHandshakerStdlib works as intended", func(t *testing.T) { + mockedErr := errors.New("mocked") + tx := NewTrace(0, time.Now()) + thx := tx.NewTLSHandshakerStdlib(model.DiscardLogger) + tcpConn := &mocks.Conn{ + MockSetDeadline: func(t time.Time) error { + return nil + }, + MockRemoteAddr: func() net.Addr { + return &mocks.Addr{ + MockNetwork: func() string { + return "tcp" + }, + MockString: func() string { + return "1.1.1.1:443" + }, + } + }, + MockWrite: func(b []byte) (int, error) { + return 0, mockedErr + }, + MockClose: func() error { + return nil + }, + } + tlsConfig := &tls.Config{ + InsecureSkipVerify: true, + } + ctx := context.Background() + conn, err := thx.Handshake(ctx, tcpConn, tlsConfig) + if !errors.Is(err, mockedErr) { + t.Fatal("unexpected err", err) + } + if conn != nil { + t.Fatal("expected nil conn") + } }) - t.Run("NewTLSHandshakerUTLSFn works as intended", func(t *testing.T) { - t.Run("when not nil", func(t *testing.T) { - mockedErr := errors.New("mocked") - tx := &Trace{ - newTLSHandshakerUTLSFn: func(dl model.DebugLogger, id *utls.ClientHelloID) model.TLSHandshaker { - return &mocks.TLSHandshaker{ - MockHandshake: func(ctx context.Context, conn net.Conn, config *tls.Config) (net.Conn, tls.ConnectionState, error) { - return nil, tls.ConnectionState{}, mockedErr - }, - } - }, - } - thx := tx.NewTLSHandshakerUTLS(model.DiscardLogger, &utls.HelloGolang) - ctx := context.Background() - conn, state, err := thx.Handshake(ctx, &mocks.Conn{}, &tls.Config{}) - if !errors.Is(err, mockedErr) { - t.Fatal("unexpected err", err) - } - if !reflect.ValueOf(state).IsZero() { - t.Fatal("state is not a zero value") - } - if conn != nil { - t.Fatal("expected nil conn") - } - }) - - t.Run("when nil", func(t *testing.T) { - mockedErr := errors.New("mocked") - tx := &Trace{ - newTLSHandshakerStdlibFn: nil, - } - thx := tx.newTLSHandshakerUTLS(model.DiscardLogger, &utls.HelloGolang) - tcpConn := &mocks.Conn{ - MockSetDeadline: func(t time.Time) error { - return nil - }, - MockRemoteAddr: func() net.Addr { - return &mocks.Addr{ - MockNetwork: func() string { - return "tcp" - }, - MockString: func() string { - return "1.1.1.1:443" - }, - } - }, - MockWrite: func(b []byte) (int, error) { - return 0, mockedErr - }, - MockClose: func() error { - return nil - }, - } - tlsConfig := &tls.Config{ - InsecureSkipVerify: true, - } - ctx := context.Background() - conn, state, err := thx.Handshake(ctx, tcpConn, tlsConfig) - if !errors.Is(err, mockedErr) { - t.Fatal("unexpected err", err) - } - if !reflect.ValueOf(state).IsZero() { - t.Fatal("state is not a zero value") - } - if conn != nil { - t.Fatal("expected nil conn") - } - }) + t.Run("NewTLSHandshakerUTLS works as intended", func(t *testing.T) { + mockedErr := errors.New("mocked") + tx := NewTrace(0, time.Now()) + thx := tx.NewTLSHandshakerUTLS(model.DiscardLogger, &utls.HelloGolang) + tcpConn := &mocks.Conn{ + MockSetDeadline: func(t time.Time) error { + return nil + }, + MockRemoteAddr: func() net.Addr { + return &mocks.Addr{ + MockNetwork: func() string { + return "tcp" + }, + MockString: func() string { + return "1.1.1.1:443" + }, + } + }, + MockWrite: func(b []byte) (int, error) { + return 0, mockedErr + }, + MockClose: func() error { + return nil + }, + } + tlsConfig := &tls.Config{ + InsecureSkipVerify: true, + } + ctx := context.Background() + conn, err := thx.Handshake(ctx, tcpConn, tlsConfig) + if !errors.Is(err, mockedErr) { + t.Fatal("unexpected err", err) + } + if conn != nil { + t.Fatal("expected nil conn") + } }) - t.Run("NewQUICDialerWithoutResolverFn works as intended", func(t *testing.T) { - t.Run("when not nil", func(t *testing.T) { - mockedErr := errors.New("mocked") - tx := &Trace{ - newQUICDialerWithoutResolverFn: func(listener model.QUICListener, dl model.DebugLogger) model.QUICDialer { - return &mocks.QUICDialer{ - MockDialContext: func(ctx context.Context, address string, - tlsConfig *tls.Config, quicConfig *quic.Config) (quic.EarlyConnection, error) { - return nil, mockedErr - }, - } - }, - } - qdx := tx.newQUICDialerWithoutResolver(&mocks.QUICListener{}, model.DiscardLogger) - ctx := context.Background() - qconn, err := qdx.DialContext(ctx, "1.1.1.1:443", &tls.Config{}, &quic.Config{}) - if !errors.Is(err, mockedErr) { - t.Fatal("unexpected err", err) - } - if qconn != nil { - t.Fatal("expected nil conn") - } - }) - - t.Run("when nil", func(t *testing.T) { - mockedErr := errors.New("mocked") - tx := &Trace{ - newQUICDialerWithoutResolverFn: nil, // explicit - } - pconn := &mocks.UDPLikeConn{ - MockLocalAddr: func() net.Addr { - return &net.UDPAddr{ - Port: 0, - } - }, - MockRemoteAddr: func() net.Addr { - return &net.UDPAddr{ - Port: 0, - } - }, - MockSyscallConn: func() (syscall.RawConn, error) { - return nil, mockedErr - }, - MockClose: func() error { - return nil - }, - } - listener := &mocks.QUICListener{ - MockListen: func(addr *net.UDPAddr) (model.UDPLikeConn, error) { - return pconn, nil - }, - } - tlsConfig := &tls.Config{ - InsecureSkipVerify: true, - } - dialer := tx.newQUICDialerWithoutResolver(listener, model.DiscardLogger) - ctx := context.Background() - qconn, err := dialer.DialContext(ctx, "1.1.1.1:443", tlsConfig, &quic.Config{}) - if !errors.Is(err, mockedErr) { - t.Fatal("unexpected err", err) - } - if qconn != nil { - t.Fatal("expected nil conn") - } - }) + t.Run("NewQUICDialerWithoutResolver works as intended", func(t *testing.T) { + mockedErr := errors.New("mocked") + tx := NewTrace(0, time.Now()) + pconn := &mocks.UDPLikeConn{ + MockLocalAddr: func() net.Addr { + return &net.UDPAddr{ + // quic-go does not allow the use of the same net.PacketConn for multiple "Dial" + // calls (unless a quic.Transport is used), so we have to make sure to mock local + // addresses with different ports, as tests run in parallel. + Port: 0, + } + }, + MockRemoteAddr: func() net.Addr { + return &net.UDPAddr{ + Port: 0, + } + }, + MockSyscallConn: func() (syscall.RawConn, error) { + return nil, mockedErr + }, + MockClose: func() error { + return nil + }, + MockSetReadBuffer: func(n int) error { + return nil + }, + } + listener := &mocks.UDPListener{ + MockListen: func(addr *net.UDPAddr) (model.UDPLikeConn, error) { + return pconn, nil + }, + } + tlsConfig := &tls.Config{ + InsecureSkipVerify: true, + } + dialer := tx.NewQUICDialerWithoutResolver(listener, model.DiscardLogger) + ctx := context.Background() + qconn, err := dialer.DialContext(ctx, "1.1.1.1:443", tlsConfig, &quic.Config{}) + if !errors.Is(err, mockedErr) { + t.Fatal("unexpected err", err) + } + if qconn != nil { + t.Fatal("expected nil conn") + } }) t.Run("TimeNowFn works as intended", func(t *testing.T) { diff --git a/pkg/measurexlite/udp.go b/pkg/measurexlite/udp.go new file mode 100644 index 000000000..46147b834 --- /dev/null +++ b/pkg/measurexlite/udp.go @@ -0,0 +1,7 @@ +package measurexlite + +import "github.com/ooni/probe-engine/pkg/model" + +func (tx *Trace) NewUDPListener() model.UDPListener { + return tx.Netx.NewUDPListener() +} diff --git a/pkg/measurexlite/udp_test.go b/pkg/measurexlite/udp_test.go new file mode 100644 index 000000000..4950cfdc9 --- /dev/null +++ b/pkg/measurexlite/udp_test.go @@ -0,0 +1,24 @@ +package measurexlite + +import ( + "testing" + "time" + + "github.com/ooni/probe-engine/pkg/mocks" + "github.com/ooni/probe-engine/pkg/model" +) + +func TestNewUDPListener(t *testing.T) { + // Make sure that we're forwarding the call to the measuring network. + expectListener := &mocks.UDPListener{} + trace := NewTrace(0, time.Now()) + trace.Netx = &mocks.MeasuringNetwork{ + MockNewUDPListener: func() model.UDPListener { + return expectListener + }, + } + listener := trace.NewUDPListener() + if listener != expectListener { + t.Fatal("unexpected listener") + } +} diff --git a/pkg/measurexlite/utls.go b/pkg/measurexlite/utls.go index 730b1e099..15f5d71ef 100644 --- a/pkg/measurexlite/utls.go +++ b/pkg/measurexlite/utls.go @@ -13,7 +13,7 @@ import ( // except that it returns a model.TLSHandshaker that uses this trace. func (tx *Trace) NewTLSHandshakerUTLS(dl model.DebugLogger, id *utls.ClientHelloID) model.TLSHandshaker { return &tlsHandshakerTrace{ - thx: tx.newTLSHandshakerUTLS(dl, id), + thx: tx.Netx.NewTLSHandshakerUTLS(dl, id), tx: tx, } } diff --git a/pkg/measurexlite/utls_test.go b/pkg/measurexlite/utls_test.go index 6b0d6e4cc..7ee2f4cd0 100644 --- a/pkg/measurexlite/utls_test.go +++ b/pkg/measurexlite/utls_test.go @@ -14,8 +14,10 @@ func TestNewTLSHandshakerUTLS(t *testing.T) { underlying := &mocks.TLSHandshaker{} zeroTime := time.Now() trace := NewTrace(0, zeroTime) - trace.newTLSHandshakerUTLSFn = func(dl model.DebugLogger, id *utls.ClientHelloID) model.TLSHandshaker { - return underlying + trace.Netx = &mocks.MeasuringNetwork{ + MockNewTLSHandshakerUTLS: func(logger model.DebugLogger, id *utls.ClientHelloID) model.TLSHandshaker { + return underlying + }, } thx := trace.NewTLSHandshakerUTLS(model.DiscardLogger, &utls.HelloGolang) thxt := thx.(*tlsHandshakerTrace) diff --git a/pkg/mlablocatev2/mlablocatev2_test.go b/pkg/mlablocatev2/mlablocatev2_test.go index 7596a75ca..df733cec5 100644 --- a/pkg/mlablocatev2/mlablocatev2_test.go +++ b/pkg/mlablocatev2/mlablocatev2_test.go @@ -14,7 +14,9 @@ import ( ) func TestQueryNDT7Success(t *testing.T) { - // this integration test is ~0.5 s, so we can always run it + if testing.Short() { + t.Skip("skip test in short mode") + } client := NewClient(http.DefaultClient, model.DiscardLogger, "miniooni/0.1.0-dev") result, err := client.QueryNDT7(context.Background()) @@ -48,7 +50,9 @@ func TestQueryNDT7Success(t *testing.T) { } func TestQueryDashSuccess(t *testing.T) { - // this integration test is ~0.5 s, so we can always run it + if testing.Short() { + t.Skip("skip test in short mode") + } client := NewClient(http.DefaultClient, model.DiscardLogger, "miniooni/0.1.0-dev") result, err := client.QueryDash(context.Background()) @@ -82,7 +86,9 @@ func TestQueryDashSuccess(t *testing.T) { } func TestQuery404Response(t *testing.T) { - // this integration test is ~0.5 s, so we can always run it + if testing.Short() { + t.Skip("skip test in short mode") + } client := NewClient(http.DefaultClient, model.DiscardLogger, "miniooni/0.1.0-dev") result, err := client.query(context.Background(), "nonexistent") diff --git a/pkg/mocks/measuringnetwork.go b/pkg/mocks/measuringnetwork.go new file mode 100644 index 000000000..4c94d08ea --- /dev/null +++ b/pkg/mocks/measuringnetwork.go @@ -0,0 +1,67 @@ +package mocks + +import ( + "github.com/ooni/probe-engine/pkg/model" + utls "gitlab.com/yawning/utls.git" +) + +// MeasuringNetwork allows mocking [model.MeasuringNetwork]. +type MeasuringNetwork struct { + MockNewDialerWithoutResolver func(dl model.DebugLogger, w ...model.DialerWrapper) model.Dialer + + MockNewParallelDNSOverHTTPSResolver func(logger model.DebugLogger, URL string) model.Resolver + + MockNewParallelUDPResolver func(logger model.DebugLogger, dialer model.Dialer, address string) model.Resolver + + MockNewQUICDialerWithoutResolver func(listener model.UDPListener, logger model.DebugLogger, w ...model.QUICDialerWrapper) model.QUICDialer + + MockNewStdlibResolver func(logger model.DebugLogger) model.Resolver + + MockNewTLSHandshakerStdlib func(logger model.DebugLogger) model.TLSHandshaker + + MockNewTLSHandshakerUTLS func(logger model.DebugLogger, id *utls.ClientHelloID) model.TLSHandshaker + + MockNewUDPListener func() model.UDPListener +} + +var _ model.MeasuringNetwork = &MeasuringNetwork{} + +// NewDialerWithoutResolver implements model.MeasuringNetwork. +func (mn *MeasuringNetwork) NewDialerWithoutResolver(dl model.DebugLogger, w ...model.DialerWrapper) model.Dialer { + return mn.MockNewDialerWithoutResolver(dl, w...) +} + +// NewParallelDNSOverHTTPSResolver implements model.MeasuringNetwork. +func (mn *MeasuringNetwork) NewParallelDNSOverHTTPSResolver(logger model.DebugLogger, URL string) model.Resolver { + return mn.MockNewParallelDNSOverHTTPSResolver(logger, URL) +} + +// NewParallelUDPResolver implements model.MeasuringNetwork. +func (mn *MeasuringNetwork) NewParallelUDPResolver(logger model.DebugLogger, dialer model.Dialer, address string) model.Resolver { + return mn.MockNewParallelUDPResolver(logger, dialer, address) +} + +// NewQUICDialerWithoutResolver implements model.MeasuringNetwork. +func (mn *MeasuringNetwork) NewQUICDialerWithoutResolver(listener model.UDPListener, logger model.DebugLogger, w ...model.QUICDialerWrapper) model.QUICDialer { + return mn.MockNewQUICDialerWithoutResolver(listener, logger, w...) +} + +// NewStdlibResolver implements model.MeasuringNetwork. +func (mn *MeasuringNetwork) NewStdlibResolver(logger model.DebugLogger) model.Resolver { + return mn.MockNewStdlibResolver(logger) +} + +// NewTLSHandshakerStdlib implements model.MeasuringNetwork. +func (mn *MeasuringNetwork) NewTLSHandshakerStdlib(logger model.DebugLogger) model.TLSHandshaker { + return mn.MockNewTLSHandshakerStdlib(logger) +} + +// NewTLSHandshakerUTLS implements model.MeasuringNetwork. +func (mn *MeasuringNetwork) NewTLSHandshakerUTLS(logger model.DebugLogger, id *utls.ClientHelloID) model.TLSHandshaker { + return mn.MockNewTLSHandshakerUTLS(logger, id) +} + +// NewUDPListener implements model.MeasuringNetwork. +func (mn *MeasuringNetwork) NewUDPListener() model.UDPListener { + return mn.MockNewUDPListener() +} diff --git a/pkg/mocks/measuringnetwork_test.go b/pkg/mocks/measuringnetwork_test.go new file mode 100644 index 000000000..75d260fac --- /dev/null +++ b/pkg/mocks/measuringnetwork_test.go @@ -0,0 +1,114 @@ +package mocks + +import ( + "testing" + + "github.com/ooni/probe-engine/pkg/model" + utls "gitlab.com/yawning/utls.git" +) + +func TestMeasuringN(t *testing.T) { + t.Run("MockNewDialerWithoutResolver", func(t *testing.T) { + expected := &Dialer{} + mn := &MeasuringNetwork{ + MockNewDialerWithoutResolver: func(dl model.DebugLogger, w ...model.DialerWrapper) model.Dialer { + return expected + }, + } + got := mn.NewDialerWithoutResolver(nil, nil) + if expected != got { + t.Fatal("unexpected result") + } + }) + + t.Run("MockNewParallelDNSOverHTTPSResolver", func(t *testing.T) { + expected := &Resolver{} + mn := &MeasuringNetwork{ + MockNewParallelDNSOverHTTPSResolver: func(logger model.DebugLogger, URL string) model.Resolver { + return expected + }, + } + got := mn.NewParallelDNSOverHTTPSResolver(nil, "") + if expected != got { + t.Fatal("unexpected result") + } + }) + + t.Run("MockNewParallelUDPResolver", func(t *testing.T) { + expected := &Resolver{} + mn := &MeasuringNetwork{ + MockNewParallelUDPResolver: func(logger model.DebugLogger, dialer model.Dialer, address string) model.Resolver { + return expected + }, + } + got := mn.NewParallelUDPResolver(nil, nil, "") + if expected != got { + t.Fatal("unexpected result") + } + }) + + t.Run("MockNewQUICDialerWithoutResolver", func(t *testing.T) { + expected := &QUICDialer{} + mn := &MeasuringNetwork{ + MockNewQUICDialerWithoutResolver: func(listener model.UDPListener, logger model.DebugLogger, w ...model.QUICDialerWrapper) model.QUICDialer { + return expected + }, + } + got := mn.NewQUICDialerWithoutResolver(nil, nil, nil) + if expected != got { + t.Fatal("unexpected result") + } + }) + + t.Run("MockNewStdlibResolver", func(t *testing.T) { + expected := &Resolver{} + mn := &MeasuringNetwork{ + MockNewStdlibResolver: func(logger model.DebugLogger) model.Resolver { + return expected + }, + } + got := mn.NewStdlibResolver(nil) + if expected != got { + t.Fatal("unexpected result") + } + }) + + t.Run("MockNewTLSHandshakerStdlib", func(t *testing.T) { + expected := &TLSHandshaker{} + mn := &MeasuringNetwork{ + MockNewTLSHandshakerStdlib: func(logger model.DebugLogger) model.TLSHandshaker { + return expected + }, + } + got := mn.NewTLSHandshakerStdlib(nil) + if expected != got { + t.Fatal("unexpected result") + } + }) + + t.Run("MockNewTLSHandshakerUTLS", func(t *testing.T) { + expected := &TLSHandshaker{} + mn := &MeasuringNetwork{ + MockNewTLSHandshakerUTLS: func(logger model.DebugLogger, id *utls.ClientHelloID) model.TLSHandshaker { + return expected + }, + } + got := mn.NewTLSHandshakerUTLS(nil, nil) + if expected != got { + t.Fatal("unexpected result") + } + }) + + t.Run("MockNewUDPListener", func(t *testing.T) { + expected := &UDPListener{} + mn := &MeasuringNetwork{ + MockNewUDPListener: func() model.UDPListener { + return expected + }, + } + got := mn.NewUDPListener() + if expected != got { + t.Fatal("unexpected result") + } + }) +} diff --git a/pkg/mocks/quic.go b/pkg/mocks/quic.go index 0248c47e4..ac6a0f700 100644 --- a/pkg/mocks/quic.go +++ b/pkg/mocks/quic.go @@ -11,16 +11,6 @@ import ( "github.com/quic-go/quic-go" ) -// QUICListener is a mockable netxlite.QUICListener. -type QUICListener struct { - MockListen func(addr *net.UDPAddr) (model.UDPLikeConn, error) -} - -// Listen calls MockListen. -func (ql *QUICListener) Listen(addr *net.UDPAddr) (model.UDPLikeConn, error) { - return ql.MockListen(addr) -} - // QUICDialer is a mockable netxlite.QUICDialer. type QUICDialer struct { // MockDialContext allows mocking DialContext. @@ -57,10 +47,10 @@ type QUICEarlyConnection struct { MockCloseWithError func(code quic.ApplicationErrorCode, reason string) error MockContext func() context.Context MockConnectionState func() quic.ConnectionState - MockHandshakeComplete func() context.Context + MockHandshakeComplete func() <-chan struct{} MockNextConnection func() quic.Connection MockSendMessage func(b []byte) error - MockReceiveMessage func() ([]byte, error) + MockReceiveMessage func(ctx context.Context) ([]byte, error) } var _ quic.EarlyConnection = &QUICEarlyConnection{} @@ -122,7 +112,7 @@ func (s *QUICEarlyConnection) ConnectionState() quic.ConnectionState { } // HandshakeComplete calls MockHandshakeComplete. -func (s *QUICEarlyConnection) HandshakeComplete() context.Context { +func (s *QUICEarlyConnection) HandshakeComplete() <-chan struct{} { return s.MockHandshakeComplete() } @@ -137,8 +127,8 @@ func (s *QUICEarlyConnection) SendMessage(b []byte) error { } // ReceiveMessage calls MockReceiveMessage. -func (s *QUICEarlyConnection) ReceiveMessage() ([]byte, error) { - return s.MockReceiveMessage() +func (s *QUICEarlyConnection) ReceiveMessage(ctx context.Context) ([]byte, error) { + return s.MockReceiveMessage(ctx) } // UDPLikeConn is an UDP conn used by QUIC. diff --git a/pkg/mocks/quic_test.go b/pkg/mocks/quic_test.go index a37e600f5..93fc00f93 100644 --- a/pkg/mocks/quic_test.go +++ b/pkg/mocks/quic_test.go @@ -11,28 +11,9 @@ import ( "time" "github.com/google/go-cmp/cmp" - "github.com/ooni/probe-engine/pkg/model" "github.com/quic-go/quic-go" ) -func TestQUICListenerListen(t *testing.T) { - t.Run("Listen", func(t *testing.T) { - expected := errors.New("mocked error") - ql := &QUICListener{ - MockListen: func(addr *net.UDPAddr) (model.UDPLikeConn, error) { - return nil, expected - }, - } - pconn, err := ql.Listen(&net.UDPAddr{}) - if !errors.Is(err, expected) { - t.Fatal("not the error we expected", expected) - } - if pconn != nil { - t.Fatal("expected nil conn here") - } - }) -} - func TestQUICDialer(t *testing.T) { t.Run("DialContext", func(t *testing.T) { expected := errors.New("mocked error") @@ -235,13 +216,13 @@ func TestQUICEarlyConnection(t *testing.T) { t.Run("HandshakeComplete", func(t *testing.T) { ctx := context.Background() qconn := &QUICEarlyConnection{ - MockHandshakeComplete: func() context.Context { - return ctx + MockHandshakeComplete: func() <-chan struct{} { + return ctx.Done() }, } out := qconn.HandshakeComplete() - if !reflect.DeepEqual(ctx, out) { - t.Fatal("not the context we expected") + if !reflect.DeepEqual(ctx.Done(), out) { + t.Fatal("not the channel we expected") } }) @@ -274,12 +255,13 @@ func TestQUICEarlyConnection(t *testing.T) { t.Run("ReceiveMessage", func(t *testing.T) { expected := errors.New("mocked error") + ctx := context.Background() qconn := &QUICEarlyConnection{ - MockReceiveMessage: func() ([]byte, error) { + MockReceiveMessage: func(ctx context.Context) ([]byte, error) { return nil, expected }, } - b, err := qconn.ReceiveMessage() + b, err := qconn.ReceiveMessage(ctx) if !errors.Is(err, expected) { t.Fatal("not the error we expected", err) } diff --git a/pkg/mocks/tls.go b/pkg/mocks/tls.go index 7c418520e..b89a53f68 100644 --- a/pkg/mocks/tls.go +++ b/pkg/mocks/tls.go @@ -4,17 +4,17 @@ import ( "context" "crypto/tls" "net" + + "github.com/ooni/probe-engine/pkg/model" ) // TLSHandshaker is a mockable TLS handshaker. type TLSHandshaker struct { - MockHandshake func(ctx context.Context, conn net.Conn, config *tls.Config) ( - net.Conn, tls.ConnectionState, error) + MockHandshake func(ctx context.Context, conn net.Conn, config *tls.Config) (model.TLSConn, error) } // Handshake calls MockHandshake. -func (th *TLSHandshaker) Handshake(ctx context.Context, conn net.Conn, config *tls.Config) ( - net.Conn, tls.ConnectionState, error) { +func (th *TLSHandshaker) Handshake(ctx context.Context, conn net.Conn, config *tls.Config) (model.TLSConn, error) { return th.MockHandshake(ctx, conn, config) } diff --git a/pkg/mocks/tls_test.go b/pkg/mocks/tls_test.go index 03bb8c04a..43392b50c 100644 --- a/pkg/mocks/tls_test.go +++ b/pkg/mocks/tls_test.go @@ -7,6 +7,8 @@ import ( "net" "reflect" "testing" + + "github.com/ooni/probe-engine/pkg/model" ) func TestTLSHandshaker(t *testing.T) { @@ -16,18 +18,14 @@ func TestTLSHandshaker(t *testing.T) { ctx := context.Background() config := &tls.Config{} th := &TLSHandshaker{ - MockHandshake: func(ctx context.Context, conn net.Conn, - config *tls.Config) (net.Conn, tls.ConnectionState, error) { - return nil, tls.ConnectionState{}, expected + MockHandshake: func(ctx context.Context, conn net.Conn, config *tls.Config) (model.TLSConn, error) { + return nil, expected }, } - tlsConn, connState, err := th.Handshake(ctx, conn, config) + tlsConn, err := th.Handshake(ctx, conn, config) if !errors.Is(err, expected) { t.Fatal("not the error we expected", err) } - if !reflect.ValueOf(connState).IsZero() { - t.Fatal("expected zero ConnectionState here") - } if tlsConn != nil { t.Fatal("expected nil conn here") } diff --git a/pkg/mocks/udplistener.go b/pkg/mocks/udplistener.go new file mode 100644 index 000000000..cc4f637d3 --- /dev/null +++ b/pkg/mocks/udplistener.go @@ -0,0 +1,17 @@ +package mocks + +import ( + "net" + + "github.com/ooni/probe-engine/pkg/model" +) + +// UDPListener is a mockable netxlite.UDPListener. +type UDPListener struct { + MockListen func(addr *net.UDPAddr) (model.UDPLikeConn, error) +} + +// Listen calls MockListen. +func (ql *UDPListener) Listen(addr *net.UDPAddr) (model.UDPLikeConn, error) { + return ql.MockListen(addr) +} diff --git a/pkg/mocks/udplistener_test.go b/pkg/mocks/udplistener_test.go new file mode 100644 index 000000000..0be5ae892 --- /dev/null +++ b/pkg/mocks/udplistener_test.go @@ -0,0 +1,27 @@ +package mocks + +import ( + "errors" + "net" + "testing" + + "github.com/ooni/probe-engine/pkg/model" +) + +func TestUDPListener(t *testing.T) { + t.Run("Listen", func(t *testing.T) { + expected := errors.New("mocked error") + ql := &UDPListener{ + MockListen: func(addr *net.UDPAddr) (model.UDPLikeConn, error) { + return nil, expected + }, + } + pconn, err := ql.Listen(&net.UDPAddr{}) + if !errors.Is(err, expected) { + t.Fatal("not the error we expected", expected) + } + if pconn != nil { + t.Fatal("expected nil conn here") + } + }) +} diff --git a/pkg/mocks/underlyingnetwork.go b/pkg/mocks/underlyingnetwork.go index 5ead2b0e8..909b4ba7d 100644 --- a/pkg/mocks/underlyingnetwork.go +++ b/pkg/mocks/underlyingnetwork.go @@ -17,6 +17,8 @@ type UnderlyingNetwork struct { MockDialContext func(ctx context.Context, network, address string) (net.Conn, error) + MockListenTCP func(network string, addr *net.TCPAddr) (net.Listener, error) + MockListenUDP func(network string, addr *net.UDPAddr) (model.UDPLikeConn, error) MockGetaddrinfoLookupANY func(ctx context.Context, domain string) ([]string, string, error) @@ -38,6 +40,10 @@ func (un *UnderlyingNetwork) DialContext(ctx context.Context, network, address s return un.MockDialContext(ctx, network, address) } +func (un *UnderlyingNetwork) ListenTCP(network string, addr *net.TCPAddr) (net.Listener, error) { + return un.MockListenTCP(network, addr) +} + func (un *UnderlyingNetwork) ListenUDP(network string, addr *net.UDPAddr) (model.UDPLikeConn, error) { return un.MockListenUDP(network, addr) } diff --git a/pkg/mocks/underlyingnetwork_test.go b/pkg/mocks/underlyingnetwork_test.go index ea78265ea..7c0f0f5b5 100644 --- a/pkg/mocks/underlyingnetwork_test.go +++ b/pkg/mocks/underlyingnetwork_test.go @@ -55,6 +55,22 @@ func TestUnderlyingNetwork(t *testing.T) { } }) + t.Run("ListenTCP", func(t *testing.T) { + expect := errors.New("mocked error") + un := &UnderlyingNetwork{ + MockListenTCP: func(network string, addr *net.TCPAddr) (net.Listener, error) { + return nil, expect + }, + } + listener, err := un.ListenTCP("tcp", &net.TCPAddr{}) + if !errors.Is(err, expect) { + t.Fatal("unexpected err", err) + } + if listener != nil { + t.Fatal("expected nil listener") + } + }) + t.Run("ListenUDP", func(t *testing.T) { expect := errors.New("mocked error") un := &UnderlyingNetwork{ diff --git a/pkg/model/archival.go b/pkg/model/archival.go index 9dfb532ae..eca6ed1d6 100644 --- a/pkg/model/archival.go +++ b/pkg/model/archival.go @@ -1,12 +1,5 @@ package model -import ( - "encoding/base64" - "encoding/json" - "errors" - "unicode/utf8" -) - // // Archival format for individual measurement results // such as TCP connect, TLS handshake, DNS lookup. @@ -17,6 +10,18 @@ import ( // See https://github.com/ooni/spec/tree/master/data-formats. // +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "net/http" + "sort" + "unicode/utf8" + + "github.com/ooni/probe-engine/pkg/scrubber" +) + // // Data format extension specification // @@ -59,48 +64,110 @@ var ( // Base types // -// ArchivalMaybeBinaryData is a possibly binary string. We use this helper class -// to define a custom JSON encoder that allows us to choose the proper -// representation depending on whether the Value field is valid UTF-8 or not. +// ArchivalBinaryData is a wrapper for bytes that serializes the enclosed +// data using the specific ooni/spec data format for binary data. // -// See https://github.com/ooni/spec/blob/master/data-formats/df-001-httpt.md#maybebinarydata -type ArchivalMaybeBinaryData struct { - Value string +// See https://github.com/ooni/spec/blob/master/data-formats/df-001-httpt.md#maybebinarydata. +type ArchivalBinaryData []byte + +// archivalBinaryDataRepr is the wire representation of binary data according to +// https://github.com/ooni/spec/blob/master/data-formats/df-001-httpt.md#maybebinarydata. +type archivalBinaryDataRepr struct { + Data []byte `json:"data"` + Format string `json:"format"` } -// MarshalJSON marshals a string-like to JSON following the OONI spec that -// says that UTF-8 content is represented as string and non-UTF-8 content is -// instead represented using `{"format":"base64","data":"..."}`. -func (hb ArchivalMaybeBinaryData) MarshalJSON() ([]byte, error) { - if utf8.ValidString(hb.Value) { - return json.Marshal(hb.Value) +var ( + _ json.Marshaler = ArchivalBinaryData{} + _ json.Unmarshaler = &ArchivalBinaryData{} +) + +// MarshalJSON implements json.Marshaler. +func (value ArchivalBinaryData) MarshalJSON() ([]byte, error) { + // special case: we need to marshal the empty data as the null value + if len(value) <= 0 { + return json.Marshal(nil) } - er := make(map[string]string) - er["format"] = "base64" - er["data"] = base64.StdEncoding.EncodeToString([]byte(hb.Value)) - return json.Marshal(er) + + // construct and serialize the OONI representation + repr := &archivalBinaryDataRepr{Format: "base64", Data: value} + return json.Marshal(repr) } -// UnmarshalJSON is the opposite of MarshalJSON. -func (hb *ArchivalMaybeBinaryData) UnmarshalJSON(d []byte) error { - if err := json.Unmarshal(d, &hb.Value); err == nil { +// ErrInvalidBinaryDataFormat is the format returned when marshaling and +// unmarshaling binary data and the value of "format" is unknown. +var ErrInvalidBinaryDataFormat = errors.New("model: invalid binary data format") + +// UnmarshalJSON implements json.Unmarshaler. +func (value *ArchivalBinaryData) UnmarshalJSON(raw []byte) error { + // handle the case where input is a literal null + if bytes.Equal(raw, []byte("null")) { + *value = nil return nil } - er := make(map[string]string) - if err := json.Unmarshal(d, &er); err != nil { + + // attempt to unmarshal into the archival representation + var repr archivalBinaryDataRepr + if err := json.Unmarshal(raw, &repr); err != nil { return err } - if v, ok := er["format"]; !ok || v != "base64" { - return errors.New("missing or invalid format field") + + // make sure the data format is "base64" + if repr.Format != "base64" { + return fmt.Errorf("%w: '%s'", ErrInvalidBinaryDataFormat, repr.Format) + } + + // we're good because Go uses base64 for []byte automatically + *value = repr.Data + return nil +} + +// ArchivalScrubbedMaybeBinaryString is a possibly-binary string. When the string is valid UTF-8 +// we serialize it as itself. Otherwise, we use the binary data format defined by +// https://github.com/ooni/spec/blob/master/data-formats/df-001-httpt.md#maybebinarydata +// +// As the name implies, the data contained by this type is scrubbed to remove IPv4 and IPv6 +// addresses and endpoints during JSON serialization, to make it less likely that OONI leaks +// IP addresses in textual or binary fields such as HTTP headers and bodies. +type ArchivalScrubbedMaybeBinaryString string + +var ( + _ json.Marshaler = ArchivalScrubbedMaybeBinaryString("") + _ json.Unmarshaler = (func() *ArchivalScrubbedMaybeBinaryString { return nil }()) +) + +// MarshalJSON implements json.Marshaler. +func (value ArchivalScrubbedMaybeBinaryString) MarshalJSON() ([]byte, error) { + // convert value to a string + str := string(value) + + // make sure we get rid of IPv4 and IPv6 addresses and endpoints + str = scrubber.ScrubString(str) + + // if we can serialize as UTF-8 string, do that + if utf8.ValidString(str) { + return json.Marshal(str) } - if _, ok := er["data"]; !ok { - return errors.New("missing data field") + + // otherwise fallback to the serialization of ArchivalBinaryData + return json.Marshal(ArchivalBinaryData(str)) +} + +// UnmarshalJSON implements json.Unmarshaler. +func (value *ArchivalScrubbedMaybeBinaryString) UnmarshalJSON(rawData []byte) error { + // first attempt to decode as a string + var s string + if err := json.Unmarshal(rawData, &s); err == nil { + *value = ArchivalScrubbedMaybeBinaryString(s) + return nil } - b64, err := base64.StdEncoding.DecodeString(er["data"]) - if err != nil { + + // then attempt to decode as ArchivalBinaryData + var d ArchivalBinaryData + if err := json.Unmarshal(rawData, &d); err != nil { return err } - hb.Value = string(b64) + *value = ArchivalScrubbedMaybeBinaryString(d) return nil } @@ -172,20 +239,20 @@ type ArchivalTCPConnectStatus struct { // // See https://github.com/ooni/spec/blob/master/data-formats/df-006-tlshandshake.md type ArchivalTLSOrQUICHandshakeResult struct { - Network string `json:"network"` - Address string `json:"address"` - CipherSuite string `json:"cipher_suite"` - Failure *string `json:"failure"` - SoError *string `json:"so_error,omitempty"` - NegotiatedProtocol string `json:"negotiated_protocol"` - NoTLSVerify bool `json:"no_tls_verify"` - PeerCertificates []ArchivalMaybeBinaryData `json:"peer_certificates"` - ServerName string `json:"server_name"` - T0 float64 `json:"t0,omitempty"` - T float64 `json:"t"` - Tags []string `json:"tags"` - TLSVersion string `json:"tls_version"` - TransactionID int64 `json:"transaction_id,omitempty"` + Network string `json:"network"` + Address string `json:"address"` + CipherSuite string `json:"cipher_suite"` + Failure *string `json:"failure"` + SoError *string `json:"so_error,omitempty"` + NegotiatedProtocol string `json:"negotiated_protocol"` + NoTLSVerify bool `json:"no_tls_verify"` + PeerCertificates []ArchivalBinaryData `json:"peer_certificates"` + ServerName string `json:"server_name"` + T0 float64 `json:"t0,omitempty"` + T float64 `json:"t"` + Tags []string `json:"tags"` + TLSVersion string `json:"tls_version"` + TransactionID int64 `json:"transaction_id,omitempty"` } // @@ -213,14 +280,14 @@ type ArchivalHTTPRequestResult struct { // Headers are a map in Web Connectivity data format but // we have added support for a list since January 2020. type ArchivalHTTPRequest struct { - Body ArchivalHTTPBody `json:"body"` - BodyIsTruncated bool `json:"body_is_truncated"` - HeadersList []ArchivalHTTPHeader `json:"headers_list"` - Headers map[string]ArchivalMaybeBinaryData `json:"headers"` - Method string `json:"method"` - Tor ArchivalHTTPTor `json:"tor"` - Transport string `json:"x_transport"` - URL string `json:"url"` + Body ArchivalScrubbedMaybeBinaryString `json:"body"` + BodyIsTruncated bool `json:"body_is_truncated"` + HeadersList []ArchivalHTTPHeader `json:"headers_list"` + Headers map[string]ArchivalScrubbedMaybeBinaryString `json:"headers"` + Method string `json:"method"` + Tor ArchivalHTTPTor `json:"tor"` + Transport string `json:"x_transport"` + URL string `json:"url"` } // ArchivalHTTPResponse contains an HTTP response. @@ -228,79 +295,73 @@ type ArchivalHTTPRequest struct { // Headers are a map in Web Connectivity data format but // we have added support for a list since January 2020. type ArchivalHTTPResponse struct { - Body ArchivalHTTPBody `json:"body"` - BodyIsTruncated bool `json:"body_is_truncated"` - Code int64 `json:"code"` - HeadersList []ArchivalHTTPHeader `json:"headers_list"` - Headers map[string]ArchivalMaybeBinaryData `json:"headers"` + Body ArchivalScrubbedMaybeBinaryString `json:"body"` + BodyIsTruncated bool `json:"body_is_truncated"` + Code int64 `json:"code"` + HeadersList []ArchivalHTTPHeader `json:"headers_list"` + Headers map[string]ArchivalScrubbedMaybeBinaryString `json:"headers"` // The following fields are not serialised but are useful to simplify // analysing the measurements in telegram, whatsapp, etc. Locations []string `json:"-"` } -// ArchivalHTTPBody is an HTTP body. As an implementation note, this type must -// be an alias for the MaybeBinaryValue type, otherwise the specific serialisation -// mechanism implemented by MaybeBinaryValue is not working. -type ArchivalHTTPBody = ArchivalMaybeBinaryData +// ArchivalNewHTTPHeadersList constructs a new ArchivalHTTPHeader list given HTTP headers. +func ArchivalNewHTTPHeadersList(source http.Header) (out []ArchivalHTTPHeader) { + out = []ArchivalHTTPHeader{} -// ArchivalHTTPHeader is a single HTTP header. -type ArchivalHTTPHeader struct { - Key string - Value ArchivalMaybeBinaryData + // obtain the header keys + keys := []string{} + for key := range source { + keys = append(keys, key) + } + + // ensure the output is consistent, which helps with testing; + // for an example of why we need to sort headers, see + // https://github.com/ooni/probe-engine/pull/751/checks?check_run_id=853562310 + sort.Strings(keys) + + // insert into the output list + for _, key := range keys { + for _, value := range source[key] { + out = append(out, ArchivalHTTPHeader{ + ArchivalScrubbedMaybeBinaryString(key), + ArchivalScrubbedMaybeBinaryString(value), + }) + } + } + return } -// MarshalJSON marshals a single HTTP header to a tuple where the first -// element is a string and the second element is maybe-binary data. -func (hh ArchivalHTTPHeader) MarshalJSON() ([]byte, error) { - if utf8.ValidString(hh.Value.Value) { - return json.Marshal([]string{hh.Key, hh.Value.Value}) +// ArchivalNewHTTPHeadersMap creates a map representation of HTTP headers +func ArchivalNewHTTPHeadersMap(header http.Header) (out map[string]ArchivalScrubbedMaybeBinaryString) { + out = make(map[string]ArchivalScrubbedMaybeBinaryString) + for key, values := range header { + for _, value := range values { + out[key] = ArchivalScrubbedMaybeBinaryString(value) + break // just the first header + } } - value := make(map[string]string) - value["format"] = "base64" - value["data"] = base64.StdEncoding.EncodeToString([]byte(hh.Value.Value)) - return json.Marshal([]interface{}{hh.Key, value}) + return } -// UnmarshalJSON is the opposite of MarshalJSON. -func (hh *ArchivalHTTPHeader) UnmarshalJSON(d []byte) error { - var pair []interface{} - if err := json.Unmarshal(d, &pair); err != nil { +// ArchivalHTTPHeader is a single HTTP header. +type ArchivalHTTPHeader [2]ArchivalScrubbedMaybeBinaryString + +// errCannotParseArchivalHTTPHeader indicates that we cannot parse an ArchivalHTTPHeader. +var errCannotParseArchivalHTTPHeader = errors.New("invalid ArchivalHTTPHeader") + +// UnmarshalJSON implements json.Unmarshaler. +func (ahh *ArchivalHTTPHeader) UnmarshalJSON(data []byte) error { + var helper []ArchivalScrubbedMaybeBinaryString + if err := json.Unmarshal(data, &helper); err != nil { return err } - if len(pair) != 2 { - return errors.New("unexpected pair length") - } - key, ok := pair[0].(string) - if !ok { - return errors.New("the key is not a string") - } - value, ok := pair[1].(string) - if !ok { - mapvalue, ok := pair[1].(map[string]interface{}) - if !ok { - return errors.New("the value is neither a string nor a map[string]interface{}") - } - if _, ok := mapvalue["format"]; !ok { - return errors.New("missing format") - } - if v, ok := mapvalue["format"].(string); !ok || v != "base64" { - return errors.New("invalid format") - } - if _, ok := mapvalue["data"]; !ok { - return errors.New("missing data field") - } - v, ok := mapvalue["data"].(string) - if !ok { - return errors.New("the data field is not a string") - } - b64, err := base64.StdEncoding.DecodeString(v) - if err != nil { - return err - } - value = string(b64) + if len(helper) != 2 { + return fmt.Errorf("%w: expected 2 elements, got %d", errCannotParseArchivalHTTPHeader, len(helper)) } - hh.Key, hh.Value = key, ArchivalMaybeBinaryData{Value: value} + (*ahh)[0] = helper[0] + (*ahh)[1] = helper[1] return nil } diff --git a/pkg/model/archival_test.go b/pkg/model/archival_test.go index a0212ab35..6f9434ee3 100644 --- a/pkg/model/archival_test.go +++ b/pkg/model/archival_test.go @@ -1,11 +1,15 @@ package model_test import ( + "encoding/json" + "errors" + "net/http" "testing" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" "github.com/ooni/probe-engine/pkg/model" - "github.com/ooni/probe-engine/pkg/testingx" + "github.com/ooni/probe-engine/pkg/netxlite" ) func TestArchivalExtSpec(t *testing.T) { @@ -37,92 +41,516 @@ var archivalBinaryInput = []uint8{ // we use this value below to test we can handle binary data var archivalEncodedBinaryInput = []byte(`{"data":"V+V5+6a7DbzOvaeguqR4eBJZ7mg5pAeYxT68Vcv+NDx+G1qzIp3BLW7KW/EQJUceROItYAjqsArMBUig9Xg48Ns/nZ8lb4kAlpOvQ6xNyawT2yK+en3ZJKJSadiJwdFXqgQrotixGfbVETm7gM+G+V+djKv1xXQkOqLUQE7XEB8=","format":"base64"}`) -func TestMaybeBinaryValue(t *testing.T) { +func TestArchivalBinaryData(t *testing.T) { + // This test verifies that we correctly serialize binary data to JSON by + // producing null | {"format":"base64","data":""} t.Run("MarshalJSON", func(t *testing.T) { - tests := []struct { - name string // test name - input string // value to marshal - want []byte // expected result - wantErr bool // whether we expect an error - }{{ - name: "with string input", - input: "antani", - want: []byte(`"antani"`), - wantErr: false, + // testcase is a test case defined by this function + type testcase struct { + // name is the name of the test case + name string + + // input is the binary input + input model.ArchivalBinaryData + + // expectErr is the error we expect to see or nil + expectErr error + + // expectData is the data we expect to see + expectData []byte + } + + cases := []testcase{{ + name: "with nil value", + input: nil, + expectErr: nil, + expectData: []byte("null"), }, { - name: "with binary input", - input: string(archivalBinaryInput), - want: archivalEncodedBinaryInput, - wantErr: false, + name: "with zero length value", + input: []byte{}, + expectErr: nil, + expectData: []byte("null"), + }, { + name: "with value being a simple binary string", + input: []byte("Elliot"), + expectErr: nil, + expectData: []byte(`{"data":"RWxsaW90","format":"base64"}`), + }, { + name: "with value being a long binary string", + input: archivalBinaryInput, + expectErr: nil, + expectData: archivalEncodedBinaryInput, }} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - hb := model.ArchivalMaybeBinaryData{ - Value: tt.input, + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // serialize to JSON + output, err := json.Marshal(tc.input) + + t.Log("got this error", err) + t.Log("got this binary data", output) + t.Logf("converted to string: %s", string(output)) + + // handle errors + switch { + case err == nil && tc.expectErr != nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr == nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr != nil: + if err.Error() != tc.expectErr.Error() { + t.Fatal("expected", tc.expectErr, "got", err) + } + + case err == nil && tc.expectErr == nil: + // all good--fallthrough } - got, err := hb.MarshalJSON() - if (err != nil) != tt.wantErr { - t.Fatalf("ArchivalMaybeBinaryData.MarshalJSON() error = %v, wantErr %v", err, tt.wantErr) + + if diff := cmp.Diff(tc.expectData, output); diff != "" { + t.Fatal(diff) } - if diff := cmp.Diff(tt.want, got); diff != "" { + }) + } + }) + + // This test verifies that we correctly parse binary data to JSON by + // reading from null | {"format":"base64","data":""} + t.Run("UnmarshalJSON", func(t *testing.T) { + // testcase is a test case defined by this function + type testcase struct { + // name is the name of the test case + name string + + // input is the binary input + input []byte + + // expectErr is the error we expect to see or nil + expectErr error + + // expectData is the data we expect to see + expectData model.ArchivalBinaryData + } + + cases := []testcase{{ + name: "with nil input array", + input: nil, + expectErr: errors.New("unexpected end of JSON input"), + expectData: nil, + }, { + name: "with zero-length input array", + input: []byte{}, + expectErr: errors.New("unexpected end of JSON input"), + expectData: nil, + }, { + name: "with binary input that is not a complete JSON", + input: []byte("{"), + expectErr: errors.New("unexpected end of JSON input"), + expectData: nil, + }, { + name: "with ~random binary data as input", + input: archivalBinaryInput, + expectErr: errors.New("invalid character 'W' looking for beginning of value"), + expectData: nil, + }, { + name: "with valid JSON of the wrong type (array)", + input: []byte("[]"), + expectErr: errors.New("json: cannot unmarshal array into Go value of type model.archivalBinaryDataRepr"), + expectData: nil, + }, { + name: "with valid JSON of the wrong type (number)", + input: []byte("1.17"), + expectErr: errors.New("json: cannot unmarshal number into Go value of type model.archivalBinaryDataRepr"), + expectData: nil, + }, { + name: "with input being the liternal null", + input: []byte(`null`), + expectErr: nil, + expectData: nil, + }, { + name: "with empty JSON object", + input: []byte("{}"), + expectErr: errors.New("model: invalid binary data format: ''"), + expectData: nil, + }, { + name: "with correct data model but invalid format", + input: []byte(`{"data":"","format":"antani"}`), + expectErr: errors.New("model: invalid binary data format: 'antani'"), + expectData: nil, + }, { + name: "with correct data model and format but invalid base64 string", + input: []byte(`{"data":"x","format":"base64"}`), + expectErr: errors.New("illegal base64 data at input byte 0"), + expectData: nil, + }, { + name: "with correct data model and format but empty base64 string", + input: []byte(`{"data":"","format":"base64"}`), + expectErr: nil, + expectData: []byte{}, + }, { + name: "with the encoding of a simple binary string", + input: []byte(`{"data":"RWxsaW90","format":"base64"}`), + expectErr: nil, + expectData: []byte("Elliot"), + }, { + name: "with the encoding of a complex binary string", + input: archivalEncodedBinaryInput, + expectErr: nil, + expectData: archivalBinaryInput, + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // unmarshal the raw input into an ArchivalBinaryData type + var abd model.ArchivalBinaryData + err := json.Unmarshal(tc.input, &abd) + + t.Log("got this error", err) + t.Log("got this []byte-like value", abd) + t.Logf("converted to string: %s", string(abd)) + + // handle errors + switch { + case err == nil && tc.expectErr != nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr == nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr != nil: + if err.Error() != tc.expectErr.Error() { + t.Fatal("expected", tc.expectErr, "got", err) + } + + case err == nil && tc.expectErr == nil: + // all good--fallthrough + } + + if diff := cmp.Diff(tc.expectData, abd); diff != "" { + t.Fatal(diff) + } + }) + } + }) + + // This test verifies that we correctly round trip through JSON + t.Run("MarshalJSON then UnmarshalJSON", func(t *testing.T) { + // testcase is a test case defined by this function + type testcase struct { + // name is the name of the test case + name string + + // input is the binary input + input model.ArchivalBinaryData + } + + cases := []testcase{{ + name: "with nil value", + input: nil, + }, { + name: "with zero length value", + input: []byte{}, + }, { + name: "with value being a simple binary string", + input: []byte("Elliot"), + }, { + name: "with value being a long binary string", + input: archivalBinaryInput, + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // serialize to JSON + output, err := json.Marshal(tc.input) + + t.Log("got this error", err) + t.Log("got this binary data", output) + t.Logf("converted to string: %s", string(output)) + + if err != nil { + t.Fatal(err) + } + + // parse from JSON + var abc model.ArchivalBinaryData + if err := json.Unmarshal(output, &abc); err != nil { + t.Fatal(err) + } + + // make sure we round tripped + // + // Note: the round trip is not perfect because the zero length value, + // which originally is []byte{}, unmarshals to a nil value. + // + // Because the two are ~equivalent in Go most intents and purposes + // and the wire representation does not change, this is OK(TM) + diffOptions := []cmp.Option{cmpopts.EquateEmpty()} + if diff := cmp.Diff(tc.input, abc, diffOptions...); diff != "" { t.Fatal(diff) } }) } }) +} + +func TestArchivalScrubbedMaybeBinaryString(t *testing.T) { + t.Run("Supports assignment from a nil byte array", func(t *testing.T) { + var data []byte = nil // explicit + casted := model.ArchivalScrubbedMaybeBinaryString(data) + if casted != "" { + t.Fatal("unexpected value") + } + }) + + // This test verifies that we correctly serialize a string to JSON by + // producing "" | {"format":"base64","data":""} + t.Run("MarshalJSON", func(t *testing.T) { + // testcase is a test case defined by this function + type testcase struct { + // name is the name of the test case + name string + + // input is the possibly-binary input + input model.ArchivalScrubbedMaybeBinaryString + // expectErr is the error we expect to see or nil + expectErr error + + // expectData is the data we expect to see + expectData []byte + } + + cases := []testcase{{ + name: "with empty string value", + input: "", + expectErr: nil, + expectData: []byte(`""`), + }, { + name: "with value being a textual string", + input: "Elliot", + expectErr: nil, + expectData: []byte(`"Elliot"`), + }, { + name: "with value being a long binary string", + input: model.ArchivalScrubbedMaybeBinaryString(archivalBinaryInput), + expectErr: nil, + expectData: archivalEncodedBinaryInput, + }, { + name: "with string containing IP addresses and endpoints", + input: "a 130.192.91.211 b ::1 c [::1]:443 d 130.192.91.211:80", + expectErr: nil, + expectData: []byte(`"a [scrubbed] b [scrubbed] c [scrubbed] d [scrubbed]"`), + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // serialize to JSON + output, err := json.Marshal(tc.input) + + t.Log("got this error", err) + t.Log("got this binary data", output) + t.Logf("converted to string: %s", string(output)) + + // handle errors + switch { + case err == nil && tc.expectErr != nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr == nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr != nil: + if err.Error() != tc.expectErr.Error() { + t.Fatal("expected", tc.expectErr, "got", err) + } + + case err == nil && tc.expectErr == nil: + // all good--fallthrough + } + + if diff := cmp.Diff(tc.expectData, output); diff != "" { + t.Fatal(diff) + } + }) + } + }) + + // This test verifies that we correctly parse binary data to JSON by + // reading from "" | {"format":"base64","data":""} t.Run("UnmarshalJSON", func(t *testing.T) { - tests := []struct { - name string // test name - input []byte // value to unmarshal - want string // expected result - wantErr bool // whether we want an error - }{{ - name: "with string input", - input: []byte(`"xo"`), - want: "xo", - wantErr: false, + // testcase is a test case defined by this function + type testcase struct { + // name is the name of the test case + name string + + // input is the binary input + input []byte + + // expectErr is the error we expect to see or nil + expectErr error + + // expectData is the data we expect + expectData model.ArchivalScrubbedMaybeBinaryString + } + + cases := []testcase{{ + name: "with nil input array", + input: nil, + expectErr: errors.New("unexpected end of JSON input"), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), }, { - name: "with nil input", - input: nil, - want: "", - wantErr: true, - }, { - name: "with missing/invalid format", - input: []byte(`{"format": "foo"}`), - want: "", - wantErr: true, - }, { - name: "with missing data", - input: []byte(`{"format": "base64"}`), - want: "", - wantErr: true, - }, { - name: "with invalid base64 data", - input: []byte(`{"format": "base64", "data": "x"}`), - want: "", - wantErr: true, - }, { - name: "with valid base64 data", - input: archivalEncodedBinaryInput, - want: string(archivalBinaryInput), - wantErr: false, + name: "with zero-length input array", + input: []byte{}, + expectErr: errors.New("unexpected end of JSON input"), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), + }, { + name: "with binary input that is not a complete JSON", + input: []byte("{"), + expectErr: errors.New("unexpected end of JSON input"), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), + }, { + name: "with ~random binary data as input", + input: archivalBinaryInput, + expectErr: errors.New("invalid character 'W' looking for beginning of value"), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), + }, { + name: "with valid JSON of the wrong type (array)", + input: []byte("[]"), + expectErr: errors.New("json: cannot unmarshal array into Go value of type model.archivalBinaryDataRepr"), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), + }, { + name: "with valid JSON of the wrong type (number)", + input: []byte("1.17"), + expectErr: errors.New("json: cannot unmarshal number into Go value of type model.archivalBinaryDataRepr"), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), + }, { + name: "with input being the liternal null", + input: []byte(`null`), + expectErr: nil, + expectData: model.ArchivalScrubbedMaybeBinaryString(""), + }, { + name: "with empty JSON object", + input: []byte("{}"), + expectErr: errors.New("model: invalid binary data format: ''"), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), + }, { + name: "with correct data model but invalid format", + input: []byte(`{"data":"","format":"antani"}`), + expectErr: errors.New("model: invalid binary data format: 'antani'"), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), + }, { + name: "with correct data model and format but invalid base64 string", + input: []byte(`{"data":"x","format":"base64"}`), + expectErr: errors.New("illegal base64 data at input byte 0"), + expectData: model.ArchivalScrubbedMaybeBinaryString(""), + }, { + name: "with correct data model and format but empty base64 string", + input: []byte(`{"data":"","format":"base64"}`), + expectErr: nil, + expectData: model.ArchivalScrubbedMaybeBinaryString(""), + }, { + name: "with the a string", + input: []byte(`"Elliot"`), + expectErr: nil, + expectData: model.ArchivalScrubbedMaybeBinaryString("Elliot"), + }, { + name: "with the encoding of a string", + input: []byte(`{"data":"RWxsaW90","format":"base64"}`), + expectErr: nil, + expectData: model.ArchivalScrubbedMaybeBinaryString("Elliot"), + }, { + name: "with the encoding of a complex binary string", + input: archivalEncodedBinaryInput, + expectErr: nil, + expectData: model.ArchivalScrubbedMaybeBinaryString(archivalBinaryInput), }} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - hb := &model.ArchivalMaybeBinaryData{} - if err := hb.UnmarshalJSON(tt.input); (err != nil) != tt.wantErr { - t.Fatalf("ArchivalMaybeBinaryData.UnmarshalJSON() error = %v, wantErr %v", err, tt.wantErr) + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // unmarshal the raw input into an ArchivalBinaryData type + var abd model.ArchivalScrubbedMaybeBinaryString + err := json.Unmarshal(tc.input, &abd) + + t.Log("got this error", err) + t.Log("got this maybe-binary-string value", abd) + t.Logf("converted to string: %s", string(abd)) + + // handle errors + switch { + case err == nil && tc.expectErr != nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr == nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr != nil: + if err.Error() != tc.expectErr.Error() { + t.Fatal("expected", tc.expectErr, "got", err) + } + + case err == nil && tc.expectErr == nil: + // all good--fallthrough } - if d := cmp.Diff(tt.want, hb.Value); d != "" { - t.Fatal(d) + + if diff := cmp.Diff(tc.expectData, abd); diff != "" { + t.Fatal(diff) + } + }) + } + }) + + // This test verifies that we correctly round trip through JSON + t.Run("MarshalJSON then UnmarshalJSON", func(t *testing.T) { + // testcase is a test case defined by this function + type testcase struct { + // name is the name of the test case + name string + + // input is the maybe-binary input + input model.ArchivalScrubbedMaybeBinaryString + } + + cases := []testcase{{ + name: "with empty value", + input: "", + }, { + name: "with value being a simple textual string", + input: "Elliot", + }, { + name: "with value being a long binary string", + input: model.ArchivalScrubbedMaybeBinaryString(archivalBinaryInput), + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // serialize to JSON + output, err := json.Marshal(tc.input) + + t.Log("got this error", err) + t.Log("got this binary data", output) + t.Logf("converted to string: %s", string(output)) + + if err != nil { + t.Fatal(err) + } + + // parse from JSON + var abc model.ArchivalScrubbedMaybeBinaryString + if err := json.Unmarshal(output, &abc); err != nil { + t.Fatal(err) + } + + // make sure we round tripped + if diff := cmp.Diff(tc.input, abc); diff != "" { + t.Fatal(diff) } }) } }) } -func TestHTTPHeader(t *testing.T) { +func TestArchivalHTTPHeader(t *testing.T) { t.Run("MarshalJSON", func(t *testing.T) { tests := []struct { name string // test name @@ -132,29 +560,25 @@ func TestHTTPHeader(t *testing.T) { }{{ name: "with string value", input: model.ArchivalHTTPHeader{ - Key: "Content-Type", - Value: model.ArchivalMaybeBinaryData{ - Value: "text/plain", - }, + model.ArchivalScrubbedMaybeBinaryString("Content-Type"), + model.ArchivalScrubbedMaybeBinaryString("text/plain"), }, want: []byte(`["Content-Type","text/plain"]`), wantErr: false, }, { name: "with binary value", input: model.ArchivalHTTPHeader{ - Key: "Content-Type", - Value: model.ArchivalMaybeBinaryData{ - Value: string(archivalBinaryInput), - }, + model.ArchivalScrubbedMaybeBinaryString("Content-Type"), + model.ArchivalScrubbedMaybeBinaryString(archivalBinaryInput), }, want: []byte(`["Content-Type",` + string(archivalEncodedBinaryInput) + `]`), wantErr: false, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := tt.input.MarshalJSON() + got, err := json.Marshal(tt.input) if (err != nil) != tt.wantErr { - t.Fatalf("ArchivalHTTPHeader.MarshalJSON() error = %v, wantErr %v", err, tt.wantErr) + t.Fatalf("json.Marshal() error = %v, wantErr %v", err, tt.wantErr) } if diff := cmp.Diff(tt.want, got); diff != "" { t.Fatal(diff) @@ -168,144 +592,1523 @@ func TestHTTPHeader(t *testing.T) { name string // test name input []byte // input for the test want model.ArchivalHTTPHeader // expected output - wantErr bool // whether we want an error + wantErr error // whether we want an error }{{ name: "with invalid input", input: []byte(`{}`), want: model.ArchivalHTTPHeader{ - Key: "", - Value: model.ArchivalMaybeBinaryData{Value: ""}, + model.ArchivalScrubbedMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), }, - wantErr: true, + wantErr: errors.New("json: cannot unmarshal object into Go value of type []model.ArchivalScrubbedMaybeBinaryString"), }, { - name: "with unexpected number of items", + name: "with zero items", input: []byte(`[]`), want: model.ArchivalHTTPHeader{ - Key: "", - Value: model.ArchivalMaybeBinaryData{Value: ""}, + model.ArchivalScrubbedMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), }, - wantErr: true, + wantErr: errors.New("invalid ArchivalHTTPHeader: expected 2 elements, got 0"), + }, { + name: "with just one item", + input: []byte(`["x"]`), + want: [2]model.ArchivalScrubbedMaybeBinaryString{}, + wantErr: errors.New("invalid ArchivalHTTPHeader: expected 2 elements, got 1"), + }, { + name: "with three items", + input: []byte(`["x","x","x"]`), + want: [2]model.ArchivalScrubbedMaybeBinaryString{}, + wantErr: errors.New("invalid ArchivalHTTPHeader: expected 2 elements, got 3"), }, { name: "with first item not being a string", input: []byte(`[0,0]`), want: model.ArchivalHTTPHeader{ - Key: "", - Value: model.ArchivalMaybeBinaryData{Value: ""}, + model.ArchivalScrubbedMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), }, - wantErr: true, + wantErr: errors.New("json: cannot unmarshal number into Go value of type model.archivalBinaryDataRepr"), }, { name: "with both items being a string", input: []byte(`["x","y"]`), want: model.ArchivalHTTPHeader{ - Key: "x", - Value: model.ArchivalMaybeBinaryData{ - Value: "y", - }, + model.ArchivalScrubbedMaybeBinaryString("x"), + model.ArchivalScrubbedMaybeBinaryString("y"), }, - wantErr: false, + wantErr: nil, }, { name: "with second item not being a map[string]interface{}", input: []byte(`["x",[]]`), want: model.ArchivalHTTPHeader{ - Key: "", - Value: model.ArchivalMaybeBinaryData{ - Value: "", - }, + model.ArchivalScrubbedMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), }, - wantErr: true, + wantErr: errors.New("json: cannot unmarshal array into Go value of type model.archivalBinaryDataRepr"), }, { name: "with missing format key in second item", input: []byte(`["x",{}]`), want: model.ArchivalHTTPHeader{ - Key: "", - Value: model.ArchivalMaybeBinaryData{ - Value: "", - }, + model.ArchivalScrubbedMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), }, - wantErr: true, + wantErr: errors.New("model: invalid binary data format: ''"), }, { name: "with format value not being base64", input: []byte(`["x",{"format":1}]`), want: model.ArchivalHTTPHeader{ - Key: "", - Value: model.ArchivalMaybeBinaryData{ - Value: "", - }, + model.ArchivalScrubbedMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), }, - wantErr: true, + wantErr: errors.New("json: cannot unmarshal number into Go struct field archivalBinaryDataRepr.format of type string"), }, { name: "with missing data field", input: []byte(`["x",{"format":"base64"}]`), want: model.ArchivalHTTPHeader{ - Key: "", - Value: model.ArchivalMaybeBinaryData{ - Value: "", - }, + model.ArchivalScrubbedMaybeBinaryString("x"), + model.ArchivalScrubbedMaybeBinaryString(""), }, - wantErr: true, + wantErr: nil, }, { name: "with data not being a string", input: []byte(`["x",{"format":"base64","data":1}]`), want: model.ArchivalHTTPHeader{ - Key: "", - Value: model.ArchivalMaybeBinaryData{ - Value: "", - }, + model.ArchivalScrubbedMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), }, - wantErr: true, + wantErr: errors.New("json: cannot unmarshal number into Go struct field archivalBinaryDataRepr.data of type []uint8"), }, { name: "with data not being base64", input: []byte(`["x",{"format":"base64","data":"xx"}]`), want: model.ArchivalHTTPHeader{ - Key: "", - Value: model.ArchivalMaybeBinaryData{ - Value: "", - }, + model.ArchivalScrubbedMaybeBinaryString(""), + model.ArchivalScrubbedMaybeBinaryString(""), }, - wantErr: true, + wantErr: errors.New("illegal base64 data at input byte 0"), }, { name: "with correctly encoded base64 data", input: []byte(`["x",` + string(archivalEncodedBinaryInput) + `]`), want: model.ArchivalHTTPHeader{ - Key: "x", - Value: model.ArchivalMaybeBinaryData{ - Value: string(archivalBinaryInput), - }, + model.ArchivalScrubbedMaybeBinaryString("x"), + model.ArchivalScrubbedMaybeBinaryString(archivalBinaryInput), }, - wantErr: false, + wantErr: nil, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - hh := &model.ArchivalHTTPHeader{} - if err := hh.UnmarshalJSON(tt.input); (err != nil) != tt.wantErr { - t.Fatalf("ArchivalHTTPHeader.UnmarshalJSON() error = %v, wantErr %v", err, tt.wantErr) + var hh model.ArchivalHTTPHeader + + err := json.Unmarshal(tt.input, &hh) + + switch { + case err != nil && tt.wantErr != nil: + if err.Error() != tt.wantErr.Error() { + t.Fatal("expected", tt.wantErr, "got", err) + } + + case err != nil && tt.wantErr == nil: + t.Fatal("expected", tt.wantErr, "got", err) + case err == nil && tt.wantErr != nil: + t.Fatal("expected", tt.wantErr, "got", err) + + case err == nil && tt.wantErr == nil: + // note: only check the result when there is no error + if diff := cmp.Diff(tt.want, hh); diff != "" { + t.Error(diff) + } } - if diff := cmp.Diff(&tt.want, hh); diff != "" { - t.Error(diff) + }) + } + }) +} + +// This test ensures that ArchivalDNSLookupResult is WAI +func TestArchivalDNSLookupResult(t *testing.T) { + + // This test ensures that we correctly serialize to JSON. + t.Run("MarshalJSON", func(t *testing.T) { + // testcase is a test case defined by this function + type testcase struct { + // name is the name of the test case + name string + + // input is the input struct + input model.ArchivalDNSLookupResult + + // expectErr is the error we expect to see or nil + expectErr error + + // expectData is the data we expect to see + expectData []byte + } + + cases := []testcase{{ + name: "serialization of a successful DNS lookup", + input: model.ArchivalDNSLookupResult{ + Answers: []model.ArchivalDNSAnswer{{ + ASN: 15169, + ASOrgName: "Google LLC", + AnswerType: "A", + Hostname: "", + IPv4: "8.8.8.8", + IPv6: "", + TTL: nil, + }, { + ASN: 15169, + ASOrgName: "Google LLC", + AnswerType: "AAAA", + Hostname: "", + IPv4: "", + IPv6: "2001:4860:4860::8888", + TTL: nil, + }}, + Engine: "getaddrinfo", + Failure: nil, + GetaddrinfoError: 0, + Hostname: "dns.google", + QueryType: "ANY", + RawResponse: nil, + Rcode: 0, + ResolverHostname: nil, + ResolverPort: nil, + ResolverAddress: "", + T0: 0.5, + T: 0.7, + Tags: []string{"dns"}, + TransactionID: 44, + }, + expectErr: nil, + expectData: []byte(`{"answers":[{"asn":15169,"as_org_name":"Google LLC","answer_type":"A","ipv4":"8.8.8.8","ttl":null},{"asn":15169,"as_org_name":"Google LLC","answer_type":"AAAA","ipv6":"2001:4860:4860::8888","ttl":null}],"engine":"getaddrinfo","failure":null,"hostname":"dns.google","query_type":"ANY","resolver_hostname":null,"resolver_port":null,"resolver_address":"","t0":0.5,"t":0.7,"tags":["dns"],"transaction_id":44}`), + }, { + name: "serialization of a failed DNS lookup", + input: model.ArchivalDNSLookupResult{ + Answers: nil, + Engine: "getaddrinfo", + Failure: (func() *string { + s := netxlite.FailureDNSNXDOMAINError + return &s + }()), + GetaddrinfoError: 5, + Hostname: "dns.googlex", + QueryType: "ANY", + RawResponse: nil, + Rcode: 0, + ResolverHostname: nil, + ResolverPort: nil, + ResolverAddress: "", + T0: 0.5, + T: 0.77, + Tags: []string{"dns"}, + TransactionID: 43, + }, + expectErr: nil, + expectData: []byte(`{"answers":null,"engine":"getaddrinfo","failure":"dns_nxdomain_error","getaddrinfo_error":5,"hostname":"dns.googlex","query_type":"ANY","resolver_hostname":null,"resolver_port":null,"resolver_address":"","t0":0.5,"t":0.77,"tags":["dns"],"transaction_id":43}`), + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // serialize to JSON + data, err := json.Marshal(tc.input) + + t.Log("got this error", err) + t.Log("got this raw data", data) + t.Logf("converted to string: %s", string(data)) + + // handle errors + switch { + case err == nil && tc.expectErr != nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr == nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr != nil: + if err.Error() != tc.expectErr.Error() { + t.Fatal("expected", tc.expectErr, "got", err) + } + + case err == nil && tc.expectErr == nil: + // all good--fallthrough + } + + // make sure the serialization is OK + if diff := cmp.Diff(tc.expectData, data); diff != "" { + t.Fatal(diff) + } + }) + } + }) + + // This test ensures that we can unmarshal from the JSON representation + t.Run("UnmarshalJSON", func(t *testing.T) { + // testcase is a test case defined by this function + type testcase struct { + // name is the name of the test case + name string + + // input is the binary input + input []byte + + // expectErr is the error we expect to see or nil + expectErr error + + // expectStruct is the struct we expect to see + expectStruct model.ArchivalDNSLookupResult + } + + cases := []testcase{{ + name: "deserialization of a successful DNS lookup", + expectErr: nil, + input: []byte(`{"answers":[{"asn":15169,"as_org_name":"Google LLC","answer_type":"A","ipv4":"8.8.8.8","ttl":null},{"asn":15169,"as_org_name":"Google LLC","answer_type":"AAAA","ipv6":"2001:4860:4860::8888","ttl":null}],"engine":"getaddrinfo","failure":null,"hostname":"dns.google","query_type":"ANY","resolver_hostname":null,"resolver_port":null,"resolver_address":"","t0":0.5,"t":0.7,"tags":["dns"],"transaction_id":44}`), + expectStruct: model.ArchivalDNSLookupResult{ + Answers: []model.ArchivalDNSAnswer{{ + ASN: 15169, + ASOrgName: "Google LLC", + AnswerType: "A", + Hostname: "", + IPv4: "8.8.8.8", + IPv6: "", + TTL: nil, + }, { + ASN: 15169, + ASOrgName: "Google LLC", + AnswerType: "AAAA", + Hostname: "", + IPv4: "", + IPv6: "2001:4860:4860::8888", + TTL: nil, + }}, + Engine: "getaddrinfo", + Failure: nil, + GetaddrinfoError: 0, + Hostname: "dns.google", + QueryType: "ANY", + RawResponse: nil, + Rcode: 0, + ResolverHostname: nil, + ResolverPort: nil, + ResolverAddress: "", + T0: 0.5, + T: 0.7, + Tags: []string{"dns"}, + TransactionID: 44, + }, + }, { + name: "deserialization of a failed DNS lookup", + input: []byte(`{"answers":null,"engine":"getaddrinfo","failure":"dns_nxdomain_error","getaddrinfo_error":5,"hostname":"dns.googlex","query_type":"ANY","resolver_hostname":null,"resolver_port":null,"resolver_address":"","t0":0.5,"t":0.77,"tags":["dns"],"transaction_id":43}`), + expectErr: nil, + expectStruct: model.ArchivalDNSLookupResult{ + Answers: nil, + Engine: "getaddrinfo", + Failure: (func() *string { + s := netxlite.FailureDNSNXDOMAINError + return &s + }()), + GetaddrinfoError: 5, + Hostname: "dns.googlex", + QueryType: "ANY", + RawResponse: nil, + Rcode: 0, + ResolverHostname: nil, + ResolverPort: nil, + ResolverAddress: "", + T0: 0.5, + T: 0.77, + Tags: []string{"dns"}, + TransactionID: 43, + }, + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // parse the JSON + var data model.ArchivalDNSLookupResult + err := json.Unmarshal(tc.input, &data) + + t.Log("got this error", err) + t.Logf("got this struct %+v", data) + + // handle errors + switch { + case err == nil && tc.expectErr != nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr == nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr != nil: + if err.Error() != tc.expectErr.Error() { + t.Fatal("expected", tc.expectErr, "got", err) + } + + case err == nil && tc.expectErr == nil: + // all good--fallthrough + } + + // make sure the deserialization is OK + if diff := cmp.Diff(tc.expectStruct, data); diff != "" { + t.Fatal(diff) } }) } }) } -func TestHTTPBody(t *testing.T) { - // Implementation note: the content is always going to be the same - // even if we modify the implementation to become: - // - // type ArchivalHTTPBody ArchivalMaybeBinaryData - // - // instead of the correct: - // - // type ArchivalHTTPBody = ArchivalMaybeBinaryData - // - // However, cmp.Diff also takes into account the data type. Hence, if - // we make a mistake and apply the above change (which will in turn - // break correct JSON serialization), the this test will fail. - var body model.ArchivalHTTPBody - ff := &testingx.FakeFiller{} - ff.Fill(&body) - data := model.ArchivalMaybeBinaryData(body) - if diff := cmp.Diff(body, data); diff != "" { - t.Fatal(diff) +// This test ensures that ArchivalTCPConnectResult is WAI +func TestArchivalTCPConnectResult(t *testing.T) { + + // This test ensures that we correctly serialize to JSON. + t.Run("MarshalJSON", func(t *testing.T) { + // testcase is a test case defined by this function + type testcase struct { + // name is the name of the test case + name string + + // input is the input struct + input model.ArchivalTCPConnectResult + + // expectErr is the error we expect to see or nil + expectErr error + + // expectData is the data we expect to see + expectData []byte + } + + cases := []testcase{{ + name: "serialization of a successful TCP connect", + input: model.ArchivalTCPConnectResult{ + IP: "8.8.8.8", + Port: 443, + Status: model.ArchivalTCPConnectStatus{ + Blocked: nil, + Failure: nil, + Success: true, + }, + T0: 4, + T: 7, + Tags: []string{"tcp"}, + TransactionID: 99, + }, + expectErr: nil, + expectData: []byte(`{"ip":"8.8.8.8","port":443,"status":{"failure":null,"success":true},"t0":4,"t":7,"tags":["tcp"],"transaction_id":99}`), + }, { + name: "serialization of a failed TCP connect", + input: model.ArchivalTCPConnectResult{ + IP: "8.8.8.8", + Port: 443, + Status: model.ArchivalTCPConnectStatus{ + Blocked: nil, + Failure: (func() *string { + s := netxlite.FailureGenericTimeoutError + return &s + }()), + Success: false, + }, + T0: 4, + T: 7, + Tags: []string{"tcp"}, + TransactionID: 99, + }, + expectErr: nil, + expectData: []byte(`{"ip":"8.8.8.8","port":443,"status":{"failure":"generic_timeout_error","success":false},"t0":4,"t":7,"tags":["tcp"],"transaction_id":99}`), + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // serialize to JSON + data, err := json.Marshal(tc.input) + + t.Log("got this error", err) + t.Log("got this raw data", data) + t.Logf("converted to string: %s", string(data)) + + // handle errors + switch { + case err == nil && tc.expectErr != nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr == nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr != nil: + if err.Error() != tc.expectErr.Error() { + t.Fatal("expected", tc.expectErr, "got", err) + } + + case err == nil && tc.expectErr == nil: + // all good--fallthrough + } + + // make sure the serialization is OK + if diff := cmp.Diff(tc.expectData, data); diff != "" { + t.Fatal(diff) + } + }) + } + }) + + // This test ensures that we can unmarshal from the JSON representation + t.Run("UnmarshalJSON", func(t *testing.T) { + // testcase is a test case defined by this function + type testcase struct { + // name is the name of the test case + name string + + // input is the binary input + input []byte + + // expectErr is the error we expect to see or nil + expectErr error + + // expectStruct is the struct we expect to see + expectStruct model.ArchivalTCPConnectResult + } + + cases := []testcase{{ + name: "deserialization of a successful TCP connect", + expectErr: nil, + input: []byte(`{"ip":"8.8.8.8","port":443,"status":{"failure":null,"success":true},"t0":4,"t":7,"tags":["tcp"],"transaction_id":99}`), + expectStruct: model.ArchivalTCPConnectResult{ + IP: "8.8.8.8", + Port: 443, + Status: model.ArchivalTCPConnectStatus{ + Blocked: nil, + Failure: nil, + Success: true, + }, + T0: 4, + T: 7, + Tags: []string{"tcp"}, + TransactionID: 99, + }, + }, { + name: "deserialization of a failed TCP connect", + input: []byte(`{"ip":"8.8.8.8","port":443,"status":{"failure":"generic_timeout_error","success":false},"t0":4,"t":7,"tags":["tcp"],"transaction_id":99}`), + expectErr: nil, + expectStruct: model.ArchivalTCPConnectResult{ + IP: "8.8.8.8", + Port: 443, + Status: model.ArchivalTCPConnectStatus{ + Blocked: nil, + Failure: (func() *string { + s := netxlite.FailureGenericTimeoutError + return &s + }()), + Success: false, + }, + T0: 4, + T: 7, + Tags: []string{"tcp"}, + TransactionID: 99, + }, + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // parse the JSON + var data model.ArchivalTCPConnectResult + err := json.Unmarshal(tc.input, &data) + + t.Log("got this error", err) + t.Logf("got this struct %+v", data) + + // handle errors + switch { + case err == nil && tc.expectErr != nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr == nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr != nil: + if err.Error() != tc.expectErr.Error() { + t.Fatal("expected", tc.expectErr, "got", err) + } + + case err == nil && tc.expectErr == nil: + // all good--fallthrough + } + + // make sure the deserialization is OK + if diff := cmp.Diff(tc.expectStruct, data); diff != "" { + t.Fatal(diff) + } + }) + } + }) +} + +// This test ensures that ArchivalTLSOrQUICHandshakeResult is WAI +func TestArchivalTLSOrQUICHandshakeResult(t *testing.T) { + + // This test ensures that we correctly serialize to JSON. + t.Run("MarshalJSON", func(t *testing.T) { + // testcase is a test case defined by this function + type testcase struct { + // name is the name of the test case + name string + + // input is the input struct + input model.ArchivalTLSOrQUICHandshakeResult + + // expectErr is the error we expect to see or nil + expectErr error + + // expectData is the data we expect to see + expectData []byte + } + + cases := []testcase{{ + name: "serialization of a successful TLS handshake", + input: model.ArchivalTLSOrQUICHandshakeResult{ + Network: "tcp", + Address: "8.8.8.8:443", + CipherSuite: "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + Failure: nil, + SoError: nil, + NegotiatedProtocol: "http/1.1", + NoTLSVerify: false, + PeerCertificates: []model.ArchivalBinaryData{ + model.ArchivalBinaryData(archivalBinaryInput), + }, + ServerName: "dns.google", + T0: 1.0, + T: 2.0, + Tags: []string{"tls"}, + TLSVersion: "TLSv1.3", + TransactionID: 14, + }, + expectErr: nil, + expectData: []byte(`{"network":"tcp","address":"8.8.8.8:443","cipher_suite":"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256","failure":null,"negotiated_protocol":"http/1.1","no_tls_verify":false,"peer_certificates":[{"data":"V+V5+6a7DbzOvaeguqR4eBJZ7mg5pAeYxT68Vcv+NDx+G1qzIp3BLW7KW/EQJUceROItYAjqsArMBUig9Xg48Ns/nZ8lb4kAlpOvQ6xNyawT2yK+en3ZJKJSadiJwdFXqgQrotixGfbVETm7gM+G+V+djKv1xXQkOqLUQE7XEB8=","format":"base64"}],"server_name":"dns.google","t0":1,"t":2,"tags":["tls"],"tls_version":"TLSv1.3","transaction_id":14}`), + }, { + name: "serialization of a failed TLS handshake", + input: model.ArchivalTLSOrQUICHandshakeResult{ + Network: "tcp", + Address: "8.8.8.8:443", + CipherSuite: "", + Failure: (func() *string { + s := netxlite.FailureConnectionReset + return &s + })(), + SoError: (func() *string { + s := "connection reset by peer" + return &s + })(), + NegotiatedProtocol: "", + NoTLSVerify: false, + PeerCertificates: []model.ArchivalBinaryData{}, + ServerName: "dns.google", + T0: 1.0, + T: 2.0, + Tags: []string{"tls"}, + TLSVersion: "", + TransactionID: 4, + }, + expectErr: nil, + expectData: []byte(`{"network":"tcp","address":"8.8.8.8:443","cipher_suite":"","failure":"connection_reset","so_error":"connection reset by peer","negotiated_protocol":"","no_tls_verify":false,"peer_certificates":[],"server_name":"dns.google","t0":1,"t":2,"tags":["tls"],"tls_version":"","transaction_id":4}`), + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // serialize to JSON + data, err := json.Marshal(tc.input) + + t.Log("got this error", err) + t.Log("got this raw data", data) + t.Logf("converted to string: %s", string(data)) + + // handle errors + switch { + case err == nil && tc.expectErr != nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr == nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr != nil: + if err.Error() != tc.expectErr.Error() { + t.Fatal("expected", tc.expectErr, "got", err) + } + + case err == nil && tc.expectErr == nil: + // all good--fallthrough + } + + // make sure the serialization is OK + if diff := cmp.Diff(tc.expectData, data); diff != "" { + t.Fatal(diff) + } + }) + } + }) + + // This test ensures that we can unmarshal from the JSON representation + t.Run("UnmarshalJSON", func(t *testing.T) { + // testcase is a test case defined by this function + type testcase struct { + // name is the name of the test case + name string + + // input is the binary input + input []byte + + // expectErr is the error we expect to see or nil + expectErr error + + // expectStruct is the struct we expect to see + expectStruct model.ArchivalTLSOrQUICHandshakeResult + } + + cases := []testcase{{ + name: "deserialization of a successful TLS handshake", + expectErr: nil, + input: []byte(`{"network":"tcp","address":"8.8.8.8:443","cipher_suite":"TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256","failure":null,"negotiated_protocol":"http/1.1","no_tls_verify":false,"peer_certificates":[{"data":"V+V5+6a7DbzOvaeguqR4eBJZ7mg5pAeYxT68Vcv+NDx+G1qzIp3BLW7KW/EQJUceROItYAjqsArMBUig9Xg48Ns/nZ8lb4kAlpOvQ6xNyawT2yK+en3ZJKJSadiJwdFXqgQrotixGfbVETm7gM+G+V+djKv1xXQkOqLUQE7XEB8=","format":"base64"}],"server_name":"dns.google","t0":1,"t":2,"tags":["tls"],"tls_version":"TLSv1.3","transaction_id":14}`), + expectStruct: model.ArchivalTLSOrQUICHandshakeResult{ + Network: "tcp", + Address: "8.8.8.8:443", + CipherSuite: "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + Failure: nil, + SoError: nil, + NegotiatedProtocol: "http/1.1", + NoTLSVerify: false, + PeerCertificates: []model.ArchivalBinaryData{ + model.ArchivalBinaryData(archivalBinaryInput), + }, + ServerName: "dns.google", + T0: 1.0, + T: 2.0, + Tags: []string{"tls"}, + TLSVersion: "TLSv1.3", + TransactionID: 14, + }, + }, { + name: "deserialization of a failed TLS handshake", + input: []byte(`{"network":"tcp","address":"8.8.8.8:443","cipher_suite":"","failure":"connection_reset","so_error":"connection reset by peer","negotiated_protocol":"","no_tls_verify":false,"peer_certificates":[],"server_name":"dns.google","t0":1,"t":2,"tags":["tls"],"tls_version":"","transaction_id":4}`), + expectErr: nil, + expectStruct: model.ArchivalTLSOrQUICHandshakeResult{ + Network: "tcp", + Address: "8.8.8.8:443", + CipherSuite: "", + Failure: (func() *string { + s := netxlite.FailureConnectionReset + return &s + })(), + SoError: (func() *string { + s := "connection reset by peer" + return &s + })(), + NegotiatedProtocol: "", + NoTLSVerify: false, + PeerCertificates: []model.ArchivalBinaryData{}, + ServerName: "dns.google", + T0: 1.0, + T: 2.0, + Tags: []string{"tls"}, + TLSVersion: "", + TransactionID: 4, + }, + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // parse the JSON + var data model.ArchivalTLSOrQUICHandshakeResult + err := json.Unmarshal(tc.input, &data) + + t.Log("got this error", err) + t.Logf("got this struct %+v", data) + + // handle errors + switch { + case err == nil && tc.expectErr != nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr == nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr != nil: + if err.Error() != tc.expectErr.Error() { + t.Fatal("expected", tc.expectErr, "got", err) + } + + case err == nil && tc.expectErr == nil: + // all good--fallthrough + } + + // make sure the deserialization is OK + if diff := cmp.Diff(tc.expectStruct, data); diff != "" { + t.Fatal(diff) + } + }) + } + }) +} + +// This test ensures that ArchivalHTTPRequestResult is WAI +func TestArchivalHTTPRequestResult(t *testing.T) { + + // This test ensures that we correctly serialize to JSON. + t.Run("MarshalJSON", func(t *testing.T) { + // testcase is a test case defined by this function + type testcase struct { + // name is the name of the test case + name string + + // input is the input struct + input model.ArchivalHTTPRequestResult + + // expectErr is the error we expect to see or nil + expectErr error + + // expectData is the data we expect to see + expectData []byte + } + + cases := []testcase{ + + // This test ensures we can serialize a typical, successful HTTP measurement + { + name: "serialization of a successful HTTP request", + input: model.ArchivalHTTPRequestResult{ + Network: "tcp", + Address: "[2606:2800:220:1:248:1893:25c8:1946]:443", + ALPN: "h2", + Failure: nil, + Request: model.ArchivalHTTPRequest{ + Body: model.ArchivalScrubbedMaybeBinaryString(""), + BodyIsTruncated: false, + HeadersList: []model.ArchivalHTTPHeader{{ + model.ArchivalScrubbedMaybeBinaryString("Accept"), + model.ArchivalScrubbedMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), + }, { + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0"), + }}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "miniooni/0.1.0", + }, + Method: "GET", + Tor: model.ArchivalHTTPTor{ + ExitIP: nil, + ExitName: nil, + IsTor: false, + }, + Transport: "tcp", + URL: "https://www.example.com/", + }, + Response: model.ArchivalHTTPResponse{ + Body: model.ArchivalScrubbedMaybeBinaryString( + "Bonsoir, Elliot!", + ), + BodyIsTruncated: false, + Code: 200, + HeadersList: []model.ArchivalHTTPHeader{{ + model.ArchivalScrubbedMaybeBinaryString("Age"), + model.ArchivalScrubbedMaybeBinaryString("131833"), + }, { + model.ArchivalScrubbedMaybeBinaryString("Server"), + model.ArchivalScrubbedMaybeBinaryString("Apache"), + }}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "Age": "131833", + "Server": "Apache", + }, + Locations: nil, + }, + T0: 0.7, + T: 1.33, + Tags: []string{"http"}, + TransactionID: 5, + }, + expectErr: nil, + expectData: []byte(`{"network":"tcp","address":"[2606:2800:220:1:248:1893:25c8:1946]:443","alpn":"h2","failure":null,"request":{"body":"","body_is_truncated":false,"headers_list":[["Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"],["User-Agent","miniooni/0.1.0"]],"headers":{"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8","User-Agent":"miniooni/0.1.0"},"method":"GET","tor":{"exit_ip":null,"exit_name":null,"is_tor":false},"x_transport":"tcp","url":"https://www.example.com/"},"response":{"body":"Bonsoir, Elliot!","body_is_truncated":false,"code":200,"headers_list":[["Age","131833"],["Server","Apache"]],"headers":{"Age":"131833","Server":"Apache"}},"t0":0.7,"t":1.33,"tags":["http"],"transaction_id":5}`), + }, + + // This test ensures we can serialize a typical failed HTTP measurement + { + name: "serialization of a failed HTTP request", + input: model.ArchivalHTTPRequestResult{ + Network: "tcp", + Address: "[2606:2800:220:1:248:1893:25c8:1946]:443", + ALPN: "h2", + Failure: (func() *string { + s := netxlite.FailureGenericTimeoutError + return &s + })(), + Request: model.ArchivalHTTPRequest{ + Body: model.ArchivalScrubbedMaybeBinaryString(""), + BodyIsTruncated: false, + HeadersList: []model.ArchivalHTTPHeader{{ + model.ArchivalScrubbedMaybeBinaryString("Accept"), + model.ArchivalScrubbedMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), + }, { + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0"), + }}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "miniooni/0.1.0", + }, + Method: "GET", + Tor: model.ArchivalHTTPTor{ + ExitIP: nil, + ExitName: nil, + IsTor: false, + }, + Transport: "tcp", + URL: "https://www.example.com/", + }, + Response: model.ArchivalHTTPResponse{ + Body: model.ArchivalScrubbedMaybeBinaryString(""), + BodyIsTruncated: false, + Code: 0, + HeadersList: []model.ArchivalHTTPHeader{}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{}, + Locations: nil, + }, + T0: 0.4, + T: 1.563, + Tags: []string{"http"}, + TransactionID: 6, + }, + expectErr: nil, + expectData: []byte(`{"network":"tcp","address":"[2606:2800:220:1:248:1893:25c8:1946]:443","alpn":"h2","failure":"generic_timeout_error","request":{"body":"","body_is_truncated":false,"headers_list":[["Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"],["User-Agent","miniooni/0.1.0"]],"headers":{"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8","User-Agent":"miniooni/0.1.0"},"method":"GET","tor":{"exit_ip":null,"exit_name":null,"is_tor":false},"x_transport":"tcp","url":"https://www.example.com/"},"response":{"body":"","body_is_truncated":false,"code":0,"headers_list":[],"headers":{}},"t0":0.4,"t":1.563,"tags":["http"],"transaction_id":6}`), + }, + + // This test ensures we can correctly serialize an HTTP measurement where the + // response body and some headers contain binary data + // + // We need this test to continue to have confidence that our serialization + // code is always correctly handling how we generate JSONs + { + name: "serialization of a successful HTTP request with binary data", + input: model.ArchivalHTTPRequestResult{ + Network: "tcp", + Address: "[2606:2800:220:1:248:1893:25c8:1946]:443", + ALPN: "h2", + Failure: nil, + Request: model.ArchivalHTTPRequest{ + Body: model.ArchivalScrubbedMaybeBinaryString(""), + BodyIsTruncated: false, + HeadersList: []model.ArchivalHTTPHeader{{ + model.ArchivalScrubbedMaybeBinaryString("Accept"), + model.ArchivalScrubbedMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), + }, { + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0"), + }, { + model.ArchivalScrubbedMaybeBinaryString("Antani"), + model.ArchivalScrubbedMaybeBinaryString(string(archivalBinaryInput[:7])), + }, { + model.ArchivalScrubbedMaybeBinaryString("Antani"), + model.ArchivalScrubbedMaybeBinaryString(archivalBinaryInput[7:14]), + }}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "miniooni/0.1.0", + "Antani": model.ArchivalScrubbedMaybeBinaryString(archivalBinaryInput[:7]), + }, + Method: "GET", + Tor: model.ArchivalHTTPTor{ + ExitIP: nil, + ExitName: nil, + IsTor: false, + }, + Transport: "tcp", + URL: "https://www.example.com/", + }, + Response: model.ArchivalHTTPResponse{ + Body: model.ArchivalScrubbedMaybeBinaryString( + archivalBinaryInput[:77], + ), + BodyIsTruncated: false, + Code: 200, + HeadersList: []model.ArchivalHTTPHeader{{ + model.ArchivalScrubbedMaybeBinaryString("Age"), + model.ArchivalScrubbedMaybeBinaryString("131833"), + }, { + model.ArchivalScrubbedMaybeBinaryString("Server"), + model.ArchivalScrubbedMaybeBinaryString("Apache"), + }, { + model.ArchivalScrubbedMaybeBinaryString("Mascetti"), + model.ArchivalScrubbedMaybeBinaryString(archivalBinaryInput[14:21]), + }, { + model.ArchivalScrubbedMaybeBinaryString("Mascetti"), + model.ArchivalScrubbedMaybeBinaryString(archivalBinaryInput[21:28]), + }}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "Age": "131833", + "Server": "Apache", + "Mascetti": model.ArchivalScrubbedMaybeBinaryString(archivalEncodedBinaryInput[14:21]), + }, + Locations: nil, + }, + T0: 0.7, + T: 1.33, + Tags: []string{"http"}, + TransactionID: 5, + }, + expectErr: nil, + expectData: []byte(`{"network":"tcp","address":"[2606:2800:220:1:248:1893:25c8:1946]:443","alpn":"h2","failure":null,"request":{"body":"","body_is_truncated":false,"headers_list":[["Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"],["User-Agent","miniooni/0.1.0"],["Antani",{"data":"V+V5+6a7DQ==","format":"base64"}],["Antani",{"data":"vM69p6C6pA==","format":"base64"}]],"headers":{"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8","Antani":{"data":"V+V5+6a7DQ==","format":"base64"},"User-Agent":"miniooni/0.1.0"},"method":"GET","tor":{"exit_ip":null,"exit_name":null,"is_tor":false},"x_transport":"tcp","url":"https://www.example.com/"},"response":{"body":{"data":"V+V5+6a7DbzOvaeguqR4eBJZ7mg5pAeYxT68Vcv+NDx+G1qzIp3BLW7KW/EQJUceROItYAjqsArMBUig9Xg48Ns/nZ8lb4kAlpOvQ6w=","format":"base64"},"body_is_truncated":false,"code":200,"headers_list":[["Age","131833"],["Server","Apache"],["Mascetti",{"data":"eHgSWe5oOQ==","format":"base64"}],["Mascetti",{"data":"pAeYxT68VQ==","format":"base64"}]],"headers":{"Age":"131833","Mascetti":"6a7DbzO","Server":"Apache"}},"t0":0.7,"t":1.33,"tags":["http"],"transaction_id":5}`), + }, + + // This test ensures we can serialize an HTTP measurement containing + // IP addresses in the headers or the body. + // + // This test will fail until we implement more aggressive scrubbing, which + // is poised to happen as part of https://github.com/ooni/probe/issues/2531, + // where we implemented happy eyeballs, which may lead to surprises, so + // we want to be proactive and scrub more than before. + // + // We need this test to continue to have confidence that our serialization + // code is always correctly handling how we generate JSONs. + { + name: "serialization of a successful HTTP request with IP addresses and endpoints", + input: model.ArchivalHTTPRequestResult{ + Network: "tcp", + Address: "[2606:2800:220:1:248:1893:25c8:1946]:443", + ALPN: "h2", + Failure: nil, + Request: model.ArchivalHTTPRequest{ + Body: model.ArchivalScrubbedMaybeBinaryString(""), + BodyIsTruncated: false, + HeadersList: []model.ArchivalHTTPHeader{{ + model.ArchivalScrubbedMaybeBinaryString("Accept"), + model.ArchivalScrubbedMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), + }, { + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0"), + }, { + model.ArchivalScrubbedMaybeBinaryString("AntaniV4"), + model.ArchivalScrubbedMaybeBinaryString("130.192.91.211"), + }, { + model.ArchivalScrubbedMaybeBinaryString("AntaniV6"), + model.ArchivalScrubbedMaybeBinaryString("2606:2800:220:1:248:1893:25c8:1946"), + }, { + model.ArchivalScrubbedMaybeBinaryString("AntaniV4Epnt"), + model.ArchivalScrubbedMaybeBinaryString("130.192.91.211:443"), + }, { + model.ArchivalScrubbedMaybeBinaryString("AntaniV6Epnt"), + model.ArchivalScrubbedMaybeBinaryString("[2606:2800:220:1:248:1893:25c8:1946]:5222"), + }}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "miniooni/0.1.0", + "AntaniV4": "130.192.91.211", + "AntaniV6": "2606:2800:220:1:248:1893:25c8:1946", + "AntaniV4Epnt": "130.192.91.211:443", + "AntaniV6Epnt": "[2606:2800:220:1:248:1893:25c8:1946]:5222", + }, + Method: "GET", + Tor: model.ArchivalHTTPTor{ + ExitIP: nil, + ExitName: nil, + IsTor: false, + }, + Transport: "tcp", + URL: "https://www.example.com/", + }, + Response: model.ArchivalHTTPResponse{ + Body: model.ArchivalScrubbedMaybeBinaryString( + "Your address is 130.192.91.211 and 2606:2800:220:1:248:1893:25c8:1946 and you have endpoints [2606:2800:220:1:248:1893:25c8:1946]:5222 and 130.192.91.211:443. You're welcome.", + ), + BodyIsTruncated: false, + Code: 200, + HeadersList: []model.ArchivalHTTPHeader{{ + model.ArchivalScrubbedMaybeBinaryString("Age"), + model.ArchivalScrubbedMaybeBinaryString("131833"), + }, { + model.ArchivalScrubbedMaybeBinaryString("Server"), + model.ArchivalScrubbedMaybeBinaryString("Apache"), + }, { + model.ArchivalScrubbedMaybeBinaryString("MascettiV4"), + model.ArchivalScrubbedMaybeBinaryString("130.192.91.211"), + }, { + model.ArchivalScrubbedMaybeBinaryString("MascettiV6"), + model.ArchivalScrubbedMaybeBinaryString("2606:2800:220:1:248:1893:25c8:1946"), + }, { + model.ArchivalScrubbedMaybeBinaryString("MascettiV4Epnt"), + model.ArchivalScrubbedMaybeBinaryString("130.192.91.211:443"), + }, { + model.ArchivalScrubbedMaybeBinaryString("MascettiV6Epnt"), + model.ArchivalScrubbedMaybeBinaryString("[2606:2800:220:1:248:1893:25c8:1946]:5222"), + }}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "Age": "131833", + "Server": "Apache", + "MascettiV4": "130.192.91.211", + "MascettiV6": "2606:2800:220:1:248:1893:25c8:1946", + "MascettiV4Epnt": "130.192.91.211:443", + "MascettiV6Epnt": "[2606:2800:220:1:248:1893:25c8:1946]:5222", + }, + Locations: nil, + }, + T0: 0.7, + T: 1.33, + Tags: []string{"http"}, + TransactionID: 5, + }, + expectErr: nil, + expectData: []byte(`{"network":"tcp","address":"[2606:2800:220:1:248:1893:25c8:1946]:443","alpn":"h2","failure":null,"request":{"body":"","body_is_truncated":false,"headers_list":[["Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"],["User-Agent","miniooni/0.1.0"],["AntaniV4","[scrubbed]"],["AntaniV6","[scrubbed]"],["AntaniV4Epnt","[scrubbed]"],["AntaniV6Epnt","[scrubbed]"]],"headers":{"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8","AntaniV4":"[scrubbed]","AntaniV4Epnt":"[scrubbed]","AntaniV6":"[scrubbed]","AntaniV6Epnt":"[scrubbed]","User-Agent":"miniooni/0.1.0"},"method":"GET","tor":{"exit_ip":null,"exit_name":null,"is_tor":false},"x_transport":"tcp","url":"https://www.example.com/"},"response":{"body":"\u003cHTML\u003e\u003cBODY\u003eYour address is [scrubbed] and [scrubbed] and you have endpoints [scrubbed] and [scrubbed]. You're welcome.\u003c/BODY\u003e\u003c/HTML\u003e","body_is_truncated":false,"code":200,"headers_list":[["Age","131833"],["Server","Apache"],["MascettiV4","[scrubbed]"],["MascettiV6","[scrubbed]"],["MascettiV4Epnt","[scrubbed]"],["MascettiV6Epnt","[scrubbed]"]],"headers":{"Age":"131833","MascettiV4":"[scrubbed]","MascettiV4Epnt":"[scrubbed]","MascettiV6":"[scrubbed]","MascettiV6Epnt":"[scrubbed]","Server":"Apache"}},"t0":0.7,"t":1.33,"tags":["http"],"transaction_id":5}`), + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // serialize to JSON + data, err := json.Marshal(tc.input) + + t.Log("got this error", err) + t.Log("got this raw data", data) + t.Logf("converted to string: %s", string(data)) + + // handle errors + switch { + case err == nil && tc.expectErr != nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr == nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr != nil: + if err.Error() != tc.expectErr.Error() { + t.Fatal("expected", tc.expectErr, "got", err) + } + + case err == nil && tc.expectErr == nil: + // all good--fallthrough + } + + // make sure the serialization is OK + if diff := cmp.Diff(tc.expectData, data); diff != "" { + t.Fatal(diff) + } + }) + } + }) + + // This test ensures that we can unmarshal from the JSON representation + t.Run("UnmarshalJSON", func(t *testing.T) { + // testcase is a test case defined by this function + type testcase struct { + // name is the name of the test case + name string + + // input is the binary input + input []byte + + // expectErr is the error we expect to see or nil + expectErr error + + // expectStruct is the struct we expect to see + expectStruct model.ArchivalHTTPRequestResult + } + + cases := []testcase{{ + name: "deserialization of a successful HTTP request", + expectErr: nil, + input: []byte(`{"network":"tcp","address":"[2606:2800:220:1:248:1893:25c8:1946]:443","alpn":"h2","failure":null,"request":{"body":"","body_is_truncated":false,"headers_list":[["Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"],["User-Agent","miniooni/0.1.0"]],"headers":{"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8","User-Agent":"miniooni/0.1.0"},"method":"GET","tor":{"exit_ip":null,"exit_name":null,"is_tor":false},"x_transport":"tcp","url":"https://www.example.com/"},"response":{"body":"Bonsoir, Elliot!","body_is_truncated":false,"code":200,"headers_list":[["Age","131833"],["Server","Apache"]],"headers":{"Age":"131833","Server":"Apache"}},"t0":0.7,"t":1.33,"tags":["http"],"transaction_id":5}`), + expectStruct: model.ArchivalHTTPRequestResult{ + Network: "tcp", + Address: "[2606:2800:220:1:248:1893:25c8:1946]:443", + ALPN: "h2", + Failure: nil, + Request: model.ArchivalHTTPRequest{ + Body: model.ArchivalScrubbedMaybeBinaryString(""), + BodyIsTruncated: false, + HeadersList: []model.ArchivalHTTPHeader{{ + model.ArchivalScrubbedMaybeBinaryString("Accept"), + model.ArchivalScrubbedMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), + }, { + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0"), + }}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "miniooni/0.1.0", + }, + Method: "GET", + Tor: model.ArchivalHTTPTor{ + ExitIP: nil, + ExitName: nil, + IsTor: false, + }, + Transport: "tcp", + URL: "https://www.example.com/", + }, + Response: model.ArchivalHTTPResponse{ + Body: model.ArchivalScrubbedMaybeBinaryString( + "Bonsoir, Elliot!", + ), + BodyIsTruncated: false, + Code: 200, + HeadersList: []model.ArchivalHTTPHeader{{ + model.ArchivalScrubbedMaybeBinaryString("Age"), + model.ArchivalScrubbedMaybeBinaryString("131833"), + }, { + model.ArchivalScrubbedMaybeBinaryString("Server"), + model.ArchivalScrubbedMaybeBinaryString("Apache"), + }}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "Age": "131833", + "Server": "Apache", + }, + Locations: nil, + }, + T0: 0.7, + T: 1.33, + Tags: []string{"http"}, + TransactionID: 5, + }, + }, { + name: "deserialization of a failed HTTP request", + input: []byte(`{"network":"tcp","address":"[2606:2800:220:1:248:1893:25c8:1946]:443","alpn":"h2","failure":"generic_timeout_error","request":{"body":"","body_is_truncated":false,"headers_list":[["Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"],["User-Agent","miniooni/0.1.0"]],"headers":{"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8","User-Agent":"miniooni/0.1.0"},"method":"GET","tor":{"exit_ip":null,"exit_name":null,"is_tor":false},"x_transport":"tcp","url":"https://www.example.com/"},"response":{"body":"","body_is_truncated":false,"code":0,"headers_list":[],"headers":{}},"t0":0.4,"t":1.563,"tags":["http"],"transaction_id":6}`), + expectErr: nil, + expectStruct: model.ArchivalHTTPRequestResult{ + Network: "tcp", + Address: "[2606:2800:220:1:248:1893:25c8:1946]:443", + ALPN: "h2", + Failure: (func() *string { + s := netxlite.FailureGenericTimeoutError + return &s + })(), + Request: model.ArchivalHTTPRequest{ + Body: model.ArchivalScrubbedMaybeBinaryString(""), + BodyIsTruncated: false, + HeadersList: []model.ArchivalHTTPHeader{{ + model.ArchivalScrubbedMaybeBinaryString("Accept"), + model.ArchivalScrubbedMaybeBinaryString("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), + }, { + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0"), + }}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "User-Agent": "miniooni/0.1.0", + }, + Method: "GET", + Tor: model.ArchivalHTTPTor{ + ExitIP: nil, + ExitName: nil, + IsTor: false, + }, + Transport: "tcp", + URL: "https://www.example.com/", + }, + Response: model.ArchivalHTTPResponse{ + Body: model.ArchivalScrubbedMaybeBinaryString(""), + BodyIsTruncated: false, + Code: 0, + HeadersList: []model.ArchivalHTTPHeader{}, + Headers: map[string]model.ArchivalScrubbedMaybeBinaryString{}, + Locations: nil, + }, + T0: 0.4, + T: 1.563, + Tags: []string{"http"}, + TransactionID: 6, + }, + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // parse the JSON + var data model.ArchivalHTTPRequestResult + err := json.Unmarshal(tc.input, &data) + + t.Log("got this error", err) + t.Logf("got this struct %+v", data) + + // handle errors + switch { + case err == nil && tc.expectErr != nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr == nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr != nil: + if err.Error() != tc.expectErr.Error() { + t.Fatal("expected", tc.expectErr, "got", err) + } + + case err == nil && tc.expectErr == nil: + // all good--fallthrough + } + + // make sure the deserialization is OK + if diff := cmp.Diff(tc.expectStruct, data); diff != "" { + t.Fatal(diff) + } + }) + } + }) +} + +// This test ensures that ArchivalNetworkEvent is WAI +func TestArchivalNetworkEvent(t *testing.T) { + + // This test ensures that we correctly serialize to JSON. + t.Run("MarshalJSON", func(t *testing.T) { + // testcase is a test case defined by this function + type testcase struct { + // name is the name of the test case + name string + + // input is the input struct + input model.ArchivalNetworkEvent + + // expectErr is the error we expect to see or nil + expectErr error + + // expectData is the data we expect to see + expectData []byte + } + + cases := []testcase{{ + name: "serialization of a successful network event", + input: model.ArchivalNetworkEvent{ + Address: "8.8.8.8:443", + Failure: nil, + NumBytes: 32768, + Operation: "read", + Proto: "tcp", + T0: 1.1, + T: 1.55, + TransactionID: 77, + Tags: []string{"net"}, + }, + expectErr: nil, + expectData: []byte(`{"address":"8.8.8.8:443","failure":null,"num_bytes":32768,"operation":"read","proto":"tcp","t0":1.1,"t":1.55,"transaction_id":77,"tags":["net"]}`), + }, { + name: "serialization of a failed network event", + input: model.ArchivalNetworkEvent{ + Address: "8.8.8.8:443", + Failure: (func() *string { + s := netxlite.FailureGenericTimeoutError + return &s + })(), + NumBytes: 0, + Operation: "read", + Proto: "tcp", + T0: 1.1, + T: 7, + TransactionID: 144, + Tags: []string{"net"}, + }, + expectErr: nil, + expectData: []byte(`{"address":"8.8.8.8:443","failure":"generic_timeout_error","operation":"read","proto":"tcp","t0":1.1,"t":7,"transaction_id":144,"tags":["net"]}`), + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // serialize to JSON + data, err := json.Marshal(tc.input) + + t.Log("got this error", err) + t.Log("got this raw data", data) + t.Logf("converted to string: %s", string(data)) + + // handle errors + switch { + case err == nil && tc.expectErr != nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr == nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr != nil: + if err.Error() != tc.expectErr.Error() { + t.Fatal("expected", tc.expectErr, "got", err) + } + + case err == nil && tc.expectErr == nil: + // all good--fallthrough + } + + // make sure the serialization is OK + if diff := cmp.Diff(tc.expectData, data); diff != "" { + t.Fatal(diff) + } + }) + } + }) + + // This test ensures that we can unmarshal from the JSON representation + t.Run("UnmarshalJSON", func(t *testing.T) { + // testcase is a test case defined by this function + type testcase struct { + // name is the name of the test case + name string + + // input is the binary input + input []byte + + // expectErr is the error we expect to see or nil + expectErr error + + // expectStruct is the struct we expect to see + expectStruct model.ArchivalNetworkEvent + } + + cases := []testcase{{ + name: "deserialization of a successful network event", + expectErr: nil, + input: []byte(`{"address":"8.8.8.8:443","failure":null,"num_bytes":32768,"operation":"read","proto":"tcp","t0":1.1,"t":1.55,"transaction_id":77,"tags":["net"]}`), + expectStruct: model.ArchivalNetworkEvent{ + Address: "8.8.8.8:443", + Failure: nil, + NumBytes: 32768, + Operation: "read", + Proto: "tcp", + T0: 1.1, + T: 1.55, + TransactionID: 77, + Tags: []string{"net"}, + }, + }, { + name: "deserialization of a failed network event", + input: []byte(`{"address":"8.8.8.8:443","failure":"generic_timeout_error","operation":"read","proto":"tcp","t0":1.1,"t":7,"transaction_id":144,"tags":["net"]}`), + expectErr: nil, + expectStruct: model.ArchivalNetworkEvent{ + Address: "8.8.8.8:443", + Failure: (func() *string { + s := netxlite.FailureGenericTimeoutError + return &s + })(), + NumBytes: 0, + Operation: "read", + Proto: "tcp", + T0: 1.1, + T: 7, + TransactionID: 144, + Tags: []string{"net"}, + }, + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // parse the JSON + var data model.ArchivalNetworkEvent + err := json.Unmarshal(tc.input, &data) + + t.Log("got this error", err) + t.Logf("got this struct %+v", data) + + // handle errors + switch { + case err == nil && tc.expectErr != nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr == nil: + t.Fatal("expected", tc.expectErr, "got", err) + + case err != nil && tc.expectErr != nil: + if err.Error() != tc.expectErr.Error() { + t.Fatal("expected", tc.expectErr, "got", err) + } + + case err == nil && tc.expectErr == nil: + // all good--fallthrough + } + + // make sure the deserialization is OK + if diff := cmp.Diff(tc.expectStruct, data); diff != "" { + t.Fatal(diff) + } + }) + } + }) +} + +func TestArchivalNewHTTPHeadersList(t *testing.T) { + + // testcase is a test case run by this func + type testcase struct { + name string + input http.Header + expect []model.ArchivalHTTPHeader + } + + cases := []testcase{{ + name: "with nil input", + input: nil, + expect: []model.ArchivalHTTPHeader{}, + }, { + name: "with empty input", + input: map[string][]string{}, + expect: []model.ArchivalHTTPHeader{}, + }, { + name: "common case", + input: map[string][]string{ + "Content-Type": {"text/html; charset=utf-8"}, + "Via": {"a", "b", "c"}, + "User-Agent": {"miniooni/0.1.0"}, + }, + expect: []model.ArchivalHTTPHeader{{ + model.ArchivalScrubbedMaybeBinaryString("Content-Type"), + model.ArchivalScrubbedMaybeBinaryString("text/html; charset=utf-8"), + }, { + model.ArchivalScrubbedMaybeBinaryString("User-Agent"), + model.ArchivalScrubbedMaybeBinaryString("miniooni/0.1.0"), + }, { + model.ArchivalScrubbedMaybeBinaryString("Via"), + model.ArchivalScrubbedMaybeBinaryString("a"), + }, { + model.ArchivalScrubbedMaybeBinaryString("Via"), + model.ArchivalScrubbedMaybeBinaryString("b"), + }, { + model.ArchivalScrubbedMaybeBinaryString("Via"), + model.ArchivalScrubbedMaybeBinaryString("c"), + }}, + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := model.ArchivalNewHTTPHeadersList(tc.input) + if diff := cmp.Diff(tc.expect, got); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestArchivalNewHTTPHeadersMap(t *testing.T) { + + // testcase is a test case run by this func + type testcase struct { + name string + input http.Header + expect map[string]model.ArchivalScrubbedMaybeBinaryString + } + + cases := []testcase{{ + name: "with nil input", + input: nil, + expect: map[string]model.ArchivalScrubbedMaybeBinaryString{}, + }, { + name: "with empty input", + input: map[string][]string{}, + expect: map[string]model.ArchivalScrubbedMaybeBinaryString{}, + }, { + name: "common case", + input: map[string][]string{ + "Content-Type": {"text/html; charset=utf-8"}, + "Via": {"a", "b", "c"}, + "User-Agent": {"miniooni/0.1.0"}, + }, + expect: map[string]model.ArchivalScrubbedMaybeBinaryString{ + "Content-Type": "text/html; charset=utf-8", + "Via": "a", + "User-Agent": "miniooni/0.1.0", + }, + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := model.ArchivalNewHTTPHeadersMap(tc.input) + if diff := cmp.Diff(tc.expect, got); diff != "" { + t.Fatal(diff) + } + }) } } diff --git a/pkg/model/http.go b/pkg/model/http.go index 3d34277ef..774b90403 100644 --- a/pkg/model/http.go +++ b/pkg/model/http.go @@ -13,9 +13,9 @@ const ( HTTPHeaderAcceptLanguage = "en-US,en;q=0.9" // HTTPHeaderUserAgent is the User-Agent header used for measuring. The current header - // is 22.0% of the browser population as of Jun 06, 2023 according to the + // is 20.2% of the browser population as of 2023-10-04 according to the // https://techblog.willshouse.com/2012/01/03/most-common-user-agents/ webpage. - HTTPHeaderUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36" + HTTPHeaderUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36" ) // Additional strings used to report HTTP errors. They're currently only used by diff --git a/pkg/model/measurement_test.go b/pkg/model/measurement_test.go index be868c9b8..7b425adbb 100644 --- a/pkg/model/measurement_test.go +++ b/pkg/model/measurement_test.go @@ -6,8 +6,20 @@ import ( "errors" "fmt" "testing" + "time" ) +func TestMeasurementFormatTimeNowUTC(t *testing.T) { + t.Run("produces a string using the correct date format", func(t *testing.T) { + out := MeasurementFormatTimeNowUTC() + result, err := time.Parse(MeasurementDateFormat, out) + if err != nil { + t.Fatal(err) + } + _ = result + }) +} + func TestMeasurementTargetMarshalJSON(t *testing.T) { var mt MeasurementTarget data, err := json.Marshal(mt) diff --git a/pkg/model/netx.go b/pkg/model/netx.go index f24db2ac6..f6f4bc78f 100644 --- a/pkg/model/netx.go +++ b/pkg/model/netx.go @@ -13,7 +13,9 @@ import ( "syscall" "time" + oohttp "github.com/ooni/oohttp" "github.com/quic-go/quic-go" + utls "gitlab.com/yawning/utls.git" ) // DNSResponse is a parsed DNS response ready for further processing. @@ -105,12 +107,6 @@ type DNSEncoder interface { Encode(domain string, qtype uint16, padding bool) DNSQuery } -// DNSTransportWrapper is a type that takes in input a DNSTransport -// and returns in output a wrapped DNSTransport. -type DNSTransportWrapper interface { - WrapDNSTransport(txp DNSTransport) DNSTransport -} - // DNSTransport represents an abstract DNS transport. type DNSTransport interface { // RoundTrip sends a DNS query and receives the reply. @@ -137,8 +133,17 @@ type DialerWrapper interface { // SimpleDialer establishes network connections. type SimpleDialer interface { - // DialContext behaves like net.Dialer.DialContext. - DialContext(ctx context.Context, network, address string) (net.Conn, error) + // DialContext creates a new TCP/UDP connection like [net.DialContext] would do. + // + // The endpoint is an endpoint like the ones accepted by [net.DialContext]. For example, + // x.org:443, 130.192.91.211:443 and [::1]:443. Note that IPv6 addrs are quoted. + // + // This function MUST gracefully handle the case where the endpoint contains an IPv4 + // or IPv6 address by skipping DNS resolution and directly using the endpoint. + // + // See https://github.com/ooni/probe-cli/pull/1295#issuecomment-1731243994 for more + // details on why DialContext MUST do that. + DialContext(ctx context.Context, network, endpoint string) (net.Conn, error) } // Dialer is a SimpleDialer with the possibility of closing open connections. @@ -181,10 +186,55 @@ type HTTPSSvc struct { IPv6 []string } -// QUICListener listens for QUIC connections. -type QUICListener interface { - // Listen creates a new listening UDPLikeConn. - Listen(addr *net.UDPAddr) (UDPLikeConn, error) +// MeasuringNetwork defines the constructors required for implementing OONI experiments. All +// these constructors MUST guarantee proper error wrapping to map Go errors to OONI errors +// as documented by the [netxlite] package. The [*netxlite.Netx] type is currently the default +// implementation of this interface. This interface SHOULD always be implemented in terms of +// an [UnderlyingNetwork] that allows to switch between the host network and [netemx]. +type MeasuringNetwork interface { + // NewDialerWithoutResolver creates a [Dialer] with error wrapping and without an attached + // resolver, meaning that you MUST pass TCP or UDP endpoint addresses to this dialer. + // + // The [DialerWrapper] arguments wraps the returned dialer in such a way that we can implement + // the legacy [netx] package. New code MUST NOT use this functionality, which we'd like to remove ASAP. + NewDialerWithoutResolver(dl DebugLogger, w ...DialerWrapper) Dialer + + // NewParallelDNSOverHTTPSResolver creates a new DNS-over-HTTPS resolver with error wrapping. + NewParallelDNSOverHTTPSResolver(logger DebugLogger, URL string) Resolver + + // NewParallelUDPResolver creates a new Resolver using DNS-over-UDP + // that performs parallel A/AAAA lookups during LookupHost. + // + // The address argument is the UDP endpoint address (e.g., 1.1.1.1:53, [::1]:53). + NewParallelUDPResolver(logger DebugLogger, dialer Dialer, address string) Resolver + + // NewQUICDialerWithoutResolver creates a [QUICDialer] with error wrapping and without an attached + // resolver, meaning that you MUST pass UDP endpoint addresses to this dialer. + // + // The [QUICDialerWrapper] arguments wraps the returned dialer in such a way + // that we can implement the legacy [netx] package. New code MUST NOT + // use this functionality, which we'd like to remove ASAP. + NewQUICDialerWithoutResolver( + listener UDPListener, logger DebugLogger, w ...QUICDialerWrapper) QUICDialer + + // NewStdlibResolver creates a new Resolver with error wrapping using + // getaddrinfo or &net.Resolver{} depending on `-tags netgo`. + NewStdlibResolver(logger DebugLogger) Resolver + + // NewTLSHandshakerStdlib creates a new TLSHandshaker with error wrapping + // that is using the go standard library to manage TLS. + NewTLSHandshakerStdlib(logger DebugLogger) TLSHandshaker + + // NewTLSHandshakerUTLS creates a new TLS handshaker using + // gitlab.com/yawning/utls for TLS that implements error wrapping. + // + // The id is the address of something like utls.HelloFirefox_55. + // + // Passing a nil `id` will make this function panic. + NewTLSHandshakerUTLS(logger DebugLogger, id *utls.ClientHelloID) TLSHandshaker + + // NewUDPListener creates a new UDPListener with error wrapping. + NewUDPListener() UDPListener } // QUICDialerWrapper is a type that takes in input a QUICDialer @@ -217,7 +267,9 @@ type QUICDialer interface { // Resolver performs domain name resolutions. type Resolver interface { - // LookupHost behaves like net.Resolver.LookupHost. + // LookupHost resolves the given hostname to IP addreses. This function SHOULD handle the + // case in which hostname is an IP address by returning a 1-element list containing the hostname, + // for consistency with [net.Resolver] behaviour. LookupHost(ctx context.Context, hostname string) (addrs []string, err error) // Network returns the resolver type. It should be one of: @@ -257,6 +309,16 @@ type Resolver interface { LookupNS(ctx context.Context, domain string) ([]*net.NS, error) } +// TLSConn is the type of connection that oohttp expects from +// any library that implements TLS functionality. By using this +// kind of TLSConn we're able to use both the standard library +// and gitlab.com/yawning/utls.git to perform TLS operations. Note +// that the stdlib's tls.Conn implements this interface. +type TLSConn = oohttp.TLSConn + +// Ensures that a [*tls.Conn] implements the [TLSConn] interface. +var _ TLSConn = &tls.Conn{} + // TLSDialer is a Dialer dialing TLS connections. type TLSDialer interface { // CloseIdleConnections closes idle connections, if any. @@ -264,6 +326,15 @@ type TLSDialer interface { // DialTLSContext dials a TLS connection. This method will always return // to you a oohttp.TLSConn, so you can always safely cast to it. + // + // The endpoint is an endpoint like the ones accepted by [net.DialContext]. For example, + // x.org:443, 130.192.91.211:443 and [::1]:443. Note that IPv6 addrs are quoted. + // + // This function MUST gracefully handle the case where the endpoint contains an IPv4 + // or IPv6 address by skipping DNS resolution and directly using the endpoint. + // + // See https://github.com/ooni/probe-cli/pull/1295#issuecomment-1731243994 for more + // details on why DialTLSContext MUST do that. DialTLSContext(ctx context.Context, network, address string) (net.Conn, error) } @@ -281,12 +352,7 @@ type TLSHandshaker interface { // // - set NextProtos to []string{"h2", "http/1.1"} for HTTPS // and []string{"dot"} for DNS-over-TLS. - // - // QUIRK: The returned connection will always implement the TLSConn interface - // exposed by ooni/oohttp. A future version of this interface may instead - // return directly a TLSConn to avoid unconditional castings. - Handshake(ctx context.Context, conn net.Conn, tlsConfig *tls.Config) ( - net.Conn, tls.ConnectionState, error) + Handshake(ctx context.Context, conn net.Conn, tlsConfig *tls.Config) (TLSConn, error) } // Trace allows to collect measurement traces. A trace is injected into @@ -482,8 +548,14 @@ type UDPLikeConn interface { SyscallConn() (syscall.RawConn, error) } +// UDPListener listens for connections over UDP, e.g. QUIC. +type UDPListener interface { + // Listen creates a new listening UDPLikeConn. + Listen(addr *net.UDPAddr) (UDPLikeConn, error) +} + // UnderlyingNetwork implements the underlying network APIs on -// top of which we implement network extensions. +// top of which we implement network extensions such as [MeasuringNetwork]. type UnderlyingNetwork interface { // DefaultCertPool returns the underlying cert pool used by the // network extensions library. You MUST NOT use this function to @@ -505,6 +577,9 @@ type UnderlyingNetwork interface { // GetaddrinfoResolverNetwork returns the resolver network. GetaddrinfoResolverNetwork() string + // ListenTCP is equivalent to net.ListenTCP. + ListenTCP(network string, addr *net.TCPAddr) (net.Listener, error) + // ListenUDP is equivalent to net.ListenUDP. ListenUDP(network string, addr *net.UDPAddr) (UDPLikeConn, error) } diff --git a/pkg/netemx/badssl.go b/pkg/netemx/badssl.go index 57155251d..6d11267be 100644 --- a/pkg/netemx/badssl.go +++ b/pkg/netemx/badssl.go @@ -88,38 +88,28 @@ func (thx *tlsHandlerForBadSSLServer) GetCertificate( case "wrong.host.badssl.com": // Use the correct root CA but return a certificate for a different // host, which should cause the SNI verification to fail. - tlsConfig := thx.unet.ServerTLSConfig() - return tlsConfig.GetCertificate(&tls.ClientHelloInfo{ - CipherSuites: chi.CipherSuites, - ServerName: "wrong-host.badssl.com", // different! - SupportedCurves: chi.SupportedCurves, - SupportedPoints: chi.SupportedPoints, - SignatureSchemes: chi.SignatureSchemes, - SupportedProtos: chi.SupportedProtos, - SupportedVersions: chi.SupportedVersions, - Conn: tcpConn, - }) + tlsCert := thx.unet.MustNewTLSCertificate("wrong-host.badssl.com") // different + return tlsCert, nil case "untrusted-root.badssl.com": fallthrough default: - // Create a custom MITM config and use it to negotiate TLS. Because this would be + // Create a custom CA config and use it to negotiate TLS. Because this would be // a different root CA, validating certs will fail the handshake. // // A more advanced version of this handler could choose different behaviors // depending on the SNI provided as part of the *tls.ClientHelloInfo. - mitm := testingx.MustNewTLSMITMProviderNetem() - tlsConfig := mitm.ServerTLSConfig() - return tlsConfig.GetCertificate(chi) + ca := netem.MustNewCA() + return ca.MustNewTLSCertificate(chi.ServerName), nil case "expired.badssl.com": // Create on-the-fly a certificate with the right SNI but that is clearly expired. - mitm := thx.unet.TLSMITMConfig() - return mitm.Config.NewCertWithoutCacheWithTimeNow( - chi.ServerName, + cert := thx.unet.MustNewTLSCertificateWithTimeNow( func() time.Time { return time.Date(2017, time.July, 17, 0, 0, 0, 0, time.UTC) }, + chi.ServerName, ) + return cert, nil } } diff --git a/pkg/netemx/example_test.go b/pkg/netemx/example_test.go index d7f089ef8..859b766e0 100644 --- a/pkg/netemx/example_test.go +++ b/pkg/netemx/example_test.go @@ -23,8 +23,25 @@ func exampleNewEnvironment() *netemx.QAEnv { netemx.QAEnvOptionNetStack("8.8.4.4", &netemx.DNSOverUDPServerFactory{}), netemx.QAEnvOptionNetStack("9.9.9.9", &netemx.DNSOverUDPServerFactory{}), netemx.QAEnvOptionClientAddress(netemx.DefaultClientAddress), - netemx.QAEnvOptionHTTPServer( - netemx.AddressWwwExampleCom, netemx.ExampleWebPageHandlerFactory()), + netemx.QAEnvOptionNetStack( + netemx.AddressWwwExampleCom, + &netemx.HTTPCleartextServerFactory{ + Factory: netemx.ExampleWebPageHandlerFactory(), + Ports: []int{80}, + }, + &netemx.HTTPSecureServerFactory{ + Factory: netemx.ExampleWebPageHandlerFactory(), + Ports: []int{443}, + ServerNameMain: "www.example.com", + ServerNameExtras: []string{}, + }, + &netemx.HTTP3ServerFactory{ + Factory: netemx.ExampleWebPageHandlerFactory(), + Ports: []int{443}, + ServerNameMain: "www.example.com", + ServerNameExtras: []string{}, + }, + ), netemx.QAEnvOptionLogger(log.Log), ) } @@ -66,6 +83,7 @@ func Example_dpiRule() { reso := netxlite.NewStdlibResolver(model.DiscardLogger) // create the HTTP client + // TODO(https://github.com/ooni/probe/issues/2534): the NewHTTPClientWithResolver func has QUIRKS but we don't care. client := netxlite.NewHTTPClientWithResolver(model.DiscardLogger, reso) // create the HTTP request @@ -310,6 +328,7 @@ func Example_exampleWebServerWithInternetScenario() { defer env.Close() env.Do(func() { + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPClientStdlib has QUIRKS but they're not needed here client := netxlite.NewHTTPClientStdlib(log.Log) req, err := http.NewRequest("GET", "https://www.example.com/", nil) @@ -389,6 +408,7 @@ func Example_ooniAPIWithInternetScenario() { defer env.Close() env.Do(func() { + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPClientStdlib has QUIRKS but they're not needed here client := netxlite.NewHTTPClientStdlib(log.Log) req, err := http.NewRequest("GET", "https://api.ooni.io/api/v1/test-helpers", nil) @@ -419,6 +439,7 @@ func Example_oohelperdWithInternetScenario() { defer env.Close() env.Do(func() { + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPClientStdlib has QUIRKS but they're not needed here client := netxlite.NewHTTPClientStdlib(log.Log) thRequest := []byte(`{"http_request": "https://www.example.com/","http_request_headers":{},"tcp_connect":["93.184.216.34:443"]}`) @@ -452,6 +473,7 @@ func Example_ubuntuGeoIPWithInternetScenario() { defer env.Close() env.Do(func() { + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPClientStdlib has QUIRKS but they're not needed here client := netxlite.NewHTTPClientStdlib(log.Log) req, err := http.NewRequest("GET", "https://geoip.ubuntu.com/lookup", nil) @@ -482,6 +504,7 @@ func Example_examplePublicBlockpage() { defer env.Close() env.Do(func() { + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPClientStdlib has QUIRKS but they're not needed here client := netxlite.NewHTTPClientStdlib(log.Log) req, err := http.NewRequest("GET", "http://"+netemx.AddressPublicBlockpage+"/", nil) @@ -523,6 +546,8 @@ func Example_exampleURLShortener() { defer env.Close() env.Do(func() { + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPTransportStdlib has QUIRKS but we + // don't actually care about those QUIRKS in this context client := netxlite.NewHTTPTransportStdlib(log.Log) req, err := http.NewRequest("GET", "https://bit.ly/21645", nil) diff --git a/pkg/netemx/http3.go b/pkg/netemx/http3.go index 05f869649..bc4066c96 100644 --- a/pkg/netemx/http3.go +++ b/pkg/netemx/http3.go @@ -1,7 +1,6 @@ package netemx import ( - "crypto/tls" "io" "net" "net/http" @@ -22,8 +21,11 @@ type HTTP3ServerFactory struct { // Ports is the MANDATORY list of ports where to listen. Ports []int - // TLSConfig is the OPTIONAL TLS config to use. - TLSConfig *tls.Config + // ServerNameMain is the MANDATORY server name we should configure. + ServerNameMain string + + // ServerNameExtras contains OPTIONAL extra server names we should configure. + ServerNameExtras []string } var _ NetStackServerFactory = &HTTP3ServerFactory{} @@ -31,24 +33,26 @@ var _ NetStackServerFactory = &HTTP3ServerFactory{} // MustNewServer implements NetStackServerFactory. func (f *HTTP3ServerFactory) MustNewServer(env NetStackServerFactoryEnv, stack *netem.UNetStack) NetStackServer { return &http3Server{ - closers: []io.Closer{}, - env: env, - factory: f.Factory, - mu: sync.Mutex{}, - ports: f.Ports, - tlsConfig: f.TLSConfig, - unet: stack, + closers: []io.Closer{}, + env: env, + factory: f.Factory, + mu: sync.Mutex{}, + ports: f.Ports, + serverNameMain: f.ServerNameMain, + serverNameExtras: f.ServerNameExtras, + unet: stack, } } type http3Server struct { - closers []io.Closer - env NetStackServerFactoryEnv - factory HTTPHandlerFactory - mu sync.Mutex - ports []int - tlsConfig *tls.Config - unet *netem.UNetStack + closers []io.Closer + env NetStackServerFactoryEnv + factory HTTPHandlerFactory + mu sync.Mutex + ports []int + serverNameMain string + serverNameExtras []string + unet *netem.UNetStack } // Close implements NetStackServer. @@ -90,13 +94,8 @@ func (srv *http3Server) mustListenPortLocked(handler http.Handler, ipAddr net.IP addr := &net.UDPAddr{IP: ipAddr, Port: port} listener := runtimex.Try1(srv.unet.ListenUDP("udp", addr)) - // use the netstack TLS config or the custom one configured by the user - tlsConfig := srv.tlsConfig - if tlsConfig == nil { - tlsConfig = srv.unet.ServerTLSConfig() - } else { - tlsConfig = tlsConfig.Clone() - } + // create TLS config for the server name + tlsConfig := srv.unet.MustNewServerTLSConfig(srv.serverNameMain, srv.serverNameExtras...) // serve requests in a background goroutine srvr := &http3.Server{ diff --git a/pkg/netemx/http3_test.go b/pkg/netemx/http3_test.go index eeaa75ed0..ec21b7237 100644 --- a/pkg/netemx/http3_test.go +++ b/pkg/netemx/http3_test.go @@ -18,8 +18,8 @@ func TestHTTP3ServerFactory(t *testing.T) { Factory: HTTPHandlerFactoryFunc(func(env NetStackServerFactoryEnv, stack *netem.UNetStack) http.Handler { return ExampleWebPageHandler() }), - Ports: []int{443}, - TLSConfig: nil, // explicitly nil, let's use netem's config + Ports: []int{443}, + ServerNameMain: "www.example.com", }), ) defer env.Close() @@ -46,35 +46,4 @@ func TestHTTP3ServerFactory(t *testing.T) { } }) }) - - t.Run("when using an incompatible TLS config", func(t *testing.T) { - // we're creating a distinct MITM TLS config and we're using it, so we expect - // that we're not able to verify certificates in client code - mitmConfig := runtimex.Try1(netem.NewTLSMITMConfig()) - - env := MustNewQAEnv( - QAEnvOptionNetStack(AddressWwwExampleCom, &HTTP3ServerFactory{ - Factory: HTTPHandlerFactoryFunc(func(env NetStackServerFactoryEnv, stack *netem.UNetStack) http.Handler { - return ExampleWebPageHandler() - }), - Ports: []int{443}, - TLSConfig: mitmConfig.TLSConfig(), // custom! - }), - ) - defer env.Close() - - env.AddRecordToAllResolvers("www.example.com", "", AddressWwwExampleCom) - - env.Do(func() { - client := netxlite.NewHTTP3ClientWithResolver(log.Log, netxlite.NewStdlibResolver(log.Log)) - req := runtimex.Try1(http.NewRequest("GET", "https://www.example.com/", nil)) - resp, err := client.Do(req) - if err == nil || err.Error() != netxlite.FailureSSLInvalidCertificate { - t.Fatal("unexpected error", err) - } - if resp != nil { - t.Fatal("expected nil resp") - } - }) - }) } diff --git a/pkg/netemx/http_test.go b/pkg/netemx/http_test.go index 3d17f36a5..69f371d16 100644 --- a/pkg/netemx/http_test.go +++ b/pkg/netemx/http_test.go @@ -25,6 +25,7 @@ func TestHTTPCleartextServerFactory(t *testing.T) { env.AddRecordToAllResolvers("www.example.com", "", AddressWwwExampleCom) env.Do(func() { + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPClientStdlib has QUIRKS but they're not needed here client := netxlite.NewHTTPClientStdlib(log.Log) req := runtimex.Try1(http.NewRequest("GET", "http://www.example.com/", nil)) resp, err := client.Do(req) diff --git a/pkg/netemx/https.go b/pkg/netemx/https.go index 97c13d325..e0fd7b558 100644 --- a/pkg/netemx/https.go +++ b/pkg/netemx/https.go @@ -1,7 +1,6 @@ package netemx import ( - "crypto/tls" "io" "net" "net/http" @@ -21,8 +20,11 @@ type HTTPSecureServerFactory struct { // Ports is the MANDATORY list of ports where to listen. Ports []int - // TLSConfig is the OPTIONAL TLS config to use. - TLSConfig *tls.Config + // ServerNameMain is the MANDATORY server name we should configure. + ServerNameMain string + + // ServerNameExtras contains OPTIONAL extra server names we should configure. + ServerNameExtras []string } var _ NetStackServerFactory = &HTTPSecureServerFactory{} @@ -30,24 +32,26 @@ var _ NetStackServerFactory = &HTTPSecureServerFactory{} // MustNewServer implements NetStackServerFactory. func (f *HTTPSecureServerFactory) MustNewServer(env NetStackServerFactoryEnv, stack *netem.UNetStack) NetStackServer { return &httpSecureServer{ - closers: []io.Closer{}, - env: env, - factory: f.Factory, - mu: sync.Mutex{}, - ports: f.Ports, - tlsConfig: f.TLSConfig, - unet: stack, + closers: []io.Closer{}, + env: env, + factory: f.Factory, + mu: sync.Mutex{}, + ports: f.Ports, + serverNameMain: f.ServerNameMain, + serverNameExtras: f.ServerNameExtras, + unet: stack, } } type httpSecureServer struct { - closers []io.Closer - env NetStackServerFactoryEnv - factory HTTPHandlerFactory - mu sync.Mutex - ports []int - tlsConfig *tls.Config - unet *netem.UNetStack + closers []io.Closer + env NetStackServerFactoryEnv + factory HTTPHandlerFactory + mu sync.Mutex + ports []int + serverNameMain string + serverNameExtras []string + unet *netem.UNetStack } // Close implements NetStackServer. @@ -89,13 +93,8 @@ func (srv *httpSecureServer) mustListenPortLocked(handler http.Handler, ipAddr n addr := &net.TCPAddr{IP: ipAddr, Port: port} listener := runtimex.Try1(srv.unet.ListenTCP("tcp", addr)) - // use the netstack TLS config or the custom one configured by the user - tlsConfig := srv.tlsConfig - if tlsConfig == nil { - tlsConfig = srv.unet.ServerTLSConfig() - } else { - tlsConfig = tlsConfig.Clone() - } + // create TLS config for the server name + tlsConfig := srv.unet.MustNewServerTLSConfig(srv.serverNameMain, srv.serverNameExtras...) // serve requests in a background goroutine srvr := &http.Server{ diff --git a/pkg/netemx/https_test.go b/pkg/netemx/https_test.go index ea9ba5e2b..80f45998b 100644 --- a/pkg/netemx/https_test.go +++ b/pkg/netemx/https_test.go @@ -18,8 +18,8 @@ func TestHTTPSecureServerFactory(t *testing.T) { Factory: HTTPHandlerFactoryFunc(func(env NetStackServerFactoryEnv, stack *netem.UNetStack) http.Handler { return ExampleWebPageHandler() }), - Ports: []int{443}, - TLSConfig: nil, // explicitly nil, let's use netem's config + Ports: []int{443}, + ServerNameMain: "www.example.com", }), ) defer env.Close() @@ -27,6 +27,7 @@ func TestHTTPSecureServerFactory(t *testing.T) { env.AddRecordToAllResolvers("www.example.com", "", AddressWwwExampleCom) env.Do(func() { + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPClientStdlib has QUIRKS but they're not needed here client := netxlite.NewHTTPClientStdlib(log.Log) req := runtimex.Try1(http.NewRequest("GET", "https://www.example.com/", nil)) resp, err := client.Do(req) @@ -46,35 +47,4 @@ func TestHTTPSecureServerFactory(t *testing.T) { } }) }) - - t.Run("when using an incompatible TLS config", func(t *testing.T) { - // we're creating a distinct MITM TLS config and we're using it, so we expect - // that we're not able to verify certificates in client code - mitmConfig := runtimex.Try1(netem.NewTLSMITMConfig()) - - env := MustNewQAEnv( - QAEnvOptionNetStack(AddressWwwExampleCom, &HTTPSecureServerFactory{ - Factory: HTTPHandlerFactoryFunc(func(env NetStackServerFactoryEnv, stack *netem.UNetStack) http.Handler { - return ExampleWebPageHandler() - }), - Ports: []int{443}, - TLSConfig: mitmConfig.TLSConfig(), // custom! - }), - ) - defer env.Close() - - env.AddRecordToAllResolvers("www.example.com", "", AddressWwwExampleCom) - - env.Do(func() { - client := netxlite.NewHTTPClientStdlib(log.Log) - req := runtimex.Try1(http.NewRequest("GET", "https://www.example.com/", nil)) - resp, err := client.Do(req) - if err == nil || err.Error() != netxlite.FailureSSLUnknownAuthority { - t.Fatal("unexpected error", err) - } - if resp != nil { - t.Fatal("expected nil resp") - } - }) - }) } diff --git a/pkg/netemx/oohelperd.go b/pkg/netemx/oohelperd.go index ec0329189..5ed5c5109 100644 --- a/pkg/netemx/oohelperd.go +++ b/pkg/netemx/oohelperd.go @@ -35,7 +35,7 @@ func (f *OOHelperDFactory) NewHandler(env NetStackServerFactoryEnv, unet *netem. handler.NewQUICDialer = func(logger model.Logger) model.QUICDialer { return netx.NewQUICDialerWithResolver( - netx.NewQUICListener(), + netx.NewUDPListener(), logger, netx.NewStdlibResolver(logger), ) @@ -49,6 +49,8 @@ func (f *OOHelperDFactory) NewHandler(env NetStackServerFactoryEnv, unet *netem. cookieJar, _ := cookiejar.New(&cookiejar.Options{ PublicSuffixList: publicsuffix.List, }) + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPTransportStdlib is QUIRKY but we probably + // don't care about using a QUIRKY function here return &http.Client{ Transport: netx.NewHTTPTransportStdlib(logger), CheckRedirect: nil, diff --git a/pkg/netemx/oohelperd_test.go b/pkg/netemx/oohelperd_test.go index a98409a61..7661e8d2a 100644 --- a/pkg/netemx/oohelperd_test.go +++ b/pkg/netemx/oohelperd_test.go @@ -31,8 +31,7 @@ func TestOOHelperDHandler(t *testing.T) { } thReqRaw := runtimex.Try1(json.Marshal(thReq)) - //log.SetLevel(log.DebugLevel) - + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPClientStdlib has QUIRKS but they're not needed here httpClient := netxlite.NewHTTPClientStdlib(log.Log) req, err := http.NewRequest(http.MethodPost, "https://0.th.ooni.org/", bytes.NewReader(thReqRaw)) @@ -100,9 +99,10 @@ func TestOOHelperDHandler(t *testing.T) { Failure: nil, Title: "Default Web Page", Headers: map[string]string{ - "Alt-Svc": `h3=":443"`, - "Content-Type": "text/html; charset=utf-8", - "Date": "Thu, 24 Aug 2023 14:35:29 GMT", + "Alt-Svc": `h3=":443"`, + "Content-Length": "1533", + "Content-Type": "text/html; charset=utf-8", + "Date": "Thu, 24 Aug 2023 14:35:29 GMT", }, StatusCode: 200, }, diff --git a/pkg/netemx/qaenv.go b/pkg/netemx/qaenv.go index 5e59b64db..4bdc37fa2 100644 --- a/pkg/netemx/qaenv.go +++ b/pkg/netemx/qaenv.go @@ -59,36 +59,6 @@ func QAEnvOptionClientNICWrapper(wrapper netem.LinkNICWrapper) QAEnvOption { } } -// QAEnvOptionHTTPServer adds the given HTTP handler factory. If you do -// not set this option we will not create any HTTP server. Note that this -// option is just syntactic sugar for calling [QAEnvOptionNetStack] -// with the following three factories as argument: -// -// - [HTTPCleartextServerFactory] with port 80/tcp; -// -// - [HTTPSecureServerFactory] with port 443/tcp and nil TLSConfig; -// -// - [HTTP3ServerFactory] with port 443/udp and nil TLSConfig. -// -// We wrote this syntactic sugar factory because it covers the common case -// where you want support for HTTP, HTTPS, and HTTP3. -func QAEnvOptionHTTPServer(ipAddr string, factory HTTPHandlerFactory) QAEnvOption { - runtimex.Assert(net.ParseIP(ipAddr) != nil, "not an IP addr") - runtimex.Assert(factory != nil, "passed a nil handler factory") - return qaEnvOptionNetStack(ipAddr, &HTTPCleartextServerFactory{ - Factory: factory, - Ports: []int{80}, - }, &HTTPSecureServerFactory{ - Factory: factory, - Ports: []int{443}, - TLSConfig: nil, // use netem's default - }, &HTTP3ServerFactory{ - Factory: factory, - Ports: []int{443}, - TLSConfig: nil, // use netem's default - }) -} - // QAEnvOptionLogger sets the logger to use. If you do not set this option we // will use [model.DiscardLogger] as the logger. func QAEnvOptionLogger(logger model.Logger) QAEnvOption { @@ -140,8 +110,8 @@ type QAEnv struct { // clientNICWrapper is the OPTIONAL wrapper for the client NIC. clientNICWrapper netem.LinkNICWrapper - // clientStack is the client stack to use. - clientStack *netem.UNetStack + // ClientStack is the client stack to use. + ClientStack *netem.UNetStack // closables contains all entities where we have to take care of closing. closables []io.Closer @@ -197,18 +167,18 @@ func MustNewQAEnv(options ...QAEnvOption) *QAEnv { env := &QAEnv{ baseLogger: config.logger, clientNICWrapper: config.clientNICWrapper, - clientStack: nil, + ClientStack: nil, closables: []io.Closer{}, emulateAndroidGetaddrinfo: &atomic.Bool{}, ispResolverConfig: netem.NewDNSConfig(), dpi: netem.NewDPIEngine(prefixLogger), once: sync.Once{}, otherResolversConfig: netem.NewDNSConfig(), - topology: runtimex.Try1(netem.NewStarTopology(prefixLogger)), + topology: netem.MustNewStarTopology(prefixLogger), } // create all the required internals - env.clientStack = env.mustNewClientStack(config) + env.ClientStack = env.mustNewClientStack(config) env.closables = append(env.closables, env.mustNewNetStacks(config)...) return env @@ -306,7 +276,7 @@ func (env *QAEnv) EmulateAndroidGetaddrinfo(value bool) { // Do executes the given function such that [netxlite] code uses the // underlying clientStack rather than ordinary networking code. func (env *QAEnv) Do(function func()) { - var stack netem.UnderlyingNetwork = env.clientStack + var stack netem.UnderlyingNetwork = env.ClientStack if env.emulateAndroidGetaddrinfo.Load() { stack = &androidStack{stack} } diff --git a/pkg/netemx/qaenv_test.go b/pkg/netemx/qaenv_test.go index 70f0ddba5..c3e948f38 100644 --- a/pkg/netemx/qaenv_test.go +++ b/pkg/netemx/qaenv_test.go @@ -75,9 +75,14 @@ func TestQAEnv(t *testing.T) { t.Run("we can hijack HTTPS requests", func(t *testing.T) { // create QA env env := netemx.MustNewQAEnv( - netemx.QAEnvOptionHTTPServer( + netemx.QAEnvOptionNetStack( netemx.AddressWwwExampleCom, - netemx.ExampleWebPageHandlerFactory(), + &netemx.HTTPSecureServerFactory{ + Factory: netemx.ExampleWebPageHandlerFactory(), + Ports: []int{443}, + ServerNameMain: "www.example.com", + ServerNameExtras: []string{}, + }, ), ) defer env.Close() @@ -92,6 +97,7 @@ func TestQAEnv(t *testing.T) { env.Do(func() { // create client, which will use the underlying client stack's // DialContext method to dial connections + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPClientStdlib has QUIRKS but they're not needed here client := netxlite.NewHTTPClientStdlib(model.DiscardLogger) // create request using a domain that has been configured in the @@ -138,9 +144,14 @@ func TestQAEnv(t *testing.T) { t.Run("we can hijack HTTP3 requests", func(t *testing.T) { // create QA env env := netemx.MustNewQAEnv( - netemx.QAEnvOptionHTTPServer( + netemx.QAEnvOptionNetStack( netemx.AddressWwwExampleCom, - netemx.ExampleWebPageHandlerFactory(), + &netemx.HTTP3ServerFactory{ + Factory: netemx.ExampleWebPageHandlerFactory(), + Ports: []int{443}, + ServerNameMain: "www.example.com", + ServerNameExtras: []string{}, + }, ), ) defer env.Close() @@ -190,7 +201,15 @@ func TestQAEnv(t *testing.T) { t.Run("we can configure DPI rules", func(t *testing.T) { // create QA env env := netemx.MustNewQAEnv( - netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.ExampleWebPageHandlerFactory()), + netemx.QAEnvOptionNetStack( + "8.8.8.8", + &netemx.HTTPSecureServerFactory{ + Factory: netemx.ExampleWebPageHandlerFactory(), + Ports: []int{443}, + ServerNameMain: "quad8.com", + ServerNameExtras: []string{}, + }, + ), ) defer env.Close() @@ -211,6 +230,7 @@ func TestQAEnv(t *testing.T) { env.Do(func() { // create client, which will use the underlying client stack's // DialContext method to dial connections + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPClientStdlib has QUIRKS but they're not needed here client := netxlite.NewHTTPClientStdlib(model.DiscardLogger) // create the request @@ -242,7 +262,15 @@ func TestQAEnv(t *testing.T) { // create QA env env := netemx.MustNewQAEnv( - netemx.QAEnvOptionHTTPServer("8.8.8.8", netemx.ExampleWebPageHandlerFactory()), + netemx.QAEnvOptionNetStack( + "8.8.8.8", + &netemx.HTTPSecureServerFactory{ + Factory: netemx.ExampleWebPageHandlerFactory(), + Ports: []int{443}, + ServerNameMain: "quad8.com", + ServerNameExtras: []string{}, + }, + ), netemx.QAEnvOptionClientNICWrapper(dumper), ) defer env.Close() @@ -257,6 +285,7 @@ func TestQAEnv(t *testing.T) { env.Do(func() { // create client, which will use the underlying client stack's // DialContext method to dial connections + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPClientStdlib has QUIRKS but they're not needed here client := netxlite.NewHTTPClientStdlib(model.DiscardLogger) // create the request diff --git a/pkg/netemx/scenario.go b/pkg/netemx/scenario.go index 5445b9f22..d37762652 100644 --- a/pkg/netemx/scenario.go +++ b/pkg/netemx/scenario.go @@ -41,15 +41,21 @@ const ( // ScenarioDomainAddresses describes a domain and address used in a scenario. type ScenarioDomainAddresses struct { - // Domains contains a related set of domains domains (MANDATORY field). - Domains []string - // Addresses contains the MANDATORY list of addresses belonging to the domain. Addresses []string + // Domains contains a related set of domains domains (MANDATORY field). + Domains []string + // Role is the MANDATORY role of this domain (e.g., ScenarioRoleOONIAPI). Role uint64 + // ServerNameMain is the MANDATORY server name to use as common name for X.509 certs. + ServerNameMain string + + // ServerNameExtras contains OPTIONAL extra names to also configure into the cert. + ServerNameExtras []string + // WebServerFactory is the factory to use when Role is ScenarioRoleWebServer. WebServerFactory HTTPHandlerFactory } @@ -60,13 +66,17 @@ var InternetScenario = []*ScenarioDomainAddresses{{ Addresses: []string{ AddressApiOONIIo, }, - Role: ScenarioRoleOONIAPI, + Role: ScenarioRoleOONIAPI, + ServerNameMain: "api.ooni.io", + ServerNameExtras: []string{}, }, { Domains: []string{"geoip.ubuntu.com"}, Addresses: []string{ AddressGeoIPUbuntuCom, }, - Role: ScenarioRoleUbuntuGeoIP, + Role: ScenarioRoleUbuntuGeoIP, + ServerNameMain: "geoip.ubuntu.com", + ServerNameExtras: []string{}, }, { Domains: []string{"www.example.com", "example.com", "www.example.org", "example.org"}, Addresses: []string{ @@ -74,55 +84,73 @@ var InternetScenario = []*ScenarioDomainAddresses{{ }, Role: ScenarioRoleWebServer, WebServerFactory: ExampleWebPageHandlerFactory(), + ServerNameMain: "www.example.com", + ServerNameExtras: []string{"example.com", "www.example.org", "example.org"}, }, { Domains: []string{"0.th.ooni.org"}, Addresses: []string{ AddressZeroThOONIOrg, }, - Role: ScenarioRoleOONITestHelper, + Role: ScenarioRoleOONITestHelper, + ServerNameMain: "0.th.ooni.org", + ServerNameExtras: []string{}, }, { Domains: []string{"1.th.ooni.org"}, Addresses: []string{ AddressOneThOONIOrg, }, - Role: ScenarioRoleOONITestHelper, + Role: ScenarioRoleOONITestHelper, + ServerNameMain: "1.th.ooni.org", + ServerNameExtras: []string{}, }, { Domains: []string{"2.th.ooni.org"}, Addresses: []string{ AddressTwoThOONIOrg, }, - Role: ScenarioRoleOONITestHelper, + Role: ScenarioRoleOONITestHelper, + ServerNameMain: "2.th.ooni.org", + ServerNameExtras: []string{}, }, { Domains: []string{"3.th.ooni.org"}, Addresses: []string{ AddressThreeThOONIOrg, }, - Role: ScenarioRoleOONITestHelper, + Role: ScenarioRoleOONITestHelper, + ServerNameMain: "3.th.ooni.org", + ServerNameExtras: []string{}, }, { Domains: []string{"d33d1gs9kpq1c5.cloudfront.net"}, Addresses: []string{ AddressTHCloudfront, }, - Role: ScenarioRoleOONITestHelper, + Role: ScenarioRoleOONITestHelper, + ServerNameMain: "d33d1gs9kpq1c5.cloudfront.net", + ServerNameExtras: []string{}, }, { Domains: []string{"dns.quad9.net"}, Addresses: []string{ AddressDNSQuad9Net, }, - Role: ScenarioRolePublicDNS, + Role: ScenarioRolePublicDNS, + ServerNameMain: "dns.quad9.net", + ServerNameExtras: []string{}, }, { Domains: []string{"mozilla.cloudflare-dns.com"}, Addresses: []string{ AddressMozillaCloudflareDNSCom, }, - Role: ScenarioRolePublicDNS, + Role: ScenarioRolePublicDNS, + ServerNameMain: "mozilla.cloudflare-dns.com", + ServerNameExtras: []string{}, }, { Domains: []string{"dns.google", "dns.google.com"}, Addresses: []string{ AddressDNSGoogle8844, AddressDNSGoogle8888, }, - Role: ScenarioRolePublicDNS, + Role: ScenarioRolePublicDNS, + ServerNameMain: "dns.google", + ServerNameExtras: []string{"dns.google.com"}, }, { Domains: []string{}, Addresses: []string{ @@ -130,19 +158,24 @@ var InternetScenario = []*ScenarioDomainAddresses{{ }, Role: ScenarioRoleBlockpageServer, WebServerFactory: BlockpageHandlerFactory(), + ServerNameMain: "blockpage.local", + ServerNameExtras: []string{}, }, { Domains: []string{}, Addresses: []string{ ISPProxyAddress, }, Role: ScenarioRoleProxy, - WebServerFactory: nil, + ServerNameMain: "proxy.local", + ServerNameExtras: []string{}, }, { Domains: []string{"bit.ly", "bitly.com"}, Addresses: []string{ AddressBitly, }, - Role: ScenarioRoleURLShortener, + Role: ScenarioRoleURLShortener, + ServerNameMain: "bit.ly", + ServerNameExtras: []string{"bitly.com"}, }, { Domains: []string{ "wrong.host.badssl.com", @@ -152,7 +185,9 @@ var InternetScenario = []*ScenarioDomainAddresses{{ Addresses: []string{ AddressBadSSLCom, }, - Role: ScenarioRoleBadSSL, + Role: ScenarioRoleBadSSL, + ServerNameMain: "badssl.com", + ServerNameExtras: []string{}, }} // MustNewScenario constructs a complete testing scenario using the domains and IP @@ -169,33 +204,54 @@ func MustNewScenario(config []*ScenarioDomainAddresses) *QAEnv { addr, &DNSOverUDPServerFactory{}, &HTTPSecureServerFactory{ - Factory: &DNSOverHTTPSHandlerFactory{}, - Ports: []int{443}, - TLSConfig: nil, // use netem's default + Factory: &DNSOverHTTPSHandlerFactory{}, + Ports: []int{443}, + ServerNameMain: sad.ServerNameMain, + ServerNameExtras: sad.ServerNameExtras, }, )) } case ScenarioRoleWebServer: for _, addr := range sad.Addresses { - opts = append(opts, QAEnvOptionHTTPServer(addr, sad.WebServerFactory)) + opts = append(opts, qaEnvOptionNetStack( + addr, + &HTTPCleartextServerFactory{ + Factory: sad.WebServerFactory, + Ports: []int{80}, + }, + &HTTPSecureServerFactory{ + Factory: sad.WebServerFactory, + Ports: []int{443}, + ServerNameMain: sad.ServerNameMain, + ServerNameExtras: sad.ServerNameExtras, + }, + &HTTP3ServerFactory{ + Factory: sad.WebServerFactory, + Ports: []int{443}, + ServerNameMain: sad.ServerNameMain, + ServerNameExtras: sad.ServerNameExtras, + }, + )) } case ScenarioRoleOONIAPI: for _, addr := range sad.Addresses { opts = append(opts, QAEnvOptionNetStack(addr, &HTTPSecureServerFactory{ - Factory: &OOAPIHandlerFactory{}, - Ports: []int{443}, - TLSConfig: nil, // use netem's default + Factory: &OOAPIHandlerFactory{}, + Ports: []int{443}, + ServerNameMain: sad.ServerNameMain, + ServerNameExtras: sad.ServerNameExtras, })) } case ScenarioRoleOONITestHelper: for _, addr := range sad.Addresses { opts = append(opts, QAEnvOptionNetStack(addr, &HTTPSecureServerFactory{ - Factory: &OOHelperDFactory{}, - Ports: []int{443}, - TLSConfig: nil, // use netem's default + Factory: &OOHelperDFactory{}, + Ports: []int{443}, + ServerNameMain: sad.ServerNameMain, + ServerNameExtras: sad.ServerNameExtras, })) } @@ -205,8 +261,9 @@ func MustNewScenario(config []*ScenarioDomainAddresses) *QAEnv { Factory: &GeoIPHandlerFactoryUbuntu{ ProbeIP: DefaultClientAddress, }, - Ports: []int{443}, - TLSConfig: nil, // use netem's default + Ports: []int{443}, + ServerNameMain: sad.ServerNameMain, + ServerNameExtras: sad.ServerNameExtras, })) } @@ -223,7 +280,7 @@ func MustNewScenario(config []*ScenarioDomainAddresses) *QAEnv { opts = append(opts, QAEnvOptionNetStack(addr, &HTTPCleartextServerFactory{ Factory: HTTPHandlerFactoryFunc(func(env NetStackServerFactoryEnv, stack *netem.UNetStack) http.Handler { - return testingx.HTTPHandlerProxy(env.Logger(), &netxlite.Netx{ + return testingx.NewHTTPProxyHandler(env.Logger(), &netxlite.Netx{ Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: stack}}) }), Ports: []int{80}, @@ -234,7 +291,14 @@ func MustNewScenario(config []*ScenarioDomainAddresses) *QAEnv { case ScenarioRoleURLShortener: for _, addr := range sad.Addresses { - opts = append(opts, QAEnvOptionHTTPServer(addr, URLShortenerFactory(DefaultURLShortenerMapping))) + opts = append(opts, QAEnvOptionNetStack(addr, + &HTTPSecureServerFactory{ + Factory: URLShortenerFactory(DefaultURLShortenerMapping), + Ports: []int{443}, + ServerNameMain: sad.ServerNameMain, + ServerNameExtras: sad.ServerNameExtras, + }, + )) } case ScenarioRoleBadSSL: @@ -244,7 +308,7 @@ func MustNewScenario(config []*ScenarioDomainAddresses) *QAEnv { } } - // create the QAEnv + // create QAEnv env := MustNewQAEnv(opts...) // configure all the domain names diff --git a/pkg/netemx/tlsproxy.go b/pkg/netemx/tlsproxy.go index 60358ac15..c36c54153 100644 --- a/pkg/netemx/tlsproxy.go +++ b/pkg/netemx/tlsproxy.go @@ -79,7 +79,6 @@ func (srv *tlsProxyServer) MustStart() { srv.logger, &netxlite.Netx{Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: srv.unet}}, epnt, - srv.unet, ) // track this server as something to close later diff --git a/pkg/netxlite/certifi.go b/pkg/netxlite/certifi.go index 3b5c7c089..e3e533d31 100644 --- a/pkg/netxlite/certifi.go +++ b/pkg/netxlite/certifi.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2023-05-31 11:09:58.566952 +0200 CEST m=+0.556580876 +// 2023-10-04 17:50:10.699722 +0200 CEST m=+0.596753793 // https://curl.haxx.se/ca/cacert.pem package netxlite @@ -10,7 +10,7 @@ const pemcerts string = ` ## ## Bundle of CA Root Certificates ## -## Certificate data from Mozilla as of: Tue May 30 03:12:04 2023 GMT +## Certificate data from Mozilla as of: Tue Aug 22 03:12:04 2023 GMT ## ## This is a bundle of X.509 certificates of public Certificate Authorities ## (CA). These were automatically extracted from Mozilla's root certificates @@ -23,7 +23,7 @@ const pemcerts string = ` ## Just configure this file as the SSLCACertificateFile. ## ## Conversion done with mk-ca-bundle.pl version 1.29. -## SHA256: c47475103fb05bb562bbadff0d1e72346b03236154e1448a6ca191b740f83507 +## SHA256: 0ff137babc6a5561a9cfbe9f29558972e5b528202681b7d3803d03a3e82922bd ## @@ -3231,55 +3231,6 @@ AwMDaAAwZQIxALGOWiDDshliTd6wT99u0nCK8Z9+aozmut6Dacpps6kFtZaSF4fC0urQe87YQVt8 rgIwRt7qy12a7DLCZRawTDBcMPPaTnOGBtjOiQRINzf43TNRnXCve1XYAS59BWQOhriR -----END CERTIFICATE----- -E-Tugra Global Root CA RSA v3 -============================= ------BEGIN CERTIFICATE----- -MIIF8zCCA9ugAwIBAgIUDU3FzRYilZYIfrgLfxUGNPt5EDQwDQYJKoZIhvcNAQELBQAwgYAxCzAJ -BgNVBAYTAlRSMQ8wDQYDVQQHEwZBbmthcmExGTAXBgNVBAoTEEUtVHVncmEgRUJHIEEuUy4xHTAb -BgNVBAsTFEUtVHVncmEgVHJ1c3QgQ2VudGVyMSYwJAYDVQQDEx1FLVR1Z3JhIEdsb2JhbCBSb290 -IENBIFJTQSB2MzAeFw0yMDAzMTgwOTA3MTdaFw00NTAzMTIwOTA3MTdaMIGAMQswCQYDVQQGEwJU -UjEPMA0GA1UEBxMGQW5rYXJhMRkwFwYDVQQKExBFLVR1Z3JhIEVCRyBBLlMuMR0wGwYDVQQLExRF -LVR1Z3JhIFRydXN0IENlbnRlcjEmMCQGA1UEAxMdRS1UdWdyYSBHbG9iYWwgUm9vdCBDQSBSU0Eg -djMwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCiZvCJt3J77gnJY9LTQ91ew6aEOErx -jYG7FL1H6EAX8z3DeEVypi6Q3po61CBxyryfHUuXCscxuj7X/iWpKo429NEvx7epXTPcMHD4QGxL -sqYxYdE0PD0xesevxKenhOGXpOhL9hd87jwH7eKKV9y2+/hDJVDqJ4GohryPUkqWOmAalrv9c/SF -/YP9f4RtNGx/ardLAQO/rWm31zLZ9Vdq6YaCPqVmMbMWPcLzJmAy01IesGykNz709a/r4d+ABs8q -QedmCeFLl+d3vSFtKbZnwy1+7dZ5ZdHPOrbRsV5WYVB6Ws5OUDGAA5hH5+QYfERaxqSzO8bGwzrw -bMOLyKSRBfP12baqBqG3q+Sx6iEUXIOk/P+2UNOMEiaZdnDpwA+mdPy70Bt4znKS4iicvObpCdg6 -04nmvi533wEKb5b25Y08TVJ2Glbhc34XrD2tbKNSEhhw5oBOM/J+JjKsBY04pOZ2PJ8QaQ5tndLB -eSBrW88zjdGUdjXnXVXHt6woq0bM5zshtQoK5EpZ3IE1S0SVEgpnpaH/WwAH0sDM+T/8nzPyAPiM -bIedBi3x7+PmBvrFZhNb/FAHnnGGstpvdDDPk1Po3CLW3iAfYY2jLqN4MpBs3KwytQXk9TwzDdbg -h3cXTJ2w2AmoDVf3RIXwyAS+XF1a4xeOVGNpf0l0ZAWMowIDAQABo2MwYTAPBgNVHRMBAf8EBTAD -AQH/MB8GA1UdIwQYMBaAFLK0ruYt9ybVqnUtdkvAG1Mh0EjvMB0GA1UdDgQWBBSytK7mLfcm1ap1 -LXZLwBtTIdBI7zAOBgNVHQ8BAf8EBAMCAQYwDQYJKoZIhvcNAQELBQADggIBAImocn+M684uGMQQ -gC0QDP/7FM0E4BQ8Tpr7nym/Ip5XuYJzEmMmtcyQ6dIqKe6cLcwsmb5FJ+Sxce3kOJUxQfJ9emN4 -38o2Fi+CiJ+8EUdPdk3ILY7r3y18Tjvarvbj2l0Upq7ohUSdBm6O++96SmotKygY/r+QLHUWnw/q -ln0F7psTpURs+APQ3SPh/QMSEgj0GDSz4DcLdxEBSL9htLX4GdnLTeqjjO/98Aa1bZL0SmFQhO3s -SdPkvmjmLuMxC1QLGpLWgti2omU8ZgT5Vdps+9u1FGZNlIM7zR6mK7L+d0CGq+ffCsn99t2HVhjY -sCxVYJb6CH5SkPVLpi6HfMsg2wY+oF0Dd32iPBMbKaITVaA9FCKvb7jQmhty3QUBjYZgv6Rn7rWl -DdF/5horYmbDB7rnoEgcOMPpRfunf/ztAmgayncSd6YAVSgU7NbHEqIbZULpkejLPoeJVF3Zr52X -nGnnCv8PWniLYypMfUeUP95L6VPQMPHF9p5J3zugkaOj/s1YzOrfr28oO6Bpm4/srK4rVJ2bBLFH -IK+WEj5jlB0E5y67hscMmoi/dkfv97ALl2bSRM9gUgfh1SxKOidhd8rXj+eHDjD/DLsE4mHDosiX -YY60MGo8bcIHX0pzLz/5FooBZu+6kcpSV3uu1OYP3Qt6f4ueJiDPO++BcYNZ ------END CERTIFICATE----- - -E-Tugra Global Root CA ECC v3 -============================= ------BEGIN CERTIFICATE----- -MIICpTCCAiqgAwIBAgIUJkYZdzHhT28oNt45UYbm1JeIIsEwCgYIKoZIzj0EAwMwgYAxCzAJBgNV -BAYTAlRSMQ8wDQYDVQQHEwZBbmthcmExGTAXBgNVBAoTEEUtVHVncmEgRUJHIEEuUy4xHTAbBgNV -BAsTFEUtVHVncmEgVHJ1c3QgQ2VudGVyMSYwJAYDVQQDEx1FLVR1Z3JhIEdsb2JhbCBSb290IENB -IEVDQyB2MzAeFw0yMDAzMTgwOTQ2NThaFw00NTAzMTIwOTQ2NThaMIGAMQswCQYDVQQGEwJUUjEP -MA0GA1UEBxMGQW5rYXJhMRkwFwYDVQQKExBFLVR1Z3JhIEVCRyBBLlMuMR0wGwYDVQQLExRFLVR1 -Z3JhIFRydXN0IENlbnRlcjEmMCQGA1UEAxMdRS1UdWdyYSBHbG9iYWwgUm9vdCBDQSBFQ0MgdjMw -djAQBgcqhkjOPQIBBgUrgQQAIgNiAASOmCm/xxAeJ9urA8woLNheSBkQKczLWYHMjLiSF4mDKpL2 -w6QdTGLVn9agRtwcvHbB40fQWxPa56WzZkjnIZpKT4YKfWzqTTKACrJ6CZtpS5iB4i7sAnCWH/31 -Rs7K3IKjYzBhMA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAU/4Ixcj75xGZsrTie0bBRiKWQ -zPUwHQYDVR0OBBYEFP+CMXI++cRmbK04ntGwUYilkMz1MA4GA1UdDwEB/wQEAwIBBjAKBggqhkjO -PQQDAwNpADBmAjEA5gVYaWHlLcoNy/EZCL3W/VGSGn5jVASQkZo1kTmZ+gepZpO6yGjUij/67W4W -Aie3AjEA3VoXK3YdZUKWpqxdinlW2Iob35reX8dQj7FbcQwm32pAAOwzkSFxvmjkI6TZraE3 ------END CERTIFICATE----- - Security Communication RootCA3 ============================== -----BEGIN CERTIFICATE----- @@ -3371,4 +3322,141 @@ W9f+qdJUDkpd0m2xQNz0Q9XSSpkZElaA94M04TVOSG0ED1cxMDAtsaqdAzjbBgIxAMvMh1PLet8g UXOQwKhbYdDFUDn9hf7B43j4ptZLvZuHjw/l1lOWqzzIQNph91Oj9w== -----END CERTIFICATE----- +Sectigo Public Server Authentication Root E46 +============================================= +-----BEGIN CERTIFICATE----- +MIICOjCCAcGgAwIBAgIQQvLM2htpN0RfFf51KBC49DAKBggqhkjOPQQDAzBfMQswCQYDVQQGEwJH +QjEYMBYGA1UEChMPU2VjdGlnbyBMaW1pdGVkMTYwNAYDVQQDEy1TZWN0aWdvIFB1YmxpYyBTZXJ2 +ZXIgQXV0aGVudGljYXRpb24gUm9vdCBFNDYwHhcNMjEwMzIyMDAwMDAwWhcNNDYwMzIxMjM1OTU5 +WjBfMQswCQYDVQQGEwJHQjEYMBYGA1UEChMPU2VjdGlnbyBMaW1pdGVkMTYwNAYDVQQDEy1TZWN0 +aWdvIFB1YmxpYyBTZXJ2ZXIgQXV0aGVudGljYXRpb24gUm9vdCBFNDYwdjAQBgcqhkjOPQIBBgUr +gQQAIgNiAAR2+pmpbiDt+dd34wc7qNs9Xzjoq1WmVk/WSOrsfy2qw7LFeeyZYX8QeccCWvkEN/U0 +NSt3zn8gj1KjAIns1aeibVvjS5KToID1AZTc8GgHHs3u/iVStSBDHBv+6xnOQ6OjQjBAMB0GA1Ud +DgQWBBTRItpMWfFLXyY4qp3W7usNw/upYTAOBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB +/zAKBggqhkjOPQQDAwNnADBkAjAn7qRaqCG76UeXlImldCBteU/IvZNeWBj7LRoAasm4PdCkT0RH +lAFWovgzJQxC36oCMB3q4S6ILuH5px0CMk7yn2xVdOOurvulGu7t0vzCAxHrRVxgED1cf5kDW21U +SAGKcw== +-----END CERTIFICATE----- + +Sectigo Public Server Authentication Root R46 +============================================= +-----BEGIN CERTIFICATE----- +MIIFijCCA3KgAwIBAgIQdY39i658BwD6qSWn4cetFDANBgkqhkiG9w0BAQwFADBfMQswCQYDVQQG +EwJHQjEYMBYGA1UEChMPU2VjdGlnbyBMaW1pdGVkMTYwNAYDVQQDEy1TZWN0aWdvIFB1YmxpYyBT +ZXJ2ZXIgQXV0aGVudGljYXRpb24gUm9vdCBSNDYwHhcNMjEwMzIyMDAwMDAwWhcNNDYwMzIxMjM1 +OTU5WjBfMQswCQYDVQQGEwJHQjEYMBYGA1UEChMPU2VjdGlnbyBMaW1pdGVkMTYwNAYDVQQDEy1T +ZWN0aWdvIFB1YmxpYyBTZXJ2ZXIgQXV0aGVudGljYXRpb24gUm9vdCBSNDYwggIiMA0GCSqGSIb3 +DQEBAQUAA4ICDwAwggIKAoICAQCTvtU2UnXYASOgHEdCSe5jtrch/cSV1UgrJnwUUxDaef0rty2k +1Cz66jLdScK5vQ9IPXtamFSvnl0xdE8H/FAh3aTPaE8bEmNtJZlMKpnzSDBh+oF8HqcIStw+Kxwf +GExxqjWMrfhu6DtK2eWUAtaJhBOqbchPM8xQljeSM9xfiOefVNlI8JhD1mb9nxc4Q8UBUQvX4yMP +FF1bFOdLvt30yNoDN9HWOaEhUTCDsG3XME6WW5HwcCSrv0WBZEMNvSE6Lzzpng3LILVCJ8zab5vu +ZDCQOc2TZYEhMbUjUDM3IuM47fgxMMxF/mL50V0yeUKH32rMVhlATc6qu/m1dkmU8Sf4kaWD5Qaz +Yw6A3OASVYCmO2a0OYctyPDQ0RTp5A1NDvZdV3LFOxxHVp3i1fuBYYzMTYCQNFu31xR13NgESJ/A +wSiItOkcyqex8Va3e0lMWeUgFaiEAin6OJRpmkkGj80feRQXEgyDet4fsZfu+Zd4KKTIRJLpfSYF +plhym3kT2BFfrsU4YjRosoYwjviQYZ4ybPUHNs2iTG7sijbt8uaZFURww3y8nDnAtOFr94MlI1fZ +EoDlSfB1D++N6xybVCi0ITz8fAr/73trdf+LHaAZBav6+CuBQug4urv7qv094PPK306Xlynt8xhW +6aWWrL3DkJiy4Pmi1KZHQ3xtzwIDAQABo0IwQDAdBgNVHQ4EFgQUVnNYZJX5khqwEioEYnmhQBWI +IUkwDgYDVR0PAQH/BAQDAgGGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEMBQADggIBAC9c +mTz8Bl6MlC5w6tIyMY208FHVvArzZJ8HXtXBc2hkeqK5Duj5XYUtqDdFqij0lgVQYKlJfp/imTYp +E0RHap1VIDzYm/EDMrraQKFz6oOht0SmDpkBm+S8f74TlH7Kph52gDY9hAaLMyZlbcp+nv4fjFg4 +exqDsQ+8FxG75gbMY/qB8oFM2gsQa6H61SilzwZAFv97fRheORKkU55+MkIQpiGRqRxOF3yEvJ+M +0ejf5lG5Nkc/kLnHvALcWxxPDkjBJYOcCj+esQMzEhonrPcibCTRAUH4WAP+JWgiH5paPHxsnnVI +84HxZmduTILA7rpXDhjvLpr3Etiga+kFpaHpaPi8TD8SHkXoUsCjvxInebnMMTzD9joiFgOgyY9m +pFuiTdaBJQbpdqQACj7LzTWb4OE4y2BThihCQRxEV+ioratF4yUQvNs+ZUH7G6aXD+u5dHn5Hrwd +Vw1Hr8Mvn4dGp+smWg9WY7ViYG4A++MnESLn/pmPNPW56MORcr3Ywx65LvKRRFHQV80MNNVIIb/b +E/FmJUNS0nAiNs2fxBx1IK1jcmMGDw4nztJqDby1ORrp0XZ60Vzk50lJLVU3aPAaOpg+VBeHVOmm +J1CJeyAvP/+/oYtKR5j/K3tJPsMpRmAYQqszKbrAKbkTidOIijlBO8n9pu0f9GBj39ItVQGL +-----END CERTIFICATE----- + +SSL.com TLS RSA Root CA 2022 +============================ +-----BEGIN CERTIFICATE----- +MIIFiTCCA3GgAwIBAgIQb77arXO9CEDii02+1PdbkTANBgkqhkiG9w0BAQsFADBOMQswCQYDVQQG +EwJVUzEYMBYGA1UECgwPU1NMIENvcnBvcmF0aW9uMSUwIwYDVQQDDBxTU0wuY29tIFRMUyBSU0Eg +Um9vdCBDQSAyMDIyMB4XDTIyMDgyNTE2MzQyMloXDTQ2MDgxOTE2MzQyMVowTjELMAkGA1UEBhMC +VVMxGDAWBgNVBAoMD1NTTCBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcU1NMLmNvbSBUTFMgUlNBIFJv +b3QgQ0EgMjAyMjCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANCkCXJPQIgSYT41I57u +9nTPL3tYPc48DRAokC+X94xI2KDYJbFMsBFMF3NQ0CJKY7uB0ylu1bUJPiYYf7ISf5OYt6/wNr/y +7hienDtSxUcZXXTzZGbVXcdotL8bHAajvI9AI7YexoS9UcQbOcGV0insS657Lb85/bRi3pZ7Qcac +oOAGcvvwB5cJOYF0r/c0WRFXCsJbwST0MXMwgsadugL3PnxEX4MN8/HdIGkWCVDi1FW24IBydm5M +R7d1VVm0U3TZlMZBrViKMWYPHqIbKUBOL9975hYsLfy/7PO0+r4Y9ptJ1O4Fbtk085zx7AGL0SDG +D6C1vBdOSHtRwvzpXGk3R2azaPgVKPC506QVzFpPulJwoxJF3ca6TvvC0PeoUidtbnm1jPx7jMEW +TO6Af77wdr5BUxIzrlo4QqvXDz5BjXYHMtWrifZOZ9mxQnUjbvPNQrL8VfVThxc7wDNY8VLS+YCk +8OjwO4s4zKTGkH8PnP2L0aPP2oOnaclQNtVcBdIKQXTbYxE3waWglksejBYSd66UNHsef8JmAOSq +g+qKkK3ONkRN0VHpvB/zagX9wHQfJRlAUW7qglFA35u5CCoGAtUjHBPW6dvbxrB6y3snm/vg1UYk +7RBLY0ulBY+6uB0rpvqR4pJSvezrZ5dtmi2fgTIFZzL7SAg/2SW4BCUvAgMBAAGjYzBhMA8GA1Ud +EwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAU+y437uOEeicuzRk1sTN8/9REQrkwHQYDVR0OBBYEFPsu +N+7jhHonLs0ZNbEzfP/UREK5MA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG9w0BAQsFAAOCAgEAjYlt +hEUY8U+zoO9opMAdrDC8Z2awms22qyIZZtM7QbUQnRC6cm4pJCAcAZli05bg4vsMQtfhWsSWTVTN +j8pDU/0quOr4ZcoBwq1gaAafORpR2eCNJvkLTqVTJXojpBzOCBvfR4iyrT7gJ4eLSYwfqUdYe5by +iB0YrrPRpgqU+tvT5TgKa3kSM/tKWTcWQA673vWJDPFs0/dRa1419dvAJuoSc06pkZCmF8NsLzjU +o3KUQyxi4U5cMj29TH0ZR6LDSeeWP4+a0zvkEdiLA9z2tmBVGKaBUfPhqBVq6+AL8BQx1rmMRTqo +ENjwuSfr98t67wVylrXEj5ZzxOhWc5y8aVFjvO9nHEMaX3cZHxj4HCUp+UmZKbaSPaKDN7Egkaib +MOlqbLQjk2UEqxHzDh1TJElTHaE/nUiSEeJ9DU/1172iWD54nR4fK/4huxoTtrEoZP2wAgDHbICi +vRZQIA9ygV/MlP+7mea6kMvq+cYMwq7FGc4zoWtcu358NFcXrfA/rs3qr5nsLFR+jM4uElZI7xc7 +P0peYNLcdDa8pUNjyw9bowJWCZ4kLOGGgYz+qxcs+sjiMho6/4UIyYOf8kpIEFR3N+2ivEC+5BB0 +9+Rbu7nzifmPQdjH5FCQNYA+HLhNkNPU98OwoX6EyneSMSy4kLGCenROmxMmtNVQZlR4rmA= +-----END CERTIFICATE----- + +SSL.com TLS ECC Root CA 2022 +============================ +-----BEGIN CERTIFICATE----- +MIICOjCCAcCgAwIBAgIQFAP1q/s3ixdAW+JDsqXRxDAKBggqhkjOPQQDAzBOMQswCQYDVQQGEwJV +UzEYMBYGA1UECgwPU1NMIENvcnBvcmF0aW9uMSUwIwYDVQQDDBxTU0wuY29tIFRMUyBFQ0MgUm9v +dCBDQSAyMDIyMB4XDTIyMDgyNTE2MzM0OFoXDTQ2MDgxOTE2MzM0N1owTjELMAkGA1UEBhMCVVMx +GDAWBgNVBAoMD1NTTCBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcU1NMLmNvbSBUTFMgRUNDIFJvb3Qg +Q0EgMjAyMjB2MBAGByqGSM49AgEGBSuBBAAiA2IABEUpNXP6wrgjzhR9qLFNoFs27iosU8NgCTWy +JGYmacCzldZdkkAZDsalE3D07xJRKF3nzL35PIXBz5SQySvOkkJYWWf9lCcQZIxPBLFNSeR7T5v1 +5wj4A4j3p8OSSxlUgaNjMGEwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBSJjy+j6CugFFR7 +81a4Jl9nOAuc0DAdBgNVHQ4EFgQUiY8vo+groBRUe/NWuCZfZzgLnNAwDgYDVR0PAQH/BAQDAgGG +MAoGCCqGSM49BAMDA2gAMGUCMFXjIlbp15IkWE8elDIPDAI2wv2sdDJO4fscgIijzPvX6yv/N33w +7deedWo1dlJF4AIxAMeNb0Igj762TVntd00pxCAgRWSGOlDGxK0tk/UYfXLtqc/ErFc2KAhl3zx5 +Zn6g6g== +-----END CERTIFICATE----- + +Atos TrustedRoot Root CA ECC TLS 2021 +===================================== +-----BEGIN CERTIFICATE----- +MIICFTCCAZugAwIBAgIQPZg7pmY9kGP3fiZXOATvADAKBggqhkjOPQQDAzBMMS4wLAYDVQQDDCVB +dG9zIFRydXN0ZWRSb290IFJvb3QgQ0EgRUNDIFRMUyAyMDIxMQ0wCwYDVQQKDARBdG9zMQswCQYD +VQQGEwJERTAeFw0yMTA0MjIwOTI2MjNaFw00MTA0MTcwOTI2MjJaMEwxLjAsBgNVBAMMJUF0b3Mg +VHJ1c3RlZFJvb3QgUm9vdCBDQSBFQ0MgVExTIDIwMjExDTALBgNVBAoMBEF0b3MxCzAJBgNVBAYT +AkRFMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEloZYKDcKZ9Cg3iQZGeHkBQcfl+3oZIK59sRxUM6K +DP/XtXa7oWyTbIOiaG6l2b4siJVBzV3dscqDY4PMwL502eCdpO5KTlbgmClBk1IQ1SQ4AjJn8ZQS +b+/Xxd4u/RmAo0IwQDAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBR2KCXWfeBmmnoJsmo7jjPX +NtNPojAOBgNVHQ8BAf8EBAMCAYYwCgYIKoZIzj0EAwMDaAAwZQIwW5kp85wxtolrbNa9d+F851F+ +uDrNozZffPc8dz7kUK2o59JZDCaOMDtuCCrCp1rIAjEAmeMM56PDr9NJLkaCI2ZdyQAUEv049OGY +a3cpetskz2VAv9LcjBHo9H1/IISpQuQo +-----END CERTIFICATE----- + +Atos TrustedRoot Root CA RSA TLS 2021 +===================================== +-----BEGIN CERTIFICATE----- +MIIFZDCCA0ygAwIBAgIQU9XP5hmTC/srBRLYwiqipDANBgkqhkiG9w0BAQwFADBMMS4wLAYDVQQD +DCVBdG9zIFRydXN0ZWRSb290IFJvb3QgQ0EgUlNBIFRMUyAyMDIxMQ0wCwYDVQQKDARBdG9zMQsw +CQYDVQQGEwJERTAeFw0yMTA0MjIwOTIxMTBaFw00MTA0MTcwOTIxMDlaMEwxLjAsBgNVBAMMJUF0 +b3MgVHJ1c3RlZFJvb3QgUm9vdCBDQSBSU0EgVExTIDIwMjExDTALBgNVBAoMBEF0b3MxCzAJBgNV +BAYTAkRFMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAtoAOxHm9BYx9sKOdTSJNy/BB +l01Z4NH+VoyX8te9j2y3I49f1cTYQcvyAh5x5en2XssIKl4w8i1mx4QbZFc4nXUtVsYvYe+W/CBG +vevUez8/fEc4BKkbqlLfEzfTFRVOvV98r61jx3ncCHvVoOX3W3WsgFWZkmGbzSoXfduP9LVq6hdK +ZChmFSlsAvFr1bqjM9xaZ6cF4r9lthawEO3NUDPJcFDsGY6wx/J0W2tExn2WuZgIWWbeKQGb9Cpt +0xU6kGpn8bRrZtkh68rZYnxGEFzedUlnnkL5/nWpo63/dgpnQOPF943HhZpZnmKaau1Fh5hnstVK +PNe0OwANwI8f4UDErmwh3El+fsqyjW22v5MvoVw+j8rtgI5Y4dtXz4U2OLJxpAmMkokIiEjxQGMY +sluMWuPD0xeqqxmjLBvk1cbiZnrXghmmOxYsL3GHX0WelXOTwkKBIROW1527k2gV+p2kHYzygeBY +Br3JtuP2iV2J+axEoctr+hbxx1A9JNr3w+SH1VbxT5Aw+kUJWdo0zuATHAR8ANSbhqRAvNncTFd+ +rrcztl524WWLZt+NyteYr842mIycg5kDcPOvdO3GDjbnvezBc6eUWsuSZIKmAMFwoW4sKeFYV+xa +fJlrJaSQOoD0IJ2azsct+bJLKZWD6TWNp0lIpw9MGZHQ9b8Q4HECAwEAAaNCMEAwDwYDVR0TAQH/ +BAUwAwEB/zAdBgNVHQ4EFgQUdEmZ0f+0emhFdcN+tNzMzjkz2ggwDgYDVR0PAQH/BAQDAgGGMA0G +CSqGSIb3DQEBDAUAA4ICAQAjQ1MkYlxt/T7Cz1UAbMVWiLkO3TriJQ2VSpfKgInuKs1l+NsW4AmS +4BjHeJi78+xCUvuppILXTdiK/ORO/auQxDh1MoSf/7OwKwIzNsAQkG8dnK/haZPso0UvFJ/1TCpl +Q3IM98P4lYsU84UgYt1UU90s3BiVaU+DR3BAM1h3Egyi61IxHkzJqM7F78PRreBrAwA0JrRUITWX +AdxfG/F851X6LWh3e9NpzNMOa7pNdkTWwhWaJuywxfW70Xp0wmzNxbVe9kzmWy2B27O3Opee7c9G +slA9hGCZcbUztVdF5kJHdWoOsAgMrr3e97sPWD2PAzHoPYJQyi9eDF20l74gNAf0xBLh7tew2Vkt +afcxBPTy+av5EzH4AXcOPUIjJsyacmdRIXrMPIWo6iFqO9taPKU0nprALN+AnCng33eU0aKAQv9q +TFsR0PXNor6uzFFcw9VUewyu1rkGd4Di7wcaaMxZUa1+XGdrudviB0JbuAEFWDlN5LuYo7Ey7Nmj +1m+UI/87tyll5gfp77YZ6ufCOB0yiJA8EytuzO+rdwY0d4RPcuSBhPm5dDTedk+SKlOxJTnbPP/l +PqYO5Wue/9vsL3SD3460s6neFE3/MaNFcyT6lSnMEpcEoji2jbDwN/zIIX8/syQbPYtuzE2wFg2W +HYMfRsCbvUOZ58SWLs5fyQ== +-----END CERTIFICATE----- + ` diff --git a/pkg/netxlite/classify.go b/pkg/netxlite/classify.go index 41dc71205..3e122b78f 100644 --- a/pkg/netxlite/classify.go +++ b/pkg/netxlite/classify.go @@ -76,7 +76,7 @@ func ClassifyGenericError(err error) string { } formatted := fmt.Sprintf("%s: %s", FailureUnknown, err.Error()) - return scrubber.Scrub(formatted) // scrub IP addresses in the error + return scrubber.ScrubString(formatted) // scrub IP addresses in the error } // classifyWithStringSuffix is a subset of ClassifyGenericError that diff --git a/pkg/netxlite/dialer.go b/pkg/netxlite/dialer.go index 2dae7b9e2..3510fd558 100644 --- a/pkg/netxlite/dialer.go +++ b/pkg/netxlite/dialer.go @@ -22,10 +22,25 @@ func NewDialerWithStdlibResolver(dl model.DebugLogger) model.Dialer { return NewDialerWithResolver(dl, reso) } -// NewDialerWithResolver is equivalent to calling WrapDialer with -// the dialer argument being equal to &DialerSystem{}. +// NewDialerWithResolver creates a [Dialer] with error wrapping. +// +// This dialer will try to connect to each of the resolved IP address +// sequentially. In case of failure, such a resolver will return the first +// error that occurred. This implementation strategy is a QUIRK that is +// documented at TODO(https://github.com/ooni/probe/issues/1779). +// +// The [model.DialerWrapper] arguments wrap the returned dialer in such a way +// that we can implement the legacy [netx] package. New code MUST NOT +// use this functionality, which we'd like to remove ASAP. +func (netx *Netx) NewDialerWithResolver(dl model.DebugLogger, r model.Resolver, w ...model.DialerWrapper) model.Dialer { + return WrapDialer(dl, r, &dialerSystem{provider: netx.MaybeCustomUnderlyingNetwork()}, w...) +} + +// NewDialerWithResolver is equivalent to creating an empty [*Netx] +// and calling its NewDialerWithResolver method. func NewDialerWithResolver(dl model.DebugLogger, r model.Resolver, w ...model.DialerWrapper) model.Dialer { - return WrapDialer(dl, r, &DialerSystem{}, w...) + netx := &Netx{Underlying: nil} + return netx.NewDialerWithResolver(dl, r, w...) } // WrapDialer wraps an existing Dialer to add extra functionality @@ -102,7 +117,7 @@ func NewDialerWithResolver(dl model.DebugLogger, r model.Resolver, w ...model.Di // When the resolver is &NullResolver{} any attempt to perform DNS resolutions // in the dialer at index N+2 will fail with ErrNoResolver. // -// Otherwise, the dialer at index N+2 will try each resolver IP address +// Otherwise, the dialer at index N+2 will try each resolved IP address // sequentially. In case of failure, such a resolver will return the first // error that occurred. This implementation strategy is a QUIRK that is // documented at TODO(https://github.com/ooni/probe/issues/1779). @@ -136,30 +151,36 @@ func WrapDialer(logger model.DebugLogger, resolver model.Resolver, } } -// NewDialerWithoutResolver is equivalent to calling NewDialerWithResolver -// with the resolver argument being &NullResolver{}. +// NewDialerWithoutResolver implements [model.MeasuringNetwork]. +func (netx *Netx) NewDialerWithoutResolver(dl model.DebugLogger, w ...model.DialerWrapper) model.Dialer { + return netx.NewDialerWithResolver(dl, &NullResolver{}, w...) +} + +// NewDialerWithoutResolver is equivalent to creating an empty [*Netx] +// and calling its NewDialerWithoutResolver method. func NewDialerWithoutResolver(dl model.DebugLogger, w ...model.DialerWrapper) model.Dialer { - return NewDialerWithResolver(dl, &NullResolver{}, w...) + netx := &Netx{Underlying: nil} + return netx.NewDialerWithoutResolver(dl, w...) } -// DialerSystem is a model.Dialer that uses the stdlib's net.Dialer +// dialerSystem is a model.Dialer that uses the stdlib's net.Dialer // to construct the new SimpleDialer used for dialing. This dialer has // a fixed timeout for each connect operation equal to 15 seconds. -type DialerSystem struct { +type dialerSystem struct { // provider is the OPTIONAL nil-safe [model.UnderlyingNetwork] provider. provider *MaybeCustomUnderlyingNetwork } -var _ model.Dialer = &DialerSystem{} +var _ model.Dialer = &dialerSystem{} -func (d *DialerSystem) DialContext(ctx context.Context, network, address string) (net.Conn, error) { +func (d *dialerSystem) DialContext(ctx context.Context, network, address string) (net.Conn, error) { p := d.provider.Get() ctx, cancel := context.WithTimeout(ctx, p.DialTimeout()) defer cancel() return p.DialContext(ctx, network, address) } -func (d *DialerSystem) CloseIdleConnections() { +func (d *dialerSystem) CloseIdleConnections() { // nothing to do here } diff --git a/pkg/netxlite/dialer_test.go b/pkg/netxlite/dialer_test.go index 0ef389059..c9475197c 100644 --- a/pkg/netxlite/dialer_test.go +++ b/pkg/netxlite/dialer_test.go @@ -5,6 +5,7 @@ import ( "errors" "io" "net" + "runtime" "strings" "sync" "testing" @@ -24,14 +25,14 @@ func TestNewDialerWithStdlibResolver(t *testing.T) { } // typecheck the resolver reso := logger.Dialer.(*dialerResolverWithTracing) - typecheckForSystemResolver(t, reso.Resolver, model.DiscardLogger) + typeCheckForSystemResolver(t, reso.Resolver, model.DiscardLogger) // typecheck the dialer logger = reso.Dialer.(*dialerLogger) if logger.DebugLogger != model.DiscardLogger { t.Fatal("invalid logger") } errWrapper := logger.Dialer.(*dialerErrWrapper) - _ = errWrapper.Dialer.(*DialerSystem) + _ = errWrapper.Dialer.(*dialerSystem) } type extensionDialerFirst struct { @@ -76,19 +77,19 @@ func TestNewDialer(t *testing.T) { ext2 := logger.Dialer.(*extensionDialerSecond) ext1 := ext2.Dialer.(*extensionDialerFirst) errWrapper := ext1.Dialer.(*dialerErrWrapper) - _ = errWrapper.Dialer.(*DialerSystem) + _ = errWrapper.Dialer.(*dialerSystem) }) } func TestDialerSystem(t *testing.T) { t.Run("CloseIdleConnections", func(t *testing.T) { - d := &DialerSystem{} + d := &dialerSystem{} d.CloseIdleConnections() // to avoid missing coverage }) t.Run("DialContext", func(t *testing.T) { t.Run("with canceled context", func(t *testing.T) { - d := &DialerSystem{} + d := &dialerSystem{} ctx, cancel := context.WithCancel(context.Background()) cancel() // immediately! conn, err := d.DialContext(ctx, "tcp", "8.8.8.8:443") @@ -104,17 +105,22 @@ func TestDialerSystem(t *testing.T) { defaultTp := &DefaultTProxy{} tp := &mocks.UnderlyingNetwork{ MockDialTimeout: func() time.Duration { + // Note: this test is notoriously flaky on Windows as documented by + // TODO(https://github.com/ooni/probe/issues/2537) return time.Nanosecond }, MockDialContext: defaultTp.DialContext, } - d := &DialerSystem{provider: &MaybeCustomUnderlyingNetwork{tp}} + d := &dialerSystem{provider: &MaybeCustomUnderlyingNetwork{tp}} ctx := context.Background() start := time.Now() conn, err := d.DialContext(ctx, "tcp", "dns.google:443") stop := time.Now() if err == nil || !strings.HasSuffix(err.Error(), "i/o timeout") { - t.Fatal(err) + if runtime.GOOS == "windows" { + t.Skip("https://github.com/ooni/probe/issues/2537") + } + t.Fatal("unexpected error", err) } if conn != nil { t.Fatal("unexpected conn") @@ -134,7 +140,7 @@ func TestDialerSystem(t *testing.T) { return nil, expected }, } - d := &DialerSystem{provider: &MaybeCustomUnderlyingNetwork{proxy}} + d := &dialerSystem{provider: &MaybeCustomUnderlyingNetwork{proxy}} conn, err := d.DialContext(context.Background(), "tcp", "dns.google:443") if conn != nil { t.Fatal("unexpected conn") @@ -150,7 +156,7 @@ func TestDialerResolverWithTracing(t *testing.T) { t.Run("DialContext", func(t *testing.T) { t.Run("fails without a port", func(t *testing.T) { d := &dialerResolverWithTracing{ - Dialer: &DialerSystem{}, + Dialer: &dialerSystem{}, Resolver: NewUnwrappedStdlibResolver(), } const missingPort = "ooni.nu" @@ -497,7 +503,7 @@ func TestDialerResolverWithTracing(t *testing.T) { t.Run("lookupHost", func(t *testing.T) { t.Run("handles addresses correctly", func(t *testing.T) { dialer := &dialerResolverWithTracing{ - Dialer: &DialerSystem{}, + Dialer: &dialerSystem{}, Resolver: &NullResolver{}, } addrs, err := dialer.lookupHost(context.Background(), "1.1.1.1") @@ -511,7 +517,7 @@ func TestDialerResolverWithTracing(t *testing.T) { t.Run("fails correctly on lookup error", func(t *testing.T) { dialer := &dialerResolverWithTracing{ - Dialer: &DialerSystem{}, + Dialer: &dialerSystem{}, Resolver: &NullResolver{}, } ctx := context.Background() diff --git a/pkg/netxlite/dnsovergetaddrinfo.go b/pkg/netxlite/dnsovergetaddrinfo.go index 6648d4cc8..b9e8b4149 100644 --- a/pkg/netxlite/dnsovergetaddrinfo.go +++ b/pkg/netxlite/dnsovergetaddrinfo.go @@ -27,9 +27,14 @@ type dnsOverGetaddrinfoTransport struct { provider *MaybeCustomUnderlyingNetwork } +func (netx *Netx) newDNSOverGetaddrinfoTransport() model.DNSTransport { + return &dnsOverGetaddrinfoTransport{provider: netx.MaybeCustomUnderlyingNetwork()} +} + // NewDNSOverGetaddrinfoTransport creates a new dns-over-getaddrinfo transport. func NewDNSOverGetaddrinfoTransport() model.DNSTransport { - return &dnsOverGetaddrinfoTransport{} + netx := &Netx{Underlying: nil} + return netx.newDNSOverGetaddrinfoTransport() } var _ model.DNSTransport = &dnsOverGetaddrinfoTransport{} diff --git a/pkg/netxlite/dnsovergetaddrinfo_test.go b/pkg/netxlite/dnsovergetaddrinfo_test.go index 81762e3c2..a6bec0adf 100644 --- a/pkg/netxlite/dnsovergetaddrinfo_test.go +++ b/pkg/netxlite/dnsovergetaddrinfo_test.go @@ -13,6 +13,14 @@ import ( "github.com/ooni/probe-engine/pkg/mocks" ) +func TestNewDNSOverGetaddrinfoTransport(t *testing.T) { + txp := NewDNSOverGetaddrinfoTransport() + underlying := txp.(*dnsOverGetaddrinfoTransport) + if underlying.provider.underlying != nil { + t.Fatal("expected to see a nil underlying network") + } +} + func TestDNSOverGetaddrinfo(t *testing.T) { t.Run("RequiresPadding", func(t *testing.T) { txp := &dnsOverGetaddrinfoTransport{} diff --git a/pkg/netxlite/dnsoverhttps.go b/pkg/netxlite/dnsoverhttps.go index 200d61705..1b07396f8 100644 --- a/pkg/netxlite/dnsoverhttps.go +++ b/pkg/netxlite/dnsoverhttps.go @@ -46,13 +46,13 @@ func NewUnwrappedDNSOverHTTPSTransport(client model.HTTPClient, URL string) *DNS // NewDNSOverHTTPSTransport is like NewUnwrappedDNSOverHTTPSTransport but // returns an already wrapped DNSTransport. func NewDNSOverHTTPSTransport(client model.HTTPClient, URL string) model.DNSTransport { - return WrapDNSTransport(NewUnwrappedDNSOverHTTPSTransport(client, URL)) + return wrapDNSTransport(NewUnwrappedDNSOverHTTPSTransport(client, URL)) } // NewDNSOverHTTPSTransportWithHTTPTransport is like NewDNSOverHTTPSTransport // but takes in input an HTTPTransport rather than an HTTPClient. func NewDNSOverHTTPSTransportWithHTTPTransport(txp model.HTTPTransport, URL string) model.DNSTransport { - return WrapDNSTransport(NewUnwrappedDNSOverHTTPSTransport(NewHTTPClient(txp), URL)) + return wrapDNSTransport(NewUnwrappedDNSOverHTTPSTransport(NewHTTPClient(txp), URL)) } // NewUnwrappedDNSOverHTTPSTransportWithHostOverride creates a new DNSOverHTTPSTransport diff --git a/pkg/netxlite/dnsoverhttps_test.go b/pkg/netxlite/dnsoverhttps_test.go index 29f0c82ec..a7786f993 100644 --- a/pkg/netxlite/dnsoverhttps_test.go +++ b/pkg/netxlite/dnsoverhttps_test.go @@ -15,6 +15,7 @@ import ( func TestNewDNSOverHTTPSTransport(t *testing.T) { const URL = "https://1.1.1.1/dns-query" + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPClientStdlib has QUIRKS but they're not needed here clnt := NewHTTPClientStdlib(model.DiscardLogger) txp := NewDNSOverHTTPSTransport(clnt, URL) ew := txp.(*dnsTransportErrWrapper) @@ -29,6 +30,8 @@ func TestNewDNSOverHTTPSTransport(t *testing.T) { func TestNewDNSOverHTTPSTransportWithHTTPTransport(t *testing.T) { const URL = "https://1.1.1.1/dns-query" + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPTransportStdlib has QUIRKS but we + // don't actually care about those QUIRKS in this context httpTxp := NewHTTPTransportStdlib(model.DiscardLogger) txp := NewDNSOverHTTPSTransportWithHTTPTransport(httpTxp, URL) ew := txp.(*dnsTransportErrWrapper) diff --git a/pkg/netxlite/dnstransport.go b/pkg/netxlite/dnstransport.go index 87c110d31..e9d4f4efb 100644 --- a/pkg/netxlite/dnstransport.go +++ b/pkg/netxlite/dnstransport.go @@ -10,20 +10,11 @@ import ( "github.com/ooni/probe-engine/pkg/model" ) -// WrapDNSTransport wraps a DNSTransport to provide error wrapping. This function will -// apply all the provided wrappers around the default transport wrapping. If any of the -// wrappers is nil, we just silently and gracefully ignore it. -func WrapDNSTransport(txp model.DNSTransport, - wrappers ...model.DNSTransportWrapper) (out model.DNSTransport) { +// wrapDNSTransport wraps a DNSTransport to provide error wrapping. +func wrapDNSTransport(txp model.DNSTransport) (out model.DNSTransport) { out = &dnsTransportErrWrapper{ DNSTransport: txp, } - for _, wrapper := range wrappers { - if wrapper == nil { - continue // skip as documented - } - out = wrapper.WrapDNSTransport(out) // compose with user-provided wrappers - } return } diff --git a/pkg/netxlite/dnstransport_test.go b/pkg/netxlite/dnstransport_test.go index eaa5429c7..cbc697f59 100644 --- a/pkg/netxlite/dnstransport_test.go +++ b/pkg/netxlite/dnstransport_test.go @@ -10,37 +10,10 @@ import ( "github.com/ooni/probe-engine/pkg/model" ) -type dnsTransportExtensionFirst struct { - model.DNSTransport -} - -type dnsTransportWrapperFirst struct{} - -func (*dnsTransportWrapperFirst) WrapDNSTransport(txp model.DNSTransport) model.DNSTransport { - return &dnsTransportExtensionFirst{txp} -} - -type dnsTransportExtensionSecond struct { - model.DNSTransport -} - -type dnsTransportWrapperSecond struct{} - -func (*dnsTransportWrapperSecond) WrapDNSTransport(txp model.DNSTransport) model.DNSTransport { - return &dnsTransportExtensionSecond{txp} -} - func TestWrapDNSTransport(t *testing.T) { orig := &mocks.DNSTransport{} - extensions := []model.DNSTransportWrapper{ - &dnsTransportWrapperFirst{}, - nil, // explicitly test for documented use case - &dnsTransportWrapperSecond{}, - } - txp := WrapDNSTransport(orig, extensions...) - ext2 := txp.(*dnsTransportExtensionSecond) - ext1 := ext2.DNSTransport.(*dnsTransportExtensionFirst) - errWrapper := ext1.DNSTransport.(*dnsTransportErrWrapper) + txp := wrapDNSTransport(orig) + errWrapper := txp.(*dnsTransportErrWrapper) underlying := errWrapper.DNSTransport if orig != underlying { t.Fatal("unexpected underlying transport") diff --git a/pkg/netxlite/errno.go b/pkg/netxlite/errno.go index eb9db4cd9..71ca5777a 100644 --- a/pkg/netxlite/errno.go +++ b/pkg/netxlite/errno.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2023-05-31 11:09:59.128923 +0200 CEST m=+0.224674501 +// Generated: 2023-10-04 17:50:11.285004 +0200 CEST m=+0.213615376 package netxlite diff --git a/pkg/netxlite/errno_darwin.go b/pkg/netxlite/errno_darwin.go index 2351a2140..6226dd486 100644 --- a/pkg/netxlite/errno_darwin.go +++ b/pkg/netxlite/errno_darwin.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2023-05-31 11:09:58.904502 +0200 CEST m=+0.000247709 +// Generated: 2023-10-04 17:50:11.071975 +0200 CEST m=+0.000581043 package netxlite diff --git a/pkg/netxlite/errno_darwin_test.go b/pkg/netxlite/errno_darwin_test.go index e8c6a1078..e38f1dcdd 100644 --- a/pkg/netxlite/errno_darwin_test.go +++ b/pkg/netxlite/errno_darwin_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2023-05-31 11:09:58.962606 +0200 CEST m=+0.058353251 +// Generated: 2023-10-04 17:50:11.136114 +0200 CEST m=+0.064721709 package netxlite diff --git a/pkg/netxlite/errno_freebsd.go b/pkg/netxlite/errno_freebsd.go index f7de074f2..192a4bd5d 100644 --- a/pkg/netxlite/errno_freebsd.go +++ b/pkg/netxlite/errno_freebsd.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2023-05-31 11:09:58.986123 +0200 CEST m=+0.081871334 +// Generated: 2023-10-04 17:50:11.159 +0200 CEST m=+0.087608168 package netxlite diff --git a/pkg/netxlite/errno_freebsd_test.go b/pkg/netxlite/errno_freebsd_test.go index e75561256..6038c8363 100644 --- a/pkg/netxlite/errno_freebsd_test.go +++ b/pkg/netxlite/errno_freebsd_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2023-05-31 11:09:59.006625 +0200 CEST m=+0.102373584 +// Generated: 2023-10-04 17:50:11.179697 +0200 CEST m=+0.108305543 package netxlite diff --git a/pkg/netxlite/errno_linux.go b/pkg/netxlite/errno_linux.go index 2f2da2ca9..f285773a6 100644 --- a/pkg/netxlite/errno_linux.go +++ b/pkg/netxlite/errno_linux.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2023-05-31 11:09:59.059949 +0200 CEST m=+0.155698959 +// Generated: 2023-10-04 17:50:11.226097 +0200 CEST m=+0.154706876 package netxlite diff --git a/pkg/netxlite/errno_linux_test.go b/pkg/netxlite/errno_linux_test.go index 344e6818e..65668633a 100644 --- a/pkg/netxlite/errno_linux_test.go +++ b/pkg/netxlite/errno_linux_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2023-05-31 11:09:59.078206 +0200 CEST m=+0.173955501 +// Generated: 2023-10-04 17:50:11.242256 +0200 CEST m=+0.170866459 package netxlite diff --git a/pkg/netxlite/errno_openbsd.go b/pkg/netxlite/errno_openbsd.go index 4db2a875f..4214f616a 100644 --- a/pkg/netxlite/errno_openbsd.go +++ b/pkg/netxlite/errno_openbsd.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2023-05-31 11:09:59.024286 +0200 CEST m=+0.120034584 +// Generated: 2023-10-04 17:50:11.195049 +0200 CEST m=+0.123657543 package netxlite diff --git a/pkg/netxlite/errno_openbsd_test.go b/pkg/netxlite/errno_openbsd_test.go index 44516da2b..7d0295e57 100644 --- a/pkg/netxlite/errno_openbsd_test.go +++ b/pkg/netxlite/errno_openbsd_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2023-05-31 11:09:59.042781 +0200 CEST m=+0.138530209 +// Generated: 2023-10-04 17:50:11.211149 +0200 CEST m=+0.139758251 package netxlite diff --git a/pkg/netxlite/errno_windows.go b/pkg/netxlite/errno_windows.go index dbfb2d214..5aba24ab9 100644 --- a/pkg/netxlite/errno_windows.go +++ b/pkg/netxlite/errno_windows.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2023-05-31 11:09:59.094846 +0200 CEST m=+0.190596584 +// Generated: 2023-10-04 17:50:11.2566 +0200 CEST m=+0.185210126 package netxlite diff --git a/pkg/netxlite/errno_windows_test.go b/pkg/netxlite/errno_windows_test.go index e1377a45d..36cce00a6 100644 --- a/pkg/netxlite/errno_windows_test.go +++ b/pkg/netxlite/errno_windows_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2023-05-31 11:09:59.11131 +0200 CEST m=+0.207060334 +// Generated: 2023-10-04 17:50:11.270225 +0200 CEST m=+0.198835459 package netxlite diff --git a/pkg/netxlite/http.go b/pkg/netxlite/http.go deleted file mode 100644 index c6f8b7fdc..000000000 --- a/pkg/netxlite/http.go +++ /dev/null @@ -1,388 +0,0 @@ -package netxlite - -// -// HTTP/1.1 and HTTP2 code -// - -import ( - "context" - "errors" - "net" - "net/http" - "net/url" - "time" - - oohttp "github.com/ooni/oohttp" - "github.com/ooni/probe-engine/pkg/model" -) - -// httpTransportErrWrapper is an HTTPTransport with error wrapping. -type httpTransportErrWrapper struct { - HTTPTransport model.HTTPTransport -} - -var _ model.HTTPTransport = &httpTransportErrWrapper{} - -func (txp *httpTransportErrWrapper) RoundTrip(req *http.Request) (*http.Response, error) { - resp, err := txp.HTTPTransport.RoundTrip(req) - if err != nil { - return nil, NewTopLevelGenericErrWrapper(err) - } - return resp, nil -} - -func (txp *httpTransportErrWrapper) CloseIdleConnections() { - txp.HTTPTransport.CloseIdleConnections() -} - -func (txp *httpTransportErrWrapper) Network() string { - return txp.HTTPTransport.Network() -} - -// httpTransportLogger is an HTTPTransport with logging. -type httpTransportLogger struct { - // HTTPTransport is the underlying HTTP transport. - HTTPTransport model.HTTPTransport - - // Logger is the underlying logger. - Logger model.DebugLogger -} - -var _ model.HTTPTransport = &httpTransportLogger{} - -func (txp *httpTransportLogger) RoundTrip(req *http.Request) (*http.Response, error) { - txp.Logger.Debugf("> %s %s", req.Method, req.URL.String()) - for key, values := range req.Header { - for _, value := range values { - txp.Logger.Debugf("> %s: %s", key, value) - } - } - txp.Logger.Debug(">") - resp, err := txp.HTTPTransport.RoundTrip(req) - if err != nil { - txp.Logger.Debugf("< %s", err) - return nil, err - } - txp.Logger.Debugf("< %d", resp.StatusCode) - for key, values := range resp.Header { - for _, value := range values { - txp.Logger.Debugf("< %s: %s", key, value) - } - } - txp.Logger.Debug("<") - return resp, nil -} - -func (txp *httpTransportLogger) CloseIdleConnections() { - txp.HTTPTransport.CloseIdleConnections() -} - -func (txp *httpTransportLogger) Network() string { - return txp.HTTPTransport.Network() -} - -// httpTransportConnectionsCloser is an HTTPTransport that -// correctly forwards CloseIdleConnections calls. -type httpTransportConnectionsCloser struct { - HTTPTransport model.HTTPTransport - Dialer model.Dialer - TLSDialer model.TLSDialer -} - -var _ model.HTTPTransport = &httpTransportConnectionsCloser{} - -func (txp *httpTransportConnectionsCloser) RoundTrip(req *http.Request) (*http.Response, error) { - return txp.HTTPTransport.RoundTrip(req) -} - -func (txp *httpTransportConnectionsCloser) Network() string { - return txp.HTTPTransport.Network() -} - -// CloseIdleConnections forwards the CloseIdleConnections calls. -func (txp *httpTransportConnectionsCloser) CloseIdleConnections() { - txp.HTTPTransport.CloseIdleConnections() - txp.Dialer.CloseIdleConnections() - txp.TLSDialer.CloseIdleConnections() -} - -// NewHTTPTransportWithLoggerResolverAndOptionalProxyURL creates an HTTPTransport using -// the given logger and resolver and an optional proxy URL. -// -// Arguments: -// -// - logger is the MANDATORY logger; -// -// - resolver is the MANDATORY resolver; -// -// - purl is the OPTIONAL proxy URL. -func NewHTTPTransportWithLoggerResolverAndOptionalProxyURL( - logger model.DebugLogger, resolver model.Resolver, purl *url.URL) model.HTTPTransport { - dialer := NewDialerWithResolver(logger, resolver) - dialer = MaybeWrapWithProxyDialer(dialer, purl) - handshaker := NewTLSHandshakerStdlib(logger) - tlsDialer := NewTLSDialer(dialer, handshaker) - return NewHTTPTransport(logger, dialer, tlsDialer) -} - -// NewHTTPTransportWithResolver creates a new HTTP transport using -// the stdlib for everything but the given resolver. -func NewHTTPTransportWithResolver(logger model.DebugLogger, reso model.Resolver) model.HTTPTransport { - dialer := NewDialerWithResolver(logger, reso) - thx := NewTLSHandshakerStdlib(logger) - tlsDialer := NewTLSDialer(dialer, thx) - return NewHTTPTransport(logger, dialer, tlsDialer) -} - -// NewHTTPTransport returns a wrapped HTTP transport for HTTP2 and HTTP/1.1 -// using the given dialer and logger. -// -// The returned transport will gracefully handle TLS connections -// created using gitlab.com/yawning/utls.git, if the TLS dialer -// is a dialer using such library for TLS operations. -// -// The returned transport will not have a configured proxy, not -// even the proxy configurable from the environment. -// -// QUIRK: the returned transport will disable transparent decompression -// of compressed response bodies (and will not automatically -// ask for such compression, though you can always do that manually). -// -// The returned transport will configure TCP and TLS connections -// created using its dialer and TLS dialer to always have a -// read watchdog timeout to address https://github.com/ooni/probe/issues/1609. -// -// QUIRK: the returned transport will always enforce 1 connection per host -// and we cannot get rid of this QUIRK requirement because it is -// necessary to perform sane measurements with tracing. We will be -// able to possibly relax this requirement after we change the -// way in which we perform measurements. -// -// This factory and NewHTTPTransportStdlib are the recommended -// ways of creating a new HTTPTransport. -func NewHTTPTransport(logger model.DebugLogger, dialer model.Dialer, tlsDialer model.TLSDialer) model.HTTPTransport { - return WrapHTTPTransport(logger, newOOHTTPBaseTransport(dialer, tlsDialer)) -} - -// newOOHTTPBaseTransport is the low-level factory used by NewHTTPTransport -// to create a new, suitable HTTPTransport for HTTP2 and HTTP/1.1. -// -// This factory uses github.com/ooni/oohttp, hence its name. -func newOOHTTPBaseTransport(dialer model.Dialer, tlsDialer model.TLSDialer) model.HTTPTransport { - // Using oohttp to support any TLS library. - txp := oohttp.DefaultTransport.(*oohttp.Transport).Clone() - - // This wrapping ensures that we always have a timeout when we - // are using HTTP; see https://github.com/ooni/probe/issues/1609. - dialer = &httpDialerWithReadTimeout{dialer} - txp.DialContext = dialer.DialContext - tlsDialer = &httpTLSDialerWithReadTimeout{tlsDialer} - txp.DialTLSContext = tlsDialer.DialTLSContext - - // We are using a different strategy to implement proxy: we - // use a specific dialer that knows about proxying. - txp.Proxy = nil - - // Better for Cloudflare DNS and also better because we have less - // noisy events and we can better understand what happened. - txp.MaxConnsPerHost = 1 - - // The following (1) reduces the number of headers that Go will - // automatically send for us and (2) ensures that we always receive - // back the true headers, such as Content-Length. This change is - // functional to OONI's goal of observing the network. - txp.DisableCompression = true - - // Required to enable using HTTP/2 (which will be anyway forced - // upon us when we are using TLS parroting). - txp.ForceAttemptHTTP2 = true - - // Ensure we correctly forward CloseIdleConnections. - return &httpTransportConnectionsCloser{ - HTTPTransport: &httpTransportStdlib{&oohttp.StdlibTransport{Transport: txp}}, - Dialer: dialer, - TLSDialer: tlsDialer, - } -} - -// stdlibTransport wraps oohttp.StdlibTransport to add .Network() -type httpTransportStdlib struct { - StdlibTransport *oohttp.StdlibTransport -} - -var _ model.HTTPTransport = &httpTransportStdlib{} - -func (txp *httpTransportStdlib) CloseIdleConnections() { - txp.StdlibTransport.CloseIdleConnections() -} - -func (txp *httpTransportStdlib) RoundTrip(req *http.Request) (*http.Response, error) { - return txp.StdlibTransport.RoundTrip(req) -} - -// Network implements HTTPTransport.Network. -func (txp *httpTransportStdlib) Network() string { - return "tcp" -} - -// WrapHTTPTransport creates an HTTPTransport using the given logger -// and guarantees that returned errors are wrapped. -// -// This is a low level factory. Consider not using it directly. -func WrapHTTPTransport(logger model.DebugLogger, txp model.HTTPTransport) model.HTTPTransport { - return &httpTransportLogger{ - HTTPTransport: &httpTransportErrWrapper{txp}, - Logger: logger, - } -} - -// httpDialerWithReadTimeout enforces a read timeout for all HTTP -// connections. See https://github.com/ooni/probe/issues/1609. -type httpDialerWithReadTimeout struct { - Dialer model.Dialer -} - -var _ model.Dialer = &httpDialerWithReadTimeout{} - -func (d *httpDialerWithReadTimeout) CloseIdleConnections() { - d.Dialer.CloseIdleConnections() -} - -// DialContext implements Dialer.DialContext. -func (d *httpDialerWithReadTimeout) DialContext( - ctx context.Context, network, address string) (net.Conn, error) { - conn, err := d.Dialer.DialContext(ctx, network, address) - if err != nil { - return nil, err - } - return &httpConnWithReadTimeout{conn}, nil -} - -// httpTLSDialerWithReadTimeout enforces a read timeout for all HTTP -// connections. See https://github.com/ooni/probe/issues/1609. -type httpTLSDialerWithReadTimeout struct { - TLSDialer model.TLSDialer -} - -var _ model.TLSDialer = &httpTLSDialerWithReadTimeout{} - -func (d *httpTLSDialerWithReadTimeout) CloseIdleConnections() { - d.TLSDialer.CloseIdleConnections() -} - -// ErrNotTLSConn occur when an interface accepts a net.Conn but -// internally needs a TLSConn and you pass a net.Conn that doesn't -// implement TLSConn to such an interface. -var ErrNotTLSConn = errors.New("not a TLSConn") - -// DialTLSContext implements TLSDialer's DialTLSContext. -func (d *httpTLSDialerWithReadTimeout) DialTLSContext( - ctx context.Context, network, address string) (net.Conn, error) { - conn, err := d.TLSDialer.DialTLSContext(ctx, network, address) - if err != nil { - return nil, err - } - tconn, okay := conn.(TLSConn) // part of the contract but let's be graceful - if !okay { - conn.Close() // we own the conn here - return nil, ErrNotTLSConn - } - return &httpTLSConnWithReadTimeout{tconn}, nil -} - -// httpConnWithReadTimeout enforces a read timeout for all HTTP -// connections. See https://github.com/ooni/probe/issues/1609. -type httpConnWithReadTimeout struct { - net.Conn -} - -// httpConnReadTimeout is the read timeout we apply to all HTTP -// conns (see https://github.com/ooni/probe/issues/1609). -// -// This timeout is meant as a fallback mechanism so that a stuck -// connection will _eventually_ fail. This is why it is set to -// a large value (300 seconds when writing this note). -// -// There should be other mechanisms to ensure that the code is -// lively: the context during the RoundTrip and iox.ReadAllContext -// when reading the body. They should kick in earlier. But we -// additionally want to avoid leaking a (parked?) connection and -// the corresponding goroutine, hence this large timeout. -// -// A future @bassosimone may understand this problem even better -// and possibly apply an even better fix to this issue. This -// will happen when we'll be able to further study the anomalies -// described in https://github.com/ooni/probe/issues/1609. -const httpConnReadTimeout = 300 * time.Second - -// Read implements Conn.Read. -func (c *httpConnWithReadTimeout) Read(b []byte) (int, error) { - c.Conn.SetReadDeadline(time.Now().Add(httpConnReadTimeout)) - defer c.Conn.SetReadDeadline(time.Time{}) - return c.Conn.Read(b) -} - -// httpTLSConnWithReadTimeout enforces a read timeout for all HTTP -// connections. See https://github.com/ooni/probe/issues/1609. -type httpTLSConnWithReadTimeout struct { - TLSConn -} - -// Read implements Conn.Read. -func (c *httpTLSConnWithReadTimeout) Read(b []byte) (int, error) { - c.TLSConn.SetReadDeadline(time.Now().Add(httpConnReadTimeout)) - defer c.TLSConn.SetReadDeadline(time.Time{}) - return c.TLSConn.Read(b) -} - -// NewHTTPTransportStdlib creates a new HTTPTransport using -// the stdlib for DNS resolutions and TLS. -// -// This factory calls NewHTTPTransport with suitable dialers. -// -// This factory and NewHTTPTransport are the recommended -// ways of creating a new HTTPTransport. -func NewHTTPTransportStdlib(logger model.DebugLogger) model.HTTPTransport { - dialer := NewDialerWithResolver(logger, NewStdlibResolver(logger)) - tlsDialer := NewTLSDialer(dialer, NewTLSHandshakerStdlib(logger)) - return NewHTTPTransport(logger, dialer, tlsDialer) -} - -// NewHTTPClientStdlib creates a new HTTPClient that uses the -// standard library for TLS and DNS resolutions. -func NewHTTPClientStdlib(logger model.DebugLogger) model.HTTPClient { - txp := NewHTTPTransportStdlib(logger) - return NewHTTPClient(txp) -} - -// NewHTTPClientWithResolver creates a new HTTPTransport using the -// given resolver and then from that builds an HTTPClient. -func NewHTTPClientWithResolver(logger model.Logger, reso model.Resolver) model.HTTPClient { - return NewHTTPClient(NewHTTPTransportWithResolver(logger, reso)) -} - -// NewHTTPClient creates a new, wrapped HTTPClient using the given transport. -func NewHTTPClient(txp model.HTTPTransport) model.HTTPClient { - return WrapHTTPClient(&http.Client{Transport: txp}) -} - -// WrapHTTPClient wraps an HTTP client to add error wrapping capabilities. -func WrapHTTPClient(clnt model.HTTPClient) model.HTTPClient { - return &httpClientErrWrapper{clnt} -} - -type httpClientErrWrapper struct { - HTTPClient model.HTTPClient -} - -func (c *httpClientErrWrapper) Do(req *http.Request) (*http.Response, error) { - resp, err := c.HTTPClient.Do(req) - if err != nil { - return nil, NewTopLevelGenericErrWrapper(err) - } - return resp, nil -} - -func (c *httpClientErrWrapper) CloseIdleConnections() { - c.HTTPClient.CloseIdleConnections() -} diff --git a/pkg/netxlite/http3.go b/pkg/netxlite/http3.go index 838f5809e..f4e2b3a4d 100644 --- a/pkg/netxlite/http3.go +++ b/pkg/netxlite/http3.go @@ -64,17 +64,24 @@ func NewHTTP3Transport( // NewHTTP3TransportStdlib creates a new HTTPTransport using http3 that // uses standard functionality for everything but the logger. -func NewHTTP3TransportStdlib(logger model.DebugLogger) model.HTTPTransport { - ql := NewQUICListener() - reso := NewStdlibResolver(logger) - qd := NewQUICDialerWithResolver(ql, logger, reso) +func (netx *Netx) NewHTTP3TransportStdlib(logger model.DebugLogger) model.HTTPTransport { + ql := netx.NewUDPListener() + reso := netx.NewStdlibResolver(logger) + qd := netx.NewQUICDialerWithResolver(ql, logger, reso) return NewHTTP3Transport(logger, qd, nil) } +// NewHTTP3TransportStdlib is equivalent to creating an empty [*Netx] +// and calling its NewHTTP3TransportStdlib method. +func NewHTTP3TransportStdlib(logger model.DebugLogger) model.HTTPTransport { + netx := &Netx{Underlying: nil} + return netx.NewHTTP3TransportStdlib(logger) +} + // NewHTTPTransportWithResolver creates a new HTTPTransport using http3 // that uses the given logger and the given resolver. func NewHTTP3TransportWithResolver(logger model.DebugLogger, reso model.Resolver) model.HTTPTransport { - qd := NewQUICDialerWithResolver(NewQUICListener(), logger, reso) + qd := NewQUICDialerWithResolver(NewUDPListener(), logger, reso) return NewHTTP3Transport(logger, qd, nil) } diff --git a/pkg/netxlite/http3_test.go b/pkg/netxlite/http3_test.go index e9c6e640c..467ff78b8 100644 --- a/pkg/netxlite/http3_test.go +++ b/pkg/netxlite/http3_test.go @@ -162,7 +162,7 @@ func TestNewHTTP3ClientWithResolver(t *testing.T) { if dialerReso.Resolver != reso { t.Fatal("invalid resolver") } - if dialerQUICGo.QUICListener == nil { - t.Fatal("QUICListener should not be nil") + if dialerQUICGo.UDPListener == nil { + t.Fatal("UDPListener should not be nil") } } diff --git a/pkg/netxlite/http_test.go b/pkg/netxlite/http_test.go deleted file mode 100644 index de6c9f3be..000000000 --- a/pkg/netxlite/http_test.go +++ /dev/null @@ -1,629 +0,0 @@ -package netxlite - -import ( - "context" - "errors" - "io" - "net" - "net/http" - "net/url" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/apex/log" - "github.com/ooni/probe-engine/pkg/mocks" - "github.com/ooni/probe-engine/pkg/model" -) - -func TestNewHTTPTransportWithLoggerResolverAndOptionalProxyURL(t *testing.T) { - t.Run("without proxy URL", func(t *testing.T) { - logger := &mocks.Logger{} - resolver := &mocks.Resolver{} - txp := NewHTTPTransportWithLoggerResolverAndOptionalProxyURL(logger, resolver, nil) - txpLogger := txp.(*httpTransportLogger) - if txpLogger.Logger != logger { - t.Fatal("unexpected logger") - } - txpErrWrapper := txpLogger.HTTPTransport.(*httpTransportErrWrapper) - txpCc := txpErrWrapper.HTTPTransport.(*httpTransportConnectionsCloser) - dialer := txpCc.Dialer - dialerWithReadTimeout := dialer.(*httpDialerWithReadTimeout) - dialerLog := dialerWithReadTimeout.Dialer.(*dialerLogger) - dialerReso := dialerLog.Dialer.(*dialerResolverWithTracing) - if dialerReso.Resolver != resolver { - t.Fatal("invalid resolver") - } - }) - - t.Run("with proxy URL", func(t *testing.T) { - URL := &url.URL{} - logger := &mocks.Logger{} - resolver := &mocks.Resolver{} - txp := NewHTTPTransportWithLoggerResolverAndOptionalProxyURL(logger, resolver, URL) - txpLogger := txp.(*httpTransportLogger) - if txpLogger.Logger != logger { - t.Fatal("unexpected logger") - } - txpErrWrapper := txpLogger.HTTPTransport.(*httpTransportErrWrapper) - txpCc := txpErrWrapper.HTTPTransport.(*httpTransportConnectionsCloser) - dialer := txpCc.Dialer - dialerWithReadTimeout := dialer.(*httpDialerWithReadTimeout) - dialerProxy := dialerWithReadTimeout.Dialer.(*proxyDialer) - dialerLog := dialerProxy.Dialer.(*dialerLogger) - dialerReso := dialerLog.Dialer.(*dialerResolverWithTracing) - if dialerReso.Resolver != resolver { - t.Fatal("invalid resolver") - } - }) -} - -func TestNewHTTPTransportWithResolver(t *testing.T) { - expected := errors.New("mocked error") - reso := &mocks.Resolver{ - MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { - return nil, expected - }, - } - txp := NewHTTPTransportWithResolver(model.DiscardLogger, reso) - req, err := http.NewRequest("GET", "http://x.org", nil) - if err != nil { - t.Fatal(err) - } - resp, err := txp.RoundTrip(req) - if !errors.Is(err, expected) { - t.Fatal("unexpected err") - } - if resp != nil { - t.Fatal("expected nil resp") - } -} - -func TestHTTPTransportErrWrapper(t *testing.T) { - t.Run("RoundTrip", func(t *testing.T) { - t.Run("with failure", func(t *testing.T) { - txp := &httpTransportErrWrapper{ - HTTPTransport: &mocks.HTTPTransport{ - MockRoundTrip: func(req *http.Request) (*http.Response, error) { - return nil, io.EOF - }, - }, - } - resp, err := txp.RoundTrip(&http.Request{}) - var errWrapper *ErrWrapper - if !errors.As(err, &errWrapper) { - t.Fatal("the returned error is not an ErrWrapper") - } - if errWrapper.Failure != FailureEOFError { - t.Fatal("unexpected failure", errWrapper.Failure) - } - if resp != nil { - t.Fatal("expected nil response") - } - }) - - t.Run("with success", func(t *testing.T) { - expect := &http.Response{} - txp := &httpTransportErrWrapper{ - HTTPTransport: &mocks.HTTPTransport{ - MockRoundTrip: func(req *http.Request) (*http.Response, error) { - return expect, nil - }, - }, - } - resp, err := txp.RoundTrip(&http.Request{}) - if err != nil { - t.Fatal(err) - } - if resp != expect { - t.Fatal("not the expected response") - } - }) - }) -} - -func TestHTTPTransportLogger(t *testing.T) { - t.Run("RoundTrip", func(t *testing.T) { - t.Run("with failure", func(t *testing.T) { - var count int - lo := &mocks.Logger{ - MockDebug: func(message string) { - count++ - }, - MockDebugf: func(format string, v ...interface{}) { - count++ - }, - } - txp := &httpTransportLogger{ - Logger: lo, - HTTPTransport: &mocks.HTTPTransport{ - MockRoundTrip: func(req *http.Request) (*http.Response, error) { - return nil, io.EOF - }, - }, - } - client := &http.Client{Transport: txp} - resp, err := client.Get("https://www.google.com") - if !errors.Is(err, io.EOF) { - t.Fatal("not the error we expected") - } - if resp != nil { - t.Fatal("expected nil response here") - } - if count < 1 { - t.Fatal("no logs?!") - } - }) - - t.Run("with success", func(t *testing.T) { - var count int - lo := &mocks.Logger{ - MockDebug: func(message string) { - count++ - }, - MockDebugf: func(format string, v ...interface{}) { - count++ - }, - } - txp := &httpTransportLogger{ - Logger: lo, - HTTPTransport: &mocks.HTTPTransport{ - MockRoundTrip: func(req *http.Request) (*http.Response, error) { - return &http.Response{ - Body: io.NopCloser(strings.NewReader("")), - Header: http.Header{ - "Server": []string{"antani/0.1.0"}, - }, - StatusCode: 200, - }, nil - }, - }, - } - client := &http.Client{Transport: txp} - req, err := http.NewRequest("GET", "https://www.google.com", nil) - if err != nil { - t.Fatal(err) - } - req.Header.Set("User-Agent", "miniooni/0.1.0-dev") - resp, err := client.Do(req) - if err != nil { - t.Fatal(err) - } - ReadAllContext(context.Background(), resp.Body) - resp.Body.Close() - if count < 1 { - t.Fatal("no logs?!") - } - }) - }) - - t.Run("CloseIdleConnections", func(t *testing.T) { - calls := &atomic.Int64{} - txp := &httpTransportLogger{ - HTTPTransport: &mocks.HTTPTransport{ - MockCloseIdleConnections: func() { - calls.Add(1) - }, - }, - Logger: log.Log, - } - txp.CloseIdleConnections() - if calls.Load() != 1 { - t.Fatal("not called") - } - }) -} - -func TestHTTPTransportConnectionsCloser(t *testing.T) { - t.Run("CloseIdleConnections", func(t *testing.T) { - var ( - calledTxp bool - calledDialer bool - calledTLS bool - ) - txp := &httpTransportConnectionsCloser{ - HTTPTransport: &mocks.HTTPTransport{ - MockCloseIdleConnections: func() { - calledTxp = true - }, - }, - Dialer: &mocks.Dialer{ - MockCloseIdleConnections: func() { - calledDialer = true - }, - }, - TLSDialer: &mocks.TLSDialer{ - MockCloseIdleConnections: func() { - calledTLS = true - }, - }, - } - txp.CloseIdleConnections() - if !calledDialer || !calledTLS || !calledTxp { - t.Fatal("not called") - } - }) - - t.Run("RoundTrip", func(t *testing.T) { - expected := errors.New("mocked error") - txp := &httpTransportConnectionsCloser{ - HTTPTransport: &mocks.HTTPTransport{ - MockRoundTrip: func(req *http.Request) (*http.Response, error) { - return nil, expected - }, - }, - } - client := &http.Client{Transport: txp} - resp, err := client.Get("https://www.google.com") - if !errors.Is(err, expected) { - t.Fatal("unexpected err", err) - } - if resp != nil { - t.Fatal("unexpected resp") - } - }) -} - -func TestNewHTTPTransport(t *testing.T) { - t.Run("works as intended with failing dialer", func(t *testing.T) { - called := &atomic.Int64{} - expected := errors.New("mocked error") - d := &dialerResolverWithTracing{ - Dialer: &mocks.Dialer{ - MockDialContext: func(ctx context.Context, - network, address string) (net.Conn, error) { - return nil, expected - }, - MockCloseIdleConnections: func() { - called.Add(1) - }, - }, - Resolver: NewStdlibResolver(log.Log), - } - td := NewTLSDialer(d, NewTLSHandshakerStdlib(log.Log)) - txp := NewHTTPTransport(log.Log, d, td) - client := &http.Client{Transport: txp} - resp, err := client.Get("https://8.8.4.4/robots.txt") - if !errors.Is(err, expected) { - t.Fatal("not the error we expected", err) - } - if resp != nil { - t.Fatal("expected non-nil response here") - } - client.CloseIdleConnections() - if called.Load() < 1 { - t.Fatal("did not propagate CloseIdleConnections") - } - }) - - t.Run("creates the correct type chain", func(t *testing.T) { - d := &mocks.Dialer{} - td := &mocks.TLSDialer{} - txp := NewHTTPTransport(log.Log, d, td) - logger := txp.(*httpTransportLogger) - if logger.Logger != log.Log { - t.Fatal("invalid logger") - } - errWrapper := logger.HTTPTransport.(*httpTransportErrWrapper) - connectionsCloser := errWrapper.HTTPTransport.(*httpTransportConnectionsCloser) - withReadTimeout := connectionsCloser.Dialer.(*httpDialerWithReadTimeout) - if withReadTimeout.Dialer != d { - t.Fatal("invalid dialer") - } - tlsWithReadTimeout := connectionsCloser.TLSDialer.(*httpTLSDialerWithReadTimeout) - if tlsWithReadTimeout.TLSDialer != td { - t.Fatal("invalid tls dialer") - } - stdlib := connectionsCloser.HTTPTransport.(*httpTransportStdlib) - if !stdlib.StdlibTransport.ForceAttemptHTTP2 { - t.Fatal("invalid ForceAttemptHTTP2") - } - if !stdlib.StdlibTransport.DisableCompression { - t.Fatal("invalid DisableCompression") - } - if stdlib.StdlibTransport.MaxConnsPerHost != 1 { - t.Fatal("invalid MaxConnPerHost") - } - if stdlib.StdlibTransport.DialTLSContext == nil { - t.Fatal("invalid DialTLSContext") - } - if stdlib.StdlibTransport.DialContext == nil { - t.Fatal("invalid DialContext") - } - }) -} - -func TestHTTPDialerWithReadTimeout(t *testing.T) { - t.Run("DialContext", func(t *testing.T) { - t.Run("on success", func(t *testing.T) { - var ( - calledWithZeroTime bool - calledWithNonZeroTime bool - ) - origConn := &mocks.Conn{ - MockSetReadDeadline: func(t time.Time) error { - switch t.IsZero() { - case true: - calledWithZeroTime = true - case false: - calledWithNonZeroTime = true - } - return nil - }, - MockRead: func(b []byte) (int, error) { - return 0, io.EOF - }, - } - d := &httpDialerWithReadTimeout{ - Dialer: &mocks.Dialer{ - MockDialContext: func(ctx context.Context, network, address string) (net.Conn, error) { - return origConn, nil - }, - }, - } - ctx := context.Background() - conn, err := d.DialContext(ctx, "", "") - if err != nil { - t.Fatal(err) - } - if _, okay := conn.(*httpConnWithReadTimeout); !okay { - t.Fatal("invalid conn type") - } - if conn.(*httpConnWithReadTimeout).Conn != origConn { - t.Fatal("invalid origin conn") - } - b := make([]byte, 1024) - count, err := conn.Read(b) - if !errors.Is(err, io.EOF) { - t.Fatal("invalid error") - } - if count != 0 { - t.Fatal("invalid count") - } - if !calledWithZeroTime || !calledWithNonZeroTime { - t.Fatal("not called") - } - }) - - t.Run("on failure", func(t *testing.T) { - expected := errors.New("mocked error") - d := &httpDialerWithReadTimeout{ - Dialer: &mocks.Dialer{ - MockDialContext: func(ctx context.Context, network, address string) (net.Conn, error) { - return nil, expected - }, - }, - } - conn, err := d.DialContext(context.Background(), "", "") - if !errors.Is(err, expected) { - t.Fatal("not the error we expected") - } - if conn != nil { - t.Fatal("expected nil conn here") - } - }) - }) -} - -func TestHTTPTLSDialerWithReadTimeout(t *testing.T) { - t.Run("DialContext", func(t *testing.T) { - t.Run("on success", func(t *testing.T) { - var ( - calledWithZeroTime bool - calledWithNonZeroTime bool - ) - origConn := &mocks.TLSConn{ - Conn: mocks.Conn{ - MockSetReadDeadline: func(t time.Time) error { - switch t.IsZero() { - case true: - calledWithZeroTime = true - case false: - calledWithNonZeroTime = true - } - return nil - }, - MockRead: func(b []byte) (int, error) { - return 0, io.EOF - }, - }, - } - d := &httpTLSDialerWithReadTimeout{ - TLSDialer: &mocks.TLSDialer{ - MockDialTLSContext: func(ctx context.Context, network, address string) (net.Conn, error) { - return origConn, nil - }, - }, - } - ctx := context.Background() - conn, err := d.DialTLSContext(ctx, "", "") - if err != nil { - t.Fatal(err) - } - if _, okay := conn.(*httpTLSConnWithReadTimeout); !okay { - t.Fatal("invalid conn type") - } - if conn.(*httpTLSConnWithReadTimeout).TLSConn != origConn { - t.Fatal("invalid origin conn") - } - b := make([]byte, 1024) - count, err := conn.Read(b) - if !errors.Is(err, io.EOF) { - t.Fatal("invalid error") - } - if count != 0 { - t.Fatal("invalid count") - } - if !calledWithZeroTime || !calledWithNonZeroTime { - t.Fatal("not called") - } - }) - - t.Run("on failure", func(t *testing.T) { - expected := errors.New("mocked error") - d := &httpTLSDialerWithReadTimeout{ - TLSDialer: &mocks.TLSDialer{ - MockDialTLSContext: func(ctx context.Context, network, address string) (net.Conn, error) { - return nil, expected - }, - }, - } - conn, err := d.DialTLSContext(context.Background(), "", "") - if !errors.Is(err, expected) { - t.Fatal("not the error we expected") - } - if conn != nil { - t.Fatal("expected nil conn here") - } - }) - - t.Run("with invalid conn type", func(t *testing.T) { - var called bool - d := &httpTLSDialerWithReadTimeout{ - TLSDialer: &mocks.TLSDialer{ - MockDialTLSContext: func(ctx context.Context, network, address string) (net.Conn, error) { - return &mocks.Conn{ - MockClose: func() error { - called = true - return nil - }, - }, nil - }, - }, - } - conn, err := d.DialTLSContext(context.Background(), "", "") - if !errors.Is(err, ErrNotTLSConn) { - t.Fatal("not the error we expected") - } - if conn != nil { - t.Fatal("expected nil conn here") - } - if !called { - t.Fatal("not called") - } - }) - }) -} - -func TestNewHTTPTransportStdlib(t *testing.T) { - txp := NewHTTPTransportStdlib(log.Log) - ctx, cancel := context.WithCancel(context.Background()) - cancel() // immediately! - req, err := http.NewRequestWithContext(ctx, "GET", "http://x.org", nil) - if err != nil { - t.Fatal(err) - } - resp, err := txp.RoundTrip(req) - if !errors.Is(err, context.Canceled) { - t.Fatal("unexpected err", err) - } - if resp != nil { - t.Fatal("unexpected resp") - } - if txp.Network() != "tcp" { - t.Fatal("unexpected .Network return value") - } - txp.CloseIdleConnections() -} - -func TestHTTPClientErrWrapper(t *testing.T) { - t.Run("Do", func(t *testing.T) { - t.Run("with failure", func(t *testing.T) { - clnt := &httpClientErrWrapper{ - HTTPClient: &mocks.HTTPClient{ - MockDo: func(req *http.Request) (*http.Response, error) { - return nil, io.EOF - }, - }, - } - resp, err := clnt.Do(&http.Request{}) - var errWrapper *ErrWrapper - if !errors.As(err, &errWrapper) { - t.Fatal("the returned error is not an ErrWrapper") - } - if errWrapper.Failure != FailureEOFError { - t.Fatal("unexpected failure", errWrapper.Failure) - } - if resp != nil { - t.Fatal("expected nil response") - } - }) - - t.Run("with success", func(t *testing.T) { - expect := &http.Response{} - clnt := &httpClientErrWrapper{ - HTTPClient: &mocks.HTTPClient{ - MockDo: func(req *http.Request) (*http.Response, error) { - return expect, nil - }, - }, - } - resp, err := clnt.Do(&http.Request{}) - if err != nil { - t.Fatal(err) - } - if resp != expect { - t.Fatal("not the expected response") - } - }) - }) - - t.Run("CloseIdleConnections", func(t *testing.T) { - var called bool - child := &mocks.HTTPClient{ - MockCloseIdleConnections: func() { - called = true - }, - } - clnt := &httpClientErrWrapper{child} - clnt.CloseIdleConnections() - if !called { - t.Fatal("not called") - } - }) -} - -func TestNewHTTPClientStdlib(t *testing.T) { - clnt := NewHTTPClientStdlib(model.DiscardLogger) - ewc, ok := clnt.(*httpClientErrWrapper) - if !ok { - t.Fatal("expected *httpClientErrWrapper") - } - _, ok = ewc.HTTPClient.(*http.Client) - if !ok { - t.Fatal("expected *http.Client") - } -} - -func TestNewHTTPClientWithResolver(t *testing.T) { - reso := &mocks.Resolver{} - clnt := NewHTTPClientWithResolver(model.DiscardLogger, reso) - ewc, ok := clnt.(*httpClientErrWrapper) - if !ok { - t.Fatal("expected *httpClientErrWrapper") - } - httpClnt, ok := ewc.HTTPClient.(*http.Client) - if !ok { - t.Fatal("expected *http.Client") - } - txp := httpClnt.Transport.(*httpTransportLogger) - txpEwrap := txp.HTTPTransport.(*httpTransportErrWrapper) - txpCc := txpEwrap.HTTPTransport.(*httpTransportConnectionsCloser) - dialer := txpCc.Dialer.(*httpDialerWithReadTimeout) - dialerLogger := dialer.Dialer.(*dialerLogger) - dialerReso := dialerLogger.Dialer.(*dialerResolverWithTracing) - if dialerReso.Resolver != reso { - t.Fatal("invalid resolver") - } -} - -func TestWrapHTTPClient(t *testing.T) { - origClient := &http.Client{} - wrapped := WrapHTTPClient(origClient) - errWrapper := wrapped.(*httpClientErrWrapper) - innerClient := errWrapper.HTTPClient.(*http.Client) - if innerClient != origClient { - t.Fatal("not the inner client we expected") - } -} diff --git a/pkg/netxlite/httpcloser.go b/pkg/netxlite/httpcloser.go new file mode 100644 index 000000000..66e338862 --- /dev/null +++ b/pkg/netxlite/httpcloser.go @@ -0,0 +1,36 @@ +package netxlite + +// +// Code to ensure we forward CloseIdleConnection calls +// + +import ( + "net/http" + + "github.com/ooni/probe-engine/pkg/model" +) + +// httpTransportConnectionsCloser is an HTTPTransport that +// correctly forwards CloseIdleConnections calls. +type httpTransportConnectionsCloser struct { + HTTPTransport model.HTTPTransport + Dialer model.Dialer + TLSDialer model.TLSDialer +} + +var _ model.HTTPTransport = &httpTransportConnectionsCloser{} + +func (txp *httpTransportConnectionsCloser) RoundTrip(req *http.Request) (*http.Response, error) { + return txp.HTTPTransport.RoundTrip(req) +} + +func (txp *httpTransportConnectionsCloser) Network() string { + return txp.HTTPTransport.Network() +} + +// CloseIdleConnections forwards the CloseIdleConnections calls. +func (txp *httpTransportConnectionsCloser) CloseIdleConnections() { + txp.HTTPTransport.CloseIdleConnections() + txp.Dialer.CloseIdleConnections() + txp.TLSDialer.CloseIdleConnections() +} diff --git a/pkg/netxlite/httpcloser_test.go b/pkg/netxlite/httpcloser_test.go new file mode 100644 index 000000000..1f922f3b4 --- /dev/null +++ b/pkg/netxlite/httpcloser_test.go @@ -0,0 +1,59 @@ +package netxlite + +import ( + "errors" + "net/http" + "testing" + + "github.com/ooni/probe-engine/pkg/mocks" +) + +func TestHTTPTransportConnectionsCloser(t *testing.T) { + t.Run("CloseIdleConnections", func(t *testing.T) { + var ( + calledTxp bool + calledDialer bool + calledTLS bool + ) + txp := &httpTransportConnectionsCloser{ + HTTPTransport: &mocks.HTTPTransport{ + MockCloseIdleConnections: func() { + calledTxp = true + }, + }, + Dialer: &mocks.Dialer{ + MockCloseIdleConnections: func() { + calledDialer = true + }, + }, + TLSDialer: &mocks.TLSDialer{ + MockCloseIdleConnections: func() { + calledTLS = true + }, + }, + } + txp.CloseIdleConnections() + if !calledDialer || !calledTLS || !calledTxp { + t.Fatal("not called") + } + }) + + t.Run("RoundTrip", func(t *testing.T) { + expected := errors.New("mocked error") + txp := &httpTransportConnectionsCloser{ + HTTPTransport: &mocks.HTTPTransport{ + MockRoundTrip: func(req *http.Request) (*http.Response, error) { + return nil, expected + }, + }, + } + client := &http.Client{Transport: txp} + resp, err := client.Get("https://www.google.com") + if !errors.Is(err, expected) { + t.Fatal("unexpected err", err) + } + if resp != nil { + t.Fatal("unexpected resp") + } + }) +} diff --git a/pkg/netxlite/httperrwrap.go b/pkg/netxlite/httperrwrap.go new file mode 100644 index 000000000..6f0f68380 --- /dev/null +++ b/pkg/netxlite/httperrwrap.go @@ -0,0 +1,50 @@ +package netxlite + +// +// Code to ensure we wrap errors +// + +import ( + "net/http" + + "github.com/ooni/probe-engine/pkg/model" +) + +// httpTransportErrWrapper is an HTTPTransport with error wrapping. +type httpTransportErrWrapper struct { + HTTPTransport model.HTTPTransport +} + +var _ model.HTTPTransport = &httpTransportErrWrapper{} + +func (txp *httpTransportErrWrapper) RoundTrip(req *http.Request) (*http.Response, error) { + resp, err := txp.HTTPTransport.RoundTrip(req) + if err != nil { + return nil, NewTopLevelGenericErrWrapper(err) + } + return resp, nil +} + +func (txp *httpTransportErrWrapper) CloseIdleConnections() { + txp.HTTPTransport.CloseIdleConnections() +} + +func (txp *httpTransportErrWrapper) Network() string { + return txp.HTTPTransport.Network() +} + +type httpClientErrWrapper struct { + HTTPClient model.HTTPClient +} + +func (c *httpClientErrWrapper) Do(req *http.Request) (*http.Response, error) { + resp, err := c.HTTPClient.Do(req) + if err != nil { + return nil, NewTopLevelGenericErrWrapper(err) + } + return resp, nil +} + +func (c *httpClientErrWrapper) CloseIdleConnections() { + c.HTTPClient.CloseIdleConnections() +} diff --git a/pkg/netxlite/httperrwrap_test.go b/pkg/netxlite/httperrwrap_test.go new file mode 100644 index 000000000..330c46a8b --- /dev/null +++ b/pkg/netxlite/httperrwrap_test.go @@ -0,0 +1,110 @@ +package netxlite + +import ( + "errors" + "io" + "net/http" + "testing" + + "github.com/ooni/probe-engine/pkg/mocks" +) + +func TestHTTPTransportErrWrapper(t *testing.T) { + t.Run("RoundTrip", func(t *testing.T) { + t.Run("with failure", func(t *testing.T) { + txp := &httpTransportErrWrapper{ + HTTPTransport: &mocks.HTTPTransport{ + MockRoundTrip: func(req *http.Request) (*http.Response, error) { + return nil, io.EOF + }, + }, + } + resp, err := txp.RoundTrip(&http.Request{}) + var errWrapper *ErrWrapper + if !errors.As(err, &errWrapper) { + t.Fatal("the returned error is not an ErrWrapper") + } + if errWrapper.Failure != FailureEOFError { + t.Fatal("unexpected failure", errWrapper.Failure) + } + if resp != nil { + t.Fatal("expected nil response") + } + }) + + t.Run("with success", func(t *testing.T) { + expect := &http.Response{} + txp := &httpTransportErrWrapper{ + HTTPTransport: &mocks.HTTPTransport{ + MockRoundTrip: func(req *http.Request) (*http.Response, error) { + return expect, nil + }, + }, + } + resp, err := txp.RoundTrip(&http.Request{}) + if err != nil { + t.Fatal(err) + } + if resp != expect { + t.Fatal("not the expected response") + } + }) + }) +} + +func TestHTTPClientErrWrapper(t *testing.T) { + t.Run("Do", func(t *testing.T) { + t.Run("with failure", func(t *testing.T) { + clnt := &httpClientErrWrapper{ + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + return nil, io.EOF + }, + }, + } + resp, err := clnt.Do(&http.Request{}) + var errWrapper *ErrWrapper + if !errors.As(err, &errWrapper) { + t.Fatal("the returned error is not an ErrWrapper") + } + if errWrapper.Failure != FailureEOFError { + t.Fatal("unexpected failure", errWrapper.Failure) + } + if resp != nil { + t.Fatal("expected nil response") + } + }) + + t.Run("with success", func(t *testing.T) { + expect := &http.Response{} + clnt := &httpClientErrWrapper{ + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + return expect, nil + }, + }, + } + resp, err := clnt.Do(&http.Request{}) + if err != nil { + t.Fatal(err) + } + if resp != expect { + t.Fatal("not the expected response") + } + }) + }) + + t.Run("CloseIdleConnections", func(t *testing.T) { + var called bool + child := &mocks.HTTPClient{ + MockCloseIdleConnections: func() { + called = true + }, + } + clnt := &httpClientErrWrapper{child} + clnt.CloseIdleConnections() + if !called { + t.Fatal("not called") + } + }) +} diff --git a/pkg/netxlite/httpfactory.go b/pkg/netxlite/httpfactory.go new file mode 100644 index 000000000..41645af9f --- /dev/null +++ b/pkg/netxlite/httpfactory.go @@ -0,0 +1,110 @@ +package netxlite + +import ( + "crypto/tls" + "net/url" + + oohttp "github.com/ooni/oohttp" + "github.com/ooni/probe-engine/pkg/model" +) + +// HTTPTransportOption is an initialization option for [NewHTTPTransport]. +type HTTPTransportOption func(txp *oohttp.Transport) + +// NewHTTPTransport is the high-level factory to create a [model.HTTPTransport] using +// github.com/ooni/oohttp as the HTTP library with HTTP/1.1 and HTTP2 support. +// +// This transport is suitable for HTTP2 and HTTP/1.1 using any TLS +// library, including, e.g., github.com/ooni/oocrypto. +// +// This factory clones the default github.com/ooni/oohttp transport and +// configures the provided dialer and TLS dialer by setting the .DialContext +// and .DialTLSContext fields of the transport. We also wrap the provided +// dialers to address https://github.com/ooni/probe/issues/1609. +// +// Apart from that, the only non-default options set by this factory are these: +// +// 1. the .Proxy field is set to nil, so by default we DO NOT honour the +// HTTP_PROXY and HTTPS_PROXY environment variables, which is required if +// we want to use this code for measuring; +// +// 2. the .ForceAttemptHTTP2 field is set to true; +// +// 3. the .DisableCompression field is set to true, again required if we +// want to use this code for measuring, and we should make sure the defaults +// we're using are suitable for measuring, since the impact of making a +// mistake in measuring code is a data quality issue 😅. +// +// The returned transport supports logging and error wrapping because +// internally this function calls [WrapHTTPTransport] before we return. +// +// This factory is the RECOMMENDED way of creating a [model.HTTPTransport]. +func NewHTTPTransportWithOptions(logger model.Logger, + dialer model.Dialer, tlsDialer model.TLSDialer, options ...HTTPTransportOption) model.HTTPTransport { + // Using oohttp to support any TLS library. + txp := oohttp.DefaultTransport.(*oohttp.Transport).Clone() + + // This wrapping ensures that we always have a timeout when we + // are using HTTP; see https://github.com/ooni/probe/issues/1609. + dialer = &httpDialerWithReadTimeout{dialer} + txp.DialContext = dialer.DialContext + tlsDialer = &httpTLSDialerWithReadTimeout{tlsDialer} + txp.DialTLSContext = tlsDialer.DialTLSContext + + // As documented, disable proxies and force HTTP/2 + txp.DisableCompression = true + txp.Proxy = nil + txp.ForceAttemptHTTP2 = true + + // Apply all the required options + for _, option := range options { + option(txp) + } + + // Return a fully wrapped HTTP transport + return WrapHTTPTransport(logger, &httpTransportConnectionsCloser{ + HTTPTransport: &httpTransportStdlib{&oohttp.StdlibTransport{Transport: txp}}, + Dialer: dialer, + TLSDialer: tlsDialer, + }) +} + +// HTTPTransportOptionProxyURL configures the transport to use the given proxyURL +// or disables proxying (already the default) if the proxyURL is nil. +func HTTPTransportOptionProxyURL(proxyURL *url.URL) HTTPTransportOption { + return func(txp *oohttp.Transport) { + txp.Proxy = func(r *oohttp.Request) (*url.URL, error) { + // "If Proxy is nil or returns a nil *URL, no proxy is used." + return proxyURL, nil + } + } +} + +// HTTPTransportOptionMaxConnsPerHost configures the .MaxConnPerHosts field, which +// otherwise uses the default set in github.com/ooni/oohttp. +func HTTPTransportOptionMaxConnsPerHost(value int) HTTPTransportOption { + return func(txp *oohttp.Transport) { + txp.MaxConnsPerHost = value + } +} + +// HTTPTransportOptionDisableCompression configures the .DisableCompression field, which +// otherwise is set to true, so that this code is ready for measuring out of the box. +func HTTPTransportOptionDisableCompression(value bool) HTTPTransportOption { + return func(txp *oohttp.Transport) { + txp.DisableCompression = value + } +} + +// HTTPTransportOptionTLSClientConfig configures the .TLSClientConfig field, +// which otherwise is nil, to imply using the default config. +// +// TODO(https://github.com/ooni/probe/issues/2536): using the default config breaks +// tests using netem and this option is the workaround we're using to address +// this limitation. Future releases MIGHT use a different technique and, as such, +// we MAY remove this option when we don't need it anymore. +func HTTPTransportOptionTLSClientConfig(config *tls.Config) HTTPTransportOption { + return func(txp *oohttp.Transport) { + txp.TLSClientConfig = config + } +} diff --git a/pkg/netxlite/httpfactory_test.go b/pkg/netxlite/httpfactory_test.go new file mode 100644 index 000000000..269a54282 --- /dev/null +++ b/pkg/netxlite/httpfactory_test.go @@ -0,0 +1,169 @@ +package netxlite + +import ( + "crypto/tls" + "net/url" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + oohttp "github.com/ooni/oohttp" + "github.com/ooni/probe-engine/pkg/mocks" + "github.com/ooni/probe-engine/pkg/model" +) + +func TestNewHTTPTransportWithOptions(t *testing.T) { + + t.Run("make sure that we get the correct types and settings", func(t *testing.T) { + expectDialer := &mocks.Dialer{} + expectTLSDialer := &mocks.TLSDialer{} + expectLogger := model.DiscardLogger + txp := NewHTTPTransportWithOptions(expectLogger, expectDialer, expectTLSDialer) + + // undo the results of the netxlite.WrapTransport function + txpLogger := txp.(*httpTransportLogger) + if txpLogger.Logger != expectLogger { + t.Fatal("invalid logger") + } + txpErrWrapper := txpLogger.HTTPTransport.(*httpTransportErrWrapper) + + // make sure we correctly configured dialer and TLS dialer + txpCloser := txpErrWrapper.HTTPTransport.(*httpTransportConnectionsCloser) + timeoutDialer := txpCloser.Dialer.(*httpDialerWithReadTimeout) + childDialer := timeoutDialer.Dialer + if childDialer != expectDialer { + t.Fatal("invalid dialer") + } + timeoutTLSDialer := txpCloser.TLSDialer.(*httpTLSDialerWithReadTimeout) + childTLSDialer := timeoutTLSDialer.TLSDialer + if childTLSDialer != expectTLSDialer { + t.Fatal("invalid TLS dialer") + } + + // make sure there's the stdlib adapter + stdlibAdapter := txpCloser.HTTPTransport.(*httpTransportStdlib) + oohttpStdlibAdapter := stdlibAdapter.StdlibTransport + underlying := oohttpStdlibAdapter.Transport + + // now let's check that everything is configured as intended + expectedTxp := oohttp.DefaultTransport.(*oohttp.Transport).Clone() + diff := cmp.Diff( + expectedTxp, + underlying, + cmpopts.IgnoreUnexported(oohttp.Transport{}), + cmpopts.IgnoreUnexported(tls.Config{}), + cmpopts.IgnoreFields( + oohttp.Transport{}, + "DialContext", + "DialTLSContext", + "DisableCompression", + "Proxy", + "ForceAttemptHTTP2", + ), + ) + if diff != "" { + t.Fatal(diff) + } + + // finish checking by explicitly inspecting the fields we modify + if underlying.DialContext == nil { + t.Fatal("expected non-nil .DialContext") + } + if underlying.DialTLSContext == nil { + t.Fatal("expected non-nil .DialTLSContext") + } + if underlying.Proxy != nil { + t.Fatal("expected nil .Proxy") + } + if !underlying.ForceAttemptHTTP2 { + t.Fatal("expected true .ForceAttemptHTTP2") + } + if !underlying.DisableCompression { + t.Fatal("expected true .DisableCompression") + } + }) + + unwrap := func(txp model.HTTPTransport) *oohttp.Transport { + txpLogger := txp.(*httpTransportLogger) + txpErrWrapper := txpLogger.HTTPTransport.(*httpTransportErrWrapper) + txpCloser := txpErrWrapper.HTTPTransport.(*httpTransportConnectionsCloser) + stdlibAdapter := txpCloser.HTTPTransport.(*httpTransportStdlib) + oohttpStdlibAdapter := stdlibAdapter.StdlibTransport + return oohttpStdlibAdapter.Transport + } + + t.Run("make sure HTTPTransportOptionProxyURL is WAI", func(t *testing.T) { + runWithURL := func(expectedURL *url.URL) { + expectDialer := &mocks.Dialer{} + expectTLSDialer := &mocks.TLSDialer{} + expectLogger := model.DiscardLogger + txp := NewHTTPTransportWithOptions( + expectLogger, + expectDialer, + expectTLSDialer, + HTTPTransportOptionProxyURL(expectedURL), + ) + underlying := unwrap(txp) + if underlying.Proxy == nil { + t.Fatal("expected non-nil .Proxy") + } + got, err := underlying.Proxy(&oohttp.Request{}) + if err != nil { + t.Fatal(err) + } + if got != expectedURL { + t.Fatal("not the expected URL") + } + } + + runWithURL(&url.URL{}) + + runWithURL(nil) + }) + + t.Run("make sure HTTPTransportOptionMaxConnsPerHost is WAI", func(t *testing.T) { + runWithValue := func(expectedValue int) { + expectDialer := &mocks.Dialer{} + expectTLSDialer := &mocks.TLSDialer{} + expectLogger := model.DiscardLogger + txp := NewHTTPTransportWithOptions( + expectLogger, + expectDialer, + expectTLSDialer, + HTTPTransportOptionMaxConnsPerHost(expectedValue), + ) + underlying := unwrap(txp) + got := underlying.MaxConnsPerHost + if got != expectedValue { + t.Fatal("not the expected value") + } + } + + runWithValue(100) + + runWithValue(10) + }) + + t.Run("make sure HTTPTransportDisableCompression is WAI", func(t *testing.T) { + runWithValue := func(expectedValue bool) { + expectDialer := &mocks.Dialer{} + expectTLSDialer := &mocks.TLSDialer{} + expectLogger := model.DiscardLogger + txp := NewHTTPTransportWithOptions( + expectLogger, + expectDialer, + expectTLSDialer, + HTTPTransportOptionDisableCompression(expectedValue), + ) + underlying := unwrap(txp) + got := underlying.DisableCompression + if got != expectedValue { + t.Fatal("not the expected value") + } + } + + runWithValue(true) + + runWithValue(false) + }) +} diff --git a/pkg/netxlite/httplogger.go b/pkg/netxlite/httplogger.go new file mode 100644 index 000000000..e33803563 --- /dev/null +++ b/pkg/netxlite/httplogger.go @@ -0,0 +1,53 @@ +package netxlite + +// +// Code to ensure we log round trips +// + +import ( + "net/http" + + "github.com/ooni/probe-engine/pkg/model" +) + +// httpTransportLogger is an HTTPTransport with logging. +type httpTransportLogger struct { + // HTTPTransport is the underlying HTTP transport. + HTTPTransport model.HTTPTransport + + // Logger is the underlying logger. + Logger model.DebugLogger +} + +var _ model.HTTPTransport = &httpTransportLogger{} + +func (txp *httpTransportLogger) RoundTrip(req *http.Request) (*http.Response, error) { + txp.Logger.Debugf("> %s %s", req.Method, req.URL.String()) + for key, values := range req.Header { + for _, value := range values { + txp.Logger.Debugf("> %s: %s", key, value) + } + } + txp.Logger.Debug(">") + resp, err := txp.HTTPTransport.RoundTrip(req) + if err != nil { + txp.Logger.Debugf("< %s", err) + return nil, err + } + txp.Logger.Debugf("< %d", resp.StatusCode) + for key, values := range resp.Header { + for _, value := range values { + txp.Logger.Debugf("< %s: %s", key, value) + } + } + txp.Logger.Debug("<") + return resp, nil +} + +func (txp *httpTransportLogger) CloseIdleConnections() { + txp.HTTPTransport.CloseIdleConnections() +} + +func (txp *httpTransportLogger) Network() string { + return txp.HTTPTransport.Network() +} diff --git a/pkg/netxlite/httplogger_test.go b/pkg/netxlite/httplogger_test.go new file mode 100644 index 000000000..a015d5a2e --- /dev/null +++ b/pkg/netxlite/httplogger_test.go @@ -0,0 +1,106 @@ +package netxlite + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "sync/atomic" + "testing" + + "github.com/apex/log" + "github.com/ooni/probe-engine/pkg/mocks" +) + +func TestHTTPTransportLogger(t *testing.T) { + t.Run("RoundTrip", func(t *testing.T) { + t.Run("with failure", func(t *testing.T) { + var count int + lo := &mocks.Logger{ + MockDebug: func(message string) { + count++ + }, + MockDebugf: func(format string, v ...interface{}) { + count++ + }, + } + txp := &httpTransportLogger{ + Logger: lo, + HTTPTransport: &mocks.HTTPTransport{ + MockRoundTrip: func(req *http.Request) (*http.Response, error) { + return nil, io.EOF + }, + }, + } + client := &http.Client{Transport: txp} + resp, err := client.Get("https://www.google.com") + if !errors.Is(err, io.EOF) { + t.Fatal("not the error we expected") + } + if resp != nil { + t.Fatal("expected nil response here") + } + if count < 1 { + t.Fatal("no logs?!") + } + }) + + t.Run("with success", func(t *testing.T) { + var count int + lo := &mocks.Logger{ + MockDebug: func(message string) { + count++ + }, + MockDebugf: func(format string, v ...interface{}) { + count++ + }, + } + txp := &httpTransportLogger{ + Logger: lo, + HTTPTransport: &mocks.HTTPTransport{ + MockRoundTrip: func(req *http.Request) (*http.Response, error) { + return &http.Response{ + Body: io.NopCloser(strings.NewReader("")), + Header: http.Header{ + "Server": []string{"antani/0.1.0"}, + }, + StatusCode: 200, + }, nil + }, + }, + } + client := &http.Client{Transport: txp} + req, err := http.NewRequest("GET", "https://www.google.com", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("User-Agent", "miniooni/0.1.0-dev") + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + ReadAllContext(context.Background(), resp.Body) + resp.Body.Close() + if count < 1 { + t.Fatal("no logs?!") + } + }) + }) + + t.Run("CloseIdleConnections", func(t *testing.T) { + calls := &atomic.Int64{} + txp := &httpTransportLogger{ + HTTPTransport: &mocks.HTTPTransport{ + MockCloseIdleConnections: func() { + calls.Add(1) + }, + }, + Logger: log.Log, + } + txp.CloseIdleConnections() + if calls.Load() != 1 { + t.Fatal("not called") + } + }) +} diff --git a/pkg/netxlite/httpquirks.go b/pkg/netxlite/httpquirks.go new file mode 100644 index 000000000..8070857be --- /dev/null +++ b/pkg/netxlite/httpquirks.go @@ -0,0 +1,144 @@ +package netxlite + +// +// QUIRKy Legacy HTTP code and behavior assumed by ./legacy/netx 😅 +// +// Ideally, we should not modify this code or apply minimal and obvious changes. +// +// TODO(https://github.com/ooni/probe/issues/2534) +// + +import ( + "net/http" + + oohttp "github.com/ooni/oohttp" + "github.com/ooni/probe-engine/pkg/model" +) + +// NewHTTPTransportWithResolver creates a new HTTP transport using +// the stdlib for everything but the given resolver. +// +// This function behavior is QUIRKY as documented in [NewHTTPTransport]. +func NewHTTPTransportWithResolver(logger model.DebugLogger, reso model.Resolver) model.HTTPTransport { + dialer := NewDialerWithResolver(logger, reso) + thx := NewTLSHandshakerStdlib(logger) + tlsDialer := NewTLSDialer(dialer, thx) + return NewHTTPTransport(logger, dialer, tlsDialer) +} + +// NewHTTPTransport returns a wrapped HTTP transport for HTTP2 and HTTP/1.1 +// using the given dialer and logger. +// +// The returned transport will gracefully handle TLS connections +// created using gitlab.com/yawning/utls.git, if the TLS dialer +// is a dialer using such library for TLS operations. +// +// The returned transport will not have a configured proxy, not +// even the proxy configurable from the environment. +// +// QUIRK: the returned transport will disable transparent decompression +// of compressed response bodies (and will not automatically +// ask for such compression, though you can always do that manually). +// +// The returned transport will configure TCP and TLS connections +// created using its dialer and TLS dialer to always have a +// read watchdog timeout to address https://github.com/ooni/probe/issues/1609. +// +// QUIRK: the returned transport will always enforce 1 connection per host +// and we cannot get rid of this QUIRK requirement because it is +// necessary to perform sane measurements with tracing. We will be +// able to possibly relax this requirement after we change the +// way in which we perform measurements. +func NewHTTPTransport(logger model.DebugLogger, dialer model.Dialer, tlsDialer model.TLSDialer) model.HTTPTransport { + return WrapHTTPTransport(logger, newOOHTTPBaseTransport(dialer, tlsDialer)) +} + +// newOOHTTPBaseTransport is the low-level factory used by NewHTTPTransport +// to create a new, suitable HTTPTransport for HTTP2 and HTTP/1.1. +// +// This factory uses github.com/ooni/oohttp, hence its name. +// +// This function behavior is QUIRKY as documented in [NewHTTPTransport]. +func newOOHTTPBaseTransport(dialer model.Dialer, tlsDialer model.TLSDialer) model.HTTPTransport { + // Using oohttp to support any TLS library. + txp := oohttp.DefaultTransport.(*oohttp.Transport).Clone() + + // This wrapping ensures that we always have a timeout when we + // are using HTTP; see https://github.com/ooni/probe/issues/1609. + dialer = &httpDialerWithReadTimeout{dialer} + txp.DialContext = dialer.DialContext + tlsDialer = &httpTLSDialerWithReadTimeout{tlsDialer} + txp.DialTLSContext = tlsDialer.DialTLSContext + + // We are using a different strategy to implement proxy: we + // use a specific dialer that knows about proxying. + txp.Proxy = nil + + // Better for Cloudflare DNS and also better because we have less + // noisy events and we can better understand what happened. + txp.MaxConnsPerHost = 1 + + // The following (1) reduces the number of headers that Go will + // automatically send for us and (2) ensures that we always receive + // back the true headers, such as Content-Length. This change is + // functional to OONI's goal of observing the network. + txp.DisableCompression = true + + // Required to enable using HTTP/2 (which will be anyway forced + // upon us when we are using TLS parroting). + txp.ForceAttemptHTTP2 = true + + // Ensure we correctly forward CloseIdleConnections. + return &httpTransportConnectionsCloser{ + HTTPTransport: &httpTransportStdlib{&oohttp.StdlibTransport{Transport: txp}}, + Dialer: dialer, + TLSDialer: tlsDialer, + } +} + +// NewHTTPTransportStdlib creates a new HTTPTransport using +// the stdlib for DNS resolutions and TLS. +// +// This factory calls NewHTTPTransport with suitable dialers. +// +// This function behavior is QUIRKY as documented in [NewHTTPTransport]. +func (netx *Netx) NewHTTPTransportStdlib(logger model.DebugLogger) model.HTTPTransport { + dialer := netx.NewDialerWithResolver(logger, netx.NewStdlibResolver(logger)) + tlsDialer := NewTLSDialer(dialer, netx.NewTLSHandshakerStdlib(logger)) + return NewHTTPTransport(logger, dialer, tlsDialer) +} + +// NewHTTPTransportStdlib is equivalent to creating an empty [*Netx] +// and calling its NewHTTPTransportStdlib method. +// +// This function behavior is QUIRKY as documented in [NewHTTPTransport]. +func NewHTTPTransportStdlib(logger model.DebugLogger) model.HTTPTransport { + netx := &Netx{Underlying: nil} + return netx.NewHTTPTransportStdlib(logger) +} + +// NewHTTPClientStdlib creates a new HTTPClient that uses the +// standard library for TLS and DNS resolutions. +// +// This function behavior is QUIRKY as documented in [NewHTTPTransport]. +func NewHTTPClientStdlib(logger model.DebugLogger) model.HTTPClient { + txp := NewHTTPTransportStdlib(logger) + return NewHTTPClient(txp) +} + +// NewHTTPClientWithResolver creates a new HTTPTransport using the +// given resolver and then from that builds an HTTPClient. +// +// This function behavior is QUIRKY as documented in [NewHTTPTransport]. +func NewHTTPClientWithResolver(logger model.Logger, reso model.Resolver) model.HTTPClient { + return NewHTTPClient(NewHTTPTransportWithResolver(logger, reso)) +} + +// NewHTTPClient creates a new, wrapped HTTPClient using the given transport. +// +// This function behavior is QUIRKY because it does not configure a cookie jar, which +// is probably not the right thing to do in many cases, but legacy code MAY depend +// on this behavior. TODO(https://github.com/ooni/probe/issues/2534). +func NewHTTPClient(txp model.HTTPTransport) model.HTTPClient { + return WrapHTTPClient(&http.Client{Transport: txp}) +} diff --git a/pkg/netxlite/httpquirks_test.go b/pkg/netxlite/httpquirks_test.go new file mode 100644 index 000000000..b4e199904 --- /dev/null +++ b/pkg/netxlite/httpquirks_test.go @@ -0,0 +1,159 @@ +package netxlite + +import ( + "context" + "errors" + "net" + "net/http" + "sync/atomic" + "testing" + + "github.com/apex/log" + "github.com/ooni/probe-engine/pkg/mocks" + "github.com/ooni/probe-engine/pkg/model" +) + +func TestNewHTTPTransportWithResolver(t *testing.T) { + expected := errors.New("mocked error") + reso := &mocks.Resolver{ + MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { + return nil, expected + }, + } + txp := NewHTTPTransportWithResolver(model.DiscardLogger, reso) + req, err := http.NewRequest("GET", "http://x.org", nil) + if err != nil { + t.Fatal(err) + } + resp, err := txp.RoundTrip(req) + if !errors.Is(err, expected) { + t.Fatal("unexpected err") + } + if resp != nil { + t.Fatal("expected nil resp") + } +} + +func TestNewHTTPTransport(t *testing.T) { + t.Run("works as intended with failing dialer", func(t *testing.T) { + called := &atomic.Int64{} + expected := errors.New("mocked error") + d := &dialerResolverWithTracing{ + Dialer: &mocks.Dialer{ + MockDialContext: func(ctx context.Context, + network, address string) (net.Conn, error) { + return nil, expected + }, + MockCloseIdleConnections: func() { + called.Add(1) + }, + }, + Resolver: NewStdlibResolver(log.Log), + } + td := NewTLSDialer(d, NewTLSHandshakerStdlib(log.Log)) + txp := NewHTTPTransport(log.Log, d, td) + client := &http.Client{Transport: txp} + resp, err := client.Get("https://8.8.4.4/robots.txt") + if !errors.Is(err, expected) { + t.Fatal("not the error we expected", err) + } + if resp != nil { + t.Fatal("expected non-nil response here") + } + client.CloseIdleConnections() + if called.Load() < 1 { + t.Fatal("did not propagate CloseIdleConnections") + } + }) + + t.Run("creates the correct type chain", func(t *testing.T) { + d := &mocks.Dialer{} + td := &mocks.TLSDialer{} + txp := NewHTTPTransport(log.Log, d, td) + logger := txp.(*httpTransportLogger) + if logger.Logger != log.Log { + t.Fatal("invalid logger") + } + errWrapper := logger.HTTPTransport.(*httpTransportErrWrapper) + connectionsCloser := errWrapper.HTTPTransport.(*httpTransportConnectionsCloser) + withReadTimeout := connectionsCloser.Dialer.(*httpDialerWithReadTimeout) + if withReadTimeout.Dialer != d { + t.Fatal("invalid dialer") + } + tlsWithReadTimeout := connectionsCloser.TLSDialer.(*httpTLSDialerWithReadTimeout) + if tlsWithReadTimeout.TLSDialer != td { + t.Fatal("invalid tls dialer") + } + stdlib := connectionsCloser.HTTPTransport.(*httpTransportStdlib) + if !stdlib.StdlibTransport.ForceAttemptHTTP2 { + t.Fatal("invalid ForceAttemptHTTP2") + } + if !stdlib.StdlibTransport.DisableCompression { + t.Fatal("invalid DisableCompression") + } + if stdlib.StdlibTransport.MaxConnsPerHost != 1 { + t.Fatal("invalid MaxConnPerHost") + } + if stdlib.StdlibTransport.DialTLSContext == nil { + t.Fatal("invalid DialTLSContext") + } + if stdlib.StdlibTransport.DialContext == nil { + t.Fatal("invalid DialContext") + } + }) +} + +func TestNewHTTPTransportStdlib(t *testing.T) { + txp := NewHTTPTransportStdlib(log.Log) + ctx, cancel := context.WithCancel(context.Background()) + cancel() // immediately! + req, err := http.NewRequestWithContext(ctx, "GET", "http://x.org", nil) + if err != nil { + t.Fatal(err) + } + resp, err := txp.RoundTrip(req) + if !errors.Is(err, context.Canceled) { + t.Fatal("unexpected err", err) + } + if resp != nil { + t.Fatal("unexpected resp") + } + if txp.Network() != "tcp" { + t.Fatal("unexpected .Network return value") + } + txp.CloseIdleConnections() +} + +func TestNewHTTPClientStdlib(t *testing.T) { + clnt := NewHTTPClientStdlib(model.DiscardLogger) + ewc, ok := clnt.(*httpClientErrWrapper) + if !ok { + t.Fatal("expected *httpClientErrWrapper") + } + _, ok = ewc.HTTPClient.(*http.Client) + if !ok { + t.Fatal("expected *http.Client") + } +} + +func TestNewHTTPClientWithResolver(t *testing.T) { + reso := &mocks.Resolver{} + clnt := NewHTTPClientWithResolver(model.DiscardLogger, reso) + ewc, ok := clnt.(*httpClientErrWrapper) + if !ok { + t.Fatal("expected *httpClientErrWrapper") + } + httpClnt, ok := ewc.HTTPClient.(*http.Client) + if !ok { + t.Fatal("expected *http.Client") + } + txp := httpClnt.Transport.(*httpTransportLogger) + txpEwrap := txp.HTTPTransport.(*httpTransportErrWrapper) + txpCc := txpEwrap.HTTPTransport.(*httpTransportConnectionsCloser) + dialer := txpCc.Dialer.(*httpDialerWithReadTimeout) + dialerLogger := dialer.Dialer.(*dialerLogger) + dialerReso := dialerLogger.Dialer.(*dialerResolverWithTracing) + if dialerReso.Resolver != reso { + t.Fatal("invalid resolver") + } +} diff --git a/pkg/netxlite/httpstdlib.go b/pkg/netxlite/httpstdlib.go new file mode 100644 index 000000000..155071a44 --- /dev/null +++ b/pkg/netxlite/httpstdlib.go @@ -0,0 +1,32 @@ +package netxlite + +// +// Code to adapt oohttp to the stdlib and the stdlib to our HTTP models +// + +import ( + "net/http" + + oohttp "github.com/ooni/oohttp" + "github.com/ooni/probe-engine/pkg/model" +) + +// stdlibTransport wraps oohttp.StdlibTransport to add .Network() +type httpTransportStdlib struct { + StdlibTransport *oohttp.StdlibTransport +} + +var _ model.HTTPTransport = &httpTransportStdlib{} + +func (txp *httpTransportStdlib) CloseIdleConnections() { + txp.StdlibTransport.CloseIdleConnections() +} + +func (txp *httpTransportStdlib) RoundTrip(req *http.Request) (*http.Response, error) { + return txp.StdlibTransport.RoundTrip(req) +} + +// Network implements HTTPTransport.Network. +func (txp *httpTransportStdlib) Network() string { + return "tcp" +} diff --git a/pkg/netxlite/httptimeout.go b/pkg/netxlite/httptimeout.go new file mode 100644 index 000000000..b2c4b80e9 --- /dev/null +++ b/pkg/netxlite/httptimeout.go @@ -0,0 +1,114 @@ +package netxlite + +// +// Code to ensure we have proper read timeouts (for reliability +// as described by https://github.com/ooni/probe/issues/1609) +// + +import ( + "context" + "errors" + "net" + "time" + + "github.com/ooni/probe-engine/pkg/model" +) + +// httpDialerWithReadTimeout enforces a read timeout for all HTTP +// connections. See https://github.com/ooni/probe/issues/1609. +type httpDialerWithReadTimeout struct { + Dialer model.Dialer +} + +var _ model.Dialer = &httpDialerWithReadTimeout{} + +func (d *httpDialerWithReadTimeout) CloseIdleConnections() { + d.Dialer.CloseIdleConnections() +} + +// DialContext implements Dialer.DialContext. +func (d *httpDialerWithReadTimeout) DialContext( + ctx context.Context, network, address string) (net.Conn, error) { + conn, err := d.Dialer.DialContext(ctx, network, address) + if err != nil { + return nil, err + } + return &httpConnWithReadTimeout{conn}, nil +} + +// httpTLSDialerWithReadTimeout enforces a read timeout for all HTTP +// connections. See https://github.com/ooni/probe/issues/1609. +type httpTLSDialerWithReadTimeout struct { + TLSDialer model.TLSDialer +} + +var _ model.TLSDialer = &httpTLSDialerWithReadTimeout{} + +func (d *httpTLSDialerWithReadTimeout) CloseIdleConnections() { + d.TLSDialer.CloseIdleConnections() +} + +// ErrNotTLSConn occur when an interface accepts a net.Conn but +// internally needs a TLSConn and you pass a net.Conn that doesn't +// implement TLSConn to such an interface. +var ErrNotTLSConn = errors.New("not a TLSConn") + +// DialTLSContext implements TLSDialer's DialTLSContext. +func (d *httpTLSDialerWithReadTimeout) DialTLSContext( + ctx context.Context, network, address string) (net.Conn, error) { + conn, err := d.TLSDialer.DialTLSContext(ctx, network, address) + if err != nil { + return nil, err + } + tconn, okay := conn.(TLSConn) // part of the contract but let's be graceful + if !okay { + conn.Close() // we own the conn here + return nil, ErrNotTLSConn + } + return &httpTLSConnWithReadTimeout{tconn}, nil +} + +// httpConnWithReadTimeout enforces a read timeout for all HTTP +// connections. See https://github.com/ooni/probe/issues/1609. +type httpConnWithReadTimeout struct { + net.Conn +} + +// httpConnReadTimeout is the read timeout we apply to all HTTP +// conns (see https://github.com/ooni/probe/issues/1609). +// +// This timeout is meant as a fallback mechanism so that a stuck +// connection will _eventually_ fail. This is why it is set to +// a large value (300 seconds when writing this note). +// +// There should be other mechanisms to ensure that the code is +// lively: the context during the RoundTrip and iox.ReadAllContext +// when reading the body. They should kick in earlier. But we +// additionally want to avoid leaking a (parked?) connection and +// the corresponding goroutine, hence this large timeout. +// +// A future @bassosimone may understand this problem even better +// and possibly apply an even better fix to this issue. This +// will happen when we'll be able to further study the anomalies +// described in https://github.com/ooni/probe/issues/1609. +const httpConnReadTimeout = 300 * time.Second + +// Read implements Conn.Read. +func (c *httpConnWithReadTimeout) Read(b []byte) (int, error) { + c.Conn.SetReadDeadline(time.Now().Add(httpConnReadTimeout)) + defer c.Conn.SetReadDeadline(time.Time{}) + return c.Conn.Read(b) +} + +// httpTLSConnWithReadTimeout enforces a read timeout for all HTTP +// connections. See https://github.com/ooni/probe/issues/1609. +type httpTLSConnWithReadTimeout struct { + TLSConn +} + +// Read implements Conn.Read. +func (c *httpTLSConnWithReadTimeout) Read(b []byte) (int, error) { + c.TLSConn.SetReadDeadline(time.Now().Add(httpConnReadTimeout)) + defer c.TLSConn.SetReadDeadline(time.Time{}) + return c.TLSConn.Read(b) +} diff --git a/pkg/netxlite/httptimeout_test.go b/pkg/netxlite/httptimeout_test.go new file mode 100644 index 000000000..c748f2a0e --- /dev/null +++ b/pkg/netxlite/httptimeout_test.go @@ -0,0 +1,184 @@ +package netxlite + +import ( + "context" + "errors" + "io" + "net" + "testing" + "time" + + "github.com/ooni/probe-engine/pkg/mocks" +) + +func TestHTTPDialerWithReadTimeout(t *testing.T) { + t.Run("DialContext", func(t *testing.T) { + t.Run("on success", func(t *testing.T) { + var ( + calledWithZeroTime bool + calledWithNonZeroTime bool + ) + origConn := &mocks.Conn{ + MockSetReadDeadline: func(t time.Time) error { + switch t.IsZero() { + case true: + calledWithZeroTime = true + case false: + calledWithNonZeroTime = true + } + return nil + }, + MockRead: func(b []byte) (int, error) { + return 0, io.EOF + }, + } + d := &httpDialerWithReadTimeout{ + Dialer: &mocks.Dialer{ + MockDialContext: func(ctx context.Context, network, address string) (net.Conn, error) { + return origConn, nil + }, + }, + } + ctx := context.Background() + conn, err := d.DialContext(ctx, "", "") + if err != nil { + t.Fatal(err) + } + if _, okay := conn.(*httpConnWithReadTimeout); !okay { + t.Fatal("invalid conn type") + } + if conn.(*httpConnWithReadTimeout).Conn != origConn { + t.Fatal("invalid origin conn") + } + b := make([]byte, 1024) + count, err := conn.Read(b) + if !errors.Is(err, io.EOF) { + t.Fatal("invalid error") + } + if count != 0 { + t.Fatal("invalid count") + } + if !calledWithZeroTime || !calledWithNonZeroTime { + t.Fatal("not called") + } + }) + + t.Run("on failure", func(t *testing.T) { + expected := errors.New("mocked error") + d := &httpDialerWithReadTimeout{ + Dialer: &mocks.Dialer{ + MockDialContext: func(ctx context.Context, network, address string) (net.Conn, error) { + return nil, expected + }, + }, + } + conn, err := d.DialContext(context.Background(), "", "") + if !errors.Is(err, expected) { + t.Fatal("not the error we expected") + } + if conn != nil { + t.Fatal("expected nil conn here") + } + }) + }) +} + +func TestHTTPTLSDialerWithReadTimeout(t *testing.T) { + t.Run("DialContext", func(t *testing.T) { + t.Run("on success", func(t *testing.T) { + var ( + calledWithZeroTime bool + calledWithNonZeroTime bool + ) + origConn := &mocks.TLSConn{ + Conn: mocks.Conn{ + MockSetReadDeadline: func(t time.Time) error { + switch t.IsZero() { + case true: + calledWithZeroTime = true + case false: + calledWithNonZeroTime = true + } + return nil + }, + MockRead: func(b []byte) (int, error) { + return 0, io.EOF + }, + }, + } + d := &httpTLSDialerWithReadTimeout{ + TLSDialer: &mocks.TLSDialer{ + MockDialTLSContext: func(ctx context.Context, network, address string) (net.Conn, error) { + return origConn, nil + }, + }, + } + ctx := context.Background() + conn, err := d.DialTLSContext(ctx, "", "") + if err != nil { + t.Fatal(err) + } + if _, okay := conn.(*httpTLSConnWithReadTimeout); !okay { + t.Fatal("invalid conn type") + } + if conn.(*httpTLSConnWithReadTimeout).TLSConn != origConn { + t.Fatal("invalid origin conn") + } + b := make([]byte, 1024) + count, err := conn.Read(b) + if !errors.Is(err, io.EOF) { + t.Fatal("invalid error") + } + if count != 0 { + t.Fatal("invalid count") + } + if !calledWithZeroTime || !calledWithNonZeroTime { + t.Fatal("not called") + } + }) + + t.Run("on failure", func(t *testing.T) { + expected := errors.New("mocked error") + d := &httpTLSDialerWithReadTimeout{ + TLSDialer: &mocks.TLSDialer{ + MockDialTLSContext: func(ctx context.Context, network, address string) (net.Conn, error) { + return nil, expected + }, + }, + } + conn, err := d.DialTLSContext(context.Background(), "", "") + if !errors.Is(err, expected) { + t.Fatal("not the error we expected") + } + if conn != nil { + t.Fatal("expected nil conn here") + } + }) + + t.Run("with invalid conn type", func(t *testing.T) { + var called bool + d := &httpTLSDialerWithReadTimeout{ + TLSDialer: &mocks.TLSDialer{ + MockDialTLSContext: func(ctx context.Context, network, address string) (net.Conn, error) { + return &mocks.Conn{ + MockClose: func() error { + called = true + return nil + }, + }, nil + }, + }, + } + conn, err := d.DialTLSContext(context.Background(), "", "") + if !errors.Is(err, ErrNotTLSConn) { + t.Fatal("not the error we expected") + } + if conn != nil { + t.Fatal("expected nil conn here") + } + if !called { + t.Fatal("not called") + } + }) + }) +} diff --git a/pkg/netxlite/httpwrap.go b/pkg/netxlite/httpwrap.go new file mode 100644 index 000000000..cfd677dfb --- /dev/null +++ b/pkg/netxlite/httpwrap.go @@ -0,0 +1,27 @@ +package netxlite + +// +// Wrappers for already constructed types. +// + +import ( + "github.com/ooni/probe-engine/pkg/model" +) + +// WrapHTTPTransport creates an HTTPTransport using the given logger +// and guarantees that returned errors are wrapped. +// +// This is a low level factory. Consider not using it directly. +func WrapHTTPTransport(logger model.DebugLogger, txp model.HTTPTransport) model.HTTPTransport { + return &httpTransportLogger{ + HTTPTransport: &httpTransportErrWrapper{txp}, + Logger: logger, + } +} + +// WrapHTTPClient wraps an HTTP client to add error wrapping capabilities. +// +// This is a low level factory. Consider not using it directly. +func WrapHTTPClient(clnt model.HTTPClient) model.HTTPClient { + return &httpClientErrWrapper{clnt} +} diff --git a/pkg/netxlite/httpwrap_test.go b/pkg/netxlite/httpwrap_test.go new file mode 100644 index 000000000..eb200c973 --- /dev/null +++ b/pkg/netxlite/httpwrap_test.go @@ -0,0 +1,16 @@ +package netxlite + +import ( + "net/http" + "testing" +) + +func TestWrapHTTPClient(t *testing.T) { + origClient := &http.Client{} + wrapped := WrapHTTPClient(origClient) + errWrapper := wrapped.(*httpClientErrWrapper) + innerClient := errWrapper.HTTPClient.(*http.Client) + if innerClient != origClient { + t.Fatal("not the inner client we expected") + } +} diff --git a/pkg/netxlite/integration_test.go b/pkg/netxlite/integration_test.go index 574809e3e..91b4c1c5e 100644 --- a/pkg/netxlite/integration_test.go +++ b/pkg/netxlite/integration_test.go @@ -9,15 +9,16 @@ import ( "net/http" "net/http/httptest" "net/url" + "runtime" "testing" "time" "github.com/apex/log" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" - "github.com/ooni/probe-engine/pkg/netxlite/quictesting" "github.com/ooni/probe-engine/pkg/randx" "github.com/ooni/probe-engine/pkg/runtimex" + "github.com/ooni/probe-engine/pkg/testingquic" "github.com/ooni/probe-engine/pkg/testingx" "github.com/quic-go/quic-go" utls "gitlab.com/yawning/utls.git" @@ -78,6 +79,9 @@ func TestMeasureWithSystemResolver(t *testing.T) { // e.g. a domain containing a few random letters addrs, err := r.LookupHost(ctx, randx.Letters(7)+".ooni.nonexistent") if err == nil || err.Error() != netxlite.FailureGenericTimeoutError { + if runtime.GOOS == "windows" { + t.Skip("https://github.com/ooni/probe/issues/2535") + } t.Fatal("not the error we expected", err) } if addrs != nil { @@ -291,7 +295,7 @@ func TestMeasureWithTLSHandshaker(t *testing.T) { NextProtos: []string{"h2", "http/1.1"}, RootCAs: nil, } - tconn, _, err := th.Handshake(ctx, conn, config) + tconn, err := th.Handshake(ctx, conn, config) if err != nil { return fmt.Errorf("tls handshake failed: %w", err) } @@ -316,7 +320,7 @@ func TestMeasureWithTLSHandshaker(t *testing.T) { NextProtos: []string{"h2", "http/1.1"}, RootCAs: nil, } - tconn, _, err := th.Handshake(ctx, conn, config) + tconn, err := th.Handshake(ctx, conn, config) if err == nil { return fmt.Errorf("tls handshake succeded unexpectedly") } @@ -346,7 +350,7 @@ func TestMeasureWithTLSHandshaker(t *testing.T) { NextProtos: []string{"h2", "http/1.1"}, RootCAs: nil, } - tconn, _, err := th.Handshake(ctx, conn, config) + tconn, err := th.Handshake(ctx, conn, config) if err == nil { return fmt.Errorf("tls handshake succeded unexpectedly") } @@ -376,7 +380,7 @@ func TestMeasureWithTLSHandshaker(t *testing.T) { NextProtos: []string{"h2", "http/1.1"}, RootCAs: nil, } - tconn, _, err := th.Handshake(ctx, conn, config) + tconn, err := th.Handshake(ctx, conn, config) if err == nil { return fmt.Errorf("tls handshake succeded unexpectedly") } @@ -475,7 +479,7 @@ func TestMeasureWithQUICDialer(t *testing.T) { // t.Run("on success", func(t *testing.T) { - ql := netxlite.NewQUICListener() + ql := netxlite.NewUDPListener() d := netxlite.NewQUICDialerWithoutResolver(ql, log.Log) defer d.CloseIdleConnections() ctx := context.Background() @@ -483,11 +487,11 @@ func TestMeasureWithQUICDialer(t *testing.T) { // why we're using nil to force netxlite to use the cached // default Mozilla cert pool. config := &tls.Config{ - ServerName: quictesting.Domain, + ServerName: testingquic.MustDomain(), NextProtos: []string{"h3"}, RootCAs: nil, } - sess, err := d.DialContext(ctx, quictesting.Endpoint("443"), config, &quic.Config{}) + sess, err := d.DialContext(ctx, testingquic.MustEndpoint("443"), config, &quic.Config{}) if err != nil { t.Fatal(err) } @@ -498,7 +502,7 @@ func TestMeasureWithQUICDialer(t *testing.T) { }) t.Run("on timeout", func(t *testing.T) { - ql := netxlite.NewQUICListener() + ql := netxlite.NewUDPListener() d := netxlite.NewQUICDialerWithoutResolver(ql, log.Log) defer d.CloseIdleConnections() ctx := context.Background() @@ -506,12 +510,12 @@ func TestMeasureWithQUICDialer(t *testing.T) { // why we're using nil to force netxlite to use the cached // default Mozilla cert pool. config := &tls.Config{ - ServerName: quictesting.Domain, + ServerName: testingquic.MustDomain(), NextProtos: []string{"h3"}, RootCAs: nil, } // Here we assume :1 is filtered - sess, err := d.DialContext(ctx, quictesting.Endpoint("1"), config, &quic.Config{}) + sess, err := d.DialContext(ctx, testingquic.MustEndpoint("1"), config, &quic.Config{}) if err == nil || err.Error() != netxlite.FailureGenericTimeoutError { t.Fatal("not the error we expected", err) } @@ -551,6 +555,8 @@ func TestHTTPTransport(t *testing.T) { conn.Close() })) defer srvr.Close() + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPTransportStdlib has QUIRKS but we + // don't actually care about those QUIRKS in this context txp := netxlite.NewHTTPTransportStdlib(model.DiscardLogger) req, err := http.NewRequest("GET", srvr.URL, nil) if err != nil { @@ -576,13 +582,13 @@ func TestHTTP3Transport(t *testing.T) { t.Run("works as intended", func(t *testing.T) { d := netxlite.NewQUICDialerWithResolver( - netxlite.NewQUICListener(), + netxlite.NewUDPListener(), log.Log, netxlite.NewStdlibResolver(log.Log), ) txp := netxlite.NewHTTP3Transport(log.Log, d, &tls.Config{}) client := &http.Client{Transport: txp} - URL := (&url.URL{Scheme: "https", Host: quictesting.Domain, Path: "/"}).String() + URL := (&url.URL{Scheme: "https", Host: testingquic.MustDomain(), Path: "/"}).String() resp, err := client.Get(URL) if err != nil { t.Fatal(err) diff --git a/pkg/netxlite/internal/generrno/main.go b/pkg/netxlite/internal/generrno/main.go index 94d48ee38..e82aef56c 100644 --- a/pkg/netxlite/internal/generrno/main.go +++ b/pkg/netxlite/internal/generrno/main.go @@ -7,7 +7,7 @@ import ( "sort" "time" - "github.com/iancoleman/strcase" + "github.com/ooni/probe-engine/pkg/strcasex" "golang.org/x/sys/execabs" ) @@ -69,12 +69,12 @@ func (es *ErrorSpec) AsCanonicalErrnoName() string { // AsFailureVar returns the name of the failure var. func (es *ErrorSpec) AsFailureVar() string { - return "Failure" + strcase.ToCamel(es.failure) + return "Failure" + strcasex.ToCamel(es.failure) } // AsFailureString returns the OONI failure string. func (es *ErrorSpec) AsFailureString() string { - return strcase.ToSnake(es.failure) + return strcasex.ToSnake(es.failure) } // NewSystemError constructs a new ErrorSpec representing a system diff --git a/pkg/netxlite/maybeproxy.go b/pkg/netxlite/maybeproxy.go index 5f4ef756f..a26b11c20 100644 --- a/pkg/netxlite/maybeproxy.go +++ b/pkg/netxlite/maybeproxy.go @@ -22,6 +22,8 @@ type proxyDialer struct { // MaybeWrapWithProxyDialer returns the original dialer if the proxyURL is nil // and otherwise returns a wrapped dialer that implements proxying. +// +// Deprecated: do not use this function in new code. func MaybeWrapWithProxyDialer(dialer model.Dialer, proxyURL *url.URL) model.Dialer { if proxyURL == nil { return dialer diff --git a/pkg/netxlite/netem.go b/pkg/netxlite/netem.go index 4d5c97d4b..b5687bf36 100644 --- a/pkg/netxlite/netem.go +++ b/pkg/netxlite/netem.go @@ -8,7 +8,6 @@ import ( "github.com/ooni/netem" "github.com/ooni/probe-engine/pkg/model" - "github.com/ooni/probe-engine/pkg/runtimex" ) // NetemUnderlyingNetworkAdapter adapts [netem.UnderlyingNetwork] to [model.UnderlyingNetwork]. @@ -20,7 +19,7 @@ var _ model.UnderlyingNetwork = &NetemUnderlyingNetworkAdapter{} // DefaultCertPool implements model.UnderlyingNetwork func (a *NetemUnderlyingNetworkAdapter) DefaultCertPool() *x509.CertPool { - return runtimex.Try1(a.UNet.DefaultCertPool()) + return a.UNet.DefaultCertPool() } // DialTimeout implements model.UnderlyingNetwork @@ -43,6 +42,11 @@ func (a *NetemUnderlyingNetworkAdapter) GetaddrinfoResolverNetwork() string { return a.UNet.GetaddrinfoResolverNetwork() } +// ListenTCP implements model.UnderlyingNetwork +func (a *NetemUnderlyingNetworkAdapter) ListenTCP(network string, addr *net.TCPAddr) (net.Listener, error) { + return a.UNet.ListenTCP(network, addr) +} + // ListenUDP implements model.UnderlyingNetwork func (a *NetemUnderlyingNetworkAdapter) ListenUDP(network string, addr *net.UDPAddr) (model.UDPLikeConn, error) { return a.UNet.ListenUDP(network, addr) diff --git a/pkg/netxlite/netem_test.go b/pkg/netxlite/netem_test.go new file mode 100644 index 000000000..9e0f8213b --- /dev/null +++ b/pkg/netxlite/netem_test.go @@ -0,0 +1,63 @@ +package netxlite + +import ( + "context" + "net" + "sync" + "testing" + + "github.com/apex/log" + "github.com/ooni/netem" + "github.com/ooni/probe-engine/pkg/runtimex" +) + +func TestNetemUnderlyingNetworkAdapter(t *testing.T) { + + // This test case explicitly ensures we can use the adapter to listen for TCP + t.Run("ListenTCP", func(t *testing.T) { + // create a star network topology + topology := netem.MustNewStarTopology(log.Log) + defer topology.Close() + + // constants for the IP address we're using + const ( + clientAddress = "130.192.91.211" + serverAddress = "93.184.216.34" + ) + + // create the stacks + serverStack := runtimex.Try1(topology.AddHost(serverAddress, "0.0.0.0", &netem.LinkConfig{})) + clientStack := runtimex.Try1(topology.AddHost(clientAddress, "0.0.0.0", &netem.LinkConfig{})) + + // wrap the server stack and create listening socket + serverAdapter := &NetemUnderlyingNetworkAdapter{serverStack} + serverEndpoint := &net.TCPAddr{IP: net.ParseIP(serverAddress), Port: 54321} + listener := runtimex.Try1(serverAdapter.ListenTCP("tcp", serverEndpoint)) + defer listener.Close() + + // listen in a background goroutine + wg := &sync.WaitGroup{} + wg.Add(1) + go func() { + conn := runtimex.Try1(listener.Accept()) + conn.Close() + wg.Done() + }() + + // wrap the client stack + clientAdapter := &NetemUnderlyingNetworkAdapter{clientStack} + + // connect in a background goroutine + wg.Add(1) + go func() { + ctx := context.Background() + conn := runtimex.Try1(clientAdapter.DialContext(ctx, "tcp", serverEndpoint.String())) + conn.Close() + wg.Done() + }() + + // wait for all operations to complete + wg.Wait() + }) + +} diff --git a/pkg/netxlite/netx.go b/pkg/netxlite/netx.go index 2a1d96300..703401e19 100644 --- a/pkg/netxlite/netx.go +++ b/pkg/netxlite/netx.go @@ -5,10 +5,11 @@ package netxlite // network operations using a custom model.UnderlyingNetwork. // -import "github.com/ooni/probe-engine/pkg/model" +import ( + "net" -// TODO(bassosimone,kelmenhorst): we should gradually refactor the top-level netxlite -// functions to operate on a [Net] struct using a nil-initialized Underlying field. + "github.com/ooni/probe-engine/pkg/model" +) // Netx allows constructing netxlite data types using a specific [model.UnderlyingNetwork]. type Netx struct { @@ -17,62 +18,14 @@ type Netx struct { Underlying model.UnderlyingNetwork } -// tproxyNilSafeProvider wraps the [model.UnderlyingNetwork] using a [tproxyNilSafeProvider]. -func (n *Netx) tproxyNilSafeProvider() *MaybeCustomUnderlyingNetwork { - return &MaybeCustomUnderlyingNetwork{n.Underlying} -} - -// NewStdlibResolver is like [netxlite.NewStdlibResolver] but the constructed [model.Resolver] -// uses the [UnderlyingNetwork] configured inside the [Net] structure. -func (n *Netx) NewStdlibResolver(logger model.DebugLogger, wrappers ...model.DNSTransportWrapper) model.Resolver { - unwrapped := &resolverSystem{ - t: WrapDNSTransport(&dnsOverGetaddrinfoTransport{provider: n.tproxyNilSafeProvider()}, wrappers...), - } - return WrapResolver(logger, unwrapped) -} - -// NewDialerWithResolver is like [netxlite.NewDialerWithResolver] but the constructed [model.Dialer] -// uses the [UnderlyingNetwork] configured inside the [Net] structure. -func (n *Netx) NewDialerWithResolver(dl model.DebugLogger, r model.Resolver, w ...model.DialerWrapper) model.Dialer { - return WrapDialer(dl, r, &DialerSystem{provider: n.tproxyNilSafeProvider()}, w...) -} - -// NewQUICListener is like [netxlite.NewQUICListener] but the constructed [model.QUICListener] -// uses the [UnderlyingNetwork] configured inside the [Net] structure. -func (n *Netx) NewQUICListener() model.QUICListener { - return &quicListenerErrWrapper{&quicListenerStdlib{provider: n.tproxyNilSafeProvider()}} -} - -// NewQUICDialerWithResolver is like [netxlite.NewQUICDialerWithResolver] but the constructed -// [model.QUICDialer] uses the [UnderlyingNetwork] configured inside the [Net] structure. -func (n *Netx) NewQUICDialerWithResolver(listener model.QUICListener, logger model.DebugLogger, - resolver model.Resolver, wrappers ...model.QUICDialerWrapper) (outDialer model.QUICDialer) { - baseDialer := &quicDialerQUICGo{ - QUICListener: listener, - provider: n.tproxyNilSafeProvider(), - } - return WrapQUICDialer(logger, resolver, baseDialer, wrappers...) -} - -// NewTLSHandshakerStdlib is like [netxlite.NewTLSHandshakerStdlib] but the constructed [model.TLSHandshaker] -// uses the [UnderlyingNetwork] configured inside the [Net] structure. -func (n *Netx) NewTLSHandshakerStdlib(logger model.DebugLogger) model.TLSHandshaker { - return newTLSHandshakerLogger(&tlsHandshakerConfigurable{provider: n.tproxyNilSafeProvider()}, logger) -} +var _ model.MeasuringNetwork = &Netx{} -// NewHTTPTransportStdlib is like [netxlite.NewHTTPTransportStdlib] but the constructed [model.HTTPTransport] -// uses the [UnderlyingNetwork] configured inside the [Net] structure. -func (n *Netx) NewHTTPTransportStdlib(logger model.DebugLogger) model.HTTPTransport { - dialer := n.NewDialerWithResolver(logger, n.NewStdlibResolver(logger)) - tlsDialer := NewTLSDialer(dialer, n.NewTLSHandshakerStdlib(logger)) - return NewHTTPTransport(logger, dialer, tlsDialer) +// MaybeCustomUnderlyingNetwork wraps the [model.UnderlyingNetwork] using a [*MaybeCustomUnderlyingNetwork]. +func (netx *Netx) MaybeCustomUnderlyingNetwork() *MaybeCustomUnderlyingNetwork { + return &MaybeCustomUnderlyingNetwork{netx.Underlying} } -// NewHTTP3TransportStdlib is like [netxlite.NewHTTP3TransportStdlib] but the constructed [model.HTTPTransport] -// uses the [UnderlyingNetwork] configured inside the [Net] structure. -func (n *Netx) NewHTTP3TransportStdlib(logger model.DebugLogger) model.HTTPTransport { - ql := n.NewQUICListener() - reso := n.NewStdlibResolver(logger) - qd := n.NewQUICDialerWithResolver(ql, logger, reso) - return NewHTTP3Transport(logger, qd, nil) +// ListenTCP creates a new listening TCP socket using the given address. +func (netx *Netx) ListenTCP(network string, addr *net.TCPAddr) (net.Listener, error) { + return netx.MaybeCustomUnderlyingNetwork().Get().ListenTCP(network, addr) } diff --git a/pkg/netxlite/netx_test.go b/pkg/netxlite/netx_test.go index 7ecfdd937..af5660857 100644 --- a/pkg/netxlite/netx_test.go +++ b/pkg/netxlite/netx_test.go @@ -4,6 +4,7 @@ import ( "context" "net" "net/http" + "sync" "testing" "github.com/apex/log" @@ -14,9 +15,10 @@ import ( "github.com/quic-go/quic-go/http3" ) -func TestNetx(t *testing.T) { +// This test ensures that a Netx wrapping a netem.UNet is WAI +func TestNetxWithNetem(t *testing.T) { // create a star network topology - topology := runtimex.Try1(netem.NewStarTopology(log.Log)) + topology := netem.MustNewStarTopology(log.Log) defer topology.Close() // constants for the IP address we're using @@ -40,6 +42,9 @@ func TestNetx(t *testing.T) { w.Write(bonsoirElliot) }) + // create common certificate for HTTPS and HTTP3 + webServerTLSConfig := webServerStack.MustNewServerTLSConfig("www.example.com", "web01.example.com") + // listen for HTTPS requests using the above handler webServerTCPAddress := &net.TCPAddr{ IP: net.ParseIP(exampleComAddress), @@ -49,7 +54,7 @@ func TestNetx(t *testing.T) { webServerTCPListener := runtimex.Try1(webServerStack.ListenTCP("tcp", webServerTCPAddress)) webServerTCPServer := &http.Server{ Handler: webServerHandler, - TLSConfig: webServerStack.ServerTLSConfig(), + TLSConfig: webServerTLSConfig, } go webServerTCPServer.ServeTLS(webServerTCPListener, "", "") defer webServerTCPServer.Close() @@ -62,7 +67,7 @@ func TestNetx(t *testing.T) { } webServerUDPListener := runtimex.Try1(webServerStack.ListenUDP("udp", webServerUDPAddress)) webServerUDPServer := &http3.Server{ - TLSConfig: webServerStack.ServerTLSConfig(), + TLSConfig: webServerTLSConfig, QuicConfig: &quic.Config{}, Handler: webServerHandler, } @@ -75,6 +80,8 @@ func TestNetx(t *testing.T) { netx := &Netx{underlyingNetwork} t.Run("HTTPS fetch", func(t *testing.T) { + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPTransportStdlib is QUIRKY but we probably + // don't care about using a QUIRKY function here? txp := netx.NewHTTPTransportStdlib(log.Log) client := &http.Client{Transport: txp} resp, err := client.Get("https://www.example.com/") @@ -114,3 +121,34 @@ func TestNetx(t *testing.T) { } }) } + +// We generally do not listen here as part of other tests, since the listening +// functionality is mainly only use for testingx. So, here's a specific test for that. +func TestNetxListenTCP(t *testing.T) { + netx := &Netx{Underlying: nil} + + listener := runtimex.Try1(netx.ListenTCP("tcp", &net.TCPAddr{})) + serverEndpoint := listener.Addr().String() + + // listen in a background goroutine + wg := &sync.WaitGroup{} + wg.Add(1) + go func() { + conn := runtimex.Try1(listener.Accept()) + conn.Close() + wg.Done() + }() + + // dial in a background goroutine + wg.Add(1) + go func() { + ctx := context.Background() + dialer := netx.NewDialerWithoutResolver(log.Log) + conn := runtimex.Try1(dialer.DialContext(ctx, "tcp", serverEndpoint)) + conn.Close() + wg.Done() + }() + + // wait for the goroutines to finish + wg.Wait() +} diff --git a/pkg/netxlite/quic.go b/pkg/netxlite/quic.go index f711e1e01..0bbb649f8 100644 --- a/pkg/netxlite/quic.go +++ b/pkg/netxlite/quic.go @@ -16,53 +16,35 @@ import ( "github.com/quic-go/quic-go" ) -// NewQUICListener creates a new QUICListener using the standard -// library to create listening UDP sockets. -func NewQUICListener() model.QUICListener { - return &quicListenerErrWrapper{&quicListenerStdlib{}} -} - -// quicListenerStdlib is a QUICListener using the standard library. -type quicListenerStdlib struct { - // provider is the OPTIONAL nil-safe [model.UnderlyingNetwork] provider. - provider *MaybeCustomUnderlyingNetwork -} - -var _ model.QUICListener = &quicListenerStdlib{} - -// Listen implements QUICListener.Listen. -func (qls *quicListenerStdlib) Listen(addr *net.UDPAddr) (model.UDPLikeConn, error) { - return qls.provider.Get().ListenUDP("udp", addr) -} - -// NewQUICDialerWithResolver is the WrapDialer equivalent for QUIC where -// we return a composed QUICDialer modified by optional wrappers. -// -// The returned dialer guarantees: -// -// 1. logging; +// NewQUICDialerWithResolver creates a QUICDialer with error wrapping. // -// 2. error wrapping; -// -// 3. that we are going to use Mozilla CA if the [tls.Config] -// RootCAs field is zero initialized. -// -// Please, note that this fuunction will just ignore any nil wrapper. -// -// Unlike the dialer returned by WrapDialer, this dialer MAY attempt +// Unlike the dialer returned by NewDialerWithResolver, this dialer MAY attempt // happy eyeballs, perform parallel dial attempts, and return an error // that aggregates all the errors that occurred. -func NewQUICDialerWithResolver(listener model.QUICListener, logger model.DebugLogger, +// +// The [model.QUICDialerWrapper] arguments wraps the returned dialer in such a way +// that we can implement the legacy [netx] package. New code MUST NOT +// use this functionality, which we'd like to remove ASAP. +func (netx *Netx) NewQUICDialerWithResolver(listener model.UDPListener, logger model.DebugLogger, resolver model.Resolver, wrappers ...model.QUICDialerWrapper) (outDialer model.QUICDialer) { baseDialer := &quicDialerQUICGo{ - QUICListener: listener, + UDPListener: listener, + provider: netx.MaybeCustomUnderlyingNetwork(), } - return WrapQUICDialer(logger, resolver, baseDialer, wrappers...) + return wrapQUICDialer(logger, resolver, baseDialer, wrappers...) +} + +// NewQUICDialerWithResolver is equivalent to creating an empty [*Netx] +// and calling its NewQUICDialerWithResolver method. +func NewQUICDialerWithResolver(listener model.UDPListener, logger model.DebugLogger, + resolver model.Resolver, wrappers ...model.QUICDialerWrapper) (outDialer model.QUICDialer) { + netx := &Netx{Underlying: nil} + return netx.NewQUICDialerWithResolver(listener, logger, resolver, wrappers...) } -// WrapQUICDialer is similar to NewQUICDialerWithResolver except that it takes as +// wrapQUICDialer is similar to NewQUICDialerWithResolver except that it takes as // input an already constructed [model.QUICDialer] instead of creating one. -func WrapQUICDialer(logger model.DebugLogger, resolver model.Resolver, +func wrapQUICDialer(logger model.DebugLogger, resolver model.Resolver, baseDialer model.QUICDialer, wrappers ...model.QUICDialerWrapper) (outDialer model.QUICDialer) { outDialer = &quicDialerErrWrapper{ QUICDialer: &quicDialerHandshakeCompleter{ @@ -88,21 +70,28 @@ func WrapQUICDialer(logger model.DebugLogger, resolver model.Resolver, } } -// NewQUICDialerWithoutResolver is equivalent to calling NewQUICDialerWithResolver -// with the resolver argument set to &NullResolver{}. -func NewQUICDialerWithoutResolver(listener model.QUICListener, +// NewQUICDialerWithoutResolver implements [model.MeasuringNetwork]. +func (netx *Netx) NewQUICDialerWithoutResolver(listener model.UDPListener, + logger model.DebugLogger, wrappers ...model.QUICDialerWrapper) model.QUICDialer { + return netx.NewQUICDialerWithResolver(listener, logger, &NullResolver{}, wrappers...) +} + +// NewQUICDialerWithoutResolver is equivalent to creating an empty [*Netx] +// and calling its NewQUICDialerWithoutResolver method. +func NewQUICDialerWithoutResolver(listener model.UDPListener, logger model.DebugLogger, wrappers ...model.QUICDialerWrapper) model.QUICDialer { - return NewQUICDialerWithResolver(listener, logger, &NullResolver{}, wrappers...) + netx := &Netx{Underlying: nil} + return netx.NewQUICDialerWithoutResolver(listener, logger, wrappers...) } // quicDialerQUICGo dials using the quic-go/quic-go library. type quicDialerQUICGo struct { - // QUICListener is the underlying QUICListener to use. - QUICListener model.QUICListener + // UDPListener is the underlying UDPListener to use. + UDPListener model.UDPListener - // mockDialEarlyContext allows to mock quic.DialEarlyContext. - mockDialEarlyContext func(ctx context.Context, pconn net.PacketConn, - remoteAddr net.Addr, host string, tlsConfig *tls.Config, + // mockDialEarly allows to mock quic.DialEarly. + mockDialEarly func(ctx context.Context, pconn net.PacketConn, + remoteAddr net.Addr, tlsConfig *tls.Config, quicConfig *quic.Config) (quic.EarlyConnection, error) // provider is the OPTIONAL nil-safe [model.UnderlyingNetwork] provider. @@ -152,7 +141,7 @@ func (d *quicDialerQUICGo) DialContext(ctx context.Context, if err != nil { return nil, err } - pconn, err := d.QUICListener.Listen(&net.UDPAddr{IP: net.IPv4zero, Port: 0, Zone: ""}) + pconn, err := d.UDPListener.Listen(&net.UDPAddr{IP: net.IPv4zero, Port: 0, Zone: ""}) if err != nil { return nil, err } @@ -161,8 +150,7 @@ func (d *quicDialerQUICGo) DialContext(ctx context.Context, pconn = trace.MaybeWrapUDPLikeConn(pconn) started := trace.TimeNow() trace.OnQUICHandshakeStart(started, address, quicConfig) - qconn, err := d.dialEarlyContext( - ctx, pconn, udpAddr, address, tlsConfig, quicConfig) + qconn, err := d.dialEarly(ctx, pconn, udpAddr, tlsConfig, quicConfig) finished := trace.TimeNow() err = MaybeNewErrWrapper(ClassifyQUICHandshakeError, QUICHandshakeOperation, err) trace.OnQUICHandshakeDone(started, address, qconn, tlsConfig, err, finished) @@ -173,15 +161,15 @@ func (d *quicDialerQUICGo) DialContext(ctx context.Context, return newQUICConnectionOwnsConn(qconn, pconn), nil } -func (d *quicDialerQUICGo) dialEarlyContext(ctx context.Context, - pconn net.PacketConn, remoteAddr net.Addr, address string, +func (d *quicDialerQUICGo) dialEarly(ctx context.Context, + pconn net.PacketConn, remoteAddr net.Addr, tlsConfig *tls.Config, quicConfig *quic.Config) (quic.EarlyConnection, error) { - if d.mockDialEarlyContext != nil { - return d.mockDialEarlyContext( - ctx, pconn, remoteAddr, address, tlsConfig, quicConfig) + if d.mockDialEarly != nil { + return d.mockDialEarly( + ctx, pconn, remoteAddr, tlsConfig, quicConfig) } - return quic.DialEarlyContext( - ctx, pconn, remoteAddr, address, tlsConfig, quicConfig) + return quic.DialEarly( + ctx, pconn, remoteAddr, tlsConfig, quicConfig) } // maybeApplyTLSDefaults ensures that we're using our certificate pool, if @@ -225,7 +213,7 @@ func (d *quicDialerHandshakeCompleter) DialContext( return nil, err } select { - case <-conn.HandshakeComplete().Done(): + case <-conn.HandshakeComplete(): return conn, nil case <-ctx.Done(): conn.CloseWithError(0, "") // we own the conn @@ -399,17 +387,17 @@ func (s *quicDialerSingleUse) CloseIdleConnections() { // nothing to do } -// quicListenerErrWrapper is a QUICListener that wraps errors. -type quicListenerErrWrapper struct { - // QUICListener is the underlying listener. - QUICListener model.QUICListener +// udpListenerErrWrapper is a UDPListener that wraps errors. +type udpListenerErrWrapper struct { + // UDPListener is the underlying listener. + UDPListener model.UDPListener } -var _ model.QUICListener = &quicListenerErrWrapper{} +var _ model.UDPListener = &udpListenerErrWrapper{} -// Listen implements QUICListener.Listen. -func (qls *quicListenerErrWrapper) Listen(addr *net.UDPAddr) (model.UDPLikeConn, error) { - pconn, err := qls.QUICListener.Listen(addr) +// Listen implements UDPListener.Listen. +func (qls *udpListenerErrWrapper) Listen(addr *net.UDPAddr) (model.UDPLikeConn, error) { + pconn, err := qls.UDPListener.Listen(addr) if err != nil { return nil, NewErrWrapper(ClassifyGenericError, QUICListenOperation, err) } diff --git a/pkg/netxlite/quic_test.go b/pkg/netxlite/quic_test.go index 8cc5f3b75..790af4c9b 100644 --- a/pkg/netxlite/quic_test.go +++ b/pkg/netxlite/quic_test.go @@ -19,12 +19,6 @@ import ( "github.com/quic-go/quic-go" ) -func TestNewQUICListener(t *testing.T) { - ql := NewQUICListener() - qew := ql.(*quicListenerErrWrapper) - _ = qew.QUICListener.(*quicListenerStdlib) -} - type extensionQUICDialerFirst struct { model.QUICDialer } @@ -48,7 +42,7 @@ func (*quicDialerWrapperSecond) WrapQUICDialer(qd model.QUICDialer) model.QUICDi } func TestNewQUICDialer(t *testing.T) { - ql := NewQUICListener() + ql := NewUDPListener() extensions := []model.QUICDialerWrapper{ &quicDialerWrapperFirst{}, nil, // explicitly test for this documented case @@ -72,7 +66,7 @@ func TestNewQUICDialer(t *testing.T) { errWrapper := ext1.QUICDialer.(*quicDialerErrWrapper) handshakeCompleter := errWrapper.QUICDialer.(*quicDialerHandshakeCompleter) base := handshakeCompleter.Dialer.(*quicDialerQUICGo) - if base.QUICListener != ql { + if base.UDPListener != ql { t.Fatal("invalid quic listener") } } @@ -132,7 +126,7 @@ func TestQUICDialerQUICGo(t *testing.T) { ServerName: "www.google.com", } systemdialer := quicDialerQUICGo{ - QUICListener: &quicListenerStdlib{}, + UDPListener: &udpListenerStdlib{}, } defer systemdialer.CloseIdleConnections() // just to see it running ctx := context.Background() @@ -151,7 +145,7 @@ func TestQUICDialerQUICGo(t *testing.T) { ServerName: "www.google.com", } systemdialer := quicDialerQUICGo{ - QUICListener: &quicListenerStdlib{}, + UDPListener: &udpListenerStdlib{}, } ctx := context.Background() qconn, err := systemdialer.DialContext( @@ -169,7 +163,7 @@ func TestQUICDialerQUICGo(t *testing.T) { ServerName: "www.google.com", } systemdialer := quicDialerQUICGo{ - QUICListener: &quicListenerStdlib{}, + UDPListener: &udpListenerStdlib{}, } ctx := context.Background() qconn, err := systemdialer.DialContext( @@ -188,7 +182,7 @@ func TestQUICDialerQUICGo(t *testing.T) { ServerName: "www.google.com", } systemdialer := quicDialerQUICGo{ - QUICListener: &mocks.QUICListener{ + UDPListener: &mocks.UDPListener{ MockListen: func(addr *net.UDPAddr) (model.UDPLikeConn, error) { return nil, expected }, @@ -210,7 +204,7 @@ func TestQUICDialerQUICGo(t *testing.T) { ServerName: "dns.google", } systemdialer := quicDialerQUICGo{ - QUICListener: &quicListenerStdlib{}, + UDPListener: &udpListenerStdlib{}, } ctx, cancel := context.WithCancel(context.Background()) cancel() // fail immediately @@ -231,9 +225,9 @@ func TestQUICDialerQUICGo(t *testing.T) { ServerName: "dns.google", } systemdialer := quicDialerQUICGo{ - QUICListener: &quicListenerStdlib{}, - mockDialEarlyContext: func(ctx context.Context, pconn net.PacketConn, - remoteAddr net.Addr, host string, tlsConfig *tls.Config, + UDPListener: &udpListenerStdlib{}, + mockDialEarly: func(ctx context.Context, pconn net.PacketConn, + remoteAddr net.Addr, tlsConfig *tls.Config, quicConfig *quic.Config) (quic.EarlyConnection, error) { gotTLSConfig = tlsConfig return nil, expected @@ -272,9 +266,9 @@ func TestQUICDialerQUICGo(t *testing.T) { ServerName: "dns.google", } systemdialer := quicDialerQUICGo{ - QUICListener: &quicListenerStdlib{}, - mockDialEarlyContext: func(ctx context.Context, pconn net.PacketConn, - remoteAddr net.Addr, host string, tlsConfig *tls.Config, + UDPListener: &udpListenerStdlib{}, + mockDialEarly: func(ctx context.Context, pconn net.PacketConn, + remoteAddr net.Addr, tlsConfig *tls.Config, quicConfig *quic.Config) (quic.EarlyConnection, error) { gotTLSConfig = tlsConfig return nil, expected @@ -312,9 +306,9 @@ func TestQUICDialerQUICGo(t *testing.T) { } fakeconn := &mocks.QUICEarlyConnection{} systemdialer := quicDialerQUICGo{ - QUICListener: &quicListenerStdlib{}, - mockDialEarlyContext: func(ctx context.Context, pconn net.PacketConn, - remoteAddr net.Addr, host string, tlsConfig *tls.Config, + UDPListener: &udpListenerStdlib{}, + mockDialEarly: func(ctx context.Context, pconn net.PacketConn, + remoteAddr net.Addr, tlsConfig *tls.Config, quicConfig *quic.Config) (quic.EarlyConnection, error) { return fakeconn, nil }, @@ -346,7 +340,7 @@ func TestQUICDialerWithCustomUnderlyingNetwork(t *testing.T) { }, } systemdialer := &quicDialerQUICGo{ - QUICListener: &quicListenerStdlib{provider: &MaybeCustomUnderlyingNetwork{proxy}}, + UDPListener: &udpListenerStdlib{provider: &MaybeCustomUnderlyingNetwork{proxy}}, } qconn, err := systemdialer.DialContext(ctx, "8.8.8.8:443", tlsConf, qConf) if qconn != nil { @@ -376,9 +370,9 @@ func TestQUICDialerWithCustomUnderlyingNetwork(t *testing.T) { expected := errors.New("mocked") var gotTLSConfig *tls.Config systemdialer := &quicDialerQUICGo{ - QUICListener: &quicListenerStdlib{}, - mockDialEarlyContext: func(ctx context.Context, pconn net.PacketConn, remoteAddr net.Addr, - host string, tlsConfig *tls.Config, quicConfig *quic.Config) (quic.EarlyConnection, error) { + UDPListener: &udpListenerStdlib{}, + mockDialEarly: func(ctx context.Context, pconn net.PacketConn, remoteAddr net.Addr, + tlsConfig *tls.Config, quicConfig *quic.Config) (quic.EarlyConnection, error) { gotTLSConfig = tlsConfig return nil, expected }, @@ -428,9 +422,9 @@ func TestQUICDialerHandshakeCompleter(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) var called bool expected := &mocks.QUICEarlyConnection{ - MockHandshakeComplete: func() context.Context { + MockHandshakeComplete: func() <-chan struct{} { cancel() - return handshakeCtx + return handshakeCtx.Done() }, MockCloseWithError: func(code quic.ApplicationErrorCode, reason string) error { called = true @@ -461,9 +455,9 @@ func TestQUICDialerHandshakeCompleter(t *testing.T) { handshakeCtx, handshakeCancel := context.WithCancel(context.Background()) defer handshakeCancel() expected := &mocks.QUICEarlyConnection{ - MockHandshakeComplete: func() context.Context { + MockHandshakeComplete: func() <-chan struct{} { handshakeCancel() - return handshakeCtx + return handshakeCtx.Done() }, } d := &quicDialerHandshakeCompleter{ @@ -595,7 +589,7 @@ func TestQUICDialerResolver(t *testing.T) { dialer := &quicDialerResolver{ Resolver: NewStdlibResolver(log.Log), Dialer: &quicDialerQUICGo{ - QUICListener: &quicListenerStdlib{}, + UDPListener: &udpListenerStdlib{}, }} qconn, err := dialer.DialContext( context.Background(), "8.8.4.4:x", @@ -794,12 +788,12 @@ func TestNewSingleUseQUICDialer(t *testing.T) { } } -func TestQUICListenerErrWrapper(t *testing.T) { +func TestUDPListenerErrWrapper(t *testing.T) { t.Run("Listen", func(t *testing.T) { t.Run("on success", func(t *testing.T) { expectedConn := &mocks.UDPLikeConn{} - ql := &quicListenerErrWrapper{ - QUICListener: &mocks.QUICListener{ + ql := &udpListenerErrWrapper{ + UDPListener: &mocks.UDPListener{ MockListen: func(addr *net.UDPAddr) (model.UDPLikeConn, error) { return expectedConn, nil }, @@ -817,8 +811,8 @@ func TestQUICListenerErrWrapper(t *testing.T) { t.Run("on failure", func(t *testing.T) { expectedErr := io.EOF - ql := &quicListenerErrWrapper{ - QUICListener: &mocks.QUICListener{ + ql := &udpListenerErrWrapper{ + UDPListener: &mocks.UDPListener{ MockListen: func(addr *net.UDPAddr) (model.UDPLikeConn, error) { return nil, expectedErr }, diff --git a/pkg/netxlite/quictesting/quictesting.go b/pkg/netxlite/quictesting/quictesting.go deleted file mode 100644 index e956e1272..000000000 --- a/pkg/netxlite/quictesting/quictesting.go +++ /dev/null @@ -1,38 +0,0 @@ -// Package quictesting contains code useful to test QUIC. -package quictesting - -import ( - "context" - "net" - "strings" - "time" - - "github.com/ooni/probe-engine/pkg/runtimex" -) - -// Domain is the the domain we should be testing using QUIC. -const Domain = "www.cloudflare.com" - -// Address is the address we should be testing using QUIC. -var Address string - -// Endpoint returns the endpoint to test using QUIC by combining -// the Address variable with the given port. -func Endpoint(port string) string { - return net.JoinHostPort(Address, port) -} - -func init() { - const timeout = 10 * time.Second - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - reso := &net.Resolver{} - addrs, err := reso.LookupHost(ctx, Domain) - runtimex.PanicOnError(err, "reso.LookupHost failed") - for _, addr := range addrs { - if !strings.Contains(addr, ":") { - Address = addr - break - } - } -} diff --git a/pkg/netxlite/resolvercore.go b/pkg/netxlite/resolvercore.go index f63ec2f7c..7fccfd48e 100644 --- a/pkg/netxlite/resolvercore.go +++ b/pkg/netxlite/resolvercore.go @@ -23,30 +23,44 @@ import ( // but you are using the "stdlib" resolver instead. var ErrNoDNSTransport = errors.New("operation requires a DNS transport") -// NewStdlibResolver creates a new Resolver by combining WrapResolver -// with an internal "stdlib" resolver type. The list of optional wrappers -// allow to wrap the underlying getaddrinfo transport. Any nil wrapper -// will be silently ignored by the code that performs the wrapping. -func NewStdlibResolver(logger model.DebugLogger, wrappers ...model.DNSTransportWrapper) model.Resolver { - return WrapResolver(logger, NewUnwrappedStdlibResolver(wrappers...)) +// NewStdlibResolver implements [model.MeasuringNetwork]. +func (netx *Netx) NewStdlibResolver(logger model.DebugLogger) model.Resolver { + return WrapResolver(logger, netx.newUnwrappedStdlibResolver()) } -// NewParallelDNSOverHTTPSResolver creates a new DNS over HTTPS resolver -// that uses the standard library for all operations. This function constructs -// all the building blocks and calls WrapResolver on the returned resolver. -func NewParallelDNSOverHTTPSResolver(logger model.DebugLogger, URL string) model.Resolver { - client := &http.Client{Transport: NewHTTPTransportStdlib(logger)} - txp := WrapDNSTransport(NewUnwrappedDNSOverHTTPSTransport(client, URL)) +// NewStdlibResolver is equivalent to creating an empty [*Netx] +// and calling its NewStdlibResolver method. +func NewStdlibResolver(logger model.DebugLogger) model.Resolver { + netx := &Netx{Underlying: nil} + return netx.NewStdlibResolver(logger) +} + +// NewParallelDNSOverHTTPSResolver implements [model.MeasuringNetwork]. +func (netx *Netx) NewParallelDNSOverHTTPSResolver(logger model.DebugLogger, URL string) model.Resolver { + client := &http.Client{Transport: netx.NewHTTPTransportStdlib(logger)} + txp := wrapDNSTransport(NewUnwrappedDNSOverHTTPSTransport(client, URL)) return WrapResolver(logger, NewUnwrappedParallelResolver(txp)) } +// NewParallelDNSOverHTTPSResolver is equivalent to creating an empty [*Netx] +// and calling its NewParallelDNSOverHTTPSResolver method. +func NewParallelDNSOverHTTPSResolver(logger model.DebugLogger, URL string) model.Resolver { + netx := &Netx{Underlying: nil} + return netx.NewParallelDNSOverHTTPSResolver(logger, URL) +} + +func (netx *Netx) newUnwrappedStdlibResolver() model.Resolver { + return &resolverSystem{ + t: wrapDNSTransport(netx.newDNSOverGetaddrinfoTransport()), + } +} + // NewUnwrappedStdlibResolver returns a new, unwrapped resolver using the standard // library (i.e., getaddrinfo if possible and &net.Resolver{} otherwise). As the name // implies, this function returns an unwrapped resolver. -func NewUnwrappedStdlibResolver(wrappers ...model.DNSTransportWrapper) model.Resolver { - return &resolverSystem{ - t: WrapDNSTransport(NewDNSOverGetaddrinfoTransport(), wrappers...), - } +func NewUnwrappedStdlibResolver() model.Resolver { + netx := &Netx{Underlying: nil} + return netx.newUnwrappedStdlibResolver() } // NewSerialUDPResolver creates a new Resolver using DNS-over-UDP @@ -61,36 +75,26 @@ func NewUnwrappedStdlibResolver(wrappers ...model.DNSTransportWrapper) model.Res // - dialer is the dialer to create and connect UDP conns // // - address is the server address (e.g., 1.1.1.1:53) -// -// - wrappers is the optional list of wrappers to wrap the underlying -// transport. Any nil wrapper will be silently ignored. -func NewSerialUDPResolver(logger model.DebugLogger, dialer model.Dialer, - address string, wrappers ...model.DNSTransportWrapper) model.Resolver { +func NewSerialUDPResolver(logger model.DebugLogger, dialer model.Dialer, address string) model.Resolver { return WrapResolver(logger, NewUnwrappedSerialResolver( - WrapDNSTransport(NewUnwrappedDNSOverUDPTransport(dialer, address), wrappers...), + wrapDNSTransport(NewUnwrappedDNSOverUDPTransport(dialer, address)), )) } -// NewParallelUDPResolver creates a new Resolver using DNS-over-UDP -// that performs parallel A/AAAA lookups during LookupHost. -// -// Arguments: -// -// - logger is the logger to use -// -// - dialer is the dialer to create and connect UDP conns -// -// - address is the server address (e.g., 1.1.1.1:53) -// -// - wrappers is the optional list of wrappers to wrap the underlying -// transport. Any nil wrapper will be silently ignored. -func NewParallelUDPResolver(logger model.DebugLogger, dialer model.Dialer, - address string, wrappers ...model.DNSTransportWrapper) model.Resolver { +// NewParallelUDPResolver implements [model.MeasuringNetwork]. +func (netx *Netx) NewParallelUDPResolver(logger model.DebugLogger, dialer model.Dialer, address string) model.Resolver { return WrapResolver(logger, NewUnwrappedParallelResolver( - WrapDNSTransport(NewUnwrappedDNSOverUDPTransport(dialer, address), wrappers...), + wrapDNSTransport(NewUnwrappedDNSOverUDPTransport(dialer, address)), )) } +// NewParallelUDPResolver is equivalent to creating an empty [*Netx] +// and calling its NewParallelUDPResolver method. +func NewParallelUDPResolver(logger model.DebugLogger, dialer model.Dialer, address string) model.Resolver { + netx := &Netx{Underlying: nil} + return netx.NewParallelUDPResolver(logger, dialer, address) +} + // WrapResolver creates a new resolver that wraps an // existing resolver to add these properties: // @@ -110,7 +114,7 @@ func NewParallelUDPResolver(logger model.DebugLogger, dialer model.Dialer, func WrapResolver(logger model.DebugLogger, resolver model.Resolver) model.Resolver { return &resolverIDNA{ Resolver: &resolverLogger{ - Resolver: &resolverShortCircuitIPAddr{ + Resolver: &ResolverShortCircuitIPAddr{ Resolver: &resolverErrWrapper{ Resolver: resolver, }, @@ -279,22 +283,22 @@ func (r *resolverIDNA) LookupNS( return r.Resolver.LookupNS(ctx, host) } -// resolverShortCircuitIPAddr recognizes when the input hostname is an +// ResolverShortCircuitIPAddr recognizes when the input hostname is an // IP address and returns it immediately to the caller. -type resolverShortCircuitIPAddr struct { +type ResolverShortCircuitIPAddr struct { Resolver model.Resolver } -var _ model.Resolver = &resolverShortCircuitIPAddr{} +var _ model.Resolver = &ResolverShortCircuitIPAddr{} -func (r *resolverShortCircuitIPAddr) LookupHost(ctx context.Context, hostname string) ([]string, error) { +func (r *ResolverShortCircuitIPAddr) LookupHost(ctx context.Context, hostname string) ([]string, error) { if net.ParseIP(hostname) != nil { return []string{hostname}, nil } return r.Resolver.LookupHost(ctx, hostname) } -func (r *resolverShortCircuitIPAddr) LookupHTTPS(ctx context.Context, hostname string) (*model.HTTPSSvc, error) { +func (r *ResolverShortCircuitIPAddr) LookupHTTPS(ctx context.Context, hostname string) (*model.HTTPSSvc, error) { if net.ParseIP(hostname) != nil { https := &model.HTTPSSvc{} if isIPv6(hostname) { @@ -307,15 +311,15 @@ func (r *resolverShortCircuitIPAddr) LookupHTTPS(ctx context.Context, hostname s return r.Resolver.LookupHTTPS(ctx, hostname) } -func (r *resolverShortCircuitIPAddr) Network() string { +func (r *ResolverShortCircuitIPAddr) Network() string { return r.Resolver.Network() } -func (r *resolverShortCircuitIPAddr) Address() string { +func (r *ResolverShortCircuitIPAddr) Address() string { return r.Resolver.Address() } -func (r *resolverShortCircuitIPAddr) CloseIdleConnections() { +func (r *ResolverShortCircuitIPAddr) CloseIdleConnections() { r.Resolver.CloseIdleConnections() } @@ -323,7 +327,7 @@ func (r *resolverShortCircuitIPAddr) CloseIdleConnections() { // function that only works with domain names. var ErrDNSIPAddress = errors.New("ooresolver: expected domain, found IP address") -func (r *resolverShortCircuitIPAddr) LookupNS( +func (r *ResolverShortCircuitIPAddr) LookupNS( ctx context.Context, hostname string) ([]*net.NS, error) { if net.ParseIP(hostname) != nil { return nil, ErrDNSIPAddress diff --git a/pkg/netxlite/resolvercore_test.go b/pkg/netxlite/resolvercore_test.go index f5ad58e2f..37e2bef8e 100644 --- a/pkg/netxlite/resolvercore_test.go +++ b/pkg/netxlite/resolvercore_test.go @@ -17,13 +17,13 @@ import ( "github.com/ooni/probe-engine/pkg/testingx" ) -func typecheckForSystemResolver(t *testing.T, resolver model.Resolver, logger model.DebugLogger) { +func typeCheckForSystemResolver(t *testing.T, resolver model.Resolver, logger model.DebugLogger) { idna := resolver.(*resolverIDNA) loggerReso := idna.Resolver.(*resolverLogger) if loggerReso.Logger != logger { t.Fatal("invalid logger") } - shortCircuit := loggerReso.Resolver.(*resolverShortCircuitIPAddr) + shortCircuit := loggerReso.Resolver.(*ResolverShortCircuitIPAddr) errWrapper := shortCircuit.Resolver.(*resolverErrWrapper) reso := errWrapper.Resolver.(*resolverSystem) txpErrWrapper := reso.t.(*dnsTransportErrWrapper) @@ -32,7 +32,7 @@ func typecheckForSystemResolver(t *testing.T, resolver model.Resolver, logger mo func TestNewResolverSystem(t *testing.T) { resolver := NewStdlibResolver(model.DiscardLogger) - typecheckForSystemResolver(t, resolver, model.DiscardLogger) + typeCheckForSystemResolver(t, resolver, model.DiscardLogger) } func TestNewSerialUDPResolver(t *testing.T) { @@ -43,7 +43,7 @@ func TestNewSerialUDPResolver(t *testing.T) { if logger.Logger != log.Log { t.Fatal("invalid logger") } - shortCircuit := logger.Resolver.(*resolverShortCircuitIPAddr) + shortCircuit := logger.Resolver.(*ResolverShortCircuitIPAddr) errWrapper := shortCircuit.Resolver.(*resolverErrWrapper) serio := errWrapper.Resolver.(*SerialResolver) txp := serio.Transport().(*dnsTransportErrWrapper) @@ -61,7 +61,7 @@ func TestNewParallelUDPResolver(t *testing.T) { if logger.Logger != log.Log { t.Fatal("invalid logger") } - shortCircuit := logger.Resolver.(*resolverShortCircuitIPAddr) + shortCircuit := logger.Resolver.(*ResolverShortCircuitIPAddr) errWrapper := shortCircuit.Resolver.(*resolverErrWrapper) para := errWrapper.Resolver.(*ParallelResolver) txp := para.Transport().(*dnsTransportErrWrapper) @@ -78,7 +78,7 @@ func TestNewParallelDNSOverHTTPSResolver(t *testing.T) { if logger.Logger != log.Log { t.Fatal("invalid logger") } - shortCircuit := logger.Resolver.(*resolverShortCircuitIPAddr) + shortCircuit := logger.Resolver.(*ResolverShortCircuitIPAddr) errWrapper := shortCircuit.Resolver.(*resolverErrWrapper) para := errWrapper.Resolver.(*ParallelResolver) txp := para.Transport().(*dnsTransportErrWrapper) @@ -736,7 +736,7 @@ func TestResolverIDNA(t *testing.T) { func TestResolverShortCircuitIPAddr(t *testing.T) { t.Run("LookupHost", func(t *testing.T) { t.Run("with IP addr", func(t *testing.T) { - r := &resolverShortCircuitIPAddr{ + r := &ResolverShortCircuitIPAddr{ Resolver: &mocks.Resolver{ MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { return nil, errors.New("mocked error") @@ -754,7 +754,7 @@ func TestResolverShortCircuitIPAddr(t *testing.T) { }) t.Run("with domain", func(t *testing.T) { - r := &resolverShortCircuitIPAddr{ + r := &ResolverShortCircuitIPAddr{ Resolver: &mocks.Resolver{ MockLookupHost: func(ctx context.Context, domain string) ([]string, error) { return nil, errors.New("mocked error") @@ -774,7 +774,7 @@ func TestResolverShortCircuitIPAddr(t *testing.T) { t.Run("LookupHTTPS", func(t *testing.T) { t.Run("with IPv4 addr", func(t *testing.T) { - r := &resolverShortCircuitIPAddr{ + r := &ResolverShortCircuitIPAddr{ Resolver: &mocks.Resolver{ MockLookupHTTPS: func(ctx context.Context, domain string) (*model.HTTPSSvc, error) { return nil, errors.New("mocked error") @@ -792,7 +792,7 @@ func TestResolverShortCircuitIPAddr(t *testing.T) { }) t.Run("with IPv6 addr", func(t *testing.T) { - r := &resolverShortCircuitIPAddr{ + r := &ResolverShortCircuitIPAddr{ Resolver: &mocks.Resolver{ MockLookupHTTPS: func(ctx context.Context, domain string) (*model.HTTPSSvc, error) { return nil, errors.New("mocked error") @@ -810,7 +810,7 @@ func TestResolverShortCircuitIPAddr(t *testing.T) { }) t.Run("with domain", func(t *testing.T) { - r := &resolverShortCircuitIPAddr{ + r := &ResolverShortCircuitIPAddr{ Resolver: &mocks.Resolver{ MockLookupHTTPS: func(ctx context.Context, domain string) (*model.HTTPSSvc, error) { return nil, errors.New("mocked error") @@ -830,7 +830,7 @@ func TestResolverShortCircuitIPAddr(t *testing.T) { t.Run("LookupNS", func(t *testing.T) { t.Run("with IPv4 addr", func(t *testing.T) { - r := &resolverShortCircuitIPAddr{ + r := &ResolverShortCircuitIPAddr{ Resolver: &mocks.Resolver{ MockLookupNS: func(ctx context.Context, domain string) ([]*net.NS, error) { return nil, errors.New("mocked error") @@ -848,7 +848,7 @@ func TestResolverShortCircuitIPAddr(t *testing.T) { }) t.Run("with IPv6 addr", func(t *testing.T) { - r := &resolverShortCircuitIPAddr{ + r := &ResolverShortCircuitIPAddr{ Resolver: &mocks.Resolver{ MockLookupNS: func(ctx context.Context, domain string) ([]*net.NS, error) { return nil, errors.New("mocked error") @@ -866,7 +866,7 @@ func TestResolverShortCircuitIPAddr(t *testing.T) { }) t.Run("with domain", func(t *testing.T) { - r := &resolverShortCircuitIPAddr{ + r := &ResolverShortCircuitIPAddr{ Resolver: &mocks.Resolver{ MockLookupNS: func(ctx context.Context, domain string) ([]*net.NS, error) { return nil, errors.New("mocked error") @@ -890,7 +890,7 @@ func TestResolverShortCircuitIPAddr(t *testing.T) { return "x" }, } - reso := &resolverShortCircuitIPAddr{child} + reso := &ResolverShortCircuitIPAddr{child} if reso.Network() != "x" { t.Fatal("invalid result") } @@ -902,7 +902,7 @@ func TestResolverShortCircuitIPAddr(t *testing.T) { return "x" }, } - reso := &resolverShortCircuitIPAddr{child} + reso := &ResolverShortCircuitIPAddr{child} if reso.Address() != "x" { t.Fatal("invalid result") } @@ -915,7 +915,7 @@ func TestResolverShortCircuitIPAddr(t *testing.T) { called = true }, } - reso := &resolverShortCircuitIPAddr{child} + reso := &ResolverShortCircuitIPAddr{child} reso.CloseIdleConnections() if !called { t.Fatal("not called") diff --git a/pkg/netxlite/tls.go b/pkg/netxlite/tls.go index 727e59792..9cb2d405c 100644 --- a/pkg/netxlite/tls.go +++ b/pkg/netxlite/tls.go @@ -14,7 +14,6 @@ import ( "time" ootls "github.com/ooni/oocrypto/tls" - oohttp "github.com/ooni/oohttp" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/runtimex" ) @@ -112,6 +111,15 @@ func NewMozillaCertPool() *x509.CertPool { return pool } +// MaybeTLSConnectionState is a convenience function that returns an +// empty [tls.ConnectionState] when the [model.TLSConn] is nil. +func MaybeTLSConnectionState(conn model.TLSConn) (state tls.ConnectionState) { + if conn != nil { + state = conn.ConnectionState() + } + return +} + // ErrInvalidTLSVersion indicates that you passed us a string // that does not represent a valid TLS version. var ErrInvalidTLSVersion = errors.New("invalid TLS version") @@ -142,29 +150,23 @@ func ConfigureTLSVersion(config *tls.Config, version string) error { return nil } -// TLSConn is the type of connection that oohttp expects from -// any library that implements TLS functionality. By using this -// kind of TLSConn we're able to use both the standard library -// and gitlab.com/yawning/utls.git to perform TLS operations. Note -// that the stdlib's tls.Conn implements this interface. -type TLSConn = oohttp.TLSConn +// The TLSConn alias was originally defined here in [netxlite] and we +// want to keep it available to other packages for now. +type TLSConn = model.TLSConn -// Ensures that a tls.Conn implements the TLSConn interface. -var _ TLSConn = &tls.Conn{} +// NewTLSHandshakerStdlib implements [model.MeasuringNetwork]. +func (netx *Netx) NewTLSHandshakerStdlib(logger model.DebugLogger) model.TLSHandshaker { + return newTLSHandshakerLogger( + &tlsHandshakerConfigurable{provider: netx.MaybeCustomUnderlyingNetwork()}, + logger, + ) +} -// NewTLSHandshakerStdlib creates a new TLS handshaker using the -// go standard library to manage TLS. -// -// The handshaker guarantees: -// -// 1. logging; -// -// 2. error wrapping; -// -// 3. that we are going to use Mozilla CA if the [tls.Config] -// RootCAs field is zero initialized. +// NewTLSHandshakerStdlib is equivalent to creating an empty [*Netx] +// and calling its NewTLSHandshakerStdlib method. func NewTLSHandshakerStdlib(logger model.DebugLogger) model.TLSHandshaker { - return newTLSHandshakerLogger(&tlsHandshakerConfigurable{}, logger) + netx := &Netx{Underlying: nil} + return netx.NewTLSHandshakerStdlib(logger) } // newTLSHandshakerLogger creates a new tlsHandshakerLogger instance. @@ -210,7 +212,7 @@ func tlsMaybeConnectionState(conn TLSConn, err error) tls.ConnectionState { // This function will also emit TLS-handshake-related tracing events. func (h *tlsHandshakerConfigurable) Handshake( ctx context.Context, conn net.Conn, config *tls.Config, -) (net.Conn, tls.ConnectionState, error) { +) (model.TLSConn, error) { timeout := h.Timeout if timeout <= 0 { timeout = 10 * time.Second @@ -224,7 +226,7 @@ func (h *tlsHandshakerConfigurable) Handshake( } tlsconn, err := h.newConn(conn, config) if err != nil { - return nil, tls.ConnectionState{}, err + return nil, err } remoteAddr := conn.RemoteAddr().String() trace := ContextTraceOrDefault(ctx) @@ -236,9 +238,9 @@ func (h *tlsHandshakerConfigurable) Handshake( state := tlsMaybeConnectionState(tlsconn, err) trace.OnTLSHandshakeDone(started, remoteAddr, config, state, err, finished) if err != nil { - return nil, tls.ConnectionState{}, err + return nil, err } - return tlsconn, state, nil + return tlsconn, nil } // newConn creates a new TLSConn. @@ -260,24 +262,25 @@ var _ model.TLSHandshaker = &tlsHandshakerLogger{} // Handshake implements Handshaker.Handshake func (h *tlsHandshakerLogger) Handshake( ctx context.Context, conn net.Conn, config *tls.Config, -) (net.Conn, tls.ConnectionState, error) { +) (model.TLSConn, error) { h.DebugLogger.Debugf( - "tls {sni=%s next=%+v}...", config.ServerName, config.NextProtos) + "tls_handshake {sni=%s next=%+v}...", config.ServerName, config.NextProtos) start := time.Now() - tlsconn, state, err := h.TLSHandshaker.Handshake(ctx, conn, config) + tlsconn, err := h.TLSHandshaker.Handshake(ctx, conn, config) elapsed := time.Since(start) if err != nil { h.DebugLogger.Debugf( - "tls {sni=%s next=%+v}... %s in %s", config.ServerName, + "tls_handshake {sni=%s next=%+v}... %s in %s", config.ServerName, config.NextProtos, err, elapsed) - return nil, tls.ConnectionState{}, err + return nil, err } + state := MaybeTLSConnectionState(tlsconn) h.DebugLogger.Debugf( - "tls {sni=%s next=%+v}... ok in %s {next=%s cipher=%s v=%s}", + "tls_handshake {sni=%s next=%+v}... ok in %s {next=%s cipher=%s v=%s}", config.ServerName, config.NextProtos, elapsed, state.NegotiatedProtocol, TLSCipherSuiteString(state.CipherSuite), TLSVersionString(state.Version)) - return tlsconn, state, nil + return tlsconn, nil } // NewTLSDialer creates a new TLS dialer using the given dialer and handshaker. @@ -320,7 +323,7 @@ func (d *tlsDialer) DialTLSContext(ctx context.Context, network, address string) return nil, err } config := d.config(host, port) - tlsconn, _, err := d.TLSHandshaker.Handshake(ctx, conn, config) + tlsconn, err := d.TLSHandshaker.Handshake(ctx, conn, config) if err != nil { conn.Close() return nil, err diff --git a/pkg/netxlite/tls_test.go b/pkg/netxlite/tls_test.go index 3a5a0ce12..4f6603b12 100644 --- a/pkg/netxlite/tls_test.go +++ b/pkg/netxlite/tls_test.go @@ -17,6 +17,8 @@ import ( "github.com/apex/log" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/ooni/netem" "github.com/ooni/probe-engine/pkg/mocks" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/testingx" @@ -157,7 +159,7 @@ func TestTLSHandshakerConfigurable(t *testing.T) { }, } ctx := context.Background() - conn, state, err := h.Handshake(ctx, tcpConn, &tls.Config{ + conn, err := h.Handshake(ctx, tcpConn, &tls.Config{ ServerName: "x.org", }) if !errors.Is(err, io.EOF) { @@ -188,9 +190,6 @@ func TestTLSHandshakerConfigurable(t *testing.T) { if !times[1].IsZero() { t.Fatal("did not clear timeout on exit") } - if !reflect.ValueOf(state).IsZero() { - t.Fatal("the returned connection state is not a zero value") - } }) t.Run("with success", func(t *testing.T) { @@ -216,11 +215,12 @@ func TestTLSHandshakerConfigurable(t *testing.T) { MaxVersion: tls.VersionTLS13, ServerName: URL.Hostname(), } - tlsConn, connState, err := handshaker.Handshake(ctx, conn, config) + tlsConn, err := handshaker.Handshake(ctx, conn, config) if err != nil { t.Fatal(err) } defer tlsConn.Close() + connState := tlsConn.ConnectionState() if connState.Version != tls.VersionTLS13 { t.Fatal("unexpected TLS version") } @@ -256,13 +256,10 @@ func TestTLSHandshakerConfigurable(t *testing.T) { } }, } - tlsConn, connState, err := handshaker.Handshake(ctx, conn, config) + tlsConn, err := handshaker.Handshake(ctx, conn, config) if !errors.Is(err, expected) { t.Fatal("not the error we expected", err) } - if !reflect.ValueOf(connState).IsZero() { - t.Fatal("expected zero connState here") - } if tlsConn != nil { t.Fatal("expected nil tlsConn here") } @@ -320,13 +317,10 @@ func TestTLSHandshakerConfigurable(t *testing.T) { } }, } - tlsConn, connState, err := handshaker.Handshake(ctx, conn, config) + tlsConn, err := handshaker.Handshake(ctx, conn, config) if !errors.Is(err, expected) { t.Fatal("not the error we expected", err) } - if !reflect.ValueOf(connState).IsZero() { - t.Fatal("expected zero connState here") - } if tlsConn != nil { t.Fatal("expected nil tlsConn here") } @@ -352,13 +346,10 @@ func TestTLSHandshakerConfigurable(t *testing.T) { return nil }, } - tlsConn, connState, err := handshaker.Handshake(ctx, conn, config) + tlsConn, err := handshaker.Handshake(ctx, conn, config) if !errors.Is(err, expected) { t.Fatal("not the error we expected", err) } - if !reflect.ValueOf(connState).IsZero() { - t.Fatal("expected zero connState here") - } if tlsConn != nil { t.Fatal("expected nil tlsConn here") } @@ -381,8 +372,9 @@ func TestTLSHandshakerConfigurable(t *testing.T) { startCalled bool doneCalled bool ) - mitm := testingx.MustNewTLSMITMProviderNetem() - server := testingx.MustNewTLSServer(testingx.TLSHandlerHandshakeAndWriteText(mitm, testingx.HTTPBlockpage451)) + ca := netem.MustNewCA() + cert := ca.MustNewTLSCertificate(expectedSNI) + server := testingx.MustNewTLSServer(testingx.TLSHandlerHandshakeAndWriteText(cert, testingx.HTTPBlockpage451)) defer server.Close() zeroTime := time.Now() deterministicTime := testingx.NewTimeDeterministic(zeroTime) @@ -416,14 +408,11 @@ func TestTLSHandshakerConfigurable(t *testing.T) { InsecureSkipVerify: true, ServerName: expectedSNI, } - tlsConn, connState, err := thx.Handshake(ctx, tcpConn, tlsConfig) + tlsConn, err := thx.Handshake(ctx, tcpConn, tlsConfig) if err != nil { t.Fatal(err) } tlsConn.Close() - if reflect.ValueOf(connState).IsZero() { - t.Fatal("expected nonzero connState") - } if !startCalled { t.Fatal("start not called") } @@ -530,16 +519,13 @@ func TestTLSHandshakerConfigurable(t *testing.T) { InsecureSkipVerify: true, ServerName: expectedSNI, } - tlsConn, connState, err := thx.Handshake(ctx, tcpConn, tlsConfig) + tlsConn, err := thx.Handshake(ctx, tcpConn, tlsConfig) if !errors.Is(err, io.EOF) { t.Fatal("unexpected err", err) } if tlsConn != nil { t.Fatal("expected nil tlsConn") } - if !reflect.ValueOf(connState).IsZero() { - t.Fatal("expected zero connState") - } if !startCalled { t.Fatal("start not called") } @@ -594,8 +580,8 @@ func TestTLSHandshakerLogger(t *testing.T) { } th := &tlsHandshakerLogger{ TLSHandshaker: &mocks.TLSHandshaker{ - MockHandshake: func(ctx context.Context, conn net.Conn, config *tls.Config) (net.Conn, tls.ConnectionState, error) { - return tls.Client(conn, config), tls.ConnectionState{}, nil + MockHandshake: func(ctx context.Context, conn net.Conn, config *tls.Config) (model.TLSConn, error) { + return tls.Client(conn, config), nil }, }, DebugLogger: lo, @@ -607,16 +593,13 @@ func TestTLSHandshakerLogger(t *testing.T) { } config := &tls.Config{} ctx := context.Background() - tlsConn, connState, err := th.Handshake(ctx, conn, config) + tlsConn, err := th.Handshake(ctx, conn, config) if err != nil { t.Fatal(err) } if err := tlsConn.Close(); err != nil { t.Fatal(err) } - if !reflect.ValueOf(connState).IsZero() { - t.Fatal("expected zero ConnectionState here") - } if count != 2 { t.Fatal("invalid count") } @@ -632,8 +615,8 @@ func TestTLSHandshakerLogger(t *testing.T) { expected := errors.New("mocked error") th := &tlsHandshakerLogger{ TLSHandshaker: &mocks.TLSHandshaker{ - MockHandshake: func(ctx context.Context, conn net.Conn, config *tls.Config) (net.Conn, tls.ConnectionState, error) { - return nil, tls.ConnectionState{}, expected + MockHandshake: func(ctx context.Context, conn net.Conn, config *tls.Config) (model.TLSConn, error) { + return nil, expected }, }, DebugLogger: lo, @@ -645,16 +628,13 @@ func TestTLSHandshakerLogger(t *testing.T) { } config := &tls.Config{} ctx := context.Background() - tlsConn, connState, err := th.Handshake(ctx, conn, config) + tlsConn, err := th.Handshake(ctx, conn, config) if !errors.Is(err, expected) { t.Fatal("not the error we expected", err) } if tlsConn != nil { t.Fatal("expected nil conn here") } - if !reflect.ValueOf(connState).IsZero() { - t.Fatal("expected zero ConnectionState here") - } if count != 2 { t.Fatal("invalid count") } @@ -711,7 +691,7 @@ func TestTLSDialer(t *testing.T) { t.Run("failure dialing", func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() // immediately fail - dialer := tlsDialer{Dialer: &DialerSystem{}} + dialer := tlsDialer{Dialer: &dialerSystem{}} conn, err := dialer.DialTLSContext(ctx, "tcp", "www.google.com:443") if err == nil || !strings.HasSuffix(err.Error(), "operation was canceled") { t.Fatal("not the error we expected", err) @@ -767,8 +747,8 @@ func TestTLSDialer(t *testing.T) { }}, nil }}, TLSHandshaker: &mocks.TLSHandshaker{ - MockHandshake: func(ctx context.Context, conn net.Conn, config *tls.Config) (net.Conn, tls.ConnectionState, error) { - return tls.Client(conn, config), tls.ConnectionState{}, nil + MockHandshake: func(ctx context.Context, conn net.Conn, config *tls.Config) (model.TLSConn, error) { + return tls.Client(conn, config), nil }, }, } @@ -930,3 +910,39 @@ func TestMaybeConnectionState(t *testing.T) { } }) } + +func TestMaybeTLSConnectionState(t *testing.T) { + t.Run("when the TLSConn is nil", func(t *testing.T) { + expected := tls.ConnectionState{ /* empty */ } + got := MaybeTLSConnectionState(nil) + if diff := cmp.Diff(expected, got, cmpopts.IgnoreUnexported(tls.ConnectionState{})); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("when the TLSConn is not nil", func(t *testing.T) { + expected := tls.ConnectionState{ + Version: tls.VersionTLS13, + HandshakeComplete: true, + DidResume: false, + CipherSuite: tls.TLS_AES_128_GCM_SHA256, + NegotiatedProtocol: "h2", + NegotiatedProtocolIsMutual: true, + ServerName: "dns.google", + PeerCertificates: []*x509.Certificate{}, + VerifiedChains: [][]*x509.Certificate{}, + SignedCertificateTimestamps: [][]byte{}, + OCSPResponse: []byte{}, + TLSUnique: []byte{}, + } + conn := &mocks.TLSConn{ + MockConnectionState: func() tls.ConnectionState { + return expected + }, + } + got := MaybeTLSConnectionState(conn) + if diff := cmp.Diff(expected, got, cmpopts.IgnoreUnexported(tls.ConnectionState{})); diff != "" { + t.Fatal(diff) + } + }) +} diff --git a/pkg/netxlite/tproxy.go b/pkg/netxlite/tproxy.go index 6c1d0584a..31add58a4 100644 --- a/pkg/netxlite/tproxy.go +++ b/pkg/netxlite/tproxy.go @@ -86,6 +86,11 @@ func (tp *DefaultTProxy) DialContext(ctx context.Context, network, address strin return d.DialContext(ctx, network, address) } +// ListenTCP implements UnderlyingNetwork. +func (tp *DefaultTProxy) ListenTCP(network string, addr *net.TCPAddr) (net.Listener, error) { + return net.ListenTCP(network, addr) +} + // ListenUDP implements UnderlyingNetwork. func (tp *DefaultTProxy) ListenUDP(network string, addr *net.UDPAddr) (model.UDPLikeConn, error) { return net.ListenUDP(network, addr) diff --git a/pkg/netxlite/tproxy_test.go b/pkg/netxlite/tproxy_test.go index 4cdcc0489..50f000566 100644 --- a/pkg/netxlite/tproxy_test.go +++ b/pkg/netxlite/tproxy_test.go @@ -6,11 +6,13 @@ import ( "net" "net/http" "net/http/httptest" + "sync" "testing" "time" "github.com/ooni/probe-engine/pkg/mocks" "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/runtimex" ) func TestTproxyNilSafeProvider(t *testing.T) { @@ -83,6 +85,7 @@ func TestWithCustomTProxy(t *testing.T) { } WithCustomTProxy(tproxy, func() { + // TODO(https://github.com/ooni/probe/issues/2534): NewHTTPClientStdlib has QUIRKS but they're not needed here clnt := NewHTTPClientStdlib(model.DiscardLogger) req, err := http.NewRequestWithContext(context.Background(), "GET", srvr.URL, nil) if err != nil { @@ -98,3 +101,33 @@ func TestWithCustomTProxy(t *testing.T) { }) }) } + +// We generally do not listen here as part of other tests, since the listening +// functionality is mainly only use for testingx. So, here's a specific test for that. +func TestTproxyListenTCP(t *testing.T) { + tproxy := &DefaultTProxy{} + + listener := runtimex.Try1(tproxy.ListenTCP("tcp", &net.TCPAddr{})) + serverEndpoint := listener.Addr().String() + + // listen in a background goroutine + wg := &sync.WaitGroup{} + wg.Add(1) + go func() { + conn := runtimex.Try1(listener.Accept()) + conn.Close() + wg.Done() + }() + + // dial in a background goroutine + wg.Add(1) + go func() { + ctx := context.Background() + conn := runtimex.Try1(tproxy.DialContext(ctx, "tcp", serverEndpoint)) + conn.Close() + wg.Done() + }() + + // wait for the goroutines to finish + wg.Wait() +} diff --git a/pkg/netxlite/udp.go b/pkg/netxlite/udp.go new file mode 100644 index 000000000..047cca2e6 --- /dev/null +++ b/pkg/netxlite/udp.go @@ -0,0 +1,33 @@ +package netxlite + +import ( + "net" + + "github.com/ooni/probe-engine/pkg/model" +) + +// NewUDPListener creates a new UDPListener using the underlying +// [*Netx] structure to create listening UDP sockets. +func (netx *Netx) NewUDPListener() model.UDPListener { + return &udpListenerErrWrapper{&udpListenerStdlib{provider: netx.MaybeCustomUnderlyingNetwork()}} +} + +// NewUDPListener is equivalent to creating an empty [*Netx] +// and calling its NewUDPListener method. +func NewUDPListener() model.UDPListener { + netx := &Netx{Underlying: nil} + return netx.NewUDPListener() +} + +// udpListenerStdlib is a UDPListener using the standard library. +type udpListenerStdlib struct { + // provider is the OPTIONAL nil-safe [model.UnderlyingNetwork] provider. + provider *MaybeCustomUnderlyingNetwork +} + +var _ model.UDPListener = &udpListenerStdlib{} + +// Listen implements UDPListener.Listen. +func (qls *udpListenerStdlib) Listen(addr *net.UDPAddr) (model.UDPLikeConn, error) { + return qls.provider.Get().ListenUDP("udp", addr) +} diff --git a/pkg/netxlite/udp_test.go b/pkg/netxlite/udp_test.go new file mode 100644 index 000000000..77c9864e6 --- /dev/null +++ b/pkg/netxlite/udp_test.go @@ -0,0 +1,9 @@ +package netxlite + +import "testing" + +func TestNewUDPListener(t *testing.T) { + ql := NewUDPListener() + qew := ql.(*udpListenerErrWrapper) + _ = qew.UDPListener.(*udpListenerStdlib) +} diff --git a/pkg/netxlite/utls.go b/pkg/netxlite/utls.go index d373398fe..c86eb1c98 100644 --- a/pkg/netxlite/utls.go +++ b/pkg/netxlite/utls.go @@ -16,27 +16,21 @@ import ( utls "gitlab.com/yawning/utls.git" ) -// NewTLSHandshakerUTLS creates a new TLS handshaker using -// gitlab.com/yawning/utls for TLS. -// -// The id is the address of something like utls.HelloFirefox_55. -// -// The handshaker guarantees: -// -// 1. logging; -// -// 2. error wrapping; -// -// 3. that we are going to use Mozilla CA if the [tls.Config] -// RootCAs field is zero initialized. -// -// Passing a nil `id` will make this function panic. -func NewTLSHandshakerUTLS(logger model.DebugLogger, id *utls.ClientHelloID) model.TLSHandshaker { +// NewTLSHandshakerUTLS implements [model.MeasuringNetwork]. +func (netx *Netx) NewTLSHandshakerUTLS(logger model.DebugLogger, id *utls.ClientHelloID) model.TLSHandshaker { return newTLSHandshakerLogger(&tlsHandshakerConfigurable{ - NewConn: newUTLSConnFactory(id), + NewConn: newUTLSConnFactory(id), + provider: netx.MaybeCustomUnderlyingNetwork(), }, logger) } +// NewTLSHandshakerUTLS is equivalent to creating an empty [*Netx] +// and calling its NewTLSHandshakerUTLS method. +func NewTLSHandshakerUTLS(logger model.DebugLogger, id *utls.ClientHelloID) model.TLSHandshaker { + netx := &Netx{Underlying: nil} + return netx.NewTLSHandshakerUTLS(logger, id) +} + // UTLSConn implements TLSConn and uses a utls UConn as its underlying connection type UTLSConn struct { // We include the real UConn diff --git a/pkg/netxlite/utls_test.go b/pkg/netxlite/utls_test.go index dbf7223dc..184d28718 100644 --- a/pkg/netxlite/utls_test.go +++ b/pkg/netxlite/utls_test.go @@ -108,6 +108,10 @@ func TestUTLSConn(t *testing.T) { } func Test_newConnUTLSWithHelloID(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + tests := []struct { name string config *tls.Config diff --git a/pkg/oohelperd/dns.go b/pkg/oohelperd/dns.go index af7fd03c4..51f749df1 100644 --- a/pkg/oohelperd/dns.go +++ b/pkg/oohelperd/dns.go @@ -9,10 +9,10 @@ import ( "sync" "time" - "github.com/ooni/probe-engine/pkg/measurexlite" + "github.com/ooni/probe-engine/pkg/legacy/tracex" + "github.com/ooni/probe-engine/pkg/logx" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" - "github.com/ooni/probe-engine/pkg/tracex" ) // newfailure is a convenience shortcut to save typing. @@ -55,7 +55,7 @@ func dnsDo(ctx context.Context, config *dnsConfig) { defer reso.CloseIdleConnections() // perform and log the actual DNS lookup - ol := measurexlite.NewOperationLogger(config.Logger, "DNSLookup %s", config.Domain) + ol := logx.NewOperationLogger(config.Logger, "DNSLookup %s", config.Domain) addrs, err := reso.LookupHost(ctx, config.Domain) ol.Stop(err) diff --git a/pkg/oohelperd/handler.go b/pkg/oohelperd/handler.go index bf68fe038..a1ac456f7 100644 --- a/pkg/oohelperd/handler.go +++ b/pkg/oohelperd/handler.go @@ -72,6 +72,8 @@ func NewHandler() *Handler { Measure: measure, NewHTTPClient: func(logger model.Logger) model.HTTPClient { + // TODO(https://github.com/ooni/probe/issues/2534): the NewHTTPTransportWithResolver has QUIRKS and + // we should evaluate whether we can avoid using it here return newHTTPClientWithTransportFactory( logger, netxlite.NewHTTPTransportWithResolver, @@ -90,7 +92,7 @@ func NewHandler() *Handler { }, NewQUICDialer: func(logger model.Logger) model.QUICDialer { return netxlite.NewQUICDialerWithoutResolver( - netxlite.NewQUICListener(), + netxlite.NewUDPListener(), logger, ) }, diff --git a/pkg/oohelperd/http.go b/pkg/oohelperd/http.go index fc8f61b64..a4547a74c 100644 --- a/pkg/oohelperd/http.go +++ b/pkg/oohelperd/http.go @@ -13,11 +13,12 @@ import ( "sync" "time" + "github.com/ooni/probe-engine/pkg/legacy/tracex" + "github.com/ooni/probe-engine/pkg/logx" "github.com/ooni/probe-engine/pkg/measurexlite" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" "github.com/ooni/probe-engine/pkg/runtimex" - "github.com/ooni/probe-engine/pkg/tracex" ) // TODO(bassosimone): we should refactor the TH to use step-by-step such that we @@ -56,7 +57,7 @@ type httpConfig struct { // httpDo performs the HTTP check. func httpDo(ctx context.Context, config *httpConfig) { - ol := measurexlite.NewOperationLogger(config.Logger, "GET %s", config.URL) + ol := logx.NewOperationLogger(config.Logger, "GET %s", config.URL) const timeout = 15 * time.Second ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() diff --git a/pkg/oohelperd/quic.go b/pkg/oohelperd/quic.go index e7384ba55..3896f0516 100644 --- a/pkg/oohelperd/quic.go +++ b/pkg/oohelperd/quic.go @@ -10,6 +10,7 @@ import ( "sync" "time" + "github.com/ooni/probe-engine/pkg/logx" "github.com/ooni/probe-engine/pkg/measurexlite" "github.com/ooni/probe-engine/pkg/model" "github.com/quic-go/quic-go" @@ -68,7 +69,7 @@ func quicDo(ctx context.Context, config *quicConfig) { defer func() { config.Out <- out }() - ol := measurexlite.NewOperationLogger( + ol := logx.NewOperationLogger( config.Logger, "QUICConnect %s SNI=%s", config.Endpoint, diff --git a/pkg/oohelperd/tcptls.go b/pkg/oohelperd/tcptls.go index 5b6b17842..178fe71ee 100644 --- a/pkg/oohelperd/tcptls.go +++ b/pkg/oohelperd/tcptls.go @@ -10,6 +10,7 @@ import ( "sync" "time" + "github.com/ooni/probe-engine/pkg/logx" "github.com/ooni/probe-engine/pkg/measurexlite" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" @@ -81,7 +82,7 @@ func tcpTLSDo(ctx context.Context, config *tcpTLSConfig) { defer func() { config.Out <- out }() - ol := measurexlite.NewOperationLogger( + ol := logx.NewOperationLogger( config.Logger, "TCPConnect %s EnableTLS=%v SNI=%s", config.Endpoint, @@ -107,7 +108,7 @@ func tcpTLSDo(ctx context.Context, config *tcpTLSConfig) { ServerName: config.URLHostname, } thx := config.NewTSLHandshaker(config.Logger) - tlsConn, _, err := thx.Handshake(ctx, conn, tlsConfig) + tlsConn, err := thx.Handshake(ctx, conn, tlsConfig) ol.Stop(err) out.TLS = &ctrlTLSResult{ ServerName: config.URLHostname, diff --git a/pkg/probeservices/checkin_test.go b/pkg/probeservices/checkin_test.go index 9ec6c974e..2b50ef35c 100644 --- a/pkg/probeservices/checkin_test.go +++ b/pkg/probeservices/checkin_test.go @@ -9,6 +9,10 @@ import ( ) func TestCheckInSuccess(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + client := newclient() client.BaseURL = "https://ams-pg-test.ooni.org" config := model.OOAPICheckInConfig{ diff --git a/pkg/probeservices/collector_test.go b/pkg/probeservices/collector_test.go index 474751fc1..2ab684a1e 100644 --- a/pkg/probeservices/collector_test.go +++ b/pkg/probeservices/collector_test.go @@ -71,6 +71,10 @@ func TestNewReportTemplate(t *testing.T) { } func TestReportLifecycle(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + ctx := context.Background() template := model.OOAPIReportTemplate{ DataFormatVersion: model.OOAPIReportDefaultDataFormatVersion, @@ -101,6 +105,10 @@ func TestReportLifecycle(t *testing.T) { } func TestReportLifecycleWrongExperiment(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + ctx := context.Background() template := model.OOAPIReportTemplate{ DataFormatVersion: model.OOAPIReportDefaultDataFormatVersion, @@ -355,6 +363,10 @@ func TestOpenReportCancelledContext(t *testing.T) { } func TestSubmitMeasurementCancelledContext(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + ctx, cancel := context.WithCancel(context.Background()) defer cancel() template := model.OOAPIReportTemplate{ diff --git a/pkg/probeservices/login_test.go b/pkg/probeservices/login_test.go index eaf76139d..a2bf3345f 100644 --- a/pkg/probeservices/login_test.go +++ b/pkg/probeservices/login_test.go @@ -21,6 +21,7 @@ func TestMaybeLogin(t *testing.T) { t.Fatal(err) } }) + t.Run("when we have already registered", func(t *testing.T) { clnt := newclient() state := State{ @@ -34,6 +35,7 @@ func TestMaybeLogin(t *testing.T) { t.Fatal("expected an error here") } }) + t.Run("when the API call fails", func(t *testing.T) { clnt := newclient() clnt.BaseURL = "\t\t\t" // causes the code to fail @@ -52,6 +54,10 @@ func TestMaybeLogin(t *testing.T) { } func TestMaybeLoginIdempotent(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + clnt := newclient() ctx := context.Background() metadata := MetadataFixture() diff --git a/pkg/probeservices/measurementmeta_test.go b/pkg/probeservices/measurementmeta_test.go index b29ede6d9..79d0b2e17 100644 --- a/pkg/probeservices/measurementmeta_test.go +++ b/pkg/probeservices/measurementmeta_test.go @@ -15,6 +15,10 @@ import ( ) func TestGetMeasurementMetaWorkingAsIntended(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + client := Client{ APIClientTemplate: httpx.APIClientTemplate{ BaseURL: "https://api.ooni.io/", diff --git a/pkg/probeservices/probeservices_test.go b/pkg/probeservices/probeservices_test.go index b80a6aad4..8ee959f16 100644 --- a/pkg/probeservices/probeservices_test.go +++ b/pkg/probeservices/probeservices_test.go @@ -574,6 +574,10 @@ func TestSelectBestSelectsTheFastest(t *testing.T) { } func TestGetCredsAndAuthNotLoggedIn(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + clnt := newclient() if err := clnt.MaybeRegister(context.Background(), MetadataFixture()); err != nil { t.Fatal(err) diff --git a/pkg/probeservices/psiphon_test.go b/pkg/probeservices/psiphon_test.go index 173e06dc5..3bfacd2de 100644 --- a/pkg/probeservices/psiphon_test.go +++ b/pkg/probeservices/psiphon_test.go @@ -8,6 +8,10 @@ import ( ) func TestFetchPsiphonConfig(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + clnt := newclient() if err := clnt.MaybeRegister(context.Background(), MetadataFixture()); err != nil { t.Fatal(err) diff --git a/pkg/probeservices/register_test.go b/pkg/probeservices/register_test.go index bbdf9ca79..6e81a1782 100644 --- a/pkg/probeservices/register_test.go +++ b/pkg/probeservices/register_test.go @@ -47,6 +47,10 @@ func TestMaybeRegister(t *testing.T) { } func TestMaybeRegisterIdempotent(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + clnt := newclient() ctx := context.Background() metadata := MetadataFixture() diff --git a/pkg/probeservices/tor_test.go b/pkg/probeservices/tor_test.go index 58d8031cd..62f706514 100644 --- a/pkg/probeservices/tor_test.go +++ b/pkg/probeservices/tor_test.go @@ -59,6 +59,10 @@ func (clnt *FetchTorTargetsHTTPTransport) RoundTrip(req *http.Request) (*http.Re } func TestFetchTorTargetsSetsQueryString(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + clnt := newclient() txp := new(FetchTorTargetsHTTPTransport) clnt.HTTPClient = &http.Client{Transport: txp} diff --git a/pkg/ptx/fake_test.go b/pkg/ptx/fake_test.go index d9d074263..ed3af5548 100644 --- a/pkg/ptx/fake_test.go +++ b/pkg/ptx/fake_test.go @@ -6,6 +6,10 @@ import ( ) func TestFakeDialerWorks(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + fd := &FakeDialer{Address: "8.8.8.8:53"} conn, err := fd.DialContext(context.Background()) if err != nil { diff --git a/pkg/ptx/obfs4.go b/pkg/ptx/obfs4.go index eb9c0c114..d3a8707d1 100644 --- a/pkg/ptx/obfs4.go +++ b/pkg/ptx/obfs4.go @@ -7,11 +7,11 @@ import ( "path/filepath" "time" - pt "git.torproject.org/pluggable-transports/goptlib.git" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/runtimex" "gitlab.com/yawning/obfs4.git/transports/base" "gitlab.com/yawning/obfs4.git/transports/obfs4" + pt "gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/goptlib" ) // DefaultTestingOBFS4Bridge is a factory that returns you diff --git a/pkg/ptx/ptx.go b/pkg/ptx/ptx.go index b2a30f57e..464b7222b 100644 --- a/pkg/ptx/ptx.go +++ b/pkg/ptx/ptx.go @@ -45,10 +45,10 @@ import ( "strings" "sync" - pt "git.torproject.org/pluggable-transports/goptlib.git" "github.com/ooni/probe-engine/pkg/bytecounter" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" + pt "gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/goptlib" ) // PTDialer is a generic pluggable transports dialer. diff --git a/pkg/ptx/ptx_test.go b/pkg/ptx/ptx_test.go index 47767a2bf..ba0213e86 100644 --- a/pkg/ptx/ptx_test.go +++ b/pkg/ptx/ptx_test.go @@ -25,6 +25,10 @@ func TestListenerLoggerWorks(t *testing.T) { } func TestListenerWorksWithFakeDialer(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + // start the fake PT fd := &FakeDialer{Address: "google.com:80"} lst := &Listener{PTDialer: fd} diff --git a/pkg/ptx/snowflake.go b/pkg/ptx/snowflake.go index 2bb08606c..a9affb5de 100644 --- a/pkg/ptx/snowflake.go +++ b/pkg/ptx/snowflake.go @@ -5,8 +5,8 @@ import ( "errors" "net" - sflib "git.torproject.org/pluggable-transports/snowflake.git/v2/client/lib" "github.com/ooni/probe-engine/pkg/stuninput" + sflib "gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/client/lib" ) // SnowflakeRendezvousMethod is the method which with we perform the rendezvous. @@ -45,7 +45,7 @@ func (d *snowflakeRendezvousMethodDomainFronting) BrokerURL() string { } func (d *snowflakeRendezvousMethodDomainFronting) FrontDomain() string { - return "cdn.sstatic.net" + return "foursquare.com" } // NewSnowflakeRendezvousMethodAMP is a rendezvous method that diff --git a/pkg/ptx/snowflake_test.go b/pkg/ptx/snowflake_test.go index e53d029fd..6810b596d 100644 --- a/pkg/ptx/snowflake_test.go +++ b/pkg/ptx/snowflake_test.go @@ -7,8 +7,8 @@ import ( "sync/atomic" "testing" - sflib "git.torproject.org/pluggable-transports/snowflake.git/v2/client/lib" "github.com/ooni/probe-engine/pkg/mocks" + sflib "gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2/client/lib" ) func TestSnowflakeMethodDomainFronting(t *testing.T) { @@ -20,7 +20,7 @@ func TestSnowflakeMethodDomainFronting(t *testing.T) { if meth.BrokerURL() != brokerURL { t.Fatal("invalid broker URL") } - const frontDomain = "cdn.sstatic.net" + const frontDomain = "foursquare.com" if meth.FrontDomain() != frontDomain { t.Fatal("invalid front domain") } diff --git a/pkg/registry/factory.go b/pkg/registry/factory.go index 92da5fcc8..186d507f0 100644 --- a/pkg/registry/factory.go +++ b/pkg/registry/factory.go @@ -10,8 +10,8 @@ import ( "reflect" "strconv" - "github.com/iancoleman/strcase" "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/strcasex" ) // Factory allows to construct an experiment measurer. @@ -201,7 +201,7 @@ func (b *Factory) NewExperimentMeasurer() model.ExperimentMeasurer { // compatibility with MK, we need to add some exceptions here when // mapping (e.g., DNSCheck => dnscheck). func CanonicalizeExperimentName(name string) string { - switch name = strcase.ToSnake(name); name { + switch name = strcasex.ToSnake(name); name { case "ndt_7": name = "ndt" // since 2020-03-18, we use ndt7 to implement ndt by default case "dns_check": diff --git a/pkg/scrubber/logger.go b/pkg/scrubber/logger.go deleted file mode 100644 index 16248b1c4..000000000 --- a/pkg/scrubber/logger.go +++ /dev/null @@ -1,44 +0,0 @@ -package scrubber - -import ( - "fmt" - - "github.com/ooni/probe-engine/pkg/model" -) - -// Logger is a Logger with scrubbing. All messages are scrubbed -// including the ones that won't be emitted. As such, this logger -// is less efficient than a logger without scrubbing. -type Logger struct { - model.Logger -} - -// Debug scrubs and emits a debug message. -func (sl *Logger) Debug(message string) { - sl.Logger.Debug(Scrub(message)) -} - -// Debugf scrubs, formats, and emits a debug message. -func (sl *Logger) Debugf(format string, v ...interface{}) { - sl.Debug(fmt.Sprintf(format, v...)) -} - -// Info scrubs and emits an informational message. -func (sl *Logger) Info(message string) { - sl.Logger.Info(Scrub(message)) -} - -// Infof scrubs, formats, and emits an informational message. -func (sl *Logger) Infof(format string, v ...interface{}) { - sl.Info(fmt.Sprintf(format, v...)) -} - -// Warn scrubs and emits a warning message. -func (sl *Logger) Warn(message string) { - sl.Logger.Warn(Scrub(message)) -} - -// Warnf scrubs, formats, and emits a warning message. -func (sl *Logger) Warnf(format string, v ...interface{}) { - sl.Warn(fmt.Sprintf(format, v...)) -} diff --git a/pkg/scrubber/scrubber.go b/pkg/scrubber/scrubber.go index 022aa2472..1bf4c5e0c 100644 --- a/pkg/scrubber/scrubber.go +++ b/pkg/scrubber/scrubber.go @@ -54,7 +54,8 @@ var scrubberPatterns = []*regexp.Regexp{ var addressRegexp = regexp.MustCompile(addressPattern) -func scrub(b []byte) []byte { +// ScrubBytes scrubs bytes to remove references to IP endpoints. +func ScrubBytes(b []byte) []byte { scrubbedBytes := b for _, pattern := range scrubberPatterns { // this is a workaround since go does not yet support look ahead or look @@ -66,8 +67,7 @@ func scrub(b []byte) []byte { return scrubbedBytes } -// Scrub sanitizes a string containing an error such that -// any occurrence of IP endpoints is scrubbed. -func Scrub(s string) string { - return string(scrub([]byte(s))) +// ScrubString scrubs a string to remove references to IP endpoints. +func ScrubString(s string) string { + return string(ScrubBytes([]byte(s))) } diff --git a/pkg/scrubber/scrubber_test.go b/pkg/scrubber/scrubber_test.go index beb17d962..865110204 100644 --- a/pkg/scrubber/scrubber_test.go +++ b/pkg/scrubber/scrubber_test.go @@ -39,121 +39,122 @@ import ( // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // ================================================================================ -// Test the log scrubber on known problematic log messages -func TestLogScrubberMessages(t *testing.T) { - for _, test := range []struct { - input, expected string - }{ - { - "http: TLS handshake error from 129.97.208.23:38310: ", - "http: TLS handshake error from [scrubbed]: ", - }, - { - "http2: panic serving [2620:101:f000:780:9097:75b1:519f:dbb8]:58344: interface conversion: *http2.responseWriter is not http.Hijacker: missing method Hijack", - "http2: panic serving [scrubbed]: interface conversion: *http2.responseWriter is not http.Hijacker: missing method Hijack", - }, - { - //Make sure it doesn't scrub fingerprint - "a=fingerprint:sha-256 33:B6:FA:F6:94:CA:74:61:45:4A:D2:1F:2C:2F:75:8A:D9:EB:23:34:B2:30:E9:1B:2A:A6:A9:E0:44:72:CC:74", - "a=fingerprint:sha-256 33:B6:FA:F6:94:CA:74:61:45:4A:D2:1F:2C:2F:75:8A:D9:EB:23:34:B2:30:E9:1B:2A:A6:A9:E0:44:72:CC:74", - }, - { - //try with enclosing parens - "(1:2:3:4:c:d:e:f) {1:2:3:4:c:d:e:f}", - "([scrubbed]) {[scrubbed]}", - }, - { - //Make sure it doesn't scrub timestamps - "2019/05/08 15:37:31 starting", - "2019/05/08 15:37:31 starting", - }, - { - //Make sure ipv6 addresses where : are encoded as %3A or %3a are scrubbed - "error dialing relay: wss://snowflake.torproject.net/?client_ip=6201%3ac8%3A3004%3A%3A1234", - "error dialing relay: wss://snowflake.torproject.net/?client_ip=[scrubbed]", - }, - { - // make sure url encoded IPv6 IPs get scrubbed (%3a) - "http2: panic serving [fd00%3a111%3af000%3a777%3a9999%3abbbb%3affff%3adddd]:58344: xxx", - "http2: panic serving [scrubbed]: xxx", - }, - { - // make sure url encoded IPv6 IPs get scrubbed (%3A) - "http2: panic serving [fd00%3a111%3af000%3a777%3a9999%3abbbb%3affff%3adddd]:58344: xxx", - "http2: panic serving [scrubbed]: xxx", - }, - { - // make sure url encoded IPv6 IPs get scrubbed, different URL (%3A) - "error dialing relay: wss://snowflake.torproject.net/?client_ip=fd00%3A8888%3Abbbb%3Acccc%3Adddd%3Aeeee%3A2222%3A123 = dial tcp xxx", - "error dialing relay: wss://snowflake.torproject.net/?client_ip=[scrubbed] = dial tcp xxx", - }, - { - // make sure url encoded IPv6 IPs get scrubbed (%3A), compressed - "http2: panic serving [1%3A2%3A3%3A%3Ad%3Ae%3Af]:55: xxx", - "http2: panic serving [scrubbed]: xxx", - }, - { - // make sure url encoded IPv6 IPs get scrubbed (%3A), compressed - "error dialing relay: wss://snowflake.torproject.net/?client_ip=1%3A2%3A3%3A%3Ad%3Ae%3Af = dial tcp xxx", - "error dialing relay: wss://snowflake.torproject.net/?client_ip=[scrubbed] = dial tcp xxx", - }, - } { - if Scrub(test.input) != test.expected { - t.Error(cmp.Diff(test.input, test.expected)) +func TestScrubString(t *testing.T) { + t.Run("for problematic log messages", func(t *testing.T) { + for _, test := range []struct { + input, expected string + }{ + { + "http: TLS handshake error from 129.97.208.23:38310: ", + "http: TLS handshake error from [scrubbed]: ", + }, + { + "http2: panic serving [2620:101:f000:780:9097:75b1:519f:dbb8]:58344: interface conversion: *http2.responseWriter is not http.Hijacker: missing method Hijack", + "http2: panic serving [scrubbed]: interface conversion: *http2.responseWriter is not http.Hijacker: missing method Hijack", + }, + { + //Make sure it doesn't scrub fingerprint + "a=fingerprint:sha-256 33:B6:FA:F6:94:CA:74:61:45:4A:D2:1F:2C:2F:75:8A:D9:EB:23:34:B2:30:E9:1B:2A:A6:A9:E0:44:72:CC:74", + "a=fingerprint:sha-256 33:B6:FA:F6:94:CA:74:61:45:4A:D2:1F:2C:2F:75:8A:D9:EB:23:34:B2:30:E9:1B:2A:A6:A9:E0:44:72:CC:74", + }, + { + //try with enclosing parens + "(1:2:3:4:c:d:e:f) {1:2:3:4:c:d:e:f}", + "([scrubbed]) {[scrubbed]}", + }, + { + //Make sure it doesn't scrub timestamps + "2019/05/08 15:37:31 starting", + "2019/05/08 15:37:31 starting", + }, + { + //Make sure ipv6 addresses where : are encoded as %3A or %3a are scrubbed + "error dialing relay: wss://snowflake.torproject.net/?client_ip=6201%3ac8%3A3004%3A%3A1234", + "error dialing relay: wss://snowflake.torproject.net/?client_ip=[scrubbed]", + }, + { + // make sure url encoded IPv6 IPs get scrubbed (%3a) + "http2: panic serving [fd00%3a111%3af000%3a777%3a9999%3abbbb%3affff%3adddd]:58344: xxx", + "http2: panic serving [scrubbed]: xxx", + }, + { + // make sure url encoded IPv6 IPs get scrubbed (%3A) + "http2: panic serving [fd00%3a111%3af000%3a777%3a9999%3abbbb%3affff%3adddd]:58344: xxx", + "http2: panic serving [scrubbed]: xxx", + }, + { + // make sure url encoded IPv6 IPs get scrubbed, different URL (%3A) + "error dialing relay: wss://snowflake.torproject.net/?client_ip=fd00%3A8888%3Abbbb%3Acccc%3Adddd%3Aeeee%3A2222%3A123 = dial tcp xxx", + "error dialing relay: wss://snowflake.torproject.net/?client_ip=[scrubbed] = dial tcp xxx", + }, + { + // make sure url encoded IPv6 IPs get scrubbed (%3A), compressed + "http2: panic serving [1%3A2%3A3%3A%3Ad%3Ae%3Af]:55: xxx", + "http2: panic serving [scrubbed]: xxx", + }, + { + // make sure url encoded IPv6 IPs get scrubbed (%3A), compressed + "error dialing relay: wss://snowflake.torproject.net/?client_ip=1%3A2%3A3%3A%3Ad%3Ae%3Af = dial tcp xxx", + "error dialing relay: wss://snowflake.torproject.net/?client_ip=[scrubbed] = dial tcp xxx", + }, + } { + if ScrubString(test.input) != test.expected { + t.Error(cmp.Diff(test.input, test.expected)) + } } - } -} + }) -func TestLogScrubberGoodFormats(t *testing.T) { - for _, addr := range []string{ - // IPv4 - "1.2.3.4", - "255.255.255.255", - // IPv4 with port - "1.2.3.4:55", - "255.255.255.255:65535", - // IPv6 - "1:2:3:4:c:d:e:f", - "1111:2222:3333:4444:CCCC:DDDD:EEEE:FFFF", - // IPv6 with brackets - "[1:2:3:4:c:d:e:f]", - "[1111:2222:3333:4444:CCCC:DDDD:EEEE:FFFF]", - // IPv6 with brackets and port - "[1:2:3:4:c:d:e:f]:55", - "[1111:2222:3333:4444:CCCC:DDDD:EEEE:FFFF]:65535", - // compressed IPv6 - "::f", - "::d:e:f", - "1:2:3::", - "1:2:3::d:e:f", - "1:2:3:d:e:f::", - "::1:2:3:d:e:f", - "1111:2222:3333::DDDD:EEEE:FFFF", - // compressed IPv6 with brackets - "[::d:e:f]", - "[1:2:3::]", - "[1:2:3::d:e:f]", - "[1111:2222:3333::DDDD:EEEE:FFFF]", - "[1:2:3:4:5:6::8]", - "[1::7:8]", - // compressed IPv6 with brackets and port - "[1::]:58344", - "[::d:e:f]:55", - "[1:2:3::]:55", - "[1:2:3::d:e:f]:55", - "[1111:2222:3333::DDDD:EEEE:FFFF]:65535", - // IPv4-compatible and IPv4-mapped - "::255.255.255.255", - "::ffff:255.255.255.255", - "[::255.255.255.255]", - "[::ffff:255.255.255.255]", - "[::255.255.255.255]:65535", - "[::ffff:255.255.255.255]:65535", - "[::ffff:0:255.255.255.255]", - "[2001:db8:3:4::192.0.2.33]", - } { - if Scrub(addr) != "[scrubbed]" { - t.Error(cmp.Diff(addr, "[scrubbed]")) + t.Run("for strings containing IP addresses and endpoints", func(t *testing.T) { + for _, addr := range []string{ + // IPv4 + "1.2.3.4", + "255.255.255.255", + // IPv4 with port + "1.2.3.4:55", + "255.255.255.255:65535", + // IPv6 + "1:2:3:4:c:d:e:f", + "1111:2222:3333:4444:CCCC:DDDD:EEEE:FFFF", + // IPv6 with brackets + "[1:2:3:4:c:d:e:f]", + "[1111:2222:3333:4444:CCCC:DDDD:EEEE:FFFF]", + // IPv6 with brackets and port + "[1:2:3:4:c:d:e:f]:55", + "[1111:2222:3333:4444:CCCC:DDDD:EEEE:FFFF]:65535", + // compressed IPv6 + "::f", + "::d:e:f", + "1:2:3::", + "1:2:3::d:e:f", + "1:2:3:d:e:f::", + "::1:2:3:d:e:f", + "1111:2222:3333::DDDD:EEEE:FFFF", + // compressed IPv6 with brackets + "[::d:e:f]", + "[1:2:3::]", + "[1:2:3::d:e:f]", + "[1111:2222:3333::DDDD:EEEE:FFFF]", + "[1:2:3:4:5:6::8]", + "[1::7:8]", + // compressed IPv6 with brackets and port + "[1::]:58344", + "[::d:e:f]:55", + "[1:2:3::]:55", + "[1:2:3::d:e:f]:55", + "[1111:2222:3333::DDDD:EEEE:FFFF]:65535", + // IPv4-compatible and IPv4-mapped + "::255.255.255.255", + "::ffff:255.255.255.255", + "[::255.255.255.255]", + "[::ffff:255.255.255.255]", + "[::255.255.255.255]:65535", + "[::ffff:255.255.255.255]:65535", + "[::ffff:0:255.255.255.255]", + "[2001:db8:3:4::192.0.2.33]", + } { + if ScrubString(addr) != "[scrubbed]" { + t.Error(cmp.Diff(addr, "[scrubbed]")) + } } - } + }) } diff --git a/pkg/strcasex/acronyms.go b/pkg/strcasex/acronyms.go new file mode 100644 index 000000000..18b0644ff --- /dev/null +++ b/pkg/strcasex/acronyms.go @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: MIT + +package strcasex + +var uppercaseAcronym = map[string]string{ + "ID": "id", +} + +// ConfigureAcronym allows you to add additional words which will be considered acronyms +func ConfigureAcronym(key, val string) { + uppercaseAcronym[key] = val +} diff --git a/pkg/strcasex/camel.go b/pkg/strcasex/camel.go new file mode 100644 index 000000000..4c91a236e --- /dev/null +++ b/pkg/strcasex/camel.go @@ -0,0 +1,80 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2015 Ian Coleman + * Copyright (c) 2018 Ma_124, + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, Subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or Substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package strcasex + +import ( + "strings" +) + +// Converts a string to CamelCase +func toCamelInitCase(s string, initCase bool) string { + s = strings.TrimSpace(s) + if s == "" { + return s + } + if a, ok := uppercaseAcronym[s]; ok { + s = a + } + + n := strings.Builder{} + n.Grow(len(s)) + capNext := initCase + for i, v := range []byte(s) { + vIsCap := v >= 'A' && v <= 'Z' + vIsLow := v >= 'a' && v <= 'z' + if capNext { + if vIsLow { + v += 'A' + v -= 'a' + } + } else if i == 0 { + if vIsCap { + v += 'a' + v -= 'A' + } + } + if vIsCap || vIsLow { + n.WriteByte(v) + capNext = false + } else if vIsNum := v >= '0' && v <= '9'; vIsNum { + n.WriteByte(v) + capNext = true + } else { + capNext = v == '_' || v == ' ' || v == '-' || v == '.' + } + } + return n.String() +} + +// ToCamel converts a string to CamelCase +func ToCamel(s string) string { + return toCamelInitCase(s, true) +} + +// ToLowerCamel converts a string to lowerCamelCase +func ToLowerCamel(s string) string { + return toCamelInitCase(s, false) +} diff --git a/pkg/strcasex/camel_test.go b/pkg/strcasex/camel_test.go new file mode 100644 index 000000000..eb63288b4 --- /dev/null +++ b/pkg/strcasex/camel_test.go @@ -0,0 +1,168 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2015 Ian Coleman + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, Subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or Substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package strcasex + +import ( + "testing" +) + +func toCamel(tb testing.TB) { + cases := [][]string{ + {"test_case", "TestCase"}, + {"test.case", "TestCase"}, + {"test", "Test"}, + {"TestCase", "TestCase"}, + {" test case ", "TestCase"}, + {"", ""}, + {"many_many_words", "ManyManyWords"}, + {"AnyKind of_string", "AnyKindOfString"}, + {"odd-fix", "OddFix"}, + {"numbers2And55with000", "Numbers2And55With000"}, + {"ID", "Id"}, + } + for _, i := range cases { + in := i[0] + out := i[1] + result := ToCamel(in) + if result != out { + tb.Errorf("%q (%q != %q)", in, result, out) + } + } +} + +func TestToCamel(t *testing.T) { + toCamel(t) +} + +func BenchmarkToCamel(b *testing.B) { + benchmarkCamelTest(b, toCamel) +} + +func toLowerCamel(tb testing.TB) { + cases := [][]string{ + {"foo-bar", "fooBar"}, + {"TestCase", "testCase"}, + {"", ""}, + {"AnyKind of_string", "anyKindOfString"}, + {"AnyKind.of-string", "anyKindOfString"}, + {"ID", "id"}, + {"some string", "someString"}, + {" some string", "someString"}, + } + for _, i := range cases { + in := i[0] + out := i[1] + result := ToLowerCamel(in) + if result != out { + tb.Errorf("%q (%q != %q)", in, result, out) + } + } +} + +func TestToLowerCamel(t *testing.T) { + toLowerCamel(t) +} + +func TestCustomAcronymsToCamel(t *testing.T) { + tests := []struct { + name string + acronymKey string + acronymValue string + expected string + }{ + { + name: "API Custom Acronym", + acronymKey: "API", + acronymValue: "api", + expected: "Api", + }, + { + name: "ABCDACME Custom Acroynm", + acronymKey: "ABCDACME", + acronymValue: "AbcdAcme", + expected: "AbcdAcme", + }, + { + name: "PostgreSQL Custom Acronym", + acronymKey: "PostgreSQL", + acronymValue: "PostgreSQL", + expected: "PostgreSQL", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ConfigureAcronym(test.acronymKey, test.acronymValue) + if result := ToCamel(test.acronymKey); result != test.expected { + t.Errorf("expected custom acronym result %s, got %s", test.expected, result) + } + }) + } +} + +func TestCustomAcronymsToLowerCamel(t *testing.T) { + tests := []struct { + name string + acronymKey string + acronymValue string + expected string + }{ + { + name: "API Custom Acronym", + acronymKey: "API", + acronymValue: "api", + expected: "api", + }, + { + name: "ABCDACME Custom Acroynm", + acronymKey: "ABCDACME", + acronymValue: "AbcdAcme", + expected: "abcdAcme", + }, + { + name: "PostgreSQL Custom Acronym", + acronymKey: "PostgreSQL", + acronymValue: "PostgreSQL", + expected: "postgreSQL", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ConfigureAcronym(test.acronymKey, test.acronymValue) + if result := ToLowerCamel(test.acronymKey); result != test.expected { + t.Errorf("expected custom acronym result %s, got %s", test.expected, result) + } + }) + } +} + +func BenchmarkToLowerCamel(b *testing.B) { + benchmarkCamelTest(b, toLowerCamel) +} + +func benchmarkCamelTest(b *testing.B, fn func(testing.TB)) { + for n := 0; n < b.N; n++ { + fn(b) + } +} diff --git a/pkg/strcasex/doc.go b/pkg/strcasex/doc.go new file mode 100644 index 000000000..bf22ecfb3 --- /dev/null +++ b/pkg/strcasex/doc.go @@ -0,0 +1,17 @@ +// Package strcasex converts strings to various cases. +// +// This package forks https://github.com/iancoleman/strcase at v0.2.0. +// +// See the conversion table below: +// +// | Function | Result | +// |---------------------------------|--------------------| +// | ToSnake(s) | any_kind_of_string | +// | ToScreamingSnake(s) | ANY_KIND_OF_STRING | +// | ToKebab(s) | any-kind-of-string | +// | ToScreamingKebab(s) | ANY-KIND-OF-STRING | +// | ToDelimited(s, '.') | any.kind.of.string | +// | ToScreamingDelimited(s, '.') | ANY.KIND.OF.STRING | +// | ToCamel(s) | AnyKindOfString | +// | ToLowerCamel(s) | anyKindOfString | +package strcasex diff --git a/pkg/strcasex/snake.go b/pkg/strcasex/snake.go new file mode 100644 index 000000000..519a07a4b --- /dev/null +++ b/pkg/strcasex/snake.go @@ -0,0 +1,115 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2015 Ian Coleman + * Copyright (c) 2018 Ma_124, + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, Subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or Substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package strcasex + +import ( + "strings" +) + +// ToSnake converts a string to snake_case +func ToSnake(s string) string { + return ToDelimited(s, '_') +} + +func ToSnakeWithIgnore(s string, ignore string) string { + return ToScreamingDelimited(s, '_', ignore, false) +} + +// ToScreamingSnake converts a string to SCREAMING_SNAKE_CASE +func ToScreamingSnake(s string) string { + return ToScreamingDelimited(s, '_', "", true) +} + +// ToKebab converts a string to kebab-case +func ToKebab(s string) string { + return ToDelimited(s, '-') +} + +// ToScreamingKebab converts a string to SCREAMING-KEBAB-CASE +func ToScreamingKebab(s string) string { + return ToScreamingDelimited(s, '-', "", true) +} + +// ToDelimited converts a string to delimited.snake.case +// (in this case `delimiter = '.'`) +func ToDelimited(s string, delimiter uint8) string { + return ToScreamingDelimited(s, delimiter, "", false) +} + +// ToScreamingDelimited converts a string to SCREAMING.DELIMITED.SNAKE.CASE +// (in this case `delimiter = '.'; screaming = true`) +// or delimited.snake.case +// (in this case `delimiter = '.'; screaming = false`) +func ToScreamingDelimited(s string, delimiter uint8, ignore string, screaming bool) string { + s = strings.TrimSpace(s) + n := strings.Builder{} + n.Grow(len(s) + 2) // nominal 2 bytes of extra space for inserted delimiters + for i, v := range []byte(s) { + vIsCap := v >= 'A' && v <= 'Z' + vIsLow := v >= 'a' && v <= 'z' + if vIsLow && screaming { + v += 'A' + v -= 'a' + } else if vIsCap && !screaming { + v += 'a' + v -= 'A' + } + + // treat acronyms as words, eg for JSONData -> JSON is a whole word + if i+1 < len(s) { + next := s[i+1] + vIsNum := v >= '0' && v <= '9' + nextIsCap := next >= 'A' && next <= 'Z' + nextIsLow := next >= 'a' && next <= 'z' + nextIsNum := next >= '0' && next <= '9' + // add underscore if next letter case type is changed + if (vIsCap && (nextIsLow || nextIsNum)) || (vIsLow && (nextIsCap || nextIsNum)) || (vIsNum && (nextIsCap || nextIsLow)) { + prevIgnore := ignore != "" && i > 0 && strings.ContainsAny(string(s[i-1]), ignore) + if !prevIgnore { + if vIsCap && nextIsLow { + if prevIsCap := i > 0 && s[i-1] >= 'A' && s[i-1] <= 'Z'; prevIsCap { + n.WriteByte(delimiter) + } + } + n.WriteByte(v) + if vIsLow || vIsNum || nextIsNum { + n.WriteByte(delimiter) + } + continue + } + } + } + + if (v == ' ' || v == '_' || v == '-' || v == '.') && !strings.ContainsAny(string(v), ignore) { + // replace space/underscore/hyphen/dot with delimiter + n.WriteByte(delimiter) + } else { + n.WriteByte(v) + } + } + + return n.String() +} diff --git a/pkg/strcasex/snake_test.go b/pkg/strcasex/snake_test.go new file mode 100644 index 000000000..22894b9f5 --- /dev/null +++ b/pkg/strcasex/snake_test.go @@ -0,0 +1,270 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2015 Ian Coleman + * Copyright (c) 2018 Ma_124, + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, Subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or Substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package strcasex + +import ( + "testing" +) + +func toSnake(tb testing.TB) { + cases := [][]string{ + {"testCase", "test_case"}, + {"TestCase", "test_case"}, + {"Test Case", "test_case"}, + {" Test Case", "test_case"}, + {"Test Case ", "test_case"}, + {" Test Case ", "test_case"}, + {"test", "test"}, + {"test_case", "test_case"}, + {"Test", "test"}, + {"", ""}, + {"ManyManyWords", "many_many_words"}, + {"manyManyWords", "many_many_words"}, + {"AnyKind of_string", "any_kind_of_string"}, + {"numbers2and55with000", "numbers_2_and_55_with_000"}, + {"JSONData", "json_data"}, + {"userID", "user_id"}, + {"AAAbbb", "aa_abbb"}, + {"1A2", "1_a_2"}, + {"A1B", "a_1_b"}, + {"A1A2A3", "a_1_a_2_a_3"}, + {"A1 A2 A3", "a_1_a_2_a_3"}, + {"AB1AB2AB3", "ab_1_ab_2_ab_3"}, + {"AB1 AB2 AB3", "ab_1_ab_2_ab_3"}, + {"some string", "some_string"}, + {" some string", "some_string"}, + } + for _, i := range cases { + in := i[0] + out := i[1] + result := ToSnake(in) + if result != out { + tb.Errorf("%q (%q != %q)", in, result, out) + } + } +} + +func TestToSnake(t *testing.T) { toSnake(t) } + +func BenchmarkToSnake(b *testing.B) { + benchmarkSnakeTest(b, toSnake) +} + +func toSnakeWithIgnore(tb testing.TB) { + cases := [][]string{ + {"testCase", "test_case"}, + {"TestCase", "test_case"}, + {"Test Case", "test_case"}, + {" Test Case", "test_case"}, + {"Test Case ", "test_case"}, + {" Test Case ", "test_case"}, + {"test", "test"}, + {"test_case", "test_case"}, + {"Test", "test"}, + {"", ""}, + {"ManyManyWords", "many_many_words"}, + {"manyManyWords", "many_many_words"}, + {"AnyKind of_string", "any_kind_of_string"}, + {"numbers2and55with000", "numbers_2_and_55_with_000"}, + {"JSONData", "json_data"}, + {"AwesomeActivity.UserID", "awesome_activity.user_id", "."}, + {"AwesomeActivity.User.Id", "awesome_activity.user.id", "."}, + {"AwesomeUsername@Awesome.Com", "awesome_username@awesome.com", ".@"}, + {"lets-ignore all.of dots-and-dashes", "lets-ignore_all.of_dots-and-dashes", ".-"}, + } + for _, i := range cases { + in := i[0] + out := i[1] + var ignore string + ignore = "" + if len(i) == 3 { + ignore = i[2] + } + result := ToSnakeWithIgnore(in, ignore) + if result != out { + istr := "" + if len(i) == 3 { + istr = " ignoring '" + i[2] + "'" + } + tb.Errorf("%q (%q != %q%s)", in, result, out, istr) + } + } +} + +func TestToSnakeWithIgnore(t *testing.T) { toSnakeWithIgnore(t) } + +func BenchmarkToSnakeWithIgnore(b *testing.B) { + benchmarkSnakeTest(b, toSnakeWithIgnore) +} + +func toDelimited(tb testing.TB) { + cases := [][]string{ + {"testCase", "test@case"}, + {"TestCase", "test@case"}, + {"Test Case", "test@case"}, + {" Test Case", "test@case"}, + {"Test Case ", "test@case"}, + {" Test Case ", "test@case"}, + {"test", "test"}, + {"test_case", "test@case"}, + {"Test", "test"}, + {"", ""}, + {"ManyManyWords", "many@many@words"}, + {"manyManyWords", "many@many@words"}, + {"AnyKind of_string", "any@kind@of@string"}, + {"numbers2and55with000", "numbers@2@and@55@with@000"}, + {"JSONData", "json@data"}, + {"userID", "user@id"}, + {"AAAbbb", "aa@abbb"}, + {"test-case", "test@case"}, + } + for _, i := range cases { + in := i[0] + out := i[1] + result := ToDelimited(in, '@') + if result != out { + tb.Errorf("%q (%q != %q)", in, result, out) + } + } +} + +func TestToDelimited(t *testing.T) { toDelimited(t) } + +func BenchmarkToDelimited(b *testing.B) { + benchmarkSnakeTest(b, toDelimited) +} + +func toScreamingSnake(tb testing.TB) { + cases := [][]string{ + {"testCase", "TEST_CASE"}, + } + for _, i := range cases { + in := i[0] + out := i[1] + result := ToScreamingSnake(in) + if result != out { + tb.Errorf("%q (%q != %q)", in, result, out) + } + } +} + +func TestToScreamingSnake(t *testing.T) { toScreamingSnake(t) } + +func BenchmarkToScreamingSnake(b *testing.B) { + benchmarkSnakeTest(b, toScreamingSnake) +} + +func toKebab(tb testing.TB) { + cases := [][]string{ + {"testCase", "test-case"}, + } + for _, i := range cases { + in := i[0] + out := i[1] + result := ToKebab(in) + if result != out { + tb.Errorf("%q (%q != %q)", in, result, out) + } + } +} + +func TestToKebab(t *testing.T) { toKebab(t) } + +func BenchmarkToKebab(b *testing.B) { + benchmarkSnakeTest(b, toKebab) +} + +func toScreamingKebab(tb testing.TB) { + cases := [][]string{ + {"testCase", "TEST-CASE"}, + } + for _, i := range cases { + in := i[0] + out := i[1] + result := ToScreamingKebab(in) + if result != out { + tb.Errorf("%q (%q != %q)", in, result, out) + } + } +} + +func TestToScreamingKebab(t *testing.T) { toScreamingKebab(t) } + +func BenchmarkToScreamingKebab(b *testing.B) { + benchmarkSnakeTest(b, toScreamingKebab) +} + +func toScreamingDelimited(tb testing.TB) { + cases := [][]string{ + {"testCase", "TEST.CASE"}, + } + for _, i := range cases { + in := i[0] + out := i[1] + result := ToScreamingDelimited(in, '.', "", true) + if result != out { + tb.Errorf("%q (%q != %q)", in, result, out) + } + } +} + +func TestToScreamingDelimited(t *testing.T) { toScreamingDelimited(t) } + +func BenchmarkToScreamingDelimited(b *testing.B) { + benchmarkSnakeTest(b, toScreamingDelimited) +} + +func toScreamingDelimitedWithIgnore(tb testing.TB) { + cases := [][]string{ + {"AnyKind of_string", "ANY.KIND OF.STRING", ".", " "}, + } + for _, i := range cases { + in := i[0] + out := i[1] + delimiter := i[2][0] + ignore := i[3][0] + result := ToScreamingDelimited(in, delimiter, string(ignore), true) + if result != out { + istr := "" + if len(i) == 4 { + istr = " ignoring '" + i[3] + "'" + } + tb.Errorf("%q (%q != %q%s)", in, result, out, istr) + } + } +} + +func TestToScreamingDelimitedWithIgnore(t *testing.T) { toScreamingDelimitedWithIgnore(t) } + +func BenchmarkToScreamingDelimitedWithIgnore(b *testing.B) { + benchmarkSnakeTest(b, toScreamingDelimitedWithIgnore) +} + +func benchmarkSnakeTest(b *testing.B, fn func(testing.TB)) { + for n := 0; n < b.N; n++ { + fn(b) + } +} diff --git a/pkg/stuninput/stuninput.go b/pkg/stuninput/stuninput.go index a3ceaf14a..726ef0c7f 100644 --- a/pkg/stuninput/stuninput.go +++ b/pkg/stuninput/stuninput.go @@ -10,7 +10,7 @@ import ( // TODO(bassosimone): we need to keep this list in sync with // the list internally used by TPO's snowflake. // -// We should sync with https://gitlab.torproject.org/tpo/applications/tor-browser-build/-/blob/main/projects/common/bridges_list.snowflake.txt +// We should sync with https://gitlab.torproject.org/tpo/applications/tor-browser-build/-/blob/main/projects/tor-expert-bundle/pt_config.json var inputs = map[string]bool{ "stun.l.google.com:19302": true, "stun.antisip.com:3478": true, diff --git a/pkg/testingproxy/dialer.go b/pkg/testingproxy/dialer.go new file mode 100644 index 000000000..2c1865d56 --- /dev/null +++ b/pkg/testingproxy/dialer.go @@ -0,0 +1,49 @@ +package testingproxy + +import ( + "context" + "fmt" + "log" + "net" + + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/runtimex" +) + +// dialerWithAssertions ensures that we're dialing with the proxy address. +type dialerWithAssertions struct { + // ExpectAddress is the expected IP address to dial + ExpectAddress string + + // Dialer is the underlying dialer to use + Dialer model.Dialer +} + +var _ model.Dialer = &dialerWithAssertions{} + +// CloseIdleConnections implements model.Dialer. +func (d *dialerWithAssertions) CloseIdleConnections() { + d.Dialer.CloseIdleConnections() +} + +// DialContext implements model.Dialer. +func (d *dialerWithAssertions) DialContext(ctx context.Context, network string, address string) (net.Conn, error) { + // make sure the network is tcp + const expectNetwork = "tcp" + runtimex.Assert( + network == expectNetwork, + fmt.Sprintf("dialerWithAssertions: expected %s, got %s", expectNetwork, network), + ) + log.Printf("dialerWithAssertions: verified that the network is %s as expected", expectNetwork) + + // make sure the IP address is the expected one + ipAddr, _ := runtimex.Try2(net.SplitHostPort(address)) + runtimex.Assert( + ipAddr == d.ExpectAddress, + fmt.Sprintf("dialerWithAssertions: expected %s, got %s", d.ExpectAddress, ipAddr), + ) + log.Printf("dialerWithAssertions: verified that the address is %s as expected", d.ExpectAddress) + + // now that we're sure we're using the proxy, we can actually dial + return d.Dialer.DialContext(ctx, network, address) +} diff --git a/pkg/testingproxy/doc.go b/pkg/testingproxy/doc.go new file mode 100644 index 000000000..b7560cc81 --- /dev/null +++ b/pkg/testingproxy/doc.go @@ -0,0 +1,2 @@ +// Package testingproxy contains shared test cases for the proxies. +package testingproxy diff --git a/pkg/testingproxy/hosthttp.go b/pkg/testingproxy/hosthttp.go new file mode 100644 index 000000000..164a3c8d3 --- /dev/null +++ b/pkg/testingproxy/hosthttp.go @@ -0,0 +1,72 @@ +package testingproxy + +import ( + "fmt" + "net/http" + "net/url" + "testing" + + "github.com/apex/log" + "github.com/ooni/probe-engine/pkg/netxlite" + "github.com/ooni/probe-engine/pkg/runtimex" + "github.com/ooni/probe-engine/pkg/testingx" +) + +// WithHostNetworkHTTPProxyAndURL returns a [TestCase] where: +// +// - we fetch a URL; +// +// - using the host network; +// +// - and an HTTP proxy. +// +// Because this [TestCase] uses the host network, it does not run in -short mode. +func WithHostNetworkHTTPProxyAndURL(URL string) TestCase { + return &hostNetworkTestCaseWithHTTP{ + TargetURL: URL, + } +} + +type hostNetworkTestCaseWithHTTP struct { + TargetURL string +} + +var _ TestCase = &hostNetworkTestCaseWithHTTP{} + +// Name implements TestCase. +func (tc *hostNetworkTestCaseWithHTTP) Name() string { + return fmt.Sprintf("fetching %s using the host network and an HTTP proxy", tc.TargetURL) +} + +// Run implements TestCase. +func (tc *hostNetworkTestCaseWithHTTP) Run(t *testing.T) { + // create an instance of Netx where the underlying network is nil, + // which means we're using the host's network + netx := &netxlite.Netx{Underlying: nil} + + // create the proxy server using the host network + proxyServer := testingx.MustNewHTTPServer(testingx.NewHTTPProxyHandler(log.Log, netx)) + defer proxyServer.Close() + + // create an HTTP client configured to use the given proxy + // + // note how we use a dialer that asserts that we're using the proxy IP address + // rather than the host address, so we're sure that we're using the proxy + dialer := &dialerWithAssertions{ + ExpectAddress: "127.0.0.1", + Dialer: netx.NewDialerWithResolver(log.Log, netx.NewStdlibResolver(log.Log)), + } + tlsDialer := netxlite.NewTLSDialer(dialer, netxlite.NewTLSHandshakerStdlib(log.Log)) + txp := netxlite.NewHTTPTransportWithOptions(log.Log, dialer, tlsDialer, + netxlite.HTTPTransportOptionProxyURL(runtimex.Try1(url.Parse(proxyServer.URL)))) + client := &http.Client{Transport: txp} + defer client.CloseIdleConnections() + + // get the homepage and assert we're getting a succesful response + httpCheckResponse(t, client, tc.TargetURL) +} + +// Short implements TestCase. +func (tc *hostNetworkTestCaseWithHTTP) Short() bool { + return false +} diff --git a/pkg/testingproxy/hosthttps.go b/pkg/testingproxy/hosthttps.go new file mode 100644 index 000000000..12df05b0e --- /dev/null +++ b/pkg/testingproxy/hosthttps.go @@ -0,0 +1,89 @@ +package testingproxy + +import ( + "crypto/tls" + "fmt" + "net/http" + "net/url" + "testing" + + "github.com/apex/log" + "github.com/ooni/netem" + "github.com/ooni/probe-engine/pkg/netxlite" + "github.com/ooni/probe-engine/pkg/runtimex" + "github.com/ooni/probe-engine/pkg/testingx" +) + +// WithHostNetworkHTTPWithTLSProxyAndURL returns a [TestCase] where: +// +// - we fetch a URL; +// +// - using the host network; +// +// - and an HTTPS proxy. +// +// Because this [TestCase] uses the host network, it does not run in -short mode. +func WithHostNetworkHTTPWithTLSProxyAndURL(URL string) TestCase { + return &hostNetworkTestCaseWithHTTPWithTLS{ + TargetURL: URL, + } +} + +type hostNetworkTestCaseWithHTTPWithTLS struct { + TargetURL string +} + +var _ TestCase = &hostNetworkTestCaseWithHTTPWithTLS{} + +// Name implements TestCase. +func (tc *hostNetworkTestCaseWithHTTPWithTLS) Name() string { + return fmt.Sprintf("fetching %s using the host network and an HTTPS proxy", tc.TargetURL) +} + +// Run implements TestCase. +func (tc *hostNetworkTestCaseWithHTTPWithTLS) Run(t *testing.T) { + // create an instance of Netx where the underlying network is nil, + // which means we're using the host's network + netx := &netxlite.Netx{Underlying: nil} + + // create CA + proxyCA := netem.MustNewCA() + + // create the proxy server using the host network + proxyServer := testingx.MustNewHTTPServerTLS( + testingx.NewHTTPProxyHandler(log.Log, netx), + proxyCA, + "proxy.local", + ) + defer proxyServer.Close() + + // extend the default cert pool with the proxy's own CA + pool := netxlite.NewMozillaCertPool() + pool.AddCert(proxyServer.CACert) + tlsConfig := &tls.Config{RootCAs: pool} + + // create an HTTP client configured to use the given proxy + // + // note how we use a dialer that asserts that we're using the proxy IP address + // rather than the host address, so we're sure that we're using the proxy + dialer := &dialerWithAssertions{ + ExpectAddress: "127.0.0.1", + Dialer: netx.NewDialerWithResolver(log.Log, netx.NewStdlibResolver(log.Log)), + } + tlsDialer := netxlite.NewTLSDialerWithConfig( + dialer, netxlite.NewTLSHandshakerStdlib(log.Log), + tlsConfig, + ) + txp := netxlite.NewHTTPTransportWithOptions(log.Log, dialer, tlsDialer, + netxlite.HTTPTransportOptionProxyURL(runtimex.Try1(url.Parse(proxyServer.URL)))) + client := &http.Client{Transport: txp} + defer client.CloseIdleConnections() + + // get the homepage and assert we're getting a succesful response + httpCheckResponse(t, client, tc.TargetURL) +} + +// Short implements TestCase. +func (tc *hostNetworkTestCaseWithHTTPWithTLS) Short() bool { + return false +} diff --git a/pkg/testingproxy/httputils.go b/pkg/testingproxy/httputils.go new file mode 100644 index 000000000..f377754ed --- /dev/null +++ b/pkg/testingproxy/httputils.go @@ -0,0 +1,52 @@ +package testingproxy + +import "net/http" + +type httpClient interface { + Get(URL string) (*http.Response, error) +} + +type httpClientMock struct { + MockGet func(URL string) (*http.Response, error) +} + +var _ httpClient = &httpClientMock{} + +// Get implements httpClient. +func (c *httpClientMock) Get(URL string) (*http.Response, error) { + return c.MockGet(URL) +} + +type httpTestingT interface { + Logf(format string, v ...any) + Fatal(v ...any) +} + +type httpTestingTMock struct { + MockLogf func(format string, v ...any) + MockFatal func(v ...any) +} + +var _ httpTestingT = &httpTestingTMock{} + +// Fatal implements httpTestingT. +func (t *httpTestingTMock) Fatal(v ...any) { + t.MockFatal(v...) +} + +// Logf implements httpTestingT. +func (t *httpTestingTMock) Logf(format string, v ...any) { + t.MockLogf(format, v...) +} + +func httpCheckResponse(t httpTestingT, client httpClient, targetURL string) { + resp, err := client.Get(targetURL) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + t.Logf("%+v", resp) + if resp.StatusCode != 200 { + t.Fatal("invalid status code") + } +} diff --git a/pkg/testingproxy/httputils_test.go b/pkg/testingproxy/httputils_test.go new file mode 100644 index 000000000..a01c34cac --- /dev/null +++ b/pkg/testingproxy/httputils_test.go @@ -0,0 +1,122 @@ +package testingproxy + +import ( + "bytes" + "errors" + "io" + "net/http" + "testing" +) + +func TestHTTPClientMock(t *testing.T) { + t.Run("for Get", func(t *testing.T) { + expected := errors.New("mocked error") + c := &httpClientMock{ + MockGet: func(URL string) (*http.Response, error) { + return nil, expected + }, + } + resp, err := c.Get("https://www.google.com/") + if !errors.Is(err, expected) { + t.Fatal("unexpected error") + } + if resp != nil { + t.Fatal("expected nil response") + } + }) +} + +func TestHTTPTestingTMock(t *testing.T) { + t.Run("for Fatal", func(t *testing.T) { + var called bool + mt := &httpTestingTMock{ + MockFatal: func(v ...any) { + called = true + }, + } + mt.Fatal("antani") + if !called { + t.Fatal("not called") + } + }) + + t.Run("for Logf", func(t *testing.T) { + var called bool + mt := &httpTestingTMock{ + MockLogf: func(format string, v ...any) { + called = true + }, + } + mt.Logf("antani %v", "mascetti") + if !called { + t.Fatal("not called") + } + }) +} + +func TestHTTPCheckResponseHandlesFailures(t *testing.T) { + type testcase struct { + name string + mclient httpClient + expectLog bool + } + + testcases := []testcase{{ + name: "when HTTP round trip fails", + mclient: &httpClientMock{ + MockGet: func(URL string) (*http.Response, error) { + return nil, io.EOF + }, + }, + expectLog: false, + }, { + name: "with unexpected status code", + mclient: &httpClientMock{ + MockGet: func(URL string) (*http.Response, error) { + resp := &http.Response{ + StatusCode: 404, + Body: io.NopCloser(bytes.NewReader(nil)), + } + return resp, nil + }, + }, + expectLog: true, + }} + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + + // prepare for capturing what happened + var ( + calledLogf bool + calledFatal bool + ) + mt := &httpTestingTMock{ + MockLogf: func(format string, v ...any) { + calledLogf = true + }, + MockFatal: func(v ...any) { + calledFatal = true + panic(v) + }, + } + + // make sure we handle the panic and check what happened + defer func() { + result := recover() + if result == nil { + t.Fatal("did not panic") + } + if !calledFatal { + t.Fatal("did not actually call t.Fatal") + } + if tc.expectLog != calledLogf { + t.Fatal("tc.expectLog is", tc.expectLog, "but calledLogf is", calledLogf) + } + }() + + // invoke the function we're testing + httpCheckResponse(mt, tc.mclient, "https://www.google.com/") + }) + } +} diff --git a/pkg/testingproxy/netemhttp.go b/pkg/testingproxy/netemhttp.go new file mode 100644 index 000000000..494a45d93 --- /dev/null +++ b/pkg/testingproxy/netemhttp.go @@ -0,0 +1,135 @@ +package testingproxy + +import ( + "crypto/tls" + "fmt" + "net" + "net/http" + "net/url" + "testing" + + "github.com/apex/log" + "github.com/ooni/netem" + "github.com/ooni/probe-engine/pkg/netxlite" + "github.com/ooni/probe-engine/pkg/runtimex" + "github.com/ooni/probe-engine/pkg/testingx" +) + +// WithNetemHTTPProxyAndURL returns a [TestCase] where: +// +// - we fetch a URL; +// +// - using the github.com/ooni.netem; +// +// - and an HTTP proxy. +// +// Because this [TestCase] uses netem, it also runs in -short mode. +func WithNetemHTTPProxyAndURL(URL string) TestCase { + return &netemTestCaseWithHTTP{ + TargetURL: URL, + } +} + +type netemTestCaseWithHTTP struct { + TargetURL string +} + +var _ TestCase = &netemTestCaseWithHTTP{} + +// Name implements TestCase. +func (tc *netemTestCaseWithHTTP) Name() string { + return fmt.Sprintf("fetching %s using netem and an HTTP proxy", tc.TargetURL) +} + +// Run implements TestCase. +func (tc *netemTestCaseWithHTTP) Run(t *testing.T) { + topology := netem.MustNewStarTopology(log.Log) + defer topology.Close() + + const ( + wwwIPAddr = "93.184.216.34" + proxyIPAddr = "10.0.0.1" + clientIPAddr = "10.0.0.2" + ) + + // create: + // + // - a www stack modeling www.example.com + // + // - a proxy stack + // + // - a client stack + // + // Note that www.example.com's IP address is also the resolver used by everyone + wwwStack := runtimex.Try1(topology.AddHost(wwwIPAddr, wwwIPAddr, &netem.LinkConfig{})) + proxyStack := runtimex.Try1(topology.AddHost(proxyIPAddr, wwwIPAddr, &netem.LinkConfig{})) + clientStack := runtimex.Try1(topology.AddHost(clientIPAddr, wwwIPAddr, &netem.LinkConfig{})) + + // configure the wwwStack as the DNS resolver with proper configuration + dnsConfig := netem.NewDNSConfig() + dnsConfig.AddRecord("www.example.com.", "", wwwIPAddr) + dnsServer := runtimex.Try1(netem.NewDNSServer(log.Log, wwwStack, wwwIPAddr, dnsConfig)) + defer dnsServer.Close() + + // configure the wwwStack to respond to HTTP requests on port 80 + wwwServer80 := testingx.MustNewHTTPServerEx( + &net.TCPAddr{IP: net.ParseIP(wwwIPAddr), Port: 80}, + wwwStack, + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("Bonsoir, Elliot!\r\n")) + }), + ) + defer wwwServer80.Close() + + // configure the wwwStack to respond to HTTPS requests on port 443 + wwwServer443 := testingx.MustNewHTTPServerTLSEx( + &net.TCPAddr{IP: net.ParseIP(wwwIPAddr), Port: 443}, + wwwStack, + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("Bonsoir, Elliot!\r\n")) + }), + wwwStack, + "www.example.com", + ) + defer wwwServer443.Close() + + // configure the proxyStack to implement the HTTP proxy on port 8080 + proxyServer := testingx.MustNewHTTPServerEx( + &net.TCPAddr{IP: net.ParseIP(proxyIPAddr), Port: 8080}, + proxyStack, + testingx.NewHTTPProxyHandler(log.Log, &netxlite.Netx{ + Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: proxyStack}}), + ) + defer proxyServer.Close() + + // create the netx instance for the client + netx := &netxlite.Netx{Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: clientStack}} + + // create an HTTP client configured to use the given proxy + // + // note how we use a dialer that asserts that we're using the proxy IP address + // rather than the host address, so we're sure that we're using the proxy + dialer := &dialerWithAssertions{ + ExpectAddress: proxyIPAddr, + Dialer: netx.NewDialerWithResolver(log.Log, netx.NewStdlibResolver(log.Log)), + } + tlsDialer := netxlite.NewTLSDialer(dialer, netx.NewTLSHandshakerStdlib(log.Log)) + txp := netxlite.NewHTTPTransportWithOptions(log.Log, dialer, tlsDialer, + netxlite.HTTPTransportOptionProxyURL(runtimex.Try1(url.Parse(proxyServer.URL))), + + // TODO(https://github.com/ooni/probe/issues/2536) + netxlite.HTTPTransportOptionTLSClientConfig(&tls.Config{ + RootCAs: clientStack.DefaultCertPool(), + }), + ) + client := &http.Client{Transport: txp} + defer client.CloseIdleConnections() + + // get the homepage and assert we're getting a succesful response + httpCheckResponse(t, client, tc.TargetURL) +} + +// Short implements TestCase. +func (tc *netemTestCaseWithHTTP) Short() bool { + return true +} diff --git a/pkg/testingproxy/netemhttps.go b/pkg/testingproxy/netemhttps.go new file mode 100644 index 000000000..25a5a8a39 --- /dev/null +++ b/pkg/testingproxy/netemhttps.go @@ -0,0 +1,137 @@ +package testingproxy + +import ( + "crypto/tls" + "fmt" + "net" + "net/http" + "net/url" + "testing" + + "github.com/apex/log" + "github.com/ooni/netem" + "github.com/ooni/probe-engine/pkg/netxlite" + "github.com/ooni/probe-engine/pkg/runtimex" + "github.com/ooni/probe-engine/pkg/testingx" +) + +// WithNetemHTTPWithTLSProxyAndURL returns a [TestCase] where: +// +// - we fetch a URL; +// +// - using the github.com/ooni.netem; +// +// - and an HTTPS proxy. +// +// Because this [TestCase] uses netem, it also runs in -short mode. +func WithNetemHTTPWithTLSProxyAndURL(URL string) TestCase { + return &netemTestCaseWithHTTPWithTLS{ + TargetURL: URL, + } +} + +type netemTestCaseWithHTTPWithTLS struct { + TargetURL string +} + +var _ TestCase = &netemTestCaseWithHTTPWithTLS{} + +// Name implements TestCase. +func (tc *netemTestCaseWithHTTPWithTLS) Name() string { + return fmt.Sprintf("fetching %s using netem and an HTTPS proxy", tc.TargetURL) +} + +// Run implements TestCase. +func (tc *netemTestCaseWithHTTPWithTLS) Run(t *testing.T) { + topology := netem.MustNewStarTopology(log.Log) + defer topology.Close() + + const ( + wwwIPAddr = "93.184.216.34" + proxyIPAddr = "10.0.0.1" + clientIPAddr = "10.0.0.2" + ) + + // create: + // + // - a www stack modeling www.example.com + // + // - a proxy stack + // + // - a client stack + // + // Note that www.example.com's IP address is also the resolver used by everyone + wwwStack := runtimex.Try1(topology.AddHost(wwwIPAddr, wwwIPAddr, &netem.LinkConfig{})) + proxyStack := runtimex.Try1(topology.AddHost(proxyIPAddr, wwwIPAddr, &netem.LinkConfig{})) + clientStack := runtimex.Try1(topology.AddHost(clientIPAddr, wwwIPAddr, &netem.LinkConfig{})) + + // configure the wwwStack as the DNS resolver with proper configuration + dnsConfig := netem.NewDNSConfig() + dnsConfig.AddRecord("www.example.com.", "", wwwIPAddr) + dnsServer := runtimex.Try1(netem.NewDNSServer(log.Log, wwwStack, wwwIPAddr, dnsConfig)) + defer dnsServer.Close() + + // configure the wwwStack to respond to HTTP requests on port 80 + wwwServer80 := testingx.MustNewHTTPServerEx( + &net.TCPAddr{IP: net.ParseIP(wwwIPAddr), Port: 80}, + wwwStack, + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("Bonsoir, Elliot!\r\n")) + }), + ) + defer wwwServer80.Close() + + // configure the wwwStack to respond to HTTPS requests on port 443 + wwwServer443 := testingx.MustNewHTTPServerTLSEx( + &net.TCPAddr{IP: net.ParseIP(wwwIPAddr), Port: 443}, + wwwStack, + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("Bonsoir, Elliot!\r\n")) + }), + wwwStack, + "www.example.com", + ) + defer wwwServer443.Close() + + // configure the proxyStack to implement the HTTP proxy on port 4443 + proxyServer := testingx.MustNewHTTPServerTLSEx( + &net.TCPAddr{IP: net.ParseIP(proxyIPAddr), Port: 4443}, + proxyStack, + testingx.NewHTTPProxyHandler(log.Log, &netxlite.Netx{ + Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: proxyStack}}), + proxyStack, + "proxy.local", + ) + defer proxyServer.Close() + + // create the netx instance for the client + netx := &netxlite.Netx{Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: clientStack}} + + // create an HTTP client configured to use the given proxy + // + // note how we use a dialer that asserts that we're using the proxy IP address + // rather than the host address, so we're sure that we're using the proxy + dialer := &dialerWithAssertions{ + ExpectAddress: proxyIPAddr, + Dialer: netx.NewDialerWithResolver(log.Log, netx.NewStdlibResolver(log.Log)), + } + tlsDialer := netxlite.NewTLSDialer(dialer, netx.NewTLSHandshakerStdlib(log.Log)) + txp := netxlite.NewHTTPTransportWithOptions(log.Log, dialer, tlsDialer, + netxlite.HTTPTransportOptionProxyURL(runtimex.Try1(url.Parse(proxyServer.URL))), + + // TODO(https://github.com/ooni/probe/issues/2536) + netxlite.HTTPTransportOptionTLSClientConfig(&tls.Config{ + RootCAs: clientStack.DefaultCertPool(), + }), + ) + client := &http.Client{Transport: txp} + defer client.CloseIdleConnections() + + // get the homepage and assert we're getting a succesful response + httpCheckResponse(t, client, tc.TargetURL) +} + +// Short implements TestCase. +func (tc *netemTestCaseWithHTTPWithTLS) Short() bool { + return true +} diff --git a/pkg/testingproxy/qa_test.go b/pkg/testingproxy/qa_test.go new file mode 100644 index 000000000..38f31aa94 --- /dev/null +++ b/pkg/testingproxy/qa_test.go @@ -0,0 +1,31 @@ +package testingproxy_test + +import ( + "testing" + + "github.com/ooni/probe-engine/pkg/testingproxy" +) + +func TestHTTPProxy(t *testing.T) { + for _, testCase := range testingproxy.HTTPTestCases { + t.Run(testCase.Name(), func(t *testing.T) { + short := testCase.Short() + if !short && testing.Short() { + t.Skip("skip test in short mode") + } + testCase.Run(t) + }) + } +} + +func TestSOCKSProxy(t *testing.T) { + for _, testCase := range testingproxy.SOCKSTestCases { + t.Run(testCase.Name(), func(t *testing.T) { + short := testCase.Short() + if !short && testing.Short() { + t.Skip("skip test in short mode") + } + testCase.Run(t) + }) + } +} diff --git a/pkg/testingproxy/sockshost.go b/pkg/testingproxy/sockshost.go new file mode 100644 index 000000000..421218b93 --- /dev/null +++ b/pkg/testingproxy/sockshost.go @@ -0,0 +1,72 @@ +package testingproxy + +import ( + "fmt" + "net" + "net/http" + "testing" + + "github.com/apex/log" + "github.com/ooni/probe-engine/pkg/netxlite" + "github.com/ooni/probe-engine/pkg/testingsocks5" +) + +// WithHostNetworkSOCKSProxyAndURL returns a [TestCase] where: +// +// - we fetch a URL; +// +// - using the host network; +// +// - and a SOCKS5 proxy. +// +// Because this [TestCase] uses the host network, it does not run in -short mode. +func WithHostNetworkSOCKSProxyAndURL(URL string) TestCase { + return &hostNetworkTestCaseWithSOCKS{ + TargetURL: URL, + } +} + +type hostNetworkTestCaseWithSOCKS struct { + TargetURL string +} + +var _ TestCase = &hostNetworkTestCaseWithSOCKS{} + +// Name implements TestCase. +func (tc *hostNetworkTestCaseWithSOCKS) Name() string { + return fmt.Sprintf("fetching %s using the host network and a SOCKS5 proxy", tc.TargetURL) +} + +// Run implements TestCase. +func (tc *hostNetworkTestCaseWithSOCKS) Run(t *testing.T) { + // create an instance of Netx where the underlying network is nil, + // which means we're using the host's network + netx := &netxlite.Netx{Underlying: nil} + + // create the proxy server using the host network + endpoint := &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 0} + proxyServer := testingsocks5.MustNewServer(log.Log, netx, endpoint) + defer proxyServer.Close() + + // create an HTTP client configured to use the given proxy + // + // note how we use a dialer that asserts that we're using the proxy IP address + // rather than the host address, so we're sure that we're using the proxy + dialer := &dialerWithAssertions{ + ExpectAddress: "127.0.0.1", + Dialer: netx.NewDialerWithResolver(log.Log, netx.NewStdlibResolver(log.Log)), + } + tlsDialer := netxlite.NewTLSDialer(dialer, netxlite.NewTLSHandshakerStdlib(log.Log)) + txp := netxlite.NewHTTPTransportWithOptions(log.Log, dialer, tlsDialer, + netxlite.HTTPTransportOptionProxyURL(proxyServer.URL())) + client := &http.Client{Transport: txp} + defer client.CloseIdleConnections() + + // get the homepage and assert we're getting a succesful response + httpCheckResponse(t, client, tc.TargetURL) +} + +// Short implements TestCase. +func (tc *hostNetworkTestCaseWithSOCKS) Short() bool { + return false +} diff --git a/pkg/testingproxy/socksnetem.go b/pkg/testingproxy/socksnetem.go new file mode 100644 index 000000000..017d23d53 --- /dev/null +++ b/pkg/testingproxy/socksnetem.go @@ -0,0 +1,135 @@ +package testingproxy + +import ( + "crypto/tls" + "fmt" + "net" + "net/http" + "testing" + + "github.com/apex/log" + "github.com/ooni/netem" + "github.com/ooni/probe-engine/pkg/netxlite" + "github.com/ooni/probe-engine/pkg/runtimex" + "github.com/ooni/probe-engine/pkg/testingsocks5" + "github.com/ooni/probe-engine/pkg/testingx" +) + +// WithNetemSOCKSProxyAndURL returns a [TestCase] where: +// +// - we fetch a URL; +// +// - using the github.com/ooni.netem; +// +// - and a SOCKS5 proxy. +// +// Because this [TestCase] uses netem, it also runs in -short mode. +func WithNetemSOCKSProxyAndURL(URL string) TestCase { + return &netemTestCaseWithSOCKS{ + TargetURL: URL, + } +} + +type netemTestCaseWithSOCKS struct { + TargetURL string +} + +var _ TestCase = &netemTestCaseWithSOCKS{} + +// Name implements TestCase. +func (tc *netemTestCaseWithSOCKS) Name() string { + return fmt.Sprintf("fetching %s using netem and a SOCKS5 proxy", tc.TargetURL) +} + +// Run implements TestCase. +func (tc *netemTestCaseWithSOCKS) Run(t *testing.T) { + topology := netem.MustNewStarTopology(log.Log) + defer topology.Close() + + const ( + wwwIPAddr = "93.184.216.34" + proxyIPAddr = "10.0.0.1" + clientIPAddr = "10.0.0.2" + ) + + // create: + // + // - a www stack modeling www.example.com + // + // - a proxy stack + // + // - a client stack + // + // Note that www.example.com's IP address is also the resolver used by everyone + wwwStack := runtimex.Try1(topology.AddHost(wwwIPAddr, wwwIPAddr, &netem.LinkConfig{})) + proxyStack := runtimex.Try1(topology.AddHost(proxyIPAddr, wwwIPAddr, &netem.LinkConfig{})) + clientStack := runtimex.Try1(topology.AddHost(clientIPAddr, wwwIPAddr, &netem.LinkConfig{})) + + // configure the wwwStack as the DNS resolver with proper configuration + dnsConfig := netem.NewDNSConfig() + dnsConfig.AddRecord("www.example.com.", "", wwwIPAddr) + dnsServer := runtimex.Try1(netem.NewDNSServer(log.Log, wwwStack, wwwIPAddr, dnsConfig)) + defer dnsServer.Close() + + // configure the wwwStack to respond to HTTP requests on port 80 + wwwServer80 := testingx.MustNewHTTPServerEx( + &net.TCPAddr{IP: net.ParseIP(wwwIPAddr), Port: 80}, + wwwStack, + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("Bonsoir, Elliot!\r\n")) + }), + ) + defer wwwServer80.Close() + + // configure the wwwStack to respond to HTTPS requests on port 443 + wwwServer443 := testingx.MustNewHTTPServerTLSEx( + &net.TCPAddr{IP: net.ParseIP(wwwIPAddr), Port: 443}, + wwwStack, + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("Bonsoir, Elliot!\r\n")) + }), + wwwStack, + "www.example.com", + ) + defer wwwServer443.Close() + + // configure the proxyStack to implement the SOCKS proxy on port 9050 + proxyServer := testingsocks5.MustNewServer( + log.Log, + &netxlite.Netx{ + Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: proxyStack}}, + &net.TCPAddr{IP: net.ParseIP(proxyIPAddr), Port: 9050}, + ) + defer proxyServer.Close() + + // create the netx instance for the client + netx := &netxlite.Netx{Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: clientStack}} + + // create an HTTP client configured to use the given proxy + // + // note how we use a dialer that asserts that we're using the proxy IP address + // rather than the host address, so we're sure that we're using the proxy + dialer := &dialerWithAssertions{ + ExpectAddress: proxyIPAddr, + Dialer: netx.NewDialerWithResolver(log.Log, netx.NewStdlibResolver(log.Log)), + } + tlsDialer := netxlite.NewTLSDialer(dialer, netx.NewTLSHandshakerStdlib(log.Log)) + txp := netxlite.NewHTTPTransportWithOptions(log.Log, dialer, tlsDialer, + netxlite.HTTPTransportOptionProxyURL(proxyServer.URL()), + + // TODO(https://github.com/ooni/probe/issues/2536) + netxlite.HTTPTransportOptionTLSClientConfig(&tls.Config{ + RootCAs: clientStack.DefaultCertPool(), + }), + ) + client := &http.Client{Transport: txp} + defer client.CloseIdleConnections() + + // get the homepage and assert we're getting a succesful response + httpCheckResponse(t, client, tc.TargetURL) +} + +// Short implements TestCase. +func (tc *netemTestCaseWithSOCKS) Short() bool { + return true +} diff --git a/pkg/testingproxy/testcase.go b/pkg/testingproxy/testcase.go new file mode 100644 index 000000000..44f5f6130 --- /dev/null +++ b/pkg/testingproxy/testcase.go @@ -0,0 +1,49 @@ +package testingproxy + +import "testing" + +// TestCase is a test case implemented by this package. +type TestCase interface { + // Name returns the test case name. + Name() string + + // Run runs the test case. + Run(t *testing.T) + + // Short returns whether this is a short test. + Short() bool +} + +// SOCKSTestCases contains the SOCKS test cases. +var SOCKSTestCases = []TestCase{ + // with host network + WithHostNetworkSOCKSProxyAndURL("http://www.example.com/"), + WithHostNetworkSOCKSProxyAndURL("https://www.example.com/"), + + // with netem + WithNetemSOCKSProxyAndURL("http://www.example.com/"), + WithNetemSOCKSProxyAndURL("https://www.example.com/"), + + // with netem and IPv4 addresses so we test another SOCKS5 dialing mode + WithNetemSOCKSProxyAndURL("http://93.184.216.34/"), + WithNetemSOCKSProxyAndURL("https://93.184.216.34/"), +} + +// HTTPTestCases contains the HTTP test cases. +var HTTPTestCases = []TestCase{ + // with host network and HTTP proxy + WithHostNetworkHTTPProxyAndURL("http://www.example.com/"), + WithHostNetworkHTTPProxyAndURL("https://www.example.com/"), + + // with host network and HTTPS proxy + WithHostNetworkHTTPWithTLSProxyAndURL("http://www.example.com/"), + WithHostNetworkHTTPWithTLSProxyAndURL("https://www.example.com/"), + + // with netem and HTTP proxy + WithNetemHTTPProxyAndURL("http://www.example.com/"), + WithNetemHTTPProxyAndURL("https://www.example.com/"), + + // with netem and HTTPS proxy + WithNetemHTTPWithTLSProxyAndURL("http://www.example.com/"), + WithNetemHTTPWithTLSProxyAndURL("https://www.example.com/"), +} diff --git a/pkg/testingquic/testingquic.go b/pkg/testingquic/testingquic.go new file mode 100644 index 000000000..3d27fcfdc --- /dev/null +++ b/pkg/testingquic/testingquic.go @@ -0,0 +1,59 @@ +// Package testingquic allows to retrieve the domain and endpoint to use +// for all the integration tests that use QUIC. +// +// See https://github.com/ooni/probe/issues/1873 for context. +package testingquic + +import ( + "context" + "net" + "strings" + "sync" + "time" + + "github.com/ooni/probe-engine/pkg/runtimex" +) + +const domain = "www.cloudflare.com" + +var ( + address string + initOnce sync.Once +) + +func mustInit() { + // create a context using a reasonable timeout + const timeout = 10 * time.Second + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + // instantiate the system resolver + reso := &net.Resolver{} + + // perform the lookup and panic on failure + addrs := runtimex.Try1(reso.LookupHost(ctx, domain)) + + // use the first non IPv6 addr + for _, addr := range addrs { + if !strings.Contains(addr, ":") { + address = addr + return + } + } +} + +// MustEndpoint returns a QUIC endpoint using this package's default address and then given port. +// +// This function PANICS if we cannot find out the IP addr we should use. +func MustEndpoint(port string) string { + initOnce.Do(mustInit) + return net.JoinHostPort(address, port) +} + +// MustDomain returns the domain to use for QUIC testing. +// +// This function PANICS if we cannot find out the IP addr we should use. +func MustDomain() string { + initOnce.Do(mustInit) + return domain +} diff --git a/pkg/netxlite/quictesting/quictesting_test.go b/pkg/testingquic/testingquic_test.go similarity index 50% rename from pkg/netxlite/quictesting/quictesting_test.go rename to pkg/testingquic/testingquic_test.go index 0a8f99e39..47c67c448 100644 --- a/pkg/netxlite/quictesting/quictesting_test.go +++ b/pkg/testingquic/testingquic_test.go @@ -1,4 +1,4 @@ -package quictesting +package testingquic import ( "net" @@ -6,12 +6,17 @@ import ( ) func TestWorksAsIntended(t *testing.T) { - epnt := Endpoint("443") - addr, port, err := net.SplitHostPort(epnt) + if testing.Short() { + t.Skip("skip test in short mode") + } + + endpoint := MustEndpoint("443") + addr, port, err := net.SplitHostPort(endpoint) if err != nil { t.Fatal(err) } - if addr != Address { + + if addr != address { t.Fatal("invalid addr") } if port != "443" { diff --git a/pkg/testingsocks5/LICENSE b/pkg/testingsocks5/LICENSE new file mode 100644 index 000000000..a5df10e67 --- /dev/null +++ b/pkg/testingsocks5/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2014 Armon Dadgar + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/pkg/testingsocks5/auth.go b/pkg/testingsocks5/auth.go new file mode 100644 index 000000000..63f8200bb --- /dev/null +++ b/pkg/testingsocks5/auth.go @@ -0,0 +1,83 @@ +package testingsocks5 + +import ( + "fmt" + "io" + "net" +) + +// Codes representing authentication mechanisms +const ( + noAuth = uint8(0) + noAcceptable = uint8(255) + userPassAuth = uint8(2) + userAuthVersion = uint8(1) + authSuccess = uint8(0) + authFailure = uint8(1) +) + +var ( + errNoSupportedAuth = fmt.Errorf("no supported authentication mechanism") +) + +// A Request encapsulates authentication state provided +// during negotiation +type authContext struct { + // Provided auth method + Method uint8 + + // Payload provided during negotiation. + // Keys depend on the used auth method. + // For UserPassauth contains Username + Payload map[string]string +} + +// noAuthAuthenticator is used to handle the "No Authentication" mode +type noAuthAuthenticator struct{} + +func (a noAuthAuthenticator) Authenticate(cconn net.Conn) (*authContext, error) { + _, err := cconn.Write([]byte{socks5Version, noAuth}) + return &authContext{noAuth, nil}, err +} + +// authenticate is used to handle connection authentication +func (s *Server) authenticate(cconn net.Conn) (*authContext, error) { + // Get the methods + methods, err := readAuthMethods(cconn) + if err != nil { + return nil, fmt.Errorf("failed to get auth methods: %w", err) + } + + // Select a usable method + for _, method := range methods { + switch method { + case noAuth: + return (noAuthAuthenticator{}).Authenticate(cconn) + + default: + // nothing + } + } + + // No usable method found + return nil, noAcceptableAuth(cconn) +} + +// noAcceptableAuth is used to handle when we have no eligible authentication mechanism +func noAcceptableAuth(conn net.Conn) error { + conn.Write([]byte{socks5Version, noAcceptable}) + return errNoSupportedAuth +} + +// readAuthMethods is used to read the number of methods and proceeding auth methods +func readAuthMethods(cconn net.Conn) ([]byte, error) { + header := []byte{0} + if _, err := io.ReadFull(cconn, header); err != nil { + return nil, err + } + + numMethods := uint8(header[0]) + methods := make([]byte, numMethods) + _, err := io.ReadFull(cconn, methods) + return methods, err +} diff --git a/pkg/testingsocks5/client.go b/pkg/testingsocks5/client.go new file mode 100644 index 000000000..a1aa92a4e --- /dev/null +++ b/pkg/testingsocks5/client.go @@ -0,0 +1,45 @@ +package testingsocks5 + +import ( + "errors" + "fmt" + "io" + "net" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-engine/pkg/model" +) + +// client is a minimal client used for testing the server +type client struct { + exchanges []exchange +} + +// exchange is a byte exchange between the client and the server: the client +// sends the bytes to send and then reads and checks whether it has received +// the expected response from the server. +type exchange struct { + send []byte + expect []byte +} + +var errUnexpectedResponse = errors.New("unexpected response") + +func (ic *client) run(logger model.Logger, conn net.Conn) error { + for _, exchange := range ic.exchanges { + logger.Infof("SOCKS5_CLIENT: sending: %v", exchange.send) + if _, err := conn.Write(exchange.send); err != nil { + return err + } + logger.Infof("SOCKS5_CLIENT: expecting: %v", exchange.expect) + buffer := make([]byte, len(exchange.expect)) + if _, err := io.ReadFull(conn, buffer); err != nil { + return err + } + logger.Infof("SOCKS5_CLIENT: got: %v", buffer) + if diff := cmp.Diff(exchange.expect, buffer); diff != "" { + return fmt.Errorf("%w: %s", errUnexpectedResponse, diff) + } + } + return nil +} diff --git a/pkg/testingsocks5/client_test.go b/pkg/testingsocks5/client_test.go new file mode 100644 index 000000000..e0e14c261 --- /dev/null +++ b/pkg/testingsocks5/client_test.go @@ -0,0 +1,76 @@ +package testingsocks5 + +import ( + "errors" + "testing" + + "github.com/ooni/probe-engine/pkg/mocks" + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/runtimex" +) + +func TestClientErrorPaths(t *testing.T) { + t.Run("conn.Write fails", func(t *testing.T) { + expected := errors.New("mocked error") + conn := &mocks.Conn{ + MockWrite: func(b []byte) (int, error) { + return 0, expected + }, + } + c := &client{ + exchanges: []exchange{{ + send: []byte{1, 2, 3, 4}, + expect: []byte{}, + }}, + } + err := c.run(model.DiscardLogger, conn) + if !errors.Is(err, expected) { + t.Fatal("unexpected error", err) + } + }) + + t.Run("conn.Read fails", func(t *testing.T) { + expected := errors.New("mocked error") + conn := &mocks.Conn{ + MockWrite: func(b []byte) (int, error) { + return len(b), nil + }, + MockRead: func(b []byte) (int, error) { + return 0, expected + }, + } + c := &client{ + exchanges: []exchange{{ + send: []byte{1, 2, 3, 4}, + expect: []byte{4, 3, 2, 1}, + }}, + } + err := c.run(model.DiscardLogger, conn) + if !errors.Is(err, expected) { + t.Fatal("unexpected error", err) + } + }) + + t.Run("when we get an unexpected response", func(t *testing.T) { + conn := &mocks.Conn{ + MockWrite: func(b []byte) (int, error) { + return len(b), nil + }, + MockRead: func(b []byte) (int, error) { + runtimex.Assert(len(b) == 4, "unexpected buffer length") + copy(b, []byte{1, 2, 3, 4}) + return len(b), nil + }, + } + c := &client{ + exchanges: []exchange{{ + send: []byte{1, 2, 3, 4}, + expect: []byte{4, 3, 2, 1}, + }}, + } + err := c.run(model.DiscardLogger, conn) + if !errors.Is(err, errUnexpectedResponse) { + t.Fatal("unexpected error", err) + } + }) +} diff --git a/pkg/testingsocks5/doc.go b/pkg/testingsocks5/doc.go new file mode 100644 index 000000000..eaec83198 --- /dev/null +++ b/pkg/testingsocks5/doc.go @@ -0,0 +1,2 @@ +// Package testingsock5 is a netem-aware fork of https://github.com/armon/go-socks5. +package testingsocks5 diff --git a/pkg/testingsocks5/internal_test.go b/pkg/testingsocks5/internal_test.go new file mode 100644 index 000000000..2c27d3527 --- /dev/null +++ b/pkg/testingsocks5/internal_test.go @@ -0,0 +1,560 @@ +package testingsocks5 + +import ( + "errors" + "io" + "net" + "sync" + "testing" + + "github.com/apex/log" + "github.com/ooni/probe-engine/pkg/mocks" + "github.com/ooni/probe-engine/pkg/netxlite" + "github.com/ooni/probe-engine/pkg/runtimex" +) + +func TestReadVersionError(t *testing.T) { + server := &Server{ + closeOnce: sync.Once{}, + listener: &mocks.Listener{ + MockClose: func() error { + return nil + }, + }, + logger: log.Log, + netx: &netxlite.Netx{Underlying: nil}, + } + defer server.Close() + + conn := &mocks.Conn{ + MockClose: func() error { + return nil + }, + MockRead: func(b []byte) (int, error) { + return 0, io.EOF + }, + } + + err := server.serveConn(conn) + if !errors.Is(err, io.EOF) { + t.Fatal("unexpected error", err) + } +} + +func TestServerClosesConn(t *testing.T) { + server := &Server{ + closeOnce: sync.Once{}, + listener: &mocks.Listener{ + MockClose: func() error { + return nil + }, + }, + logger: log.Log, + netx: &netxlite.Netx{Underlying: nil}, + } + defer server.Close() + + called := false + conn := &mocks.Conn{ + MockClose: func() error { + called = true + return nil + }, + MockRead: func(b []byte) (int, error) { + return 0, io.EOF + }, + } + + err := server.serveConn(conn) + if !errors.Is(err, io.EOF) { + t.Fatal("unexpected error", err) + } + if !called { + t.Fatal("did not call close") + } +} + +func TestInvalidVersion(t *testing.T) { + server := MustNewServer( + log.Log, + &netxlite.Netx{Underlying: nil}, + &net.TCPAddr{ + IP: net.ParseIP("127.0.0.1"), + Port: 0, + }, + ) + defer server.Close() + + // Note: the protocol version must be 5 + conn := runtimex.Try1(net.Dial("tcp", server.Endpoint())) + _ = runtimex.Try1(conn.Write([]byte{17, 0, 0, 1})) + defer conn.Close() + + client := &client{ + exchanges: []exchange{{ + send: []byte{ + 17, // version + }, + expect: []byte{}, + }}, + } + if err := client.run(log.Log, conn); err != nil { + t.Skip("https://github.com/ooni/probe/issues/2538") + } +} + +func TestReadAuthMethodsFailure(t *testing.T) { + server := MustNewServer( + log.Log, + &netxlite.Netx{Underlying: nil}, + &net.TCPAddr{ + IP: net.ParseIP("127.0.0.1"), + Port: 0, + }, + ) + defer server.Close() + + // Note: the protocol expects something after we have sent the version + conn := runtimex.Try1(net.Dial("tcp", server.Endpoint())) + defer conn.Close() + + client := &client{ + exchanges: []exchange{{ + send: []byte{ + 5, // version + }, + expect: []byte{}, + }}, + } + if err := client.run(log.Log, conn); err != nil { + t.Fatal(err) + } +} + +func TestNoAcceptableAuth(t *testing.T) { + server := MustNewServer( + log.Log, + &netxlite.Netx{Underlying: nil}, + &net.TCPAddr{ + IP: net.ParseIP("127.0.0.1"), + Port: 0, + }, + ) + defer server.Close() + + // Note: we don't support username and password authentication + conn := runtimex.Try1(net.Dial("tcp", server.Endpoint())) + defer conn.Close() + + client := &client{ + exchanges: []exchange{{ + send: []byte{ + 5, // version + 1, // number of authentication methods supported + 2, // username and password + 1, // version of the username and password authentication + 3, // username length + 'f', 'o', 'o', // username + '3', // password length + 'b', 'a', 'r', // password + }, + expect: []byte{5, 255}, + }}, + } + if err := client.run(log.Log, conn); err != nil { + t.Fatal(err) + } +} + +func TestNewRequestReadError(t *testing.T) { + server := MustNewServer( + log.Log, + &netxlite.Netx{Underlying: nil}, + &net.TCPAddr{ + IP: net.ParseIP("127.0.0.1"), + Port: 0, + }, + ) + defer server.Close() + + // Note: the second message should contain something after the version + conn := runtimex.Try1(net.Dial("tcp", server.Endpoint())) + defer conn.Close() + + client := &client{ + exchanges: []exchange{{ + send: []byte{ + 5, // version + 1, // number of authentication methods supported + 0, // no authentication + }, + expect: []byte{5, 0}, + }, { + send: []byte{ + 5, // version + }, + expect: []byte{}, + }}, + } + if err := client.run(log.Log, conn); err != nil { + t.Fatal(err) + } +} + +func TestNewRequestWithIncompatibleVersion(t *testing.T) { + server := MustNewServer( + log.Log, + &netxlite.Netx{Underlying: nil}, + &net.TCPAddr{ + IP: net.ParseIP("127.0.0.1"), + Port: 0, + }, + ) + defer server.Close() + + // Note: the second message should contain again version equal to 5 + conn := runtimex.Try1(net.Dial("tcp", server.Endpoint())) + _ = runtimex.Try1(conn.Write([]byte{})) + defer conn.Close() + + client := &client{ + exchanges: []exchange{{ + send: []byte{ + 5, // version + 1, // number of authentication methods supported + 0, // no authentication + }, + expect: []byte{5, 0}, + }, { + send: []byte{ + 17, // version + 2, // bind command + 0, // reserved + }, + expect: []byte{}, + }}, + } + if err := client.run(log.Log, conn); err != nil { + t.Fatal(err) + } +} + +func TestUnsupportedCommand(t *testing.T) { + server := MustNewServer( + log.Log, + &netxlite.Netx{Underlying: nil}, + &net.TCPAddr{ + IP: net.ParseIP("127.0.0.1"), + Port: 0, + }, + ) + defer server.Close() + + // Note: we only support the connect command + conn := runtimex.Try1(net.Dial("tcp", server.Endpoint())) + _ = runtimex.Try1(conn.Write([]byte{})) + defer conn.Close() + + client := &client{ + exchanges: []exchange{{ + send: []byte{ + 5, // version + 1, // number of authentication methods supported + 0, // no authentication + }, + expect: []byte{5, 0}, + }, { + send: []byte{ + 5, // version + 2, // bind command + 0, // reserved + 1, // IPv4 + 127, 0, 0, 1, // address + 0, 80, // port + }, + expect: []byte{5, 7, 0, 1, 0, 0, 0, 0, 0, 0}, + }}, + } + if err := client.run(log.Log, conn); err != nil { + t.Fatal(err) + } +} + +func TestUnrecognizedAddrType(t *testing.T) { + server := MustNewServer( + log.Log, + &netxlite.Netx{Underlying: nil}, + &net.TCPAddr{ + IP: net.ParseIP("127.0.0.1"), + Port: 0, + }, + ) + defer server.Close() + + // Note: 55 is an invalid address type + conn := runtimex.Try1(net.Dial("tcp", server.Endpoint())) + _ = runtimex.Try1(conn.Write([]byte{})) + defer conn.Close() + + client := &client{ + exchanges: []exchange{{ + send: []byte{ + 5, // version + 1, // number of authentication methods supported + 0, // no authentication + }, + expect: []byte{5, 0}, + }, { + send: []byte{ + 5, // version + 2, // bind command + 0, // reserved + 55, // ??? + 127, 0, 0, 1, // address + 0, 80, // port + }, + expect: []byte{}, + }}, + } + if err := client.run(log.Log, conn); err != nil { + t.Fatal(err) + } +} + +func TestReadAddrSpecFailureReadingAddrType(t *testing.T) { + server := MustNewServer( + log.Log, + &netxlite.Netx{Underlying: nil}, + &net.TCPAddr{ + IP: net.ParseIP("127.0.0.1"), + Port: 0, + }, + ) + defer server.Close() + + // Note: here there is nothing after the reserved byte + conn := runtimex.Try1(net.Dial("tcp", server.Endpoint())) + _ = runtimex.Try1(conn.Write([]byte{})) + defer conn.Close() + + client := &client{ + exchanges: []exchange{{ + send: []byte{ + 5, // version + 1, // number of authentication methods supported + 0, // no authentication + }, + expect: []byte{5, 0}, + }, { + send: []byte{ + 5, // version + 2, // bind command + 0, // reserved + }, + expect: []byte{}, + }}, + } + if err := client.run(log.Log, conn); err != nil { + t.Fatal(err) + } +} + +func TestReadAddrSpecFailureReadingIPv4Address(t *testing.T) { + server := MustNewServer( + log.Log, + &netxlite.Netx{Underlying: nil}, + &net.TCPAddr{ + IP: net.ParseIP("127.0.0.1"), + Port: 0, + }, + ) + defer server.Close() + + // Note: here the IPv4 address bytes are missing + conn := runtimex.Try1(net.Dial("tcp", server.Endpoint())) + _ = runtimex.Try1(conn.Write([]byte{})) + defer conn.Close() + + client := &client{ + exchanges: []exchange{{ + send: []byte{ + 5, // version + 1, // number of authentication methods supported + 0, // no authentication + }, + expect: []byte{5, 0}, + }, { + send: []byte{ + 5, // version + 2, // bind command + 0, // reserved + 1, // IPv4 + }, + expect: []byte{}, + }}, + } + if err := client.run(log.Log, conn); err != nil { + t.Fatal(err) + } +} + +func TestReadAddrSpecFailureReadingIPv6Address(t *testing.T) { + server := MustNewServer( + log.Log, + &netxlite.Netx{Underlying: nil}, + &net.TCPAddr{ + IP: net.ParseIP("127.0.0.1"), + Port: 0, + }, + ) + defer server.Close() + + // Note: here the IPv6 address bytes are missing + conn := runtimex.Try1(net.Dial("tcp", server.Endpoint())) + _ = runtimex.Try1(conn.Write([]byte{})) + defer conn.Close() + + client := &client{ + exchanges: []exchange{{ + send: []byte{ + 5, // version + 1, // number of authentication methods supported + 0, // no authentication + }, + expect: []byte{5, 0}, + }, { + send: []byte{ + 5, // version + 2, // bind command + 0, // reserved + 4, // IPv6 + }, + expect: []byte{}, + }}, + } + if err := client.run(log.Log, conn); err != nil { + t.Fatal(err) + } +} + +func TestReadAddrSpecFailureReadingFQDNLength(t *testing.T) { + server := MustNewServer( + log.Log, + &netxlite.Netx{Underlying: nil}, + &net.TCPAddr{ + IP: net.ParseIP("127.0.0.1"), + Port: 0, + }, + ) + defer server.Close() + + // Note: here the length of the FQDN is missing + conn := runtimex.Try1(net.Dial("tcp", server.Endpoint())) + _ = runtimex.Try1(conn.Write([]byte{})) + defer conn.Close() + + client := &client{ + exchanges: []exchange{{ + send: []byte{ + 5, // version + 1, // number of authentication methods supported + 0, // no authentication + }, + expect: []byte{5, 0}, + }, { + send: []byte{ + 5, // version + 2, // bind command + 0, // reserved + 3, // FQDN + }, + expect: []byte{}, + }}, + } + if err := client.run(log.Log, conn); err != nil { + t.Fatal(err) + } +} + +func TestReadAddrSpecFailureReadingFQDNString(t *testing.T) { + server := MustNewServer( + log.Log, + &netxlite.Netx{Underlying: nil}, + &net.TCPAddr{ + IP: net.ParseIP("127.0.0.1"), + Port: 0, + }, + ) + defer server.Close() + + // Note: here the bytes of the FQDN are missing + conn := runtimex.Try1(net.Dial("tcp", server.Endpoint())) + _ = runtimex.Try1(conn.Write([]byte{})) + defer conn.Close() + + client := &client{ + exchanges: []exchange{{ + send: []byte{ + 5, // version + 1, // number of authentication methods supported + 0, // no authentication + }, + expect: []byte{5, 0}, + }, { + send: []byte{ + 5, // version + 2, // bind command + 0, // reserved + 3, // FQDN + 10, // length of FQDN + }, + expect: []byte{}, + }}, + } + if err := client.run(log.Log, conn); err != nil { + t.Fatal(err) + } +} + +func TestReadAddrSpecFailureReadingPortWithIPv6(t *testing.T) { + server := MustNewServer( + log.Log, + &netxlite.Netx{Underlying: nil}, + &net.TCPAddr{ + IP: net.ParseIP("127.0.0.1"), + Port: 0, + }, + ) + defer server.Close() + + // Note: here the ports bytes are missing + conn := runtimex.Try1(net.Dial("tcp", server.Endpoint())) + _ = runtimex.Try1(conn.Write([]byte{})) + defer conn.Close() + + client := &client{ + exchanges: []exchange{{ + send: []byte{ + 5, // version + 1, // number of authentication methods supported + 0, // no authentication + }, + expect: []byte{5, 0}, + }, { + send: []byte{ + 5, // version + 2, // bind command + 0, // reserved + 4, // IPv6, + 0, 0, 0, 0, // IPv6 addr + 0, 0, 0, 0, // IPv6 addr + 0, 0, 0, 0, // IPv6 addr + 0, 0, 0, 0, // IPv6 addr + }, + expect: []byte{}, + }}, + } + if err := client.run(log.Log, conn); err != nil { + t.Fatal(err) + } +} diff --git a/pkg/testingsocks5/qa_test.go b/pkg/testingsocks5/qa_test.go new file mode 100644 index 000000000..7e5dc13f3 --- /dev/null +++ b/pkg/testingsocks5/qa_test.go @@ -0,0 +1,94 @@ +package testingsocks5_test + +import ( + "crypto/tls" + "net" + "net/http" + "strings" + "testing" + + "github.com/apex/log" + "github.com/ooni/netem" + "github.com/ooni/probe-engine/pkg/netxlite" + "github.com/ooni/probe-engine/pkg/runtimex" + "github.com/ooni/probe-engine/pkg/testingproxy" + "github.com/ooni/probe-engine/pkg/testingsocks5" +) + +func TestNetem(t *testing.T) { + for _, testCase := range testingproxy.SOCKSTestCases { + t.Run(testCase.Name(), func(t *testing.T) { + short := testCase.Short() + if !short && testing.Short() { + t.Skip("skip test in short mode") + } + testCase.Run(t) + }) + } +} + +func TestNetemDialFailure(t *testing.T) { + topology := netem.MustNewStarTopology(log.Log) + defer topology.Close() + + const ( + wwwIPAddr = "93.184.216.34" + proxyIPAddr = "10.0.0.1" + clientIPAddr = "10.0.0.2" + ) + + // create: + // + // - a www stack modeling www.example.com (but w/o any listener, so connect will fail) + // + // - a proxy stack + // + // - a client stack + // + // Note that www.example.com's IP address is also the resolver used by everyone + wwwStack := runtimex.Try1(topology.AddHost(wwwIPAddr, wwwIPAddr, &netem.LinkConfig{})) + proxyStack := runtimex.Try1(topology.AddHost(proxyIPAddr, wwwIPAddr, &netem.LinkConfig{})) + clientStack := runtimex.Try1(topology.AddHost(clientIPAddr, wwwIPAddr, &netem.LinkConfig{})) + + // configure the wwwStack as the DNS resolver with proper configuration + dnsConfig := netem.NewDNSConfig() + dnsConfig.AddRecord("www.example.com.", "", wwwIPAddr) + dnsServer := runtimex.Try1(netem.NewDNSServer(log.Log, wwwStack, wwwIPAddr, dnsConfig)) + defer dnsServer.Close() + + // configure the proxyStack to implement the SOCKS proxy on port 9050 + proxyServer := testingsocks5.MustNewServer( + log.Log, + &netxlite.Netx{ + Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: proxyStack}}, + &net.TCPAddr{IP: net.ParseIP(proxyIPAddr), Port: 9050}, + ) + defer proxyServer.Close() + + // create the netx instance for the client + netx := &netxlite.Netx{Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: clientStack}} + + // create an HTTP client configured to use the given proxy + dialer := netx.NewDialerWithResolver(log.Log, netx.NewStdlibResolver(log.Log)) + tlsDialer := netxlite.NewTLSDialer(dialer, netx.NewTLSHandshakerStdlib(log.Log)) + txp := netxlite.NewHTTPTransportWithOptions(log.Log, dialer, tlsDialer, + netxlite.HTTPTransportOptionProxyURL(proxyServer.URL()), + + // TODO(https://github.com/ooni/probe/issues/2536) + netxlite.HTTPTransportOptionTLSClientConfig(&tls.Config{ + RootCAs: clientStack.DefaultCertPool(), + }), + ) + client := &http.Client{Transport: txp} + defer client.CloseIdleConnections() + + // because the TCP/IP stack exists but we're not listening, we should get an error (the + // SOCKS5 library has been simplified to always return "host unreachabile") + resp, err := client.Get("https://www.example.com/") + if err == nil || !strings.HasSuffix(err.Error(), "host unreachable") { + t.Fatal("unexpected error", err) + } + if resp != nil { + t.Fatal("expected nil resp") + } +} diff --git a/pkg/testingsocks5/request.go b/pkg/testingsocks5/request.go new file mode 100644 index 000000000..4d9d47210 --- /dev/null +++ b/pkg/testingsocks5/request.go @@ -0,0 +1,224 @@ +package testingsocks5 + +import ( + "fmt" + "io" + "net" + "strconv" + "sync" + + "context" +) + +const ( + connectCommand = uint8(1) + bindCommand = uint8(2) + associateCommand = uint8(3) + ipv4Address = uint8(1) + fqdnAddress = uint8(3) + ipv6Address = uint8(4) +) + +const ( + successReply uint8 = iota + serverFailure + ruleFailure + networkUnreachable + hostUnreachable + connectionRefused + ttlExpired + commandNotSupported + addrTypeNotSupported +) + +var ( + errUnrecognizedAddrType = fmt.Errorf("unrecognized address type") +) + +// addrSpec is used to return the target addrSpec +// which may be specified as IPv4, IPv6, or a FQDN +type addrSpec struct { + Address string + Port int +} + +// A request represents request received by a server +type request struct { + // Protocol version + Version uint8 + + // Requested command + Command uint8 + + // AddrSpec of the desired destination + DestAddr *addrSpec +} + +// newRequest creates a new request from the tcp connection +func newRequest(cconn net.Conn) (*request, error) { + // Read the version byte + header := []byte{0, 0, 0} + if _, err := io.ReadFull(cconn, header); err != nil { + return nil, fmt.Errorf("failed to get command version: %w", err) + } + + // Ensure we are compatible + if header[0] != socks5Version { + return nil, fmt.Errorf("unsupported command version: %v", header[0]) + } + + // Read in the destination address + dest, err := readAddrSpec(cconn) + if err != nil { + return nil, err + } + + request := &request{ + Version: socks5Version, + Command: header[1], + DestAddr: dest, + } + + return request, nil +} + +// handleRequest is used for request processing after authentication +func (s *Server) handleRequest(req *request, cconn net.Conn) error { + ctx := context.Background() + if req.Command != connectCommand { + return sendReply(cconn, commandNotSupported, &net.TCPAddr{}) + } + return s.handleConnect(ctx, cconn, req) +} + +// handleConnect is used to handle a connect command +func (s *Server) handleConnect(ctx context.Context, cconn net.Conn, req *request) error { + s.logger.Info("handling CONNECT command") + + // Attempt to connect + endpoint := net.JoinHostPort(req.DestAddr.Address, strconv.Itoa(req.DestAddr.Port)) + s.logger.Infof("endpoint: %s", endpoint) + dialer := s.netx.NewDialerWithResolver(s.logger, s.netx.NewStdlibResolver(s.logger)) + sconn, err := dialer.DialContext(ctx, "tcp", endpoint) + if err != nil { + // Note: the original go-socks5 selects the proper error but it does not + // matter for our purposes, so we always return hostUnreachable. + return sendReply(cconn, hostUnreachable, &net.TCPAddr{}) + } + defer sconn.Close() + + // Send success + local := sconn.LocalAddr().(*net.TCPAddr) + if err := sendReply(cconn, successReply, local); err != nil { + return fmt.Errorf("failed to send reply: %w", err) + } + + // Start proxying + wg := &sync.WaitGroup{} + wg.Add(2) + go func() { + _, _ = io.Copy(cconn, sconn) + wg.Done() + }() + go func() { + _, _ = io.Copy(sconn, cconn) + wg.Done() + }() + wg.Wait() + return nil +} + +// readAddrSpec is used to read AddrSpec. +// Expects an address type byte, follwed by the address and port. +func readAddrSpec(cconn net.Conn) (*addrSpec, error) { + d := &addrSpec{} + + // Get the address type + addrType := []byte{0} + if _, err := io.ReadFull(cconn, addrType); err != nil { + return nil, err + } + + // Handle on a per type basis + switch addrType[0] { + case ipv4Address: + addr := make([]byte, 4) + if _, err := io.ReadFull(cconn, addr); err != nil { + return nil, err + } + d.Address = net.IP(addr).String() + + case ipv6Address: + addr := make([]byte, 16) + if _, err := io.ReadFull(cconn, addr); err != nil { + return nil, err + } + d.Address = net.IP(addr).String() + + case fqdnAddress: + lengthBuffer := []byte{0} + if _, err := io.ReadFull(cconn, lengthBuffer); err != nil { + return nil, err + } + addrLen := int(lengthBuffer[0]) + fqdn := make([]byte, addrLen) + if _, err := io.ReadFull(cconn, fqdn); err != nil { + return nil, err + } + d.Address = string(fqdn) + + default: + return nil, errUnrecognizedAddrType + } + + // Read the port + port := []byte{0, 0} + if _, err := io.ReadFull(cconn, port); err != nil { + return nil, err + } + d.Port = (int(port[0]) << 8) | int(port[1]) + + return d, nil +} + +// sendReply is used to send a reply message +func sendReply(w io.Writer, resp uint8, addr *net.TCPAddr) error { + // Format the address + var ( + addrType uint8 + addrBody []byte + addrPort uint16 + ) + + // Note: the order of these cases matters! + switch { + case addr.IP.To4() != nil: + addrType = ipv4Address + addrBody = []byte(addr.IP.To4()) + addrPort = uint16(addr.Port) + + case addr.IP.To16() != nil: + addrType = ipv6Address + addrBody = []byte(addr.IP.To16()) + addrPort = uint16(addr.Port) + + default: + addrType = ipv4Address + addrBody = []byte{0, 0, 0, 0} + addrPort = 0 + } + + // Format the message + msg := make([]byte, 6+len(addrBody)) + msg[0] = socks5Version + msg[1] = resp + msg[2] = 0 // Reserved + msg[3] = addrType + copy(msg[4:], addrBody) + msg[4+len(addrBody)] = byte(addrPort >> 8) + msg[4+len(addrBody)+1] = byte(addrPort & 0xff) + + // Send the message + _, err := w.Write(msg) + return err +} diff --git a/pkg/testingsocks5/request_test.go b/pkg/testingsocks5/request_test.go new file mode 100644 index 000000000..56c0f3bba --- /dev/null +++ b/pkg/testingsocks5/request_test.go @@ -0,0 +1,134 @@ +package testingsocks5 + +import ( + "bytes" + "context" + "errors" + "net" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-engine/pkg/mocks" + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/netxlite" +) + +func TestServerHandleConnect(t *testing.T) { + t.Run("sendReply failure", func(t *testing.T) { + // create a connection that fails as soon as we try to send + expectedErr := errors.New("mocked error") + cconn := &mocks.Conn{ + MockWrite: func(b []byte) (int, error) { + return 0, expectedErr + }, + } + + // create a netx where we fake dialing + netx := &netxlite.Netx{ + Underlying: &mocks.UnderlyingNetwork{ + MockDialTimeout: func() time.Duration { + return 15 * time.Second + }, + MockDialContext: func(ctx context.Context, network, address string) (net.Conn, error) { + sconn := &mocks.Conn{ + MockClose: func() error { + return nil + }, + MockLocalAddr: func() net.Addr { + return &net.TCPAddr{ + IP: net.ParseIP("::17"), + Port: 54321, + } + }, + } + return sconn, nil + }, + }, + } + + // create fake server and request + server := &Server{ + closeOnce: sync.Once{}, + listener: &mocks.Listener{}, // not used + logger: model.DiscardLogger, + netx: netx, + } + req := &request{ + Version: socks5Version, + Command: connectCommand, + DestAddr: &addrSpec{ + Address: "::55", + Port: 80, + }, + } + + err := server.handleConnect(context.Background(), cconn, req) + if !errors.Is(err, expectedErr) { + t.Fatal("unexpected error", err) + } + }) +} + +func TestSendReply(t *testing.T) { + t.Run("we can serialize an IPv4 address", func(t *testing.T) { + buffer := &bytes.Buffer{} + err := sendReply(buffer, successReply, &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 80}) + if err != nil { + t.Fatal(err) + } + expected := []byte{ + 0x05, // version + 0x00, // successful response + 0x00, // reserved + 0x01, // IPv4 + 0x7f, 0x00, 0x00, 0x01, // 127.0.0.1 + 0x00, 0x50, // port 80 + } + if diff := cmp.Diff(expected, buffer.Bytes()); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("we can serialize an IPv6 address", func(t *testing.T) { + buffer := &bytes.Buffer{} + err := sendReply(buffer, successReply, &net.TCPAddr{IP: net.ParseIP("::1"), Port: 80}) + if err != nil { + t.Fatal(err) + } + expected := []byte{ + 0x05, // version + 0x00, // successful response + 0x00, // reserved + 0x04, // IPv6 + 0x00, 0x00, 0x00, 0x00, // ::1 (1/4) + 0x00, 0x00, 0x00, 0x00, // ::1 (2/4) + 0x00, 0x00, 0x00, 0x00, // ::1 (3/4) + 0x00, 0x00, 0x00, 0x01, // ::1 (4/4) + 0x00, 0x50, // port 80 + } + if diff := cmp.Diff(expected, buffer.Bytes()); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("we correctly handle the neither-IPv4-nor-IPv6 case", func(t *testing.T) { + buffer := &bytes.Buffer{} + err := sendReply(buffer, successReply, &net.TCPAddr{IP: nil, Port: 80}) + if err != nil { + t.Fatal(err) + } + expected := []byte{ + 0x05, // version + 0x00, // successful response + 0x00, // reserved + 0x01, // IPv4 + 0x00, 0x00, 0x00, 0x00, // 0.0.0.0 + 0x00, 0x00, // port 0 + } + if diff := cmp.Diff(expected, buffer.Bytes()); diff != "" { + t.Fatal(diff) + } + }) +} diff --git a/pkg/testingsocks5/server.go b/pkg/testingsocks5/server.go new file mode 100644 index 000000000..2e35da5d9 --- /dev/null +++ b/pkg/testingsocks5/server.go @@ -0,0 +1,122 @@ +package testingsocks5 + +import ( + "fmt" + "io" + "net" + "net/url" + "sync" + + "github.com/ooni/probe-engine/pkg/logx" + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/netxlite" + "github.com/ooni/probe-engine/pkg/runtimex" +) + +const ( + socks5Version = uint8(5) +) + +// Server accepts connections and implements the SOCSK5 protocol. +// +// The zero value is invalid; please, use [NewServer]. +type Server struct { + // closeOnce ensures close has "once" semantics. + closeOnce sync.Once + + // listener is the underlying listener. + listener net.Listener + + // logger is the logger to use. + logger model.Logger + + // netx is the network abstraction to use. + netx *netxlite.Netx +} + +// MustNewServer creates a new Server instance. +func MustNewServer(logger model.Logger, netx *netxlite.Netx, addr *net.TCPAddr) *Server { + listener := runtimex.Try1(netx.ListenTCP("tcp", addr)) + server := &Server{ + closeOnce: sync.Once{}, + listener: listener, + logger: &logx.PrefixLogger{ + Prefix: "SOCKS5: ", + Logger: logger, + }, + netx: netx, + } + go server.Serve() + return server +} + +// Serve is used to Serve connections from a given listener. +func (s *Server) Serve() error { + for { + cconn, err := s.listener.Accept() + if err != nil { + return err + } + go func() { + if err := s.serveConn(cconn); err != nil { + s.logger.Warnf("s.serveConn: %s", err.Error()) + } + }() + } +} + +// serveConn is used to serve SOCKS5 over a single connection. +func (s *Server) serveConn(cconn net.Conn) error { + defer cconn.Close() + + // Read the version byte + version := []byte{0} + if _, err := io.ReadFull(cconn, version); err != nil { + return fmt.Errorf("failed to get version byte: %w", err) + } + + s.logger.Infof("got version: %v", version) + + // Ensure we are compatible + if version[0] != socks5Version { + return fmt.Errorf("unsupported SOCKS version: %v", version) + } + + // Authenticate the connection + auth, err := s.authenticate(cconn) + if err != nil { + return fmt.Errorf("failed to authenticate: %w", err) + } + + s.logger.Infof("authenticated: %+v", auth) + + request, err := newRequest(cconn) + if err != nil { + return fmt.Errorf("failed to read destination address: %w", err) + } + + // Process the client request + return s.handleRequest(request, cconn) +} + +// Close closes the listener and waits for all goroutines to join +func (s *Server) Close() (err error) { + s.closeOnce.Do(func() { + err = s.listener.Close() + }) + return +} + +// Endpoint returns the server endpoint. +func (s *Server) Endpoint() string { + return s.listener.Addr().String() +} + +// URL returns a socks5 URL for the local listening address +func (s *Server) URL() *url.URL { + return &url.URL{ + Scheme: "socks5", + Host: s.Endpoint(), + Path: "/", + } +} diff --git a/pkg/testingx/closeverify.go b/pkg/testingx/closeverify.go new file mode 100644 index 000000000..8f8570241 --- /dev/null +++ b/pkg/testingx/closeverify.go @@ -0,0 +1,195 @@ +package testingx + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "sync" + + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/runtimex" +) + +// CloseVerify verifies that we're closing all connections. +// +// The zero value of this struct is ready to use. +type CloseVerify struct { + mu sync.Mutex + conns map[string]io.Closer +} + +func (cv *CloseVerify) addConn(key string, closer io.Closer) { + defer cv.mu.Unlock() + cv.mu.Lock() + if cv.conns == nil { + cv.conns = make(map[string]io.Closer) + } + _, good := cv.conns[key] + runtimex.Assert(!good, fmt.Sprintf("we're already tracking: %s", key)) + cv.conns[key] = closer +} + +func (cv *CloseVerify) removeConn(key string) { + defer cv.mu.Unlock() + cv.mu.Lock() + _, good := cv.conns[key] + runtimex.Assert(good, fmt.Sprintf("we're not tracking: %s", key)) + delete(cv.conns, key) +} + +// CheckForOpenConns returns an error if we still have some open connections. +func (cv *CloseVerify) CheckForOpenConns() error { + defer cv.mu.Unlock() + cv.mu.Lock() + var errorv []error + for key := range cv.conns { + errorv = append(errorv, fmt.Errorf("%s has not been closed", key)) + } + return errors.Join(errorv...) // returns nil if empty +} + +// WrapUnderlyingNetwork returns a [model.UnderlyingNetwork] that comunicates +// sockets open and close events to the [*CloseVerify] struct. +func (cv *CloseVerify) WrapUnderlyingNetwork(unet model.UnderlyingNetwork) model.UnderlyingNetwork { + return &closeVerifyUnderlyingNetwork{ + UnderlyingNetwork: unet, + cv: cv, + } +} + +type closeVerifyUnderlyingNetwork struct { + model.UnderlyingNetwork + cv *CloseVerify +} + +// DialContext implements model.UnderlyingNetwork. +func (unet *closeVerifyUnderlyingNetwork) DialContext( + ctx context.Context, network, address string) (net.Conn, error) { + conn, err := unet.UnderlyingNetwork.DialContext(ctx, network, address) + if err != nil { + return nil, err + } + + localAddr := conn.LocalAddr() + key := fmt.Sprintf("%s/%s", localAddr.String(), localAddr.Network()) + conn = &closeVerifyConn{ + Conn: conn, + cv: unet.cv, + key: key, + once: sync.Once{}, + } + + unet.cv.addConn(key, conn) + + return conn, nil +} + +type closeVerifyConn struct { + net.Conn + cv *CloseVerify + key string + once sync.Once +} + +func (c *closeVerifyConn) Close() (err error) { + c.once.Do(func() { + c.cv.removeConn(c.key) + err = c.Conn.Close() + }) + return +} + +// ListenTCP implements model.UnderlyingNetwork. +func (unet *closeVerifyUnderlyingNetwork) ListenTCP( + network string, addr *net.TCPAddr) (net.Listener, error) { + listener, err := unet.UnderlyingNetwork.ListenTCP(network, addr) + if err != nil { + return nil, err + } + + localAddr := listener.Addr() + key := fmt.Sprintf("%s/%s", localAddr.String(), localAddr.Network()) + listener = &closeVerifyListener{ + Listener: listener, + cv: unet.cv, + key: key, + once: sync.Once{}, + } + + unet.cv.addConn(key, listener) + + return listener, nil +} + +type closeVerifyListener struct { + net.Listener + cv *CloseVerify + key string + once sync.Once +} + +func (c *closeVerifyListener) Accept() (net.Conn, error) { + conn, err := c.Listener.Accept() + if err != nil { + return nil, err + } + + localAddr := conn.LocalAddr() + key := fmt.Sprintf("%s/%s", localAddr.String(), localAddr.Network()) + conn = &closeVerifyConn{ + Conn: conn, + cv: c.cv, + key: key, + once: sync.Once{}, + } + + c.cv.addConn(key, conn) + + return conn, nil +} + +func (c *closeVerifyListener) Close() (err error) { + c.once.Do(func() { + c.cv.removeConn(c.key) + err = c.Listener.Close() + }) + return +} + +func (unet *closeVerifyUnderlyingNetwork) ListenUDP( + network string, addr *net.UDPAddr) (model.UDPLikeConn, error) { + pconn, err := unet.UnderlyingNetwork.ListenUDP(network, addr) + if err != nil { + return nil, err + } + + localAddr := pconn.LocalAddr() + key := fmt.Sprintf("%s/%s", localAddr.String(), localAddr.Network()) + pconn = &closeVerifyUDPConn{ + UDPLikeConn: pconn, + cv: unet.cv, + key: key, + once: sync.Once{}, + } + + unet.cv.addConn(key, pconn) + + return pconn, nil +} + +type closeVerifyUDPConn struct { + model.UDPLikeConn + cv *CloseVerify + key string + once sync.Once +} + +func (c *closeVerifyUDPConn) Close() (err error) { + c.once.Do(func() { + c.cv.removeConn(c.key) + err = c.UDPLikeConn.Close() + }) + return +} diff --git a/pkg/testingx/closeverify_test.go b/pkg/testingx/closeverify_test.go new file mode 100644 index 000000000..cdb8c5cde --- /dev/null +++ b/pkg/testingx/closeverify_test.go @@ -0,0 +1,224 @@ +package testingx_test + +import ( + "context" + "errors" + "net" + "sync" + "sync/atomic" + "testing" + + "github.com/ooni/probe-engine/pkg/mocks" + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/netxlite" + "github.com/ooni/probe-engine/pkg/runtimex" + "github.com/ooni/probe-engine/pkg/testingx" +) + +func TestCloseVerifyWAI(t *testing.T) { + t.Run("when it contains no connections", func(t *testing.T) { + cv := &testingx.CloseVerify{} + if err := cv.CheckForOpenConns(); err != nil { + t.Fatal(err) + } + }) + + t.Run("when we have closed all connections", func(t *testing.T) { + cv := &testingx.CloseVerify{} + + func() { + wg := &sync.WaitGroup{} + + unet := cv.WrapUnderlyingNetwork(&netxlite.DefaultTProxy{}) + + listener := runtimex.Try1(unet.ListenTCP("tcp", &net.TCPAddr{})) + defer listener.Close() + wg.Add(1) + go func() { + defer wg.Done() + conn := runtimex.Try1(listener.Accept()) + defer conn.Close() + }() + + pconn := runtimex.Try1(unet.ListenUDP("udp", &net.UDPAddr{})) + defer pconn.Close() + + ctx := context.Background() + conn := runtimex.Try1(unet.DialContext(ctx, "tcp", listener.Addr().String())) + defer conn.Close() + + wg.Wait() + }() + + if err := cv.CheckForOpenConns(); err != nil { + t.Fatal(err) + } + }) + + t.Run("when we've not closed some connections", func(t *testing.T) { + cv := &testingx.CloseVerify{} + + func() { + wg := &sync.WaitGroup{} + + var ( + udpPort = &atomic.Int64{} + tcpPort = &atomic.Int64{} + ) + + unet := cv.WrapUnderlyingNetwork(&mocks.UnderlyingNetwork{ + MockDialContext: func(ctx context.Context, network, address string) (net.Conn, error) { + conn := &mocks.Conn{ + MockLocalAddr: func() net.Addr { + return &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: int(tcpPort.Add(1))} + }, + } + return conn, nil + }, + MockListenTCP: func(network string, addr *net.TCPAddr) (net.Listener, error) { + listener := &mocks.Listener{ + MockAccept: func() (net.Conn, error) { + conn := &mocks.Conn{ + MockLocalAddr: func() net.Addr { + return &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: int(tcpPort.Add(1))} + }, + } + return conn, nil + }, + MockAddr: func() net.Addr { + return &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: int(tcpPort.Add(1))} + }, + } + return listener, nil + }, + MockListenUDP: func(network string, addr *net.UDPAddr) (model.UDPLikeConn, error) { + pconn := &mocks.UDPLikeConn{ + MockLocalAddr: func() net.Addr { + return &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: int(udpPort.Add(1))} + }, + } + return pconn, nil + }, + }) + + listener := runtimex.Try1(unet.ListenTCP("tcp", &net.TCPAddr{})) + //defer listener.Close() // <- not closing the listener! + wg.Add(1) + go func() { + defer wg.Done() + conn := runtimex.Try1(listener.Accept()) + //defer conn.Close() // <- not closing the conn! + _ = conn + }() + + pconn := runtimex.Try1(unet.ListenUDP("udp", &net.UDPAddr{})) + //defer pconn.Close() <- not closing the pconn! + _ = pconn + + ctx := context.Background() + conn := runtimex.Try1(unet.DialContext(ctx, "tcp", listener.Addr().String())) + //defer conn.Close() // <- not closing the conn! + _ = conn + + wg.Wait() + }() + + if err := cv.CheckForOpenConns(); err == nil { + t.Fatal("expected an error here") + } + }) + + t.Run("on DialContext error", func(t *testing.T) { + cv := &testingx.CloseVerify{} + + expected := errors.New("mocked error") + + unet := cv.WrapUnderlyingNetwork(&mocks.UnderlyingNetwork{ + MockDialContext: func(ctx context.Context, network, address string) (net.Conn, error) { + return nil, expected + }, + }) + + conn, err := unet.DialContext(context.Background(), "tcp", "127.0.0.1:443") + if !errors.Is(err, expected) { + t.Fatal("unexpected error", err) + } + if conn != nil { + t.Fatal("expected nil conn") + } + }) + + t.Run("on ListenTCP error", func(t *testing.T) { + cv := &testingx.CloseVerify{} + + expected := errors.New("mocked error") + + unet := cv.WrapUnderlyingNetwork(&mocks.UnderlyingNetwork{ + MockListenTCP: func(network string, addr *net.TCPAddr) (net.Listener, error) { + return nil, expected + }, + }) + + listener, err := unet.ListenTCP("tcp", &net.TCPAddr{}) + if !errors.Is(err, expected) { + t.Fatal("unexpected error", err) + } + if listener != nil { + t.Fatal("expected nil listener") + } + }) + + t.Run("on ListenUDP error", func(t *testing.T) { + cv := &testingx.CloseVerify{} + + expected := errors.New("mocked error") + + unet := cv.WrapUnderlyingNetwork(&mocks.UnderlyingNetwork{ + MockListenUDP: func(network string, addr *net.UDPAddr) (model.UDPLikeConn, error) { + return nil, expected + }, + }) + + pconn, err := unet.ListenUDP("tcp", &net.UDPAddr{}) + if !errors.Is(err, expected) { + t.Fatal("unexpected error", err) + } + if pconn != nil { + t.Fatal("expected nil pconn") + } + }) + + t.Run("on Accept error", func(t *testing.T) { + cv := &testingx.CloseVerify{} + + expected := errors.New("mocked error") + + tcpPort := &atomic.Int64{} + unet := cv.WrapUnderlyingNetwork(&mocks.UnderlyingNetwork{ + MockListenTCP: func(network string, addr *net.TCPAddr) (net.Listener, error) { + listener := &mocks.Listener{ + MockAccept: func() (net.Conn, error) { + return nil, expected + }, + MockAddr: func() net.Addr { + return &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: int(tcpPort.Add(1))} + }, + } + return listener, nil + }, + }) + + listener, err := unet.ListenTCP("tcp", &net.TCPAddr{}) + if err != nil { + t.Fatal(err) + } + + conn, err := listener.Accept() + if !errors.Is(err, expected) { + t.Fatal("unexpected error", err) + } + if conn != nil { + t.Fatal("expected nil conn") + } + }) +} diff --git a/pkg/testingx/httpproxy.go b/pkg/testingx/httpproxy.go new file mode 100644 index 000000000..6889a55f3 --- /dev/null +++ b/pkg/testingx/httpproxy.go @@ -0,0 +1,141 @@ +package testingx + +import ( + "io" + "net/http" + "sync" + + "github.com/ooni/probe-engine/pkg/logx" + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/runtimex" +) + +// HTTPProxyHandlerNetx abstracts [*netxlite.Netx] for the [*HTTPProxyHandler]. +type HTTPProxyHandlerNetx interface { + // NewDialerWithResolver creates a new dialer using the given resolver and logger. + NewDialerWithResolver(dl model.DebugLogger, r model.Resolver, w ...model.DialerWrapper) model.Dialer + + // NewHTTPTransportStdlib creates a new HTTP transport using the stdlib. + NewHTTPTransportStdlib(dl model.DebugLogger) model.HTTPTransport + + // NewStdlibResolver creates a new resolver that tries to use the getaddrinfo libc call. + NewStdlibResolver(logger model.DebugLogger) model.Resolver +} + +// httpProxyHandler is an HTTP/HTTPS proxy. +type httpProxyHandler struct { + // Logger is the logger to use. + Logger model.Logger + + // Netx is the network to use. + Netx HTTPProxyHandlerNetx +} + +// NewHTTPProxyHandler constructs a new [*HTTPProxyHandler]. +func NewHTTPProxyHandler(logger model.Logger, netx HTTPProxyHandlerNetx) http.Handler { + return &httpProxyHandler{ + Logger: &logx.PrefixLogger{ + Prefix: "PROXY: ", + Logger: logger, + }, + Netx: netx, + } +} + +// ServeHTTP implements http.Handler. +func (ph *httpProxyHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + ph.Logger.Infof("request: %+v", req) + + switch req.Method { + case http.MethodConnect: + ph.connect(rw, req) + + case http.MethodGet: + ph.get(rw, req) + + default: + rw.WriteHeader(http.StatusNotImplemented) + } +} + +func (ph *httpProxyHandler) connect(rw http.ResponseWriter, req *http.Request) { + resolver := ph.Netx.NewStdlibResolver(ph.Logger) + dialer := ph.Netx.NewDialerWithResolver(ph.Logger, resolver) + + sconn, err := dialer.DialContext(req.Context(), "tcp", req.Host) + if err != nil { + rw.WriteHeader(http.StatusBadGateway) + return + } + defer sconn.Close() + + hijacker := rw.(http.Hijacker) + cconn, buffered := runtimex.Try2(hijacker.Hijack()) + runtimex.Assert(buffered.Reader.Buffered() <= 0, "data before finishing HTTP handshake") + defer cconn.Close() + + _, _ = cconn.Write([]byte("HTTP/1.1 200 Ok\r\n\r\n")) + + wg := &sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + _, _ = io.Copy(sconn, cconn) + }() + + wg.Add(1) + go func() { + defer wg.Done() + _, _ = io.Copy(cconn, sconn) + }() + + wg.Wait() +} + +func (ph *httpProxyHandler) get(rw http.ResponseWriter, req *http.Request) { + // reject requests that already visited the proxy and requests we cannot route + if req.Host == "" || req.Header.Get("Via") != "" { + rw.WriteHeader(http.StatusBadRequest) + return + } + + // clone the request before modifying it + req = req.Clone(req.Context()) + + // include proxy header to prevent sending requests to ourself + req.Header.Add("Via", "testingx/0.1.0") + + // fix: "http: Request.RequestURI can't be set in client requests" + req.RequestURI = "" + + // fix: `http: unsupported protocol scheme ""` + req.URL.Host = req.Host + + // fix: "http: no Host in request URL" + req.URL.Scheme = "http" + + ph.Logger.Debugf("sending request: %s", req) + + // create HTTP client using netx + txp := ph.Netx.NewHTTPTransportStdlib(ph.Logger) + defer txp.CloseIdleConnections() + + // obtain response + resp, err := txp.RoundTrip(req) + if err != nil { + ph.Logger.Warnf("request failed: %s", err.Error()) + rw.WriteHeader(http.StatusBadGateway) + return + } + + // write response + rw.WriteHeader(resp.StatusCode) + for key, values := range resp.Header { + for _, value := range values { + rw.Header().Add(key, value) + } + } + + // write response body + _, _ = io.Copy(rw, resp.Body) +} diff --git a/pkg/testingx/httpproxy_test.go b/pkg/testingx/httpproxy_test.go new file mode 100644 index 000000000..c5f524489 --- /dev/null +++ b/pkg/testingx/httpproxy_test.go @@ -0,0 +1,133 @@ +package testingx_test + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/apex/log" + "github.com/ooni/probe-engine/pkg/mocks" + "github.com/ooni/probe-engine/pkg/netxlite" + "github.com/ooni/probe-engine/pkg/testingproxy" + "github.com/ooni/probe-engine/pkg/testingx" +) + +func TestHTTPProxyHandler(t *testing.T) { + for _, testCase := range testingproxy.HTTPTestCases { + t.Run(testCase.Name(), func(t *testing.T) { + short := testCase.Short() + if !short && testing.Short() { + t.Skip("skip test in short mode") + } + testCase.Run(t) + }) + } + + t.Run("rejects requests without a host header", func(t *testing.T) { + rr := httptest.NewRecorder() + netx := &netxlite.Netx{Underlying: &mocks.UnderlyingNetwork{ + // all nil: panic if we hit the network + }} + handler := testingx.NewHTTPProxyHandler(log.Log, netx) + req := &http.Request{ + Method: http.MethodGet, + Host: "", // explicitly empty + } + handler.ServeHTTP(rr, req) + res := rr.Result() + if res.StatusCode != http.StatusBadRequest { + t.Fatal("unexpected status code", res.StatusCode) + } + }) + + t.Run("rejects requests with a via header", func(t *testing.T) { + rr := httptest.NewRecorder() + netx := &netxlite.Netx{Underlying: &mocks.UnderlyingNetwork{ + // all nil: panic if we hit the network + }} + handler := testingx.NewHTTPProxyHandler(log.Log, netx) + req := &http.Request{ + Method: http.MethodGet, + Host: "www.example.com", + Header: http.Header{ + "Via": {"antani/0.1.0"}, + }, + } + handler.ServeHTTP(rr, req) + res := rr.Result() + if res.StatusCode != http.StatusBadRequest { + t.Fatal("unexpected status code", res.StatusCode) + } + }) + + t.Run("rejects requests with a POST method", func(t *testing.T) { + rr := httptest.NewRecorder() + netx := &netxlite.Netx{Underlying: &mocks.UnderlyingNetwork{ + // all nil: panic if we hit the network + }} + handler := testingx.NewHTTPProxyHandler(log.Log, netx) + req := &http.Request{ + Method: http.MethodPost, + Host: "www.example.com", + Header: http.Header{}, + } + handler.ServeHTTP(rr, req) + res := rr.Result() + if res.StatusCode != http.StatusNotImplemented { + t.Fatal("unexpected status code", res.StatusCode) + } + }) + + t.Run("returns 502 when the round trip fails", func(t *testing.T) { + t.Run("with a GET request", func(t *testing.T) { + rr := httptest.NewRecorder() + netx := &netxlite.Netx{Underlying: &mocks.UnderlyingNetwork{ + MockGetaddrinfoLookupANY: func(ctx context.Context, domain string) ([]string, string, error) { + return nil, "", errors.New("mocked error") + }, + MockGetaddrinfoResolverNetwork: func() string { + return "antani" + }, + }} + handler := testingx.NewHTTPProxyHandler(log.Log, netx) + req := &http.Request{ + Method: http.MethodGet, + Host: "www.example.com", + Header: http.Header{}, + URL: &url.URL{}, + } + handler.ServeHTTP(rr, req) + res := rr.Result() + if res.StatusCode != http.StatusBadGateway { + t.Fatal("unexpected status code", res.StatusCode) + } + }) + + t.Run("with a CONNECT request", func(t *testing.T) { + rr := httptest.NewRecorder() + netx := &netxlite.Netx{Underlying: &mocks.UnderlyingNetwork{ + MockGetaddrinfoLookupANY: func(ctx context.Context, domain string) ([]string, string, error) { + return nil, "", errors.New("mocked error") + }, + MockGetaddrinfoResolverNetwork: func() string { + return "antani" + }, + }} + handler := testingx.NewHTTPProxyHandler(log.Log, netx) + req := &http.Request{ + Method: http.MethodConnect, + Host: "www.example.com:443", + Header: http.Header{}, + URL: &url.URL{}, + } + handler.ServeHTTP(rr, req) + res := rr.Result() + if res.StatusCode != http.StatusBadGateway { + t.Fatal("unexpected status code", res.StatusCode) + } + }) + }) +} diff --git a/pkg/testingx/httptestx.go b/pkg/testingx/httptestx.go index dc8f8d79a..80bbc345a 100644 --- a/pkg/testingx/httptestx.go +++ b/pkg/testingx/httptestx.go @@ -3,13 +3,11 @@ package testingx import ( "crypto/tls" "crypto/x509" - "io" "net" "net/http" "net/url" - "github.com/ooni/probe-engine/pkg/model" - "github.com/ooni/probe-engine/pkg/optional" + "github.com/ooni/netem" "github.com/ooni/probe-engine/pkg/runtimex" ) @@ -21,20 +19,35 @@ import ( // transitioning the code from that struct to this one. type HTTPServer struct { // Config contains the server started by the constructor. + // + // This field also exists in the [*net/http/httptest.Server] struct. Config *http.Server // Listener is the underlying [net.Listener]. + // + // This field also exists in the [*net/http/httptest.Server] struct. Listener net.Listener // TLS contains the TLS configuration used by the constructor, or nil // if you constructed a server that does not use TLS. + // + // This field also exists in the [*net/http/httptest.Server] struct. TLS *tls.Config // URL is the base URL used by the server. + // + // This field also exists in the [*net/http/httptest.Server] struct. URL string // X509CertPool is the X.509 cert pool we're using or nil. + // + // This field is an extension that is not present in the httptest package. X509CertPool *x509.CertPool + + // CACert is the CA used by this server or nil. + // + // This field is an extension that is not present in the httptest package. + CACert *x509.Certificate } // MustNewHTTPServer is morally equivalent to [httptest.NewHTTPServer]. @@ -43,51 +56,72 @@ func MustNewHTTPServer(handler http.Handler) *HTTPServer { return MustNewHTTPServerEx(addr, &TCPListenerStdlib{}, handler) } -// MustNewHTTPServerTLS is morally equivalent to [httptest.NewHTTPServerTLS]. -func MustNewHTTPServerTLS(handler http.Handler) *HTTPServer { - addr := &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0} - provider := MustNewTLSMITMProviderNetem() - return MustNewHTTPServerTLSEx(addr, &TCPListenerStdlib{}, handler, provider) -} - // MustNewHTTPServerEx creates a new [HTTPServer] using HTTP or PANICS. -func MustNewHTTPServerEx(addr *net.TCPAddr, listener TCPListener, handler http.Handler) *HTTPServer { - return mustNewHTTPServer(addr, listener, handler, optional.None[TLSMITMProvider]()) +func MustNewHTTPServerEx(addr *net.TCPAddr, httpListener TCPListener, handler http.Handler) *HTTPServer { + listener := runtimex.Try1(httpListener.ListenTCP("tcp", addr)) + + baseURL := &url.URL{ + Scheme: "http", + Host: listener.Addr().String(), + Path: "/", + } + srv := &HTTPServer{ + Config: &http.Server{Handler: handler}, + Listener: listener, + TLS: nil, + URL: baseURL.String(), + X509CertPool: nil, + CACert: nil, + } + + go srv.Config.Serve(listener) + + return srv } -// MustNewHTTPServerTLSEx creates a new [HTTPServer] using HTTPS or PANICS. -func MustNewHTTPServerTLSEx(addr *net.TCPAddr, listener TCPListener, handler http.Handler, mitm TLSMITMProvider) *HTTPServer { - return mustNewHTTPServer(addr, listener, handler, optional.Some(mitm)) +// MustNewHTTPServerTLS is morally equivalent to [httptest.NewHTTPServerTLS]. +func MustNewHTTPServerTLS( + handler http.Handler, + ca netem.CertificationAuthority, + commonName string, + extraSNIs ...string, +) *HTTPServer { + addr := &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 0} + return MustNewHTTPServerTLSEx(addr, &TCPListenerStdlib{}, handler, ca, commonName, extraSNIs...) } -// newHTTPOrHTTPSServer is an internal factory for creating a new instance. -func mustNewHTTPServer( +// MustNewHTTPServerTLSEx creates a new [HTTPServer] using HTTPS or PANICS. +func MustNewHTTPServerTLSEx( addr *net.TCPAddr, httpListener TCPListener, handler http.Handler, - tlsConfig optional.Value[TLSMITMProvider], + ca netem.CertificationAuthority, + commonName string, + extraSNIs ...string, ) *HTTPServer { listener := runtimex.Try1(httpListener.ListenTCP("tcp", addr)) + + baseURL := &url.URL{ + Scheme: "https", + Host: listener.Addr().String(), + Path: "/", + } + + otherNames := append([]string{}, addr.IP.String()) + otherNames = append(otherNames, extraSNIs...) + srv := &HTTPServer{ Config: &http.Server{Handler: handler}, Listener: listener, - TLS: nil, // the default when not using TLS - URL: "", // filled later - X509CertPool: nil, // the default when not using TLS - } - baseURL := &url.URL{Host: listener.Addr().String()} - switch !tlsConfig.IsNone() { - case true: - baseURL.Scheme = "https" - srv.TLS = tlsConfig.Unwrap().ServerTLSConfig() - srv.Config.TLSConfig = srv.TLS - srv.X509CertPool = runtimex.Try1(tlsConfig.Unwrap().DefaultCertPool()) - go srv.Config.ServeTLS(listener, "", "") // using server.TLSConfig - default: - baseURL.Scheme = "http" - go srv.Config.Serve(listener) + TLS: ca.MustNewServerTLSConfig(commonName, otherNames...), + URL: baseURL.String(), + X509CertPool: ca.DefaultCertPool(), + CACert: ca.CACert(), } - srv.URL = baseURL.String() + + srv.Config.TLSConfig = srv.TLS + go srv.Config.ServeTLS(listener, "", "") // using server.TLSConfig + return srv } @@ -160,70 +194,3 @@ func httpHandlerHijack(w http.ResponseWriter, r *http.Request, policy string) { // nothing } } - -// TODO(bassosimone): eventually we may want to have a model type -// that models the equivalent of [netxlite.Netx]. - -// HTTPHandlerProxyNetx is [netxlite.Netx] as seen by [HTTPHandlerProxy]. -type HTTPHandlerProxyNetx interface { - NewHTTPTransportStdlib(logger model.DebugLogger) model.HTTPTransport -} - -// HTTPHandlerProxy is a handler implementing an HTTP proxy using the host header -// to determine who to connect to. We additionally use the via header to avoid sending -// requests to ourself. Please, note that we designed this proxy ONLY to be used for -// testing purposes and that it's rather simplistic. -func HTTPHandlerProxy(logger model.Logger, netx HTTPHandlerProxyNetx) http.Handler { - return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - // reject requests that already visited the proxy and requests we cannot route - if req.Host == "" || req.Header.Get("Via") != "" { - rw.WriteHeader(http.StatusBadRequest) - return - } - - // be explicit about not supporting request bodies - if req.Method != http.MethodGet { - rw.WriteHeader(http.StatusNotImplemented) - return - } - - // clone the request before modifying it - req = req.Clone(req.Context()) - - // include proxy header to prevent sending requests to ourself - req.Header.Add("Via", "testingx/0.1.0") - - // fix: "http: Request.RequestURI can't be set in client requests" - req.RequestURI = "" - - // fix: `http: unsupported protocol scheme ""` - req.URL.Host = req.Host - - // fix: "http: no Host in request URL" - req.URL.Scheme = "http" - - logger.Debugf("PROXY: sending request: %s", req) - - // create HTTP client using netx - txp := netx.NewHTTPTransportStdlib(logger) - - // obtain response - resp, err := txp.RoundTrip(req) - if err != nil { - logger.Warnf("PROXY: request failed: %s", err.Error()) - rw.WriteHeader(http.StatusBadGateway) - return - } - - // write response - rw.WriteHeader(resp.StatusCode) - for key, values := range resp.Header { - for _, value := range values { - rw.Header().Add(key, value) - } - } - - // write response body - _, _ = io.Copy(rw, resp.Body) - }) -} diff --git a/pkg/testingx/httptestx_test.go b/pkg/testingx/httptestx_test.go index 735d60c4e..4576363ed 100644 --- a/pkg/testingx/httptestx_test.go +++ b/pkg/testingx/httptestx_test.go @@ -10,15 +10,12 @@ import ( "io" "net" "net/http" - "net/http/httptest" - "net/url" "testing" "time" "github.com/apex/log" "github.com/google/go-cmp/cmp" "github.com/ooni/netem" - "github.com/ooni/probe-engine/pkg/mocks" "github.com/ooni/probe-engine/pkg/netxlite" "github.com/ooni/probe-engine/pkg/runtimex" "github.com/ooni/probe-engine/pkg/testingx" @@ -46,6 +43,9 @@ func TestHTTPTestxWithStdlib(t *testing.T) { expectBody []byte } + // create server's CA + serverCA := netem.MustNewCA() + testcases := []testcase{ /* * HTTP @@ -94,7 +94,11 @@ func TestHTTPTestxWithStdlib(t *testing.T) { { name: "with HTTPS and the HTTPHandlerBlockpage451 handler", constructor: func() *testingx.HTTPServer { - return testingx.MustNewHTTPServerTLS(testingx.HTTPHandlerBlockpage451()) + return testingx.MustNewHTTPServerTLS( + testingx.HTTPHandlerBlockpage451(), + serverCA, + "webserver.local", + ) }, timeout: 10 * time.Second, expectErr: nil, @@ -103,7 +107,11 @@ func TestHTTPTestxWithStdlib(t *testing.T) { }, { name: "with HTTPS and the HTTPHandlerEOF handler", constructor: func() *testingx.HTTPServer { - return testingx.MustNewHTTPServerTLS(testingx.HTTPHandlerEOF()) + return testingx.MustNewHTTPServerTLS( + testingx.HTTPHandlerEOF(), + serverCA, + "webserver.local", + ) }, timeout: 10 * time.Second, expectErr: io.EOF, @@ -112,7 +120,11 @@ func TestHTTPTestxWithStdlib(t *testing.T) { }, { name: "with HTTPS and the HTTPHandlerReset handler", constructor: func() *testingx.HTTPServer { - return testingx.MustNewHTTPServerTLS(testingx.HTTPHandlerReset()) + return testingx.MustNewHTTPServerTLS( + testingx.HTTPHandlerReset(), + serverCA, + "webserver.local", + ) }, timeout: 10 * time.Second, expectErr: netxlite.ECONNRESET, @@ -121,7 +133,11 @@ func TestHTTPTestxWithStdlib(t *testing.T) { }, { name: "with HTTPS and the HTTPHandlerTimeout handler", constructor: func() *testingx.HTTPServer { - return testingx.MustNewHTTPServerTLS(testingx.HTTPHandlerTimeout()) + return testingx.MustNewHTTPServerTLS( + testingx.HTTPHandlerTimeout(), + serverCA, + "webserver.local", + ) }, timeout: 1 * time.Second, expectErr: context.DeadlineExceeded, @@ -149,6 +165,9 @@ func TestHTTPTestxWithStdlib(t *testing.T) { tlsHandshaker := netxlite.NewTLSHandshakerStdlib(log.Log) tlsDialer := netxlite.NewTLSDialerWithConfig( tcpDialer, tlsHandshaker, &tls.Config{RootCAs: srvr.X509CertPool}) + // TODO(https://github.com/ooni/probe/issues/2534): here we're using the QUIRKY netxlite.NewHTTPTransport + // function, but we can probably avoid using it, given that this code is + // not using tracing and does not care about those quirks. txp := netxlite.NewHTTPTransport(log.Log, tcpDialer, tlsDialer) client := netxlite.NewHTTPClient(txp) @@ -309,6 +328,7 @@ func TestHTTPTestxWithNetem(t *testing.T) { unet, testingx.HTTPHandlerBlockpage451(), unet, + "webserver.local", ) }, timeout: 10 * time.Second, @@ -326,6 +346,7 @@ func TestHTTPTestxWithNetem(t *testing.T) { unet, testingx.HTTPHandlerEOF(), unet, + "webserver.local", ) }, timeout: 10 * time.Second, @@ -344,6 +365,7 @@ func TestHTTPTestxWithNetem(t *testing.T) { unet, testingx.HTTPHandlerReset(), unet, + "webserver.local", ) }, timeout: 10 * time.Second, @@ -361,6 +383,7 @@ func TestHTTPTestxWithNetem(t *testing.T) { unet, testingx.HTTPHandlerTimeout(), unet, + "webserver.local", ) }, timeout: 1 * time.Second, @@ -378,7 +401,7 @@ func TestHTTPTestxWithNetem(t *testing.T) { } // create a star topology for hosting the test - topology := runtimex.Try1(netem.NewStarTopology(log.Log)) + topology := netem.MustNewStarTopology(log.Log) defer topology.Close() // create a common link config @@ -421,6 +444,9 @@ func TestHTTPTestxWithNetem(t *testing.T) { tlsHandshaker := netxlite.NewTLSHandshakerStdlib(log.Log) tlsDialer := netxlite.NewTLSDialerWithConfig( tcpDialer, tlsHandshaker, &tls.Config{RootCAs: srvr.X509CertPool}) + // TODO(https://github.com/ooni/probe/issues/2534): here we're using the QUIRKY netxlite.NewHTTPTransport + // function, but we can probably avoid using it, given that this code is + // not using tracing and does not care about those quirks. txp := netxlite.NewHTTPTransport(log.Log, tcpDialer, tlsDialer) client := netxlite.NewHTTPClient(txp) @@ -470,198 +496,3 @@ func TestHTTPTestxWithNetem(t *testing.T) { }) } } - -func TestHTTPHandlerProxy(t *testing.T) { - expectedBody := []byte("Google is built by a large team of engineers, designers, researchers, robots, and others in many different sites across the globe. It is updated continuously, and built with more tools and technologies than we can shake a stick at. If you'd like to help us out, see careers.google.com.\n") - - type testcase struct { - name string - construct func() (*netxlite.Netx, string, []io.Closer) - short bool - } - - testcases := []testcase{ - { - name: "using the real network", - construct: func() (*netxlite.Netx, string, []io.Closer) { - var closers []io.Closer - - netx := &netxlite.Netx{ - Underlying: nil, // so we're using the real network - } - - proxyServer := testingx.MustNewHTTPServer(testingx.HTTPHandlerProxy(log.Log, netx)) - closers = append(closers, proxyServer) - - return netx, proxyServer.URL, closers - }, - short: false, - }, - - { - name: "using netem", - construct: func() (*netxlite.Netx, string, []io.Closer) { - var closers []io.Closer - - topology := runtimex.Try1(netem.NewStarTopology(log.Log)) - closers = append(closers, topology) - - wwwStack := runtimex.Try1(topology.AddHost("142.251.209.14", "142.251.209.14", &netem.LinkConfig{})) - proxyStack := runtimex.Try1(topology.AddHost("10.0.0.1", "142.251.209.14", &netem.LinkConfig{})) - clientStack := runtimex.Try1(topology.AddHost("10.0.0.2", "142.251.209.14", &netem.LinkConfig{})) - - dnsConfig := netem.NewDNSConfig() - dnsConfig.AddRecord("www.google.com", "", "142.251.209.14") - dnsServer := runtimex.Try1(netem.NewDNSServer(log.Log, wwwStack, "142.251.209.14", dnsConfig)) - closers = append(closers, dnsServer) - - wwwServer := testingx.MustNewHTTPServerEx( - &net.TCPAddr{IP: net.IPv4(142, 251, 209, 14), Port: 80}, - wwwStack, - http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Write(expectedBody) - }), - ) - closers = append(closers, wwwServer) - - proxyServer := testingx.MustNewHTTPServerEx( - &net.TCPAddr{IP: net.IPv4(10, 0, 0, 1), Port: 80}, - proxyStack, - testingx.HTTPHandlerProxy(log.Log, &netxlite.Netx{ - Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: proxyStack}, - }), - ) - closers = append(closers, proxyServer) - - clientNet := &netxlite.Netx{Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: clientStack}} - return clientNet, proxyServer.URL, closers - }, - short: true, - }} - - for _, tc := range testcases { - t.Run(tc.name, func(t *testing.T) { - if !tc.short && testing.Short() { - t.Skip("skip test in short mode") - } - - netx, proxyURL, closers := tc.construct() - defer func() { - for _, closer := range closers { - closer.Close() - } - }() - - URL := runtimex.Try1(url.Parse(proxyURL)) - URL.Path = "/humans.txt" - - req := runtimex.Try1(http.NewRequest("GET", URL.String(), nil)) - req.Host = "www.google.com" - - //log.SetLevel(log.DebugLevel) - - txp := netx.NewHTTPTransportStdlib(log.Log) - client := netxlite.NewHTTPClient(txp) - - resp, err := client.Do(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - t.Fatal("expected to see 200, got", resp.StatusCode) - } - - t.Logf("%+v", resp) - - body, err := netxlite.ReadAllContext(req.Context(), resp.Body) - if err != nil { - t.Fatal(err) - } - - t.Logf("%s", string(body)) - - if diff := cmp.Diff(expectedBody, body); diff != "" { - t.Fatal(diff) - } - }) - } - - t.Run("rejects requests without a host header", func(t *testing.T) { - rr := httptest.NewRecorder() - netx := &netxlite.Netx{Underlying: &mocks.UnderlyingNetwork{ - // all nil: panic if we hit the network - }} - handler := testingx.HTTPHandlerProxy(log.Log, netx) - req := &http.Request{ - Host: "", // explicitly empty - } - handler.ServeHTTP(rr, req) - res := rr.Result() - if res.StatusCode != http.StatusBadRequest { - t.Fatal("unexpected status code", res.StatusCode) - } - }) - - t.Run("rejects requests with a via header", func(t *testing.T) { - rr := httptest.NewRecorder() - netx := &netxlite.Netx{Underlying: &mocks.UnderlyingNetwork{ - // all nil: panic if we hit the network - }} - handler := testingx.HTTPHandlerProxy(log.Log, netx) - req := &http.Request{ - Host: "www.example.com", - Header: http.Header{ - "Via": {"antani/0.1.0"}, - }, - } - handler.ServeHTTP(rr, req) - res := rr.Result() - if res.StatusCode != http.StatusBadRequest { - t.Fatal("unexpected status code", res.StatusCode) - } - }) - - t.Run("rejects requests with a POST method", func(t *testing.T) { - rr := httptest.NewRecorder() - netx := &netxlite.Netx{Underlying: &mocks.UnderlyingNetwork{ - // all nil: panic if we hit the network - }} - handler := testingx.HTTPHandlerProxy(log.Log, netx) - req := &http.Request{ - Host: "www.example.com", - Header: http.Header{}, - Method: http.MethodPost, - } - handler.ServeHTTP(rr, req) - res := rr.Result() - if res.StatusCode != http.StatusNotImplemented { - t.Fatal("unexpected status code", res.StatusCode) - } - }) - - t.Run("returns 502 when the round trip fails", func(t *testing.T) { - rr := httptest.NewRecorder() - netx := &netxlite.Netx{Underlying: &mocks.UnderlyingNetwork{ - MockGetaddrinfoLookupANY: func(ctx context.Context, domain string) ([]string, string, error) { - return nil, "", errors.New("mocked error") - }, - MockGetaddrinfoResolverNetwork: func() string { - return "antani" - }, - }} - handler := testingx.HTTPHandlerProxy(log.Log, netx) - req := &http.Request{ - Host: "www.example.com", - Header: http.Header{}, - Method: http.MethodGet, - URL: &url.URL{}, - } - handler.ServeHTTP(rr, req) - res := rr.Result() - if res.StatusCode != http.StatusBadGateway { - t.Fatal("unexpected status code", res.StatusCode) - } - }) -} diff --git a/pkg/testingx/tlssniproxy.go b/pkg/testingx/tlssniproxy.go index a74039a50..10a0f5c91 100644 --- a/pkg/testingx/tlssniproxy.go +++ b/pkg/testingx/tlssniproxy.go @@ -16,8 +16,9 @@ import ( // TLSSNIProxyNetx is how [TLSSNIProxy] views [*netxlite.Netx]. type TLSSNIProxyNetx interface { + ListenTCP(network string, addr *net.TCPAddr) (net.Listener, error) NewDialerWithResolver(dl model.DebugLogger, r model.Resolver, w ...model.DialerWrapper) model.Dialer - NewStdlibResolver(logger model.DebugLogger, wrappers ...model.DNSTransportWrapper) model.Resolver + NewStdlibResolver(logger model.DebugLogger) model.Resolver } // TLSSNIProxy is a proxy using the SNI to figure out where to connect to. @@ -38,13 +39,10 @@ type TLSSNIProxy struct { wg *sync.WaitGroup } -// TODO(bassosimone): MustNewTLSSNIProxyEx prototype would be simpler if -// netxlite.Netx was also able to create listening TCP connections - // MustNewTLSSNIProxyEx creates a new [*TLSSNIProxy]. func MustNewTLSSNIProxyEx( - logger model.Logger, netx TLSSNIProxyNetx, tcpAddr *net.TCPAddr, tcpListener TCPListener) *TLSSNIProxy { - listener := runtimex.Try1(tcpListener.ListenTCP("tcp", tcpAddr)) + logger model.Logger, netx TLSSNIProxyNetx, tcpAddr *net.TCPAddr) *TLSSNIProxy { + listener := runtimex.Try1(netx.ListenTCP("tcp", tcpAddr)) proxy := &TLSSNIProxy{ closeOnce: sync.Once{}, listener: listener, diff --git a/pkg/testingx/tlssniproxy_test.go b/pkg/testingx/tlssniproxy_test.go index 68864f331..a5349b75d 100644 --- a/pkg/testingx/tlssniproxy_test.go +++ b/pkg/testingx/tlssniproxy_test.go @@ -32,9 +32,8 @@ func TestTLSSNIProxy(t *testing.T) { Underlying: nil, // use the network } tcpAddr := &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1)} - tcpListener := &testingx.TCPListenerStdlib{} - proxy := testingx.MustNewTLSSNIProxyEx(log.Log, netxProxy, tcpAddr, tcpListener) + proxy := testingx.MustNewTLSSNIProxyEx(log.Log, netxProxy, tcpAddr) closers = append(closers, proxy) netxClient := &netxlite.Netx{ @@ -49,7 +48,7 @@ func TestTLSSNIProxy(t *testing.T) { construct: func() (*testingx.TLSSNIProxy, *netxlite.Netx, []io.Closer) { var closers []io.Closer - topology := runtimex.Try1(netem.NewStarTopology(log.Log)) + topology := netem.MustNewStarTopology(log.Log) closers = append(closers, topology) wwwStack := runtimex.Try1(topology.AddHost("142.251.209.14", "142.251.209.14", &netem.LinkConfig{})) @@ -68,6 +67,7 @@ func TestTLSSNIProxy(t *testing.T) { w.Write([]byte("Bonsoir, Elliot!")) }), wwwStack, + "www.google.com", ) closers = append(closers, wwwServer) @@ -75,7 +75,6 @@ func TestTLSSNIProxy(t *testing.T) { log.Log, &netxlite.Netx{Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: proxyStack}}, &net.TCPAddr{IP: net.IPv4(10, 0, 0, 1), Port: 443}, - proxyStack, ) closers = append(closers, proxy) @@ -98,8 +97,6 @@ func TestTLSSNIProxy(t *testing.T) { } }() - //log.SetLevel(log.DebugLevel) - tlsConfig := &tls.Config{ ServerName: "www.google.com", } @@ -113,7 +110,7 @@ func TestTLSSNIProxy(t *testing.T) { } defer conn.Close() - tconn := conn.(netxlite.TLSConn) + tconn := conn.(netxlite.TLSConn) // cast safe according to documentation connstate := tconn.ConnectionState() t.Logf("%+v", connstate) }) diff --git a/pkg/testingx/tlsx.go b/pkg/testingx/tlsx.go index d8b7e8d26..c75d82755 100644 --- a/pkg/testingx/tlsx.go +++ b/pkg/testingx/tlsx.go @@ -3,56 +3,15 @@ package testingx import ( "context" "crypto/tls" - "crypto/x509" "errors" "net" "sync" "time" "github.com/apex/log" - "github.com/ooni/netem" "github.com/ooni/probe-engine/pkg/runtimex" ) -// TLSMITMProvider provides TLS MITM capabilities. Two structs are known -// to implement this interface: -// -// 1. a [*netem.UNetStack] instance. -// -// 2. the one returned by [MustNewTLSMITMProviderNetem]. -// -// Both use [github.com/google/martian/v3/mitm] under the hood. -// -// Use the former when you're using netem; the latter when using the stdlib. -type TLSMITMProvider interface { - // DefaultCertPool returns the default cert pool to use. - DefaultCertPool() (*x509.CertPool, error) - - // ServerTLSConfig returns ready to use server TLS configuration. - ServerTLSConfig() *tls.Config -} - -var _ TLSMITMProvider = &netem.UNetStack{} - -// MustNewTLSMITMProviderNetem uses [github.com/ooni/netem] to implement [TLSMITMProvider]. -func MustNewTLSMITMProviderNetem() TLSMITMProvider { - return &netemTLSMITMProvider{runtimex.Try1(netem.NewTLSMITMConfig())} -} - -type netemTLSMITMProvider struct { - cfg *netem.TLSMITMConfig -} - -// DefaultCertPool implements TLSMITMProvider. -func (p *netemTLSMITMProvider) DefaultCertPool() (*x509.CertPool, error) { - return p.cfg.CertPool() -} - -// ServerTLSConfig implements TLSMITMProvider. -func (p *netemTLSMITMProvider) ServerTLSConfig() *tls.Config { - return p.cfg.TLSConfig() -} - // TLSHandler handles TLS connections. A handler should first handle the TLS handshake // in the GetCertificate method. If GetCertificate did not return an error, and the // handler implements [TLSConnHandler], its HandleTLSConn method will be called after @@ -277,24 +236,20 @@ func (*tlsHandlerReset) GetCertificate(ctx context.Context, tcpConn net.Conn, ch // TLSHandlerHandshakeAndWriteText returns a [TLSHandler] that attempts to // complete the handshake and returns the given text to the caller. -func TLSHandlerHandshakeAndWriteText(mitm TLSMITMProvider, text []byte) TLSHandler { - return &tlsHandlerHandshakeAndWriteText{mitm, text} +func TLSHandlerHandshakeAndWriteText(cert *tls.Certificate, text []byte) TLSHandler { + return &tlsHandlerHandshakeAndWriteText{cert, text} } var _ TLSConnHandler = &tlsHandlerHandshakeAndWriteText{} type tlsHandlerHandshakeAndWriteText struct { - mitm TLSMITMProvider + cert *tls.Certificate text []byte } // GetCertificate implements TLSHandler. func (thx *tlsHandlerHandshakeAndWriteText) GetCertificate(ctx context.Context, tcpConn net.Conn, chi *tls.ClientHelloInfo) (*tls.Certificate, error) { - // Implementation note: under the assumption that we're using github.com/ooni/netem in one way or - // another here, the ServerTLSConfig method returns a suitable GetCertificate implementation. Since - // the primary use case is that of using netem, this code is going to be WAI most of the times. - config := thx.mitm.ServerTLSConfig() - return config.GetCertificate(chi) + return thx.cert, nil } // HandleTLSConn implements TLSHandler. diff --git a/pkg/testingx/tlsx_test.go b/pkg/testingx/tlsx_test.go index 7c4d50070..46425a88c 100644 --- a/pkg/testingx/tlsx_test.go +++ b/pkg/testingx/tlsx_test.go @@ -39,8 +39,9 @@ func TestTLSHandlerWithStdlib(t *testing.T) { expectBody []byte } - // create MITM config - mitm := testingx.MustNewTLSMITMProviderNetem() + // create server's CA and leaf certificate + serverCA := netem.MustNewCA() + serverCert := serverCA.MustNewTLSCertificate("www.example.com") testcases := []testcase{{ name: "with TLSHandlerTimeout", @@ -77,7 +78,7 @@ func TestTLSHandlerWithStdlib(t *testing.T) { }, { name: "with TLSHandlerHandshakeAndWriteText", newHandler: func() testingx.TLSHandler { - return testingx.TLSHandlerHandshakeAndWriteText(mitm, testingx.HTTPBlockpage451) + return testingx.TLSHandlerHandshakeAndWriteText(serverCert, testingx.HTTPBlockpage451) }, timeout: 10 * time.Second, expectErr: nil, @@ -92,7 +93,7 @@ func TestTLSHandlerWithStdlib(t *testing.T) { // create TLS config with a specific SNI tlsConfig := &tls.Config{ - RootCAs: runtimex.Try1(mitm.DefaultCertPool()), + RootCAs: serverCA.DefaultCertPool(), ServerName: "www.example.com", } @@ -163,8 +164,9 @@ func TestTLSHandlerWithNetem(t *testing.T) { expectBody []byte } - // create MITM config - mitm := testingx.MustNewTLSMITMProviderNetem() + // create server's CA and leaf certificate + serverCA := netem.MustNewCA() + serverCert := serverCA.MustNewTLSCertificate("www.example.com") testcases := []testcase{{ name: "with TLSHandlerTimeout", @@ -202,7 +204,7 @@ func TestTLSHandlerWithNetem(t *testing.T) { }, { name: "with TLSHandlerHandshakeAndWriteText", newHandler: func() testingx.TLSHandler { - return testingx.TLSHandlerHandshakeAndWriteText(mitm, testingx.HTTPBlockpage451) + return testingx.TLSHandlerHandshakeAndWriteText(serverCert, testingx.HTTPBlockpage451) }, timeout: 10 * time.Second, expectErr: nil, @@ -215,10 +217,8 @@ func TestTLSHandlerWithNetem(t *testing.T) { t.Skip(tc.reasonToSkip) } - //log.SetLevel(log.DebugLevel) - // create a star topology for this test case - topology := runtimex.Try1(netem.NewStarTopology(log.Log)) + topology := netem.MustNewStarTopology(log.Log) defer topology.Close() // create the server @@ -237,7 +237,7 @@ func TestTLSHandlerWithNetem(t *testing.T) { netxlite.WithCustomTProxy(&netxlite.NetemUnderlyingNetworkAdapter{UNet: clientStack}, func() { // create TLS config with a specific SNI tlsConfig := &tls.Config{ - RootCAs: runtimex.Try1(mitm.DefaultCertPool()), + RootCAs: serverCA.DefaultCertPool(), ServerName: "www.example.com", } diff --git a/pkg/throttling/throttling_test.go b/pkg/throttling/throttling_test.go index 2cf417a60..cac6decfe 100644 --- a/pkg/throttling/throttling_test.go +++ b/pkg/throttling/throttling_test.go @@ -44,6 +44,9 @@ func TestSamplerWorkingAsIntended(t *testing.T) { dialer := tx.NewDialerWithoutResolver(model.DiscardLogger) // create an HTTP transport + // TODO(https://github.com/ooni/probe/issues/2534): here we're using the QUIRKY netxlite.NewHTTPTransport + // function, but we can probably avoid using it, given that this code is + // not using tracing and does not care about those quirks. txp := netxlite.NewHTTPTransport(model.DiscardLogger, dialer, netxlite.NewNullTLSDialer()) // create the HTTP request to issue diff --git a/pkg/tutorial/experiment/torsf/chapter04/README.md b/pkg/tutorial/experiment/torsf/chapter04/README.md index 7e58d1399..05048a6ed 100644 --- a/pkg/tutorial/experiment/torsf/chapter04/README.md +++ b/pkg/tutorial/experiment/torsf/chapter04/README.md @@ -38,7 +38,7 @@ The `tracex` package contains code used to format internal measurements representations to the OONI data format. ```Go - "github.com/ooni/probe-cli/v3/internal/tracex" + "github.com/ooni/probe-cli/v3/internal/legacy/tracex" ``` diff --git a/pkg/tutorial/experiment/torsf/chapter04/torsf.go b/pkg/tutorial/experiment/torsf/chapter04/torsf.go index e0d61f18e..78b4058c0 100644 --- a/pkg/tutorial/experiment/torsf/chapter04/torsf.go +++ b/pkg/tutorial/experiment/torsf/chapter04/torsf.go @@ -41,7 +41,7 @@ import ( // measurements representations to the OONI data format. // // ```Go - "github.com/ooni/probe-engine/pkg/tracex" + "github.com/ooni/probe-engine/pkg/legacy/tracex" // ``` // diff --git a/pkg/tutorial/measurex/chapter01/README.md b/pkg/tutorial/measurex/chapter01/README.md index 4c04d4ab5..c88564c6e 100644 --- a/pkg/tutorial/measurex/chapter01/README.md +++ b/pkg/tutorial/measurex/chapter01/README.md @@ -50,7 +50,7 @@ import ( "fmt" "time" - "github.com/ooni/probe-cli/v3/internal/measurex" + "github.com/ooni/probe-cli/v3/internal/legacy/measurex" "github.com/ooni/probe-cli/v3/internal/runtimex" ) diff --git a/pkg/tutorial/measurex/chapter01/main.go b/pkg/tutorial/measurex/chapter01/main.go index 440b4e351..e99dffa3e 100644 --- a/pkg/tutorial/measurex/chapter01/main.go +++ b/pkg/tutorial/measurex/chapter01/main.go @@ -51,7 +51,7 @@ import ( "fmt" "time" - "github.com/ooni/probe-engine/pkg/measurex" + "github.com/ooni/probe-engine/pkg/legacy/measurex" "github.com/ooni/probe-engine/pkg/runtimex" ) diff --git a/pkg/tutorial/measurex/chapter02/README.md b/pkg/tutorial/measurex/chapter02/README.md index 7edc3b2cd..485dce784 100644 --- a/pkg/tutorial/measurex/chapter02/README.md +++ b/pkg/tutorial/measurex/chapter02/README.md @@ -25,7 +25,7 @@ import ( "fmt" "time" - "github.com/ooni/probe-cli/v3/internal/measurex" + "github.com/ooni/probe-cli/v3/internal/legacy/measurex" "github.com/ooni/probe-cli/v3/internal/runtimex" ) diff --git a/pkg/tutorial/measurex/chapter02/main.go b/pkg/tutorial/measurex/chapter02/main.go index 0a77b9e73..32aed34a8 100644 --- a/pkg/tutorial/measurex/chapter02/main.go +++ b/pkg/tutorial/measurex/chapter02/main.go @@ -26,7 +26,7 @@ import ( "fmt" "time" - "github.com/ooni/probe-engine/pkg/measurex" + "github.com/ooni/probe-engine/pkg/legacy/measurex" "github.com/ooni/probe-engine/pkg/runtimex" ) diff --git a/pkg/tutorial/measurex/chapter03/README.md b/pkg/tutorial/measurex/chapter03/README.md index 0f0b4f302..fff80e82e 100644 --- a/pkg/tutorial/measurex/chapter03/README.md +++ b/pkg/tutorial/measurex/chapter03/README.md @@ -25,7 +25,7 @@ import ( "fmt" "time" - "github.com/ooni/probe-cli/v3/internal/measurex" + "github.com/ooni/probe-cli/v3/internal/legacy/measurex" "github.com/ooni/probe-cli/v3/internal/runtimex" ) diff --git a/pkg/tutorial/measurex/chapter03/main.go b/pkg/tutorial/measurex/chapter03/main.go index e1f678b16..b3d9efb5e 100644 --- a/pkg/tutorial/measurex/chapter03/main.go +++ b/pkg/tutorial/measurex/chapter03/main.go @@ -26,7 +26,7 @@ import ( "fmt" "time" - "github.com/ooni/probe-engine/pkg/measurex" + "github.com/ooni/probe-engine/pkg/legacy/measurex" "github.com/ooni/probe-engine/pkg/runtimex" ) diff --git a/pkg/tutorial/measurex/chapter04/README.md b/pkg/tutorial/measurex/chapter04/README.md index 5f0664e41..c308a126a 100644 --- a/pkg/tutorial/measurex/chapter04/README.md +++ b/pkg/tutorial/measurex/chapter04/README.md @@ -25,7 +25,7 @@ import ( "fmt" "time" - "github.com/ooni/probe-cli/v3/internal/measurex" + "github.com/ooni/probe-cli/v3/internal/legacy/measurex" "github.com/ooni/probe-cli/v3/internal/runtimex" ) diff --git a/pkg/tutorial/measurex/chapter04/main.go b/pkg/tutorial/measurex/chapter04/main.go index ecfa8ac81..d55a42f4d 100644 --- a/pkg/tutorial/measurex/chapter04/main.go +++ b/pkg/tutorial/measurex/chapter04/main.go @@ -26,7 +26,7 @@ import ( "fmt" "time" - "github.com/ooni/probe-engine/pkg/measurex" + "github.com/ooni/probe-engine/pkg/legacy/measurex" "github.com/ooni/probe-engine/pkg/runtimex" ) diff --git a/pkg/tutorial/measurex/chapter05/README.md b/pkg/tutorial/measurex/chapter05/README.md index 5e107d26d..7c5cb4468 100644 --- a/pkg/tutorial/measurex/chapter05/README.md +++ b/pkg/tutorial/measurex/chapter05/README.md @@ -31,7 +31,7 @@ import ( "fmt" "time" - "github.com/ooni/probe-cli/v3/internal/measurex" + "github.com/ooni/probe-cli/v3/internal/legacy/measurex" "github.com/ooni/probe-cli/v3/internal/runtimex" ) diff --git a/pkg/tutorial/measurex/chapter05/main.go b/pkg/tutorial/measurex/chapter05/main.go index 21ad28214..96a1c7f7d 100644 --- a/pkg/tutorial/measurex/chapter05/main.go +++ b/pkg/tutorial/measurex/chapter05/main.go @@ -32,7 +32,7 @@ import ( "fmt" "time" - "github.com/ooni/probe-engine/pkg/measurex" + "github.com/ooni/probe-engine/pkg/legacy/measurex" "github.com/ooni/probe-engine/pkg/runtimex" ) diff --git a/pkg/tutorial/measurex/chapter06/README.md b/pkg/tutorial/measurex/chapter06/README.md index 3184b8864..bc38714a1 100644 --- a/pkg/tutorial/measurex/chapter06/README.md +++ b/pkg/tutorial/measurex/chapter06/README.md @@ -35,7 +35,7 @@ import ( "net/url" "time" - "github.com/ooni/probe-cli/v3/internal/measurex" + "github.com/ooni/probe-cli/v3/internal/legacy/measurex" "github.com/ooni/probe-cli/v3/internal/runtimex" ) diff --git a/pkg/tutorial/measurex/chapter06/main.go b/pkg/tutorial/measurex/chapter06/main.go index 78557ab1f..71ef4e9a0 100644 --- a/pkg/tutorial/measurex/chapter06/main.go +++ b/pkg/tutorial/measurex/chapter06/main.go @@ -36,7 +36,7 @@ import ( "net/url" "time" - "github.com/ooni/probe-engine/pkg/measurex" + "github.com/ooni/probe-engine/pkg/legacy/measurex" "github.com/ooni/probe-engine/pkg/runtimex" ) diff --git a/pkg/tutorial/measurex/chapter07/README.md b/pkg/tutorial/measurex/chapter07/README.md index 2adefae5d..0fbc61b2e 100644 --- a/pkg/tutorial/measurex/chapter07/README.md +++ b/pkg/tutorial/measurex/chapter07/README.md @@ -24,7 +24,7 @@ import ( "net/url" "time" - "github.com/ooni/probe-cli/v3/internal/measurex" + "github.com/ooni/probe-cli/v3/internal/legacy/measurex" "github.com/ooni/probe-cli/v3/internal/runtimex" ) diff --git a/pkg/tutorial/measurex/chapter07/main.go b/pkg/tutorial/measurex/chapter07/main.go index cea1ed837..5d2081cc9 100644 --- a/pkg/tutorial/measurex/chapter07/main.go +++ b/pkg/tutorial/measurex/chapter07/main.go @@ -25,7 +25,7 @@ import ( "net/url" "time" - "github.com/ooni/probe-engine/pkg/measurex" + "github.com/ooni/probe-engine/pkg/legacy/measurex" "github.com/ooni/probe-engine/pkg/runtimex" ) diff --git a/pkg/tutorial/measurex/chapter08/README.md b/pkg/tutorial/measurex/chapter08/README.md index 4a7e2364f..64df7a41c 100644 --- a/pkg/tutorial/measurex/chapter08/README.md +++ b/pkg/tutorial/measurex/chapter08/README.md @@ -29,7 +29,7 @@ import ( "net/url" "time" - "github.com/ooni/probe-cli/v3/internal/measurex" + "github.com/ooni/probe-cli/v3/internal/legacy/measurex" "github.com/ooni/probe-cli/v3/internal/runtimex" ) diff --git a/pkg/tutorial/measurex/chapter08/main.go b/pkg/tutorial/measurex/chapter08/main.go index 03749e88c..db15ea68b 100644 --- a/pkg/tutorial/measurex/chapter08/main.go +++ b/pkg/tutorial/measurex/chapter08/main.go @@ -30,7 +30,7 @@ import ( "net/url" "time" - "github.com/ooni/probe-engine/pkg/measurex" + "github.com/ooni/probe-engine/pkg/legacy/measurex" "github.com/ooni/probe-engine/pkg/runtimex" ) diff --git a/pkg/tutorial/measurex/chapter09/README.md b/pkg/tutorial/measurex/chapter09/README.md index 1f4a26f1b..a696c9861 100644 --- a/pkg/tutorial/measurex/chapter09/README.md +++ b/pkg/tutorial/measurex/chapter09/README.md @@ -39,7 +39,7 @@ import ( "net/url" "time" - "github.com/ooni/probe-cli/v3/internal/measurex" + "github.com/ooni/probe-cli/v3/internal/legacy/measurex" "github.com/ooni/probe-cli/v3/internal/runtimex" ) diff --git a/pkg/tutorial/measurex/chapter09/main.go b/pkg/tutorial/measurex/chapter09/main.go index 6d68468a3..cdd7b0e08 100644 --- a/pkg/tutorial/measurex/chapter09/main.go +++ b/pkg/tutorial/measurex/chapter09/main.go @@ -40,7 +40,7 @@ import ( "net/url" "time" - "github.com/ooni/probe-engine/pkg/measurex" + "github.com/ooni/probe-engine/pkg/legacy/measurex" "github.com/ooni/probe-engine/pkg/runtimex" ) diff --git a/pkg/tutorial/measurex/chapter10/README.md b/pkg/tutorial/measurex/chapter10/README.md index a7c4f42f7..ede0c019d 100644 --- a/pkg/tutorial/measurex/chapter10/README.md +++ b/pkg/tutorial/measurex/chapter10/README.md @@ -28,7 +28,7 @@ import ( "net/url" "time" - "github.com/ooni/probe-cli/v3/internal/measurex" + "github.com/ooni/probe-cli/v3/internal/legacy/measurex" "github.com/ooni/probe-cli/v3/internal/runtimex" ) diff --git a/pkg/tutorial/measurex/chapter10/main.go b/pkg/tutorial/measurex/chapter10/main.go index f1e2608cc..15c536553 100644 --- a/pkg/tutorial/measurex/chapter10/main.go +++ b/pkg/tutorial/measurex/chapter10/main.go @@ -29,7 +29,7 @@ import ( "net/url" "time" - "github.com/ooni/probe-engine/pkg/measurex" + "github.com/ooni/probe-engine/pkg/legacy/measurex" "github.com/ooni/probe-engine/pkg/runtimex" ) diff --git a/pkg/tutorial/measurex/chapter11/README.md b/pkg/tutorial/measurex/chapter11/README.md index 0c0fec526..2824c7db6 100644 --- a/pkg/tutorial/measurex/chapter11/README.md +++ b/pkg/tutorial/measurex/chapter11/README.md @@ -30,7 +30,7 @@ import ( "fmt" "time" - "github.com/ooni/probe-cli/v3/internal/measurex" + "github.com/ooni/probe-cli/v3/internal/legacy/measurex" "github.com/ooni/probe-cli/v3/internal/runtimex" ) diff --git a/pkg/tutorial/measurex/chapter11/main.go b/pkg/tutorial/measurex/chapter11/main.go index 0fe6c85a2..79dc2fdf8 100644 --- a/pkg/tutorial/measurex/chapter11/main.go +++ b/pkg/tutorial/measurex/chapter11/main.go @@ -31,7 +31,7 @@ import ( "fmt" "time" - "github.com/ooni/probe-engine/pkg/measurex" + "github.com/ooni/probe-engine/pkg/legacy/measurex" "github.com/ooni/probe-engine/pkg/runtimex" ) diff --git a/pkg/tutorial/measurex/chapter12/README.md b/pkg/tutorial/measurex/chapter12/README.md index 07d4debec..ba7be38de 100644 --- a/pkg/tutorial/measurex/chapter12/README.md +++ b/pkg/tutorial/measurex/chapter12/README.md @@ -27,7 +27,7 @@ import ( "fmt" "time" - "github.com/ooni/probe-cli/v3/internal/measurex" + "github.com/ooni/probe-cli/v3/internal/legacy/measurex" "github.com/ooni/probe-cli/v3/internal/runtimex" ) diff --git a/pkg/tutorial/measurex/chapter12/main.go b/pkg/tutorial/measurex/chapter12/main.go index 83342b7e8..49d31c85a 100644 --- a/pkg/tutorial/measurex/chapter12/main.go +++ b/pkg/tutorial/measurex/chapter12/main.go @@ -28,7 +28,7 @@ import ( "fmt" "time" - "github.com/ooni/probe-engine/pkg/measurex" + "github.com/ooni/probe-engine/pkg/legacy/measurex" "github.com/ooni/probe-engine/pkg/runtimex" ) diff --git a/pkg/tutorial/measurex/chapter14/README.md b/pkg/tutorial/measurex/chapter14/README.md index 54b45be41..9c38fbff0 100644 --- a/pkg/tutorial/measurex/chapter14/README.md +++ b/pkg/tutorial/measurex/chapter14/README.md @@ -24,7 +24,7 @@ import ( "net/url" "time" - "github.com/ooni/probe-cli/v3/internal/measurex" + "github.com/ooni/probe-cli/v3/internal/legacy/measurex" "github.com/ooni/probe-cli/v3/internal/runtimex" ) diff --git a/pkg/tutorial/measurex/chapter14/main.go b/pkg/tutorial/measurex/chapter14/main.go index 6b1ac3769..d85a60870 100644 --- a/pkg/tutorial/measurex/chapter14/main.go +++ b/pkg/tutorial/measurex/chapter14/main.go @@ -25,7 +25,7 @@ import ( "net/url" "time" - "github.com/ooni/probe-engine/pkg/measurex" + "github.com/ooni/probe-engine/pkg/legacy/measurex" "github.com/ooni/probe-engine/pkg/runtimex" ) diff --git a/pkg/tutorial/netxlite/chapter02/README.md b/pkg/tutorial/netxlite/chapter02/README.md index ef5d9eb2f..70ea352b2 100644 --- a/pkg/tutorial/netxlite/chapter02/README.md +++ b/pkg/tutorial/netxlite/chapter02/README.md @@ -25,6 +25,7 @@ import ( "time" "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -72,7 +73,7 @@ The logic to dial and handshake have been factored into a function called `dialTLS`. ```Go - conn, state, err := dialTLS(ctx, *address, tlsConfig) + conn, err := dialTLS(ctx, *address, tlsConfig) ``` If there is an error, we bail, like before. Otherwise we @@ -84,6 +85,7 @@ like in the previous chapter, we close the connection. if err != nil { fatal(err) } + state := conn.ConnectionState() log.Infof("Conn type : %T", conn) log.Infof("Cipher suite : %s", netxlite.TLSCipherSuiteString(state.CipherSuite)) log.Infof("Negotiated protocol: %s", state.NegotiatedProtocol) @@ -124,8 +126,7 @@ chapter why this guarantee helps when writing more complex code.) ```Go -func handshakeTLS(ctx context.Context, tcpConn net.Conn, - config *tls.Config) (net.Conn, tls.ConnectionState, error) { +func handshakeTLS(ctx context.Context, tcpConn net.Conn, config *tls.Config) (model.TLSConn, error) { th := netxlite.NewTLSHandshakerStdlib(log.Log) return th.Handshake(ctx, tcpConn, config) } @@ -139,18 +140,17 @@ perform this dial+handshake operation in a single function call. ```Go -func dialTLS(ctx context.Context, address string, - config *tls.Config) (net.Conn, tls.ConnectionState, error) { +func dialTLS(ctx context.Context, address string, config *tls.Config) (model.TLSConn, error) { tcpConn, err := dialTCP(ctx, address) if err != nil { - return nil, tls.ConnectionState{}, err + return nil, err } - tlsConn, state, err := handshakeTLS(ctx, tcpConn, config) + tlsConn, err := handshakeTLS(ctx, tcpConn, config) if err != nil { tcpConn.Close() - return nil, tls.ConnectionState{}, err + return nil, err } - return tlsConn, state, nil + return tlsConn, nil } ``` diff --git a/pkg/tutorial/netxlite/chapter02/main.go b/pkg/tutorial/netxlite/chapter02/main.go index 829c98acc..d12401ca9 100644 --- a/pkg/tutorial/netxlite/chapter02/main.go +++ b/pkg/tutorial/netxlite/chapter02/main.go @@ -26,6 +26,7 @@ import ( "time" "github.com/apex/log" + "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" ) @@ -73,7 +74,7 @@ func main() { // into a function called `dialTLS`. // // ```Go - conn, state, err := dialTLS(ctx, *address, tlsConfig) + conn, err := dialTLS(ctx, *address, tlsConfig) // ``` // // If there is an error, we bail, like before. Otherwise we @@ -85,6 +86,7 @@ func main() { if err != nil { fatal(err) } + state := conn.ConnectionState() log.Infof("Conn type : %T", conn) log.Infof("Cipher suite : %s", netxlite.TLSCipherSuiteString(state.CipherSuite)) log.Infof("Negotiated protocol: %s", state.NegotiatedProtocol) @@ -125,8 +127,7 @@ func dialTCP(ctx context.Context, address string) (net.Conn, error) { // // ```Go -func handshakeTLS(ctx context.Context, tcpConn net.Conn, - config *tls.Config) (net.Conn, tls.ConnectionState, error) { +func handshakeTLS(ctx context.Context, tcpConn net.Conn, config *tls.Config) (model.TLSConn, error) { th := netxlite.NewTLSHandshakerStdlib(log.Log) return th.Handshake(ctx, tcpConn, config) } @@ -140,18 +141,17 @@ func handshakeTLS(ctx context.Context, tcpConn net.Conn, // // ```Go -func dialTLS(ctx context.Context, address string, - config *tls.Config) (net.Conn, tls.ConnectionState, error) { +func dialTLS(ctx context.Context, address string, config *tls.Config) (model.TLSConn, error) { tcpConn, err := dialTCP(ctx, address) if err != nil { - return nil, tls.ConnectionState{}, err + return nil, err } - tlsConn, state, err := handshakeTLS(ctx, tcpConn, config) + tlsConn, err := handshakeTLS(ctx, tcpConn, config) if err != nil { tcpConn.Close() - return nil, tls.ConnectionState{}, err + return nil, err } - return tlsConn, state, nil + return tlsConn, nil } // ``` diff --git a/pkg/tutorial/netxlite/chapter03/README.md b/pkg/tutorial/netxlite/chapter03/README.md index cd4074be0..482cb6350 100644 --- a/pkg/tutorial/netxlite/chapter03/README.md +++ b/pkg/tutorial/netxlite/chapter03/README.md @@ -32,6 +32,7 @@ import ( "time" "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" utls "gitlab.com/yawning/utls.git" ) @@ -49,10 +50,11 @@ func main() { NextProtos: []string{"h2", "http/1.1"}, RootCAs: nil, } - conn, state, err := dialTLS(ctx, *address, tlsConfig) + conn, err := dialTLS(ctx, *address, tlsConfig) if err != nil { fatal(err) } + state := conn.ConnectionState() log.Infof("Conn type : %T", conn) log.Infof("Cipher suite : %s", netxlite.TLSCipherSuiteString(state.CipherSuite)) log.Infof("Negotiated protocol: %s", state.NegotiatedProtocol) @@ -65,8 +67,7 @@ func dialTCP(ctx context.Context, address string) (net.Conn, error) { return d.DialContext(ctx, "tcp", address) } -func handshakeTLS(ctx context.Context, tcpConn net.Conn, - config *tls.Config) (net.Conn, tls.ConnectionState, error) { +func handshakeTLS(ctx context.Context, tcpConn net.Conn, config *tls.Config) (model.TLSConn, error) { ``` The following line of code is where we diverge from the @@ -91,18 +92,17 @@ previous chapter, so we won't add further comments. return th.Handshake(ctx, tcpConn, config) } -func dialTLS(ctx context.Context, address string, - config *tls.Config) (net.Conn, tls.ConnectionState, error) { +func dialTLS(ctx context.Context, address string, config *tls.Config) (model.TLSConn, error) { tcpConn, err := dialTCP(ctx, address) if err != nil { - return nil, tls.ConnectionState{}, err + return nil, err } - tlsConn, state, err := handshakeTLS(ctx, tcpConn, config) + tlsConn, err := handshakeTLS(ctx, tcpConn, config) if err != nil { tcpConn.Close() - return nil, tls.ConnectionState{}, err + return nil, err } - return tlsConn, state, nil + return tlsConn, nil } func fatal(err error) { diff --git a/pkg/tutorial/netxlite/chapter03/main.go b/pkg/tutorial/netxlite/chapter03/main.go index 4cc88b7fc..d364c06dd 100644 --- a/pkg/tutorial/netxlite/chapter03/main.go +++ b/pkg/tutorial/netxlite/chapter03/main.go @@ -33,6 +33,7 @@ import ( "time" "github.com/apex/log" + "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" utls "gitlab.com/yawning/utls.git" ) @@ -50,10 +51,11 @@ func main() { NextProtos: []string{"h2", "http/1.1"}, RootCAs: nil, } - conn, state, err := dialTLS(ctx, *address, tlsConfig) + conn, err := dialTLS(ctx, *address, tlsConfig) if err != nil { fatal(err) } + state := conn.ConnectionState() log.Infof("Conn type : %T", conn) log.Infof("Cipher suite : %s", netxlite.TLSCipherSuiteString(state.CipherSuite)) log.Infof("Negotiated protocol: %s", state.NegotiatedProtocol) @@ -66,8 +68,7 @@ func dialTCP(ctx context.Context, address string) (net.Conn, error) { return d.DialContext(ctx, "tcp", address) } -func handshakeTLS(ctx context.Context, tcpConn net.Conn, - config *tls.Config) (net.Conn, tls.ConnectionState, error) { +func handshakeTLS(ctx context.Context, tcpConn net.Conn, config *tls.Config) (model.TLSConn, error) { // ``` // // The following line of code is where we diverge from the @@ -92,18 +93,17 @@ func handshakeTLS(ctx context.Context, tcpConn net.Conn, return th.Handshake(ctx, tcpConn, config) } -func dialTLS(ctx context.Context, address string, - config *tls.Config) (net.Conn, tls.ConnectionState, error) { +func dialTLS(ctx context.Context, address string, config *tls.Config) (model.TLSConn, error) { tcpConn, err := dialTCP(ctx, address) if err != nil { - return nil, tls.ConnectionState{}, err + return nil, err } - tlsConn, state, err := handshakeTLS(ctx, tcpConn, config) + tlsConn, err := handshakeTLS(ctx, tcpConn, config) if err != nil { tcpConn.Close() - return nil, tls.ConnectionState{}, err + return nil, err } - return tlsConn, state, nil + return tlsConn, nil } func fatal(err error) { diff --git a/pkg/tutorial/netxlite/chapter04/README.md b/pkg/tutorial/netxlite/chapter04/README.md index 7cc4c74c7..65820b088 100644 --- a/pkg/tutorial/netxlite/chapter04/README.md +++ b/pkg/tutorial/netxlite/chapter04/README.md @@ -91,7 +91,7 @@ in the next two chapters.) ```Go func dialQUIC(ctx context.Context, address string, config *tls.Config) (quic.EarlyConnection, tls.ConnectionState, error) { - ql := netxlite.NewQUICListener() + ql := netxlite.NewUDPListener() d := netxlite.NewQUICDialerWithoutResolver(ql, log.Log) qconn, err := d.DialContext(ctx, address, config, &quic.Config{}) if err != nil { @@ -104,7 +104,7 @@ QUIC code to be of the same type of the ConnectionState that we returned in the previous chapters. ```Go - return qconn, qconn.ConnectionState().TLS.ConnectionState, nil + return qconn, qconn.ConnectionState().TLS, nil } ``` diff --git a/pkg/tutorial/netxlite/chapter04/main.go b/pkg/tutorial/netxlite/chapter04/main.go index 352406d99..2fa747166 100644 --- a/pkg/tutorial/netxlite/chapter04/main.go +++ b/pkg/tutorial/netxlite/chapter04/main.go @@ -92,7 +92,7 @@ func main() { // ```Go func dialQUIC(ctx context.Context, address string, config *tls.Config) (quic.EarlyConnection, tls.ConnectionState, error) { - ql := netxlite.NewQUICListener() + ql := netxlite.NewUDPListener() d := netxlite.NewQUICDialerWithoutResolver(ql, log.Log) qconn, err := d.DialContext(ctx, address, config, &quic.Config{}) if err != nil { @@ -105,7 +105,7 @@ func dialQUIC(ctx context.Context, address string, // we returned in the previous chapters. // // ```Go - return qconn, qconn.ConnectionState().TLS.ConnectionState, nil + return qconn, qconn.ConnectionState().TLS, nil } // ``` diff --git a/pkg/tutorial/netxlite/chapter07/README.md b/pkg/tutorial/netxlite/chapter07/README.md index 92737b402..0fd924835 100644 --- a/pkg/tutorial/netxlite/chapter07/README.md +++ b/pkg/tutorial/netxlite/chapter07/README.md @@ -33,6 +33,7 @@ import ( "time" "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" utls "gitlab.com/yawning/utls.git" ) @@ -50,7 +51,7 @@ func main() { NextProtos: []string{"h2", "http/1.1"}, RootCAs: nil, } - conn, _, err := dialTLS(ctx, *address, config) + conn, err := dialTLS(ctx, *address, config) if err != nil { fatal(err) } @@ -81,10 +82,13 @@ a single request using the given TLS conn. uses a cleartext TCP connection. In the next chapter we'll see how to do the same using QUIC.) +TODO(https://github.com/ooni/probe/issues/2534): here we're using the QUIRKY netxlite.NewHTTPTransport +function, but we can probably avoid using it, given that this code is +not using tracing and does not care about those quirks. ```Go clnt := &http.Client{Transport: netxlite.NewHTTPTransport( log.Log, netxlite.NewNullDialer(), - netxlite.NewSingleUseTLSDialer(conn.(netxlite.TLSConn)), + netxlite.NewSingleUseTLSDialer(conn), )} ``` @@ -116,24 +120,22 @@ func dialTCP(ctx context.Context, address string) (net.Conn, error) { return d.DialContext(ctx, "tcp", address) } -func handshakeTLS(ctx context.Context, tcpConn net.Conn, - config *tls.Config) (net.Conn, tls.ConnectionState, error) { +func handshakeTLS(ctx context.Context, tcpConn net.Conn, config *tls.Config) (model.TLSConn, error) { th := netxlite.NewTLSHandshakerUTLS(log.Log, &utls.HelloFirefox_55) return th.Handshake(ctx, tcpConn, config) } -func dialTLS(ctx context.Context, address string, - config *tls.Config) (net.Conn, tls.ConnectionState, error) { +func dialTLS(ctx context.Context, address string, config *tls.Config) (model.TLSConn, error) { tcpConn, err := dialTCP(ctx, address) if err != nil { - return nil, tls.ConnectionState{}, err + return nil, err } - tlsConn, state, err := handshakeTLS(ctx, tcpConn, config) + tlsConn, err := handshakeTLS(ctx, tcpConn, config) if err != nil { tcpConn.Close() - return nil, tls.ConnectionState{}, err + return nil, err } - return tlsConn, state, nil + return tlsConn, nil } func fatal(err error) { diff --git a/pkg/tutorial/netxlite/chapter07/main.go b/pkg/tutorial/netxlite/chapter07/main.go index 7ae5e544a..8cf61898a 100644 --- a/pkg/tutorial/netxlite/chapter07/main.go +++ b/pkg/tutorial/netxlite/chapter07/main.go @@ -34,6 +34,7 @@ import ( "time" "github.com/apex/log" + "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/netxlite" utls "gitlab.com/yawning/utls.git" ) @@ -51,7 +52,7 @@ func main() { NextProtos: []string{"h2", "http/1.1"}, RootCAs: nil, } - conn, _, err := dialTLS(ctx, *address, config) + conn, err := dialTLS(ctx, *address, config) if err != nil { fatal(err) } @@ -82,10 +83,13 @@ func main() { // uses a cleartext TCP connection. In the next chapter we'll // see how to do the same using QUIC.) // + // TODO(https://github.com/ooni/probe/issues/2534): here we're using the QUIRKY netxlite.NewHTTPTransport + // function, but we can probably avoid using it, given that this code is + // not using tracing and does not care about those quirks. // ```Go clnt := &http.Client{Transport: netxlite.NewHTTPTransport( log.Log, netxlite.NewNullDialer(), - netxlite.NewSingleUseTLSDialer(conn.(netxlite.TLSConn)), + netxlite.NewSingleUseTLSDialer(conn), )} // ``` // @@ -117,24 +121,22 @@ func dialTCP(ctx context.Context, address string) (net.Conn, error) { return d.DialContext(ctx, "tcp", address) } -func handshakeTLS(ctx context.Context, tcpConn net.Conn, - config *tls.Config) (net.Conn, tls.ConnectionState, error) { +func handshakeTLS(ctx context.Context, tcpConn net.Conn, config *tls.Config) (model.TLSConn, error) { th := netxlite.NewTLSHandshakerUTLS(log.Log, &utls.HelloFirefox_55) return th.Handshake(ctx, tcpConn, config) } -func dialTLS(ctx context.Context, address string, - config *tls.Config) (net.Conn, tls.ConnectionState, error) { +func dialTLS(ctx context.Context, address string, config *tls.Config) (model.TLSConn, error) { tcpConn, err := dialTCP(ctx, address) if err != nil { - return nil, tls.ConnectionState{}, err + return nil, err } - tlsConn, state, err := handshakeTLS(ctx, tcpConn, config) + tlsConn, err := handshakeTLS(ctx, tcpConn, config) if err != nil { tcpConn.Close() - return nil, tls.ConnectionState{}, err + return nil, err } - return tlsConn, state, nil + return tlsConn, nil } func fatal(err error) { diff --git a/pkg/tutorial/netxlite/chapter08/README.md b/pkg/tutorial/netxlite/chapter08/README.md index 77aa77ec0..646f12b19 100644 --- a/pkg/tutorial/netxlite/chapter08/README.md +++ b/pkg/tutorial/netxlite/chapter08/README.md @@ -104,13 +104,13 @@ exactly like what we've seen in chapter04. func dialQUIC(ctx context.Context, address string, config *tls.Config) (quic.EarlyConnection, tls.ConnectionState, error) { - ql := netxlite.NewQUICListener() + ql := netxlite.NewUDPListener() d := netxlite.NewQUICDialerWithoutResolver(ql, log.Log) qconn, err := d.DialContext(ctx, address, config, &quic.Config{}) if err != nil { return nil, tls.ConnectionState{}, err } - return qconn, qconn.ConnectionState().TLS.ConnectionState, nil + return qconn, qconn.ConnectionState().TLS, nil } func fatal(err error) { diff --git a/pkg/tutorial/netxlite/chapter08/main.go b/pkg/tutorial/netxlite/chapter08/main.go index 54f18eafb..8655ac7b9 100644 --- a/pkg/tutorial/netxlite/chapter08/main.go +++ b/pkg/tutorial/netxlite/chapter08/main.go @@ -105,13 +105,13 @@ func main() { func dialQUIC(ctx context.Context, address string, config *tls.Config) (quic.EarlyConnection, tls.ConnectionState, error) { - ql := netxlite.NewQUICListener() + ql := netxlite.NewUDPListener() d := netxlite.NewQUICDialerWithoutResolver(ql, log.Log) qconn, err := d.DialContext(ctx, address, config, &quic.Config{}) if err != nil { return nil, tls.ConnectionState{}, err } - return qconn, qconn.ConnectionState().TLS.ConnectionState, nil + return qconn, qconn.ConnectionState().TLS, nil } func fatal(err error) { diff --git a/pkg/version/version.go b/pkg/version/version.go index 3394cd41c..463a17507 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -2,4 +2,4 @@ package version // Version is the ooniprobe version. -const Version = "3.19.0-alpha" +const Version = "3.20.0-alpha"