From d567f040bc08806034db680417e037280155f0b8 Mon Sep 17 00:00:00 2001 From: decfox Date: Thu, 8 Aug 2024 15:33:03 +0530 Subject: [PATCH] chore: sync with probe-cli v3.23.0 --- .github/workflows/build.yml | 2 +- UPSTREAM | 2 +- go.mod | 30 +- go.sum | 78 ++-- pkg/cmd/buildtool/android_test.go | 32 +- pkg/cmd/buildtool/cdepsopenssl.go | 10 +- pkg/cmd/buildtool/cdepstor.go | 10 +- pkg/cmd/buildtool/ios_test.go | 24 +- pkg/cmd/buildtool/linuxcdeps_test.go | 8 +- pkg/cmd/miniooni/main.go | 11 +- pkg/cmd/miniooni/oonirun.go | 1 + pkg/engine/experiment.go | 305 +++++++------ pkg/engine/experiment_integration_test.go | 16 +- pkg/engine/experiment_test.go | 88 +++- pkg/engine/experimentbuilder.go | 38 +- pkg/engine/experimentbuilder_test.go | 168 +++++++ ...est.go => inputloader_integration_test.go} | 15 +- pkg/engine/session.go | 39 +- pkg/engine/session_integration_test.go | 10 +- pkg/engine/session_internal_test.go | 13 + pkg/experiment/dnscheck/dnscheck.go | 54 ++- pkg/experiment/dnscheck/dnscheck_test.go | 105 +++-- pkg/experiment/dnscheck/richerinput.go | 129 ++++++ pkg/experiment/dnscheck/richerinput_test.go | 325 ++++++++++++++ pkg/experiment/dnscheck/testdata/input.txt | 2 + pkg/experiment/dnsping/dnsping_test.go | 2 +- pkg/experiment/example/example.go | 13 +- pkg/experiment/example/example_test.go | 18 +- pkg/experiment/openvpn/endpoint.go | 199 +++++++++ pkg/experiment/openvpn/endpoint_test.go | 351 +++++++++++++++ pkg/experiment/openvpn/openvpn.go | 271 ++++++++++++ pkg/experiment/openvpn/openvpn_test.go | 384 ++++++++++++++++ pkg/experiment/openvpn/richerinput.go | 183 ++++++++ pkg/experiment/openvpn/richerinput_test.go | 212 +++++++++ pkg/experiment/openvpn/targets.go | 78 ++++ pkg/experiment/openvpn/targets_test.go | 81 ++++ pkg/experiment/psiphon/psiphon_test.go | 2 +- pkg/experiment/quicping/quicping_test.go | 16 +- .../simplequicping/simplequicping_test.go | 2 +- pkg/experiment/sniblocking/sniblocking.go | 4 +- .../sniblocking/sniblocking_test.go | 2 +- .../stunreachability/stunreachability_test.go | 16 +- pkg/experiment/tcpping/tcpping_test.go | 2 +- pkg/experiment/tlsmiddlebox/measurer_test.go | 2 +- pkg/experiment/tlsping/tlsping_test.go | 2 +- .../webconnectivity/webconnectivity_test.go | 2 +- pkg/experimentconfig/experimentconfig.go | 84 ++++ pkg/experimentconfig/experimentconfig_test.go | 175 ++++++++ pkg/experimentname/experimentname.go | 25 ++ pkg/experimentname/experimentname_test.go | 55 +++ pkg/inputparser/inputparser.go | 2 +- pkg/inputparser/inputparser_test.go | 2 +- pkg/legacy/mockable/mockable.go | 8 + pkg/mocks/experiment.go | 13 +- pkg/mocks/experiment_test.go | 24 +- pkg/mocks/experimentbuilder.go | 20 +- pkg/mocks/experimentbuilder_test.go | 26 ++ pkg/mocks/experimentinputloader.go | 19 - pkg/mocks/experimenttargetloader.go | 19 + ...test.go => experimenttargetloader_test.go} | 4 +- pkg/mocks/session.go | 38 ++ pkg/mocks/session_test.go | 77 ++++ pkg/model/archival.go | 28 ++ pkg/model/experiment.go | 205 +++++---- pkg/model/http.go | 2 +- pkg/model/measurement.go | 10 +- pkg/model/measurement_test.go | 6 +- pkg/model/ooapi.go | 97 ++++ pkg/model/ooapi_test.go | 28 ++ pkg/netxlite/certifi.go | 53 +-- pkg/netxlite/errno.go | 2 +- pkg/netxlite/errno_darwin.go | 2 +- pkg/netxlite/errno_darwin_test.go | 2 +- pkg/netxlite/errno_freebsd.go | 2 +- pkg/netxlite/errno_freebsd_test.go | 2 +- pkg/netxlite/errno_linux.go | 2 +- pkg/netxlite/errno_linux_test.go | 2 +- pkg/netxlite/errno_openbsd.go | 2 +- pkg/netxlite/errno_openbsd_test.go | 2 +- pkg/netxlite/errno_windows.go | 2 +- pkg/netxlite/errno_windows_test.go | 2 +- pkg/oonirun/experiment.go | 123 +++--- pkg/oonirun/experiment_test.go | 236 ++++++---- pkg/oonirun/inputprocessor.go | 54 +-- pkg/oonirun/inputprocessor_test.go | 56 +-- pkg/oonirun/link.go | 4 + pkg/oonirun/session.go | 6 +- pkg/oonirun/v1.go | 2 +- pkg/oonirun/v1_test.go | 30 +- pkg/oonirun/v2.go | 72 ++- pkg/oonirun/v2_test.go | 305 ++++++++++++- pkg/probeservices/benchselect.go | 23 +- pkg/probeservices/checkin_test.go | 2 +- pkg/probeservices/openvpn.go | 45 ++ pkg/probeservices/openvpn_test.go | 105 +++++ pkg/probeservices/probeservices.go | 20 +- pkg/probeservices/probeservices_test.go | 118 ++--- pkg/probeservices/tor_test.go | 2 +- pkg/reflectx/reflectx.go | 31 ++ pkg/reflectx/reflectx_test.go | 87 ++++ pkg/registry/allexperiments.go | 2 +- pkg/registry/dash.go | 24 +- pkg/registry/dnscheck.go | 22 +- pkg/registry/dnsping.go | 22 +- pkg/registry/dslxtutorial.go | 20 +- pkg/registry/echcheck.go | 20 +- pkg/registry/example.go | 35 +- pkg/registry/factory.go | 118 +++-- pkg/registry/factory_test.go | 344 ++++++++++++++- pkg/registry/fbmessenger.go | 22 +- pkg/registry/hhfm.go | 22 +- pkg/registry/hirl.go | 22 +- pkg/registry/httphostheader.go | 22 +- pkg/registry/ndt.go | 24 +- pkg/registry/openvpn.go | 27 ++ pkg/registry/portfiltering.go | 24 +- pkg/registry/psiphon.go | 22 +- pkg/registry/quicping.go | 22 +- pkg/registry/riseupvpn.go | 20 +- pkg/registry/signal.go | 22 +- pkg/registry/simplequicping.go | 22 +- pkg/registry/sniblocking.go | 22 +- pkg/registry/stunreachability.go | 22 +- pkg/registry/tcpping.go | 22 +- pkg/registry/telegram.go | 24 +- pkg/registry/tlsmiddlebox.go | 22 +- pkg/registry/tlsping.go | 22 +- pkg/registry/tlstool.go | 22 +- pkg/registry/tor.go | 22 +- pkg/registry/torsf.go | 22 +- pkg/registry/urlgetter.go | 22 +- pkg/registry/vanillator.go | 28 +- pkg/registry/webconnectivity.go | 24 +- pkg/registry/webconnectivityv05.go | 24 +- pkg/registry/whatsapp.go | 22 +- .../targetloading.go} | 171 ++++--- .../targetloading_test.go} | 416 +++++++++++------- .../testdata/loader1.txt} | 0 .../testdata/loader2.txt} | 0 .../testdata/loader3.txt} | 0 pkg/testingx/fakefill.go | 7 + pkg/testingx/fakefill_test.go | 55 ++- pkg/testingx/oonibackendwithlogin.go | 28 ++ pkg/version/version.go | 2 +- pkg/webconnectivityqa/measurement.go | 2 +- 145 files changed, 6177 insertions(+), 1422 deletions(-) rename pkg/engine/{inputloader_network_test.go => inputloader_integration_test.go} (52%) create mode 100644 pkg/experiment/dnscheck/richerinput.go create mode 100644 pkg/experiment/dnscheck/richerinput_test.go create mode 100644 pkg/experiment/dnscheck/testdata/input.txt create mode 100644 pkg/experiment/openvpn/endpoint.go create mode 100644 pkg/experiment/openvpn/endpoint_test.go create mode 100644 pkg/experiment/openvpn/openvpn.go create mode 100644 pkg/experiment/openvpn/openvpn_test.go create mode 100644 pkg/experiment/openvpn/richerinput.go create mode 100644 pkg/experiment/openvpn/richerinput_test.go create mode 100644 pkg/experiment/openvpn/targets.go create mode 100644 pkg/experiment/openvpn/targets_test.go create mode 100644 pkg/experimentconfig/experimentconfig.go create mode 100644 pkg/experimentconfig/experimentconfig_test.go create mode 100644 pkg/experimentname/experimentname.go create mode 100644 pkg/experimentname/experimentname_test.go delete mode 100644 pkg/mocks/experimentinputloader.go create mode 100644 pkg/mocks/experimenttargetloader.go rename pkg/mocks/{experimentinputloader_test.go => experimenttargetloader_test.go} (80%) create mode 100644 pkg/probeservices/openvpn.go create mode 100644 pkg/probeservices/openvpn_test.go create mode 100644 pkg/reflectx/reflectx.go create mode 100644 pkg/reflectx/reflectx_test.go create mode 100644 pkg/registry/openvpn.go rename pkg/{engine/inputloader.go => targetloading/targetloading.go} (68%) rename pkg/{engine/inputloader_test.go => targetloading/targetloading_test.go} (54%) rename pkg/{engine/testdata/inputloader1.txt => targetloading/testdata/loader1.txt} (100%) rename pkg/{engine/testdata/inputloader2.txt => targetloading/testdata/loader2.txt} (100%) rename pkg/{engine/testdata/inputloader3.txt => targetloading/testdata/loader3.txt} (100%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 30bfb3ad5..1874ea9b7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,5 +11,5 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v3 with: - go-version: "1.21.10" + go-version: "1.21.11" - run: go build ./... diff --git a/UPSTREAM b/UPSTREAM index 981feb4ab..b1ef60824 100644 --- a/UPSTREAM +++ b/UPSTREAM @@ -1 +1 @@ -v3.22.0-alpha +v3.23.0 diff --git a/go.mod b/go.mod index 5660405e9..d4c6727ba 100644 --- a/go.mod +++ b/go.mod @@ -2,11 +2,11 @@ module github.com/ooni/probe-engine go 1.21 -toolchain go1.21.10 +toolchain go1.21.11 require ( filippo.io/age v1.1.1 - github.com/Psiphon-Labs/psiphon-tunnel-core v1.0.11-0.20240424194431-3612a5a6fb4c + github.com/Psiphon-Labs/psiphon-tunnel-core v1.0.11-0.20240705153833-eea9ace08cd1 github.com/apex/log v1.9.0 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/cloudflare/circl v1.3.8 @@ -21,25 +21,26 @@ require ( github.com/hexops/gotextdiff v1.0.3 github.com/miekg/dns v1.1.59 github.com/montanaflynn/stats v0.7.1 + github.com/ooni/minivpn v0.0.6 github.com/ooni/netem v0.0.0-20240208095707-608dcbcd82b8 - github.com/ooni/oocrypto v0.6.1 - github.com/ooni/oohttp v0.7.2 - github.com/ooni/probe-assets v0.23.0 + github.com/ooni/oocrypto v0.6.2 + github.com/ooni/oohttp v0.7.3 + github.com/ooni/probe-assets v0.24.0 github.com/pborman/getopt/v2 v2.1.0 github.com/pion/stun v0.6.1 github.com/pkg/errors v0.9.1 github.com/quic-go/quic-go v0.43.1 github.com/rogpeppe/go-internal v1.12.0 - github.com/rubenv/sql-migrate v1.6.1 + github.com/rubenv/sql-migrate v1.7.0 github.com/schollz/progressbar/v3 v3.14.2 - github.com/upper/db/v4 v4.7.0 + github.com/upper/db/v4 v4.8.0 gitlab.com/yawning/obfs4.git v0.0.0-20231012084234-c3e2d44b1033 gitlab.com/yawning/utls.git v0.0.12-1 gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/goptlib v1.5.0 gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake/v2 v2.6.1 - golang.org/x/crypto v0.23.0 + golang.org/x/crypto v0.24.0 golang.org/x/net v0.25.0 - golang.org/x/sys v0.20.0 + golang.org/x/sys v0.21.0 ) require ( @@ -57,6 +58,7 @@ require ( github.com/gaukas/godicttls v0.0.4 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/btree v1.1.2 // indirect github.com/google/pprof v0.0.0-20240509144519-723abb6459b7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -105,13 +107,11 @@ require ( github.com/golang/glog v1.2.1 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/grafov/m3u8 v0.12.0 // indirect - github.com/hashicorp/golang-lru v1.0.2 // indirect - github.com/juju/ratelimit v1.0.2 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/klauspost/reedsolomon v1.11.8 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-sqlite3 v1.14.22 - github.com/oschwald/maxminddb-golang v1.12.0 + github.com/oschwald/maxminddb-golang v1.13.1 github.com/pion/datachannel v1.5.6 // indirect github.com/pion/dtls/v2 v2.2.11 // indirect github.com/pion/ice/v2 v2.3.24 // indirect @@ -145,8 +145,8 @@ require ( github.com/xtaci/smux v1.5.24 // indirect gitlab.com/yawning/bsaes.git v0.0.0-20190805113838-0a714cd429ec // indirect golang.org/x/mod v0.17.0 // indirect - golang.org/x/term v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect - golang.org/x/tools v0.21.0 // indirect + golang.org/x/term v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect google.golang.org/protobuf v1.34.1 // indirect ) diff --git a/go.sum b/go.sum index c13619004..dffa64fd3 100644 --- a/go.sum +++ b/go.sum @@ -22,12 +22,14 @@ github.com/Psiphon-Inc/rotate-safe-writer v0.0.0-20210303140923-464a7a37606e h1: github.com/Psiphon-Inc/rotate-safe-writer v0.0.0-20210303140923-464a7a37606e/go.mod h1:ZdY5pBfat/WVzw3eXbIf7N1nZN0XD5H5+X8ZMDWbCs4= github.com/Psiphon-Labs/bolt v0.0.0-20200624191537-23cedaef7ad7 h1:Hx/NCZTnvoKZuIBwSmxE58KKoNLXIGG6hBJYN7pj9Ag= github.com/Psiphon-Labs/bolt v0.0.0-20200624191537-23cedaef7ad7/go.mod h1:alTtZBo3j4AWFvUrAH6F5ZaHcTj4G5Y01nHz8dkU6vU= +github.com/Psiphon-Labs/consistent v0.0.0-20240322131436-20aaa4e05737 h1:QTMy7Uc2Xc7fz6O/Khy1xi0VBND13GqzLUE2mHw6HUU= +github.com/Psiphon-Labs/consistent v0.0.0-20240322131436-20aaa4e05737/go.mod h1:Enj/Gszv2zCbuRbHbabmNvfO9EM+5kmaGj8CyjwNPlY= github.com/Psiphon-Labs/goptlib v0.0.0-20200406165125-c0e32a7a3464 h1:VmnMMMheFXwLV0noxYhbJbLmkV4iaVW3xNnj6xcCNHo= github.com/Psiphon-Labs/goptlib v0.0.0-20200406165125-c0e32a7a3464/go.mod h1:Pe5BqN2DdIdChorAXl6bDaQd/wghpCleJfid2NoSli0= github.com/Psiphon-Labs/psiphon-tls v0.0.0-20240424193802-52b2602ec60c h1:+SEszyxW7yu+smufzSlAszj/WmOYJ054DJjb5jllulc= github.com/Psiphon-Labs/psiphon-tls v0.0.0-20240424193802-52b2602ec60c/go.mod h1:AaKKoshr8RI1LZTheeNDtNuZ39qNVPWVK4uir2c2XIs= -github.com/Psiphon-Labs/psiphon-tunnel-core v1.0.11-0.20240424194431-3612a5a6fb4c h1:fGq255KuBSpc9Odea+VhbMSbQcW07uWfypbUU0QWjVM= -github.com/Psiphon-Labs/psiphon-tunnel-core v1.0.11-0.20240424194431-3612a5a6fb4c/go.mod h1:M5yXGzsfrz2dPQtdjssbwYNpnWlhAeIDXwX7FrG5uv8= +github.com/Psiphon-Labs/psiphon-tunnel-core v1.0.11-0.20240705153833-eea9ace08cd1 h1:8aJm5EFss3dxEjBpD0DF2B0GDiv30E+pgAvwg3T2Bz0= +github.com/Psiphon-Labs/psiphon-tunnel-core v1.0.11-0.20240705153833-eea9ace08cd1/go.mod h1:OMEMJwRTN9F/nH9K8Q+CO3bwrMYRJXK/CFP6soAkyYg= github.com/Psiphon-Labs/quic-go v0.0.0-20240424181006-45545f5e1536 h1:pM5ex1QufkHV8lDR6Tc1Crk1bW5lYZjrFIJGZNBWE9k= github.com/Psiphon-Labs/quic-go v0.0.0-20240424181006-45545f5e1536/go.mod h1:2MTiPsgoOqWs3Bo6Xr3ElMBX6zzfjd3YkDFpQJLwHdQ= github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI= @@ -49,6 +51,7 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bifurcation/mint v0.0.0-20180306135233-198357931e61 h1:BU+NxuoaYPIvvp8NNkNlLr8aA0utGyuunf4Q3LJ0bh0= github.com/bifurcation/mint v0.0.0-20180306135233-198357931e61/go.mod h1:zVt7zX3K/aDCk9Tj+VM7YymsX66ERvzCJzw8rFCX2JU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -140,8 +143,8 @@ github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= -github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= -github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= @@ -182,6 +185,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= github.com/google/pprof v0.0.0-20240509144519-723abb6459b7 h1:velgFPYr1X9TDwLIfkV7fWqsFlf7TeP11M/7kPd/dVI= github.com/google/pprof v0.0.0-20240509144519-723abb6459b7/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= @@ -219,8 +224,7 @@ github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsU github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgconn v1.14.0/go.mod h1:9mBNlny0UvkgJdCDvdVHYSjI+8tD2rnKK69Wz8ti++E= -github.com/jackc/pgconn v1.14.1/go.mod h1:9mBNlny0UvkgJdCDvdVHYSjI+8tD2rnKK69Wz8ti++E= +github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= @@ -233,19 +237,22 @@ github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvW github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.3.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgtype v1.14.3/go.mod h1:aKeozOde08iifGosdJpz9MBZonJOUJxqNpPBcMJTlVA= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= -github.com/jackc/pgx/v4 v4.18.1/go.mod h1:FydWkUyadDmdNH/mHnGob881GawxeEm7TcMCzkb+qQE= +github.com/jackc/pgx/v4 v4.18.2/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= +github.com/jackc/pgx/v4 v4.18.3/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= @@ -256,8 +263,6 @@ github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2C github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/juju/ratelimit v1.0.2 h1:sRxmtRiajbvrcLQT7S+JbqU0ntsb9W2yhSdNN8tWfaI= -github.com/juju/ratelimit v1.0.2/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= @@ -307,7 +312,6 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mdlayher/netlink v1.4.2-0.20210930205308-a81a8c23d40a h1:yk5OmRew64lWdeNanQ3l0hDgUt1E8MfipPhh/GO9Tuw= @@ -332,18 +336,20 @@ github.com/onsi/ginkgo/v2 v2.17.3/go.mod h1:nP2DPOQoNsQmsVyv5rDA8JkXQoCs6goXIvr/ github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.33.0 h1:snPCflnZrpMsy94p4lXVEkHo12lmPnc3vY5XBbreexE= github.com/onsi/gomega v1.33.0/go.mod h1:+925n5YtiFsLzzafLUHzVMBpvvRAzrydIBiSIxjX3wY= +github.com/ooni/minivpn v0.0.6 h1:pGTsYRtofEupMrJL28f1IXO1LJslSI7dEHxSadNgGik= +github.com/ooni/minivpn v0.0.6/go.mod h1:0KNwmK2Wg9lDbk936XjtxvCq4tPNbK4C3IJvyLwIMrE= github.com/ooni/netem v0.0.0-20240208095707-608dcbcd82b8 h1:kJ2wn19lIP/y9ng85BbFRdWKHK6Er116Bbt5uhqHVD4= github.com/ooni/netem v0.0.0-20240208095707-608dcbcd82b8/go.mod h1:b/wAvTR5n92Vk2b0SBmuMU0xO4ZGVrsXtU7zjTby7vw= -github.com/ooni/oocrypto v0.6.1 h1:D0fGokmHoVKGBy39RxPxK77ov0Ob9Z5pdx4vKA6vpWk= -github.com/ooni/oocrypto v0.6.1/go.mod h1:mGlPZeI3jV1gnVQ3xs5WYNo8IoYlyB/p/x79P58hhog= -github.com/ooni/oohttp v0.7.2 h1:MtZFqq2vd0/ppYTjhKMSO5K6sj2NxVjrO7ldElWtHq4= -github.com/ooni/oohttp v0.7.2/go.mod h1:6mjMEE8uA2wODu93EABmtmbjy0/YuORNNcRY9Dw2ncw= -github.com/ooni/probe-assets v0.23.0 h1:QNagQEgyRpM5tC0yHhUU3x0+L4gjr8p8nzMl+TUI3Vo= -github.com/ooni/probe-assets v0.23.0/go.mod h1:m0k2FFzcLfFm7dhgyYkLCUR3R0CoRPr0jcjctDS2+gU= +github.com/ooni/oocrypto v0.6.2 h1:gAg24bVP03PNsOkMYGxllxmvlKlBOvyHmFAsdBlFJag= +github.com/ooni/oocrypto v0.6.2/go.mod h1:mGlPZeI3jV1gnVQ3xs5WYNo8IoYlyB/p/x79P58hhog= +github.com/ooni/oohttp v0.7.3 h1:/EjyY5kgy9zq68YTSYtjd0rkJv5+/U6bzHM3fYqYHXY= +github.com/ooni/oohttp v0.7.3/go.mod h1:6mjMEE8uA2wODu93EABmtmbjy0/YuORNNcRY9Dw2ncw= +github.com/ooni/probe-assets v0.24.0 h1:9y6bF9PyXrPBHu/RmyRZY8JOXHC6W2ZNRC7kaPcuHHk= +github.com/ooni/probe-assets v0.24.0/go.mod h1:m0k2FFzcLfFm7dhgyYkLCUR3R0CoRPr0jcjctDS2+gU= github.com/oschwald/geoip2-golang v1.9.0 h1:uvD3O6fXAXs+usU+UGExshpdP13GAqp4GBrzN7IgKZc= github.com/oschwald/geoip2-golang v1.9.0/go.mod h1:BHK6TvDyATVQhKNbQBdrj9eAvuwOMi2zSFXizL3K81Y= -github.com/oschwald/maxminddb-golang v1.12.0 h1:9FnTOD0YOhP7DGxGsq4glzpGy5+w7pq50AS6wALUMYs= -github.com/oschwald/maxminddb-golang v1.12.0/go.mod h1:q0Nob5lTCqyQ8WT6FYgS1L7PXKVVbgiymefNwIjPzgY= +github.com/oschwald/maxminddb-golang v1.13.1 h1:G3wwjdN9JmIK2o/ermkHM+98oX5fS+k5MbwsmL4MRQE= +github.com/oschwald/maxminddb-golang v1.13.1/go.mod h1:K4pgV9N/GcK694KSTmVSDTODk4IsCNThNdTmnaBZ/F8= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pborman/getopt/v2 v2.1.0 h1:eNfR+r+dWLdWmV8g5OlpyrTYHkhVNxHBdN2cCrJmOEA= @@ -441,8 +447,8 @@ github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= -github.com/rubenv/sql-migrate v1.6.1 h1:bo6/sjsan9HaXAsNxYP/jCEDUGibHp8JmOBw7NTGRos= -github.com/rubenv/sql-migrate v1.6.1/go.mod h1:tPzespupJS0jacLfhbwto/UjSX+8h2FdWB7ar+QlHa0= +github.com/rubenv/sql-migrate v1.7.0 h1:HtQq1xyTN2ISmQDggnh0c9U3JlP8apWh8YO2jzlXpTI= +github.com/rubenv/sql-migrate v1.7.0/go.mod h1:S4wtDEG1CKn+0ShpTtzWhFpHHI5PvCUtiGI+C+Z2THE= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735 h1:7YvPJVmEeFHR1Tj9sZEYsmarJEQfMVYpd/Vyy/A8dqE= @@ -520,8 +526,8 @@ github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho= github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= -github.com/upper/db/v4 v4.7.0 h1:GNOxFAR8S3r0ITTWUq1LbTvvxipmwgSP4yxSCyJdim4= -github.com/upper/db/v4 v4.7.0/go.mod h1:EO/sQ5p41YroLxv2Z2CIxRBAtEeSG4ZOTksc+KA9VfY= +github.com/upper/db/v4 v4.8.0 h1:EsWvjEV1V024HT1687usnCLgTG0IH+rFYybYIMINsdM= +github.com/upper/db/v4 v4.8.0/go.mod h1:lT4YVnTjXb9dGuo/Ilx8iP/p6M0n6hCwelC42Ns3C9U= github.com/wader/filtertransport v0.0.0-20200316221534-bdd9e61eee78 h1:9sreu9e9KOihf2Y0NbpyfWhd1XFDcL4GTkPYL4IvMrg= github.com/wader/filtertransport v0.0.0-20200316221534-bdd9e61eee78/go.mod h1:HazXTRLhXFyq80TQp7PUXi6BKE6mS+ydEdzEqNBKopQ= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= @@ -576,16 +582,17 @@ golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20181106170214-d68db9428509/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= @@ -601,6 +608,8 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -624,6 +633,7 @@ golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= @@ -635,6 +645,8 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -676,8 +688,10 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -690,8 +704,9 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= -golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -705,8 +720,9 @@ golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -724,8 +740,9 @@ golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= -golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -791,6 +808,7 @@ modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q modernc.org/fileutil v1.1.1/go.mod h1:HdjlliqRHrMAI4nVOvvpYVzVgvRSK7WnoCiG0GUWJNo= modernc.org/fileutil v1.1.2/go.mod h1:HdjlliqRHrMAI4nVOvvpYVzVgvRSK7WnoCiG0GUWJNo= modernc.org/fileutil v1.2.0/go.mod h1:0rLMFc17WSz6Bm/GtHeme7TOX8pNRhFN2NkfBlOZhrQ= +modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= modernc.org/golex v1.0.5/go.mod h1:pTY7KKjdvZbv2ROjfp6FFX5BXMM9QWZEnmCsl60aCfI= modernc.org/golex v1.1.0/go.mod h1:2pVlfqApurXhR1m0N+WDYu6Twnc4QuvO4+U8HnwoiRA= modernc.org/internal v1.0.3/go.mod h1:dvHFQEGEd33HZar0OdSYIm6yen/77eukCqffWSAwQUc= diff --git a/pkg/cmd/buildtool/android_test.go b/pkg/cmd/buildtool/android_test.go index 010889802..f1eae1fd7 100644 --- a/pkg/cmd/buildtool/android_test.go +++ b/pkg/cmd/buildtool/android_test.go @@ -702,12 +702,12 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) { expect: []buildtooltest.ExecExpectations{{ Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.3.0.tar.gz", + "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.3.1.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "openssl-3.3.0.tar.gz", + "tar", "-xf", "openssl-3.3.1.tar.gz", }, }, { Env: []string{}, @@ -757,12 +757,12 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) { }, { Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.3.0.tar.gz", + "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.3.1.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "openssl-3.3.0.tar.gz", + "tar", "-xf", "openssl-3.3.1.tar.gz", }, }, { Env: []string{}, @@ -812,12 +812,12 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) { }, { Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.3.0.tar.gz", + "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.3.1.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "openssl-3.3.0.tar.gz", + "tar", "-xf", "openssl-3.3.1.tar.gz", }, }, { Env: []string{}, @@ -867,12 +867,12 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) { }, { Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.3.0.tar.gz", + "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.3.1.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "openssl-3.3.0.tar.gz", + "tar", "-xf", "openssl-3.3.1.tar.gz", }, }, { Env: []string{}, @@ -1738,12 +1738,12 @@ func TestAndroidBuildCdepsTor(t *testing.T) { expect: []buildtooltest.ExecExpectations{{ Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.11.tar.gz", + "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.12.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "tor-0.4.8.11.tar.gz", + "tar", "-xf", "tor-0.4.8.12.tar.gz", }, }, { Env: []string{}, @@ -1827,12 +1827,12 @@ func TestAndroidBuildCdepsTor(t *testing.T) { }, { Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.11.tar.gz", + "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.12.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "tor-0.4.8.11.tar.gz", + "tar", "-xf", "tor-0.4.8.12.tar.gz", }, }, { Env: []string{}, @@ -1916,12 +1916,12 @@ func TestAndroidBuildCdepsTor(t *testing.T) { }, { Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.11.tar.gz", + "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.12.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "tor-0.4.8.11.tar.gz", + "tar", "-xf", "tor-0.4.8.12.tar.gz", }, }, { Env: []string{}, @@ -2005,12 +2005,12 @@ func TestAndroidBuildCdepsTor(t *testing.T) { }, { Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.11.tar.gz", + "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.12.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "tor-0.4.8.11.tar.gz", + "tar", "-xf", "tor-0.4.8.12.tar.gz", }, }, { Env: []string{}, diff --git a/pkg/cmd/buildtool/cdepsopenssl.go b/pkg/cmd/buildtool/cdepsopenssl.go index 8f24439b1..9cf19f217 100644 --- a/pkg/cmd/buildtool/cdepsopenssl.go +++ b/pkg/cmd/buildtool/cdepsopenssl.go @@ -27,13 +27,13 @@ func cdepsOpenSSLBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependencie defer restore() // See https://github.com/Homebrew/homebrew-core/blob/master/Formula/o/openssl@3.rb - cdepsMustFetch("https://www.openssl.org/source/openssl-3.3.0.tar.gz") + cdepsMustFetch("https://www.openssl.org/source/openssl-3.3.1.tar.gz") deps.VerifySHA256( // must be mockable - "53e66b043322a606abf0087e7699a0e033a37fa13feb9742df35c3a33b18fb02", - "openssl-3.3.0.tar.gz", + "777cd596284c883375a2a7a11bf5d2786fc5413255efab20c50d6ffe6d020b7e", + "openssl-3.3.1.tar.gz", ) - must.Run(log.Log, "tar", "-xf", "openssl-3.3.0.tar.gz") - _ = deps.MustChdir("openssl-3.3.0") // must be mockable + must.Run(log.Log, "tar", "-xf", "openssl-3.3.1.tar.gz") + _ = deps.MustChdir("openssl-3.3.1") // must be mockable mydir := filepath.Join(topdir, "CDEPS", "openssl") for _, patch := range cdepsMustListPatches(mydir) { diff --git a/pkg/cmd/buildtool/cdepstor.go b/pkg/cmd/buildtool/cdepstor.go index 8e7ca852f..85339b6d9 100644 --- a/pkg/cmd/buildtool/cdepstor.go +++ b/pkg/cmd/buildtool/cdepstor.go @@ -27,13 +27,13 @@ func cdepsTorBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependencies) { defer restore() // See https://github.com/Homebrew/homebrew-core/blob/master/Formula/t/tor.rb - cdepsMustFetch("https://www.torproject.org/dist/tor-0.4.8.11.tar.gz") + cdepsMustFetch("https://www.torproject.org/dist/tor-0.4.8.12.tar.gz") deps.VerifySHA256( // must be mockable - "8f2bdf90e63380781235aa7d604e159570f283ecee674670873d8bb7052c8e07", - "tor-0.4.8.11.tar.gz", + "ca7cc735d98e3747b58f2f3cc14f804dd789fa0fb333a84dcb6bd70adbb8c874", + "tor-0.4.8.12.tar.gz", ) - must.Run(log.Log, "tar", "-xf", "tor-0.4.8.11.tar.gz") - _ = deps.MustChdir("tor-0.4.8.11") // must be mockable + must.Run(log.Log, "tar", "-xf", "tor-0.4.8.12.tar.gz") + _ = deps.MustChdir("tor-0.4.8.12") // must be mockable mydir := filepath.Join(topdir, "CDEPS", "tor") for _, patch := range cdepsMustListPatches(mydir) { diff --git a/pkg/cmd/buildtool/ios_test.go b/pkg/cmd/buildtool/ios_test.go index c7eed181f..f1bc989de 100644 --- a/pkg/cmd/buildtool/ios_test.go +++ b/pkg/cmd/buildtool/ios_test.go @@ -349,12 +349,12 @@ func TestIOSBuildCdepsOpenSSL(t *testing.T) { expect: []buildtooltest.ExecExpectations{{ Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.3.0.tar.gz", + "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.3.1.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "openssl-3.3.0.tar.gz", + "tar", "-xf", "openssl-3.3.1.tar.gz", }, }, { Env: []string{}, @@ -399,12 +399,12 @@ func TestIOSBuildCdepsOpenSSL(t *testing.T) { }, { Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.3.0.tar.gz", + "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.3.1.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "openssl-3.3.0.tar.gz", + "tar", "-xf", "openssl-3.3.1.tar.gz", }, }, { Env: []string{}, @@ -449,12 +449,12 @@ func TestIOSBuildCdepsOpenSSL(t *testing.T) { }, { Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.3.0.tar.gz", + "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.3.1.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "openssl-3.3.0.tar.gz", + "tar", "-xf", "openssl-3.3.1.tar.gz", }, }, { Env: []string{}, @@ -1142,12 +1142,12 @@ func TestIOSBuildCdepsTor(t *testing.T) { expect: []buildtooltest.ExecExpectations{{ Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.11.tar.gz", + "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.12.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "tor-0.4.8.11.tar.gz", + "tar", "-xf", "tor-0.4.8.12.tar.gz", }, }, { Env: []string{}, @@ -1232,12 +1232,12 @@ func TestIOSBuildCdepsTor(t *testing.T) { }, { Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.11.tar.gz", + "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.12.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "tor-0.4.8.11.tar.gz", + "tar", "-xf", "tor-0.4.8.12.tar.gz", }, }, { Env: []string{}, @@ -1322,12 +1322,12 @@ func TestIOSBuildCdepsTor(t *testing.T) { }, { Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.11.tar.gz", + "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.12.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "tor-0.4.8.11.tar.gz", + "tar", "-xf", "tor-0.4.8.12.tar.gz", }, }, { Env: []string{}, diff --git a/pkg/cmd/buildtool/linuxcdeps_test.go b/pkg/cmd/buildtool/linuxcdeps_test.go index 58879dd10..50e0f5add 100644 --- a/pkg/cmd/buildtool/linuxcdeps_test.go +++ b/pkg/cmd/buildtool/linuxcdeps_test.go @@ -92,12 +92,12 @@ func TestLinuxCdepsBuildMain(t *testing.T) { expect: []buildtooltest.ExecExpectations{{ Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.3.0.tar.gz", + "curl", "-fsSLO", "https://www.openssl.org/source/openssl-3.3.1.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "openssl-3.3.0.tar.gz", + "tar", "-xf", "openssl-3.3.1.tar.gz", }, }, { Env: []string{}, @@ -322,12 +322,12 @@ func TestLinuxCdepsBuildMain(t *testing.T) { expect: []buildtooltest.ExecExpectations{{ Env: []string{}, Argv: []string{ - "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.11.tar.gz", + "curl", "-fsSLO", "https://www.torproject.org/dist/tor-0.4.8.12.tar.gz", }, }, { Env: []string{}, Argv: []string{ - "tar", "-xf", "tor-0.4.8.11.tar.gz", + "tar", "-xf", "tor-0.4.8.12.tar.gz", }, }, { Env: []string{}, diff --git a/pkg/cmd/miniooni/main.go b/pkg/cmd/miniooni/main.go index 626111802..137cccfbe 100644 --- a/pkg/cmd/miniooni/main.go +++ b/pkg/cmd/miniooni/main.go @@ -26,6 +26,7 @@ import ( // Options contains the options you can set from the CLI. type Options struct { Annotations []string + AuthFile string Emoji bool ExtraOptions []string HomeDir string @@ -228,11 +229,18 @@ func registerOONIRun(rootCmd *cobra.Command, globalOptions *Options) { []string{}, "Path to the OONI Run v2 descriptor to run (may be specified multiple times)", ) + flags.StringVarP( + &globalOptions.AuthFile, + "bearer-token-file", + "", + "", + "Path to a file containing a bearer token for fetching a remote OONI Run v2 descriptor", + ) } // registerAllExperiments registers a subcommand for each experiment func registerAllExperiments(rootCmd *cobra.Command, globalOptions *Options) { - for name, factory := range registry.AllExperiments { + for name, ff := range registry.AllExperiments { subCmd := &cobra.Command{ Use: name, Short: fmt.Sprintf("Runs the %s experiment", name), @@ -243,6 +251,7 @@ func registerAllExperiments(rootCmd *cobra.Command, globalOptions *Options) { } rootCmd.AddCommand(subCmd) flags := subCmd.Flags() + factory := ff() switch factory.InputPolicy() { case model.InputOrQueryBackend, diff --git a/pkg/cmd/miniooni/oonirun.go b/pkg/cmd/miniooni/oonirun.go index c1a4972e9..030c83341 100644 --- a/pkg/cmd/miniooni/oonirun.go +++ b/pkg/cmd/miniooni/oonirun.go @@ -21,6 +21,7 @@ func ooniRunMain(ctx context.Context, logger := sess.Logger() cfg := &oonirun.LinkConfig{ AcceptChanges: currentOptions.Yes, + AuthFile: currentOptions.AuthFile, Annotations: annotations, KVStore: sess.KeyValueStore(), MaxRuntime: currentOptions.MaxRuntime, diff --git a/pkg/engine/experiment.go b/pkg/engine/experiment.go index 6560d9019..827f22cfa 100644 --- a/pkg/engine/experiment.go +++ b/pkg/engine/experiment.go @@ -9,6 +9,7 @@ import ( "errors" "net/http" "runtime" + "sync" "time" "github.com/ooni/probe-engine/pkg/bytecounter" @@ -18,24 +19,58 @@ import ( "github.com/ooni/probe-engine/pkg/version" ) -// experiment implements Experiment. +// experimentMutableReport is the mutable experiment.report field. +// +// We isolate this into a separate data structure to ease code management. By using this +// pattern, we don't need to be concerned with locking mutexes multiple times and it's just +// a matter of using public methods exported by this struct, which are goroutine safe. +type experimentMutableReport struct { + mu sync.Mutex + report probeservices.ReportChannel +} + +// Set atomically sets the report possibly overriding a previously set report. +// +// This method is goroutine safe. +func (emr *experimentMutableReport) Set(report probeservices.ReportChannel) { + emr.mu.Lock() + emr.report = report + emr.mu.Unlock() +} + +// Get atomically gets the report possibly returning nil. +func (emr *experimentMutableReport) Get() (report probeservices.ReportChannel) { + emr.mu.Lock() + report = emr.report + emr.mu.Unlock() + return +} + +// TODO(bassosimone,DecFox): it would be nice if `*experiment` depended +// on an interface rather than depending on the concrete session, because +// that will allow us to write tests using mocks much more easily. + +// experiment implements [model.Experiment]. type experiment struct { byteCounter *bytecounter.Counter callbacks model.ExperimentCallbacks measurer model.ExperimentMeasurer - report probeservices.ReportChannel + mrep *experimentMutableReport session *Session testName string testStartTime string testVersion string } -// newExperiment creates a new experiment given a measurer. +var _ model.Experiment = &experiment{} + +// newExperiment creates a new [*experiment] given a [model.ExperimentMeasurer]. func newExperiment(sess *Session, measurer model.ExperimentMeasurer) *experiment { return &experiment{ byteCounter: bytecounter.New(), callbacks: model.NewPrinterCallbacks(sess.Logger()), measurer: measurer, + mrep: &experimentMutableReport{}, session: sess, testName: measurer.ExperimentName(), testStartTime: model.MeasurementFormatTimeNowUTC(), @@ -43,161 +78,48 @@ func newExperiment(sess *Session, measurer model.ExperimentMeasurer) *experiment } } -// KibiBytesReceived implements Experiment.KibiBytesReceived. +// KibiBytesReceived implements [model.Experiment]. func (e *experiment) KibiBytesReceived() float64 { return e.byteCounter.KibiBytesReceived() } -// KibiBytesSent implements Experiment.KibiBytesSent. +// KibiBytesSent implements [model.Experiment]. func (e *experiment) KibiBytesSent() float64 { return e.byteCounter.KibiBytesSent() } -// Name implements Experiment.Name. +// Name implements [model.Experiment]. func (e *experiment) Name() string { return e.testName } -// ExperimentMeasurementSummaryKeysNotImplemented is the [model.MeasurementSummary] we use when -// the experiment TestKeys do not provide an implementation of [model.MeasurementSummary]. -type ExperimentMeasurementSummaryKeysNotImplemented struct{} - -var _ model.MeasurementSummaryKeys = &ExperimentMeasurementSummaryKeysNotImplemented{} - -// IsAnomaly implements MeasurementSummary. -func (*ExperimentMeasurementSummaryKeysNotImplemented) Anomaly() bool { - return false -} - -// MeasurementSummaryKeys returns the [model.MeasurementSummaryKeys] associated with a given measurement. -func MeasurementSummaryKeys(m *model.Measurement) model.MeasurementSummaryKeys { - if tk, ok := m.TestKeys.(model.MeasurementSummaryKeysProvider); ok { - return tk.MeasurementSummaryKeys() - } - return &ExperimentMeasurementSummaryKeysNotImplemented{} -} - -// ReportID implements Experiment.ReportID. +// ReportID implements [model.Experiment]. func (e *experiment) ReportID() string { - if e.report == nil { + report := e.mrep.Get() + if report == nil { return "" } - return e.report.ReportID() + return report.ReportID() } -// experimentAsyncWrapper makes a sync experiment behave like it was async -type experimentAsyncWrapper struct { - *experiment -} - -var _ model.ExperimentMeasurerAsync = &experimentAsyncWrapper{} - -// RunAsync implements ExperimentMeasurerAsync.RunAsync. -func (eaw *experimentAsyncWrapper) RunAsync( - ctx context.Context, sess model.ExperimentSession, input string, - callbacks model.ExperimentCallbacks) (<-chan *model.ExperimentAsyncTestKeys, error) { - out := make(chan *model.ExperimentAsyncTestKeys) - measurement := eaw.experiment.newMeasurement(input) - start := time.Now() - args := &model.ExperimentArgs{ - Callbacks: eaw.callbacks, - Measurement: measurement, - Session: eaw.session, - } - err := eaw.experiment.measurer.Run(ctx, args) - stop := time.Now() - if err != nil { - return nil, err - } - go func() { - defer close(out) // signal the reader we're done! - out <- &model.ExperimentAsyncTestKeys{ - Extensions: measurement.Extensions, - Input: measurement.Input, - MeasurementRuntime: stop.Sub(start).Seconds(), - TestKeys: measurement.TestKeys, - TestHelpers: measurement.TestHelpers, - } - }() - return out, nil -} - -// MeasureAsync implements Experiment.MeasureAsync. -func (e *experiment) MeasureAsync( - ctx context.Context, input string) (<-chan *model.Measurement, error) { - err := e.session.MaybeLookupLocationContext(ctx) // this already tracks session bytes - if err != nil { - return nil, err - } - ctx = bytecounter.WithSessionByteCounter(ctx, e.session.byteCounter) - ctx = bytecounter.WithExperimentByteCounter(ctx, e.byteCounter) - var async model.ExperimentMeasurerAsync - if v, okay := e.measurer.(model.ExperimentMeasurerAsync); okay { - async = v - } else { - async = &experimentAsyncWrapper{e} - } - in, err := async.RunAsync(ctx, e.session, input, e.callbacks) - if err != nil { - return nil, err - } - out := make(chan *model.Measurement) - go func() { - defer close(out) // we need to signal the consumer we're done - for tk := range in { - measurement := e.newMeasurement(input) - measurement.Extensions = tk.Extensions - measurement.Input = tk.Input - measurement.MeasurementRuntime = tk.MeasurementRuntime - measurement.TestHelpers = tk.TestHelpers - measurement.TestKeys = tk.TestKeys - if err := model.ScrubMeasurement(measurement, e.session.ProbeIP()); err != nil { - // If we fail to scrub the measurement then we are not going to - // submit it. Most likely causes of error here are unlikely, - // e.g., the TestKeys being not serializable. - e.session.Logger().Warnf("can't scrub measurement: %s", err.Error()) - continue - } - out <- measurement - } - }() - return out, nil -} - -// MeasureWithContext implements Experiment.MeasureWithContext. -func (e *experiment) MeasureWithContext( - ctx context.Context, input string, -) (measurement *model.Measurement, err error) { - out, err := e.MeasureAsync(ctx, input) - if err != nil { - return nil, err - } - for m := range out { - if measurement == nil { - measurement = m // as documented just return the first one - } - } - if measurement == nil { - err = errors.New("experiment returned no measurements") - } - return -} - -// SubmitAndUpdateMeasurementContext implements Experiment.SubmitAndUpdateMeasurementContext. +// SubmitAndUpdateMeasurementContext implements [model.Experiment]. func (e *experiment) SubmitAndUpdateMeasurementContext( ctx context.Context, measurement *model.Measurement) error { - if e.report == nil { + report := e.mrep.Get() + if report == nil { return errors.New("report is not open") } - return e.report.SubmitMeasurement(ctx, measurement) + return report.SubmitMeasurement(ctx, measurement) } // newMeasurement creates a new measurement for this experiment with the given input. -func (e *experiment) newMeasurement(input string) *model.Measurement { +func (e *experiment) newMeasurement(target model.ExperimentTarget) *model.Measurement { utctimenow := time.Now().UTC() + m := &model.Measurement{ DataFormatVersion: model.OOAPIReportDefaultDataFormatVersion, - Input: model.MeasurementTarget(input), + Options: target.Options(), + Input: model.MeasurementInput(target.Input()), MeasurementStartTime: utctimenow.Format(model.MeasurementDateFormat), MeasurementStartTimeSaved: utctimenow, ProbeIP: model.DefaultProbeIP, @@ -214,6 +136,7 @@ func (e *experiment) newMeasurement(input string) *model.Measurement { TestStartTime: e.testStartTime, TestVersion: e.testVersion, } + m.AddAnnotation("architecture", runtime.GOARCH) m.AddAnnotation("engine_name", "ooniprobe-engine") m.AddAnnotation("engine_version", version.Version) @@ -223,14 +146,18 @@ func (e *experiment) newMeasurement(input string) *model.Measurement { m.AddAnnotation("vcs_revision", runtimex.BuildInfo.VcsRevision) m.AddAnnotation("vcs_time", runtimex.BuildInfo.VcsTime) m.AddAnnotation("vcs_tool", runtimex.BuildInfo.VcsTool) + return m } // OpenReportContext implements Experiment.OpenReportContext. func (e *experiment) OpenReportContext(ctx context.Context) error { - if e.report != nil { + // handle the case where we already opened the report + report := e.mrep.Get() + if report != nil { return nil // already open } + // use custom client to have proper byte accounting httpClient := &http.Client{ Transport: bytecounter.WrapHTTPTransport( @@ -244,15 +171,108 @@ func (e *experiment) OpenReportContext(ctx context.Context) error { return err } client.HTTPClient = httpClient // patch HTTP client to use + + // create the report template to open the report template := e.newReportTemplate() - e.report, err = client.OpenReport(ctx, template) + + // attempt to open the report + report, err = client.OpenReport(ctx, template) + + // handle the error case if err != nil { e.session.logger.Debugf("experiment: probe services error: %s", err.Error()) return err } + + // on success, assign the new report + e.mrep.Set(report) return nil } +// MeasureWithContext implements [model.Experiment]. +func (e *experiment) MeasureWithContext( + ctx context.Context, target model.ExperimentTarget) (*model.Measurement, error) { + // Here we ensure that we have already looked up the probe location + // information such that we correctly populate the measurement and also + // VERY IMPORTANTLY to scrub the IP address from the measurement. + // + // Also, this SHOULD happen before wrapping the context for byte counting + // since MaybeLookupLocationContext already accounts for bytes I/O. + // + // TODO(bassosimone,DecFox): historically we did this only for measuring + // and not for opening a report, which probably is not correct. Because the + // function call is idempotent, call it also when opening a report? + if err := e.session.MaybeLookupLocationContext(ctx); err != nil { + return nil, err + } + + // Tweak the context such that the bytes sent and received are accounted + // to both the session's byte counter and to the experiment's byte counter. + ctx = bytecounter.WithSessionByteCounter(ctx, e.session.byteCounter) + ctx = bytecounter.WithExperimentByteCounter(ctx, e.byteCounter) + + // Create a new measurement that the experiment measurer will finish filling + // by adding the test keys etc. Please, note that, as of 2024-06-06: + // + // 1. Experiments using richer input receive input via the Target field + // and ignore (*Measurement).Input, which however contains the same value + // that would be returned by the Target.Input method. + // + // 2. Other experiments use (*Measurement).Input. + // + // Here we're passing the whole target to newMeasurement such that we're able + // to record options values in addition to the input value. + measurement := e.newMeasurement(target) + + // Record when we started the experiment, to compute the runtime. + start := time.Now() + + // Prepare the arguments for the experiment measurer. + // + // Only richer-input-aware experiments honour the Target field. + args := &model.ExperimentArgs{ + Callbacks: e.callbacks, + Measurement: measurement, + Session: e.session, + Target: target, + } + + // Invoke the measurer. Conventionally, an error being returned here + // indicates that something went wrong during the measurement. For example, + // it could be that the user provided us with a malformed input. In case + // there's censorship, by all means the experiment should return a nil error + // and fill the measurement accordingly. + err := e.measurer.Run(ctx, args) + + // Record when the experiment finished running. + stop := time.Now() + + // Handle the case where there was a fundamental error. + if err != nil { + return nil, err + } + + // Make sure we record the measurement runtime. + measurement.MeasurementRuntime = stop.Sub(start).Seconds() + + // Scrub the measurement removing the probe IP addr from it. We are 100% sure we know + // our own IP addr, since we called MaybeLookupLocation above. Obviously, we aren't + // going to submit the measurement in case we can't scrub it, so we just return an error + // if this specific corner case happens. + // + // TODO(bassosimone,DecFox): a dual stack client MAY be such that we discover its IPv4 + // address but the IPv6 address is present inside the measurement. We should ensure that + // we improve our discovering capabilities to also cover this specific case. + if err := model.ScrubMeasurement(measurement, e.session.ProbeIP()); err != nil { + e.session.Logger().Warnf("can't scrub measurement: %s", err.Error()) + return nil, err + } + + // We're all good! Let us return the measurement to the caller, which will + // addtionally take care that we're submitting it, if needed. + return measurement, nil +} + func (e *experiment) newReportTemplate() model.OOAPIReportTemplate { return model.OOAPIReportTemplate{ DataFormatVersion: model.OOAPIReportDefaultDataFormatVersion, @@ -266,3 +286,22 @@ func (e *experiment) newReportTemplate() model.OOAPIReportTemplate { TestVersion: e.testVersion, } } + +// ExperimentMeasurementSummaryKeysNotImplemented is the [model.MeasurementSummary] we use when +// the experiment TestKeys do not provide an implementation of [model.MeasurementSummary]. +type ExperimentMeasurementSummaryKeysNotImplemented struct{} + +var _ model.MeasurementSummaryKeys = &ExperimentMeasurementSummaryKeysNotImplemented{} + +// IsAnomaly implements MeasurementSummary. +func (*ExperimentMeasurementSummaryKeysNotImplemented) Anomaly() bool { + return false +} + +// MeasurementSummaryKeys returns the [model.MeasurementSummaryKeys] associated with a given measurement. +func MeasurementSummaryKeys(m *model.Measurement) model.MeasurementSummaryKeys { + if tk, ok := m.TestKeys.(model.MeasurementSummaryKeysProvider); ok { + return tk.MeasurementSummaryKeys() + } + return &ExperimentMeasurementSummaryKeysNotImplemented{} +} diff --git a/pkg/engine/experiment_integration_test.go b/pkg/engine/experiment_integration_test.go index 9bf34c571..9befd4c1a 100644 --- a/pkg/engine/experiment_integration_test.go +++ b/pkg/engine/experiment_integration_test.go @@ -3,6 +3,7 @@ package engine import ( "context" "encoding/json" + "errors" "net/http" "net/http/httptest" "os" @@ -11,6 +12,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/probeservices" "github.com/ooni/probe-engine/pkg/registry" ) @@ -177,7 +179,8 @@ func TestSetCallbacks(t *testing.T) { } register := ®isterCallbacksCalled{} builder.SetCallbacks(register) - if _, err := builder.NewExperiment().MeasureWithContext(context.Background(), ""); err != nil { + target := model.NewOOAPIURLInfoWithDefaultCategoryAndCountry("") + if _, err := builder.NewExperiment().MeasureWithContext(context.Background(), target); err != nil { t.Fatal(err) } if register.onProgressCalled == false { @@ -221,7 +224,8 @@ func TestMeasurementFailure(t *testing.T) { if err := builder.SetOptionAny("ReturnError", true); err != nil { t.Fatal(err) } - measurement, err := builder.NewExperiment().MeasureWithContext(context.Background(), "") + target := model.NewOOAPIURLInfoWithDefaultCategoryAndCountry("") + measurement, err := builder.NewExperiment().MeasureWithContext(context.Background(), target) if err == nil { t.Fatal("expected an error here") } @@ -255,7 +259,8 @@ func runexperimentflow(t *testing.T, experiment model.Experiment, input string) if experiment.ReportID() == "" { t.Fatal("reportID should not be empty here") } - measurement, err := experiment.MeasureWithContext(ctx, input) + target := model.NewOOAPIURLInfoWithDefaultCategoryAndCountry(input) + measurement, err := experiment.MeasureWithContext(ctx, target) if err != nil { t.Fatal(err) } @@ -381,7 +386,7 @@ func TestOpenReportNewClientFailure(t *testing.T) { Type: "antani", } err = exp.OpenReportContext(context.Background()) - if err.Error() != "probe services: unsupported endpoint type" { + if !errors.Is(err, probeservices.ErrUnsupportedServiceType) { t.Fatal(err) } } @@ -413,7 +418,8 @@ func TestMeasureLookupLocationFailure(t *testing.T) { exp := newExperiment(sess, new(antaniMeasurer)) ctx, cancel := context.WithCancel(context.Background()) cancel() // so we fail immediately - if _, err := exp.MeasureWithContext(ctx, "xx"); err == nil { + target := model.NewOOAPIURLInfoWithDefaultCategoryAndCountry("xx") + if _, err := exp.MeasureWithContext(ctx, target); err == nil { t.Fatal("expected an error here") } } diff --git a/pkg/engine/experiment_test.go b/pkg/engine/experiment_test.go index 613b0ab88..0d4613313 100644 --- a/pkg/engine/experiment_test.go +++ b/pkg/engine/experiment_test.go @@ -1,9 +1,14 @@ package engine import ( + "sync" "testing" + "time" + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-engine/pkg/bytecounter" "github.com/ooni/probe-engine/pkg/enginelocate" + "github.com/ooni/probe-engine/pkg/experiment/dnscheck" "github.com/ooni/probe-engine/pkg/experiment/example" "github.com/ooni/probe-engine/pkg/experiment/signal" "github.com/ooni/probe-engine/pkg/model" @@ -17,7 +22,7 @@ func TestExperimentHonoursSharingDefaults(t *testing.T) { t.Fatal(err) } exp := builder.NewExperiment().(*experiment) - return exp.newMeasurement("") + return exp.newMeasurement(model.NewOOAPIURLInfoWithDefaultCategoryAndCountry("")) } type spec struct { name string @@ -105,3 +110,84 @@ func TestExperimentMeasurementSummaryKeys(t *testing.T) { } }) } + +// This test ensures that (*experiment).newMeasurement is working as intended. +func TestExperimentNewMeasurement(t *testing.T) { + // create a session for testing that does not use the network at all + sess := newSessionForTestingNoLookups(t) + + // create a conventional time for starting the experiment + t0 := time.Date(2024, 6, 27, 10, 33, 0, 0, time.UTC) + + // create the experiment + exp := &experiment{ + byteCounter: bytecounter.New(), + callbacks: model.NewPrinterCallbacks(model.DiscardLogger), + measurer: &dnscheck.Measurer{}, + mrep: &experimentMutableReport{ + mu: sync.Mutex{}, + report: nil, + }, + session: sess, + testName: "dnscheck", + testStartTime: t0.Format(model.MeasurementDateFormat), + testVersion: "0.1.0", + } + + // create the richer input target + target := &dnscheck.Target{ + Config: &dnscheck.Config{ + DefaultAddrs: "8.8.8.8 2001:4860:4860::8888", + HTTP3Enabled: true, + }, + URL: "https://dns.google/dns-query", + } + + // create measurement + meas := exp.newMeasurement(target) + + // make sure the input is correctly serialized + t.Run("Input", func(t *testing.T) { + if meas.Input != "https://dns.google/dns-query" { + t.Fatal("unexpected meas.Input") + } + }) + + // make sure the options are correctly serialized + t.Run("Options", func(t *testing.T) { + expectOptions := []string{`DefaultAddrs=8.8.8.8 2001:4860:4860::8888`, `HTTP3Enabled=true`} + if diff := cmp.Diff(expectOptions, meas.Options); diff != "" { + t.Fatal(diff) + } + }) + + // make sure we've got the expected annotation keys + t.Run("Annotations", func(t *testing.T) { + const ( + expected = 1 << iota + got + ) + m := map[string]int{ + "architecture": expected, + "engine_name": expected, + "engine_version": expected, + "go_version": expected, + "platform": expected, + "vcs_modified": expected, + "vcs_revision": expected, + "vcs_time": expected, + "vcs_tool": expected, + } + for key := range meas.Annotations { + m[key] |= got + } + for key, value := range m { + if value != expected|got { + t.Fatal("expected", expected|got, "for", key, "got", value) + } + } + }) + + // TODO(bassosimone,DecFox): this is the correct place where to + // add more tests regarding how we create measurements. +} diff --git a/pkg/engine/experimentbuilder.go b/pkg/engine/experimentbuilder.go index 7a8f4affa..aff87763e 100644 --- a/pkg/engine/experimentbuilder.go +++ b/pkg/engine/experimentbuilder.go @@ -5,11 +5,19 @@ package engine // import ( + "encoding/json" + "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/registry" ) -// experimentBuilder implements ExperimentBuilder. +// TODO(bassosimone,DecFox): we should eventually finish merging the code in +// file with the code inside the ./internal/registry package. +// +// If there's time, this could happen at the end of the current (as of 2024-06-27) +// richer input work, otherwise any time in the future is actually fine. + +// experimentBuilder implements [model.ExperimentBuilder]. // // This type is now just a tiny wrapper around registry.Factory. type experimentBuilder struct { @@ -22,37 +30,44 @@ type experimentBuilder struct { session *Session } -// Interruptible implements ExperimentBuilder.Interruptible. +var _ model.ExperimentBuilder = &experimentBuilder{} + +// Interruptible implements [model.ExperimentBuilder]. func (b *experimentBuilder) Interruptible() bool { return b.factory.Interruptible() } -// InputPolicy implements ExperimentBuilder.InputPolicy. +// InputPolicy implements [model.ExperimentBuilder]. func (b *experimentBuilder) InputPolicy() model.InputPolicy { return b.factory.InputPolicy() } -// Options implements ExperimentBuilder.Options. +// Options implements [model.ExperimentBuilder]. func (b *experimentBuilder) Options() (map[string]model.ExperimentOptionInfo, error) { return b.factory.Options() } -// SetOptionAny implements ExperimentBuilder.SetOptionAny. +// SetOptionAny implements [model.ExperimentBuilder]. func (b *experimentBuilder) SetOptionAny(key string, value any) error { return b.factory.SetOptionAny(key, value) } -// SetOptionsAny implements ExperimentBuilder.SetOptionsAny. +// SetOptionsAny implements [model.ExperimentBuilder]. func (b *experimentBuilder) SetOptionsAny(options map[string]any) error { return b.factory.SetOptionsAny(options) } -// SetCallbacks implements ExperimentBuilder.SetCallbacks. +// SetOptionsJSON implements [model.ExperimentBuilder]. +func (b *experimentBuilder) SetOptionsJSON(value json.RawMessage) error { + return b.factory.SetOptionsJSON(value) +} + +// SetCallbacks implements [model.ExperimentBuilder]. func (b *experimentBuilder) SetCallbacks(callbacks model.ExperimentCallbacks) { b.callbacks = callbacks } -// NewExperiment creates the experiment +// NewExperiment creates a new [model.Experiment] instance. func (b *experimentBuilder) NewExperiment() model.Experiment { measurer := b.factory.NewExperimentMeasurer() experiment := newExperiment(b.session, measurer) @@ -60,7 +75,12 @@ func (b *experimentBuilder) NewExperiment() model.Experiment { return experiment } -// newExperimentBuilder creates a new experimentBuilder instance. +// NewTargetLoader creates a new [model.ExperimentTargetLoader] instance. +func (b *experimentBuilder) NewTargetLoader(config *model.ExperimentTargetLoaderConfig) model.ExperimentTargetLoader { + return b.factory.NewTargetLoader(config) +} + +// newExperimentBuilder creates a new [*experimentBuilder] instance. func newExperimentBuilder(session *Session, name string) (*experimentBuilder, error) { factory, err := registry.NewFactory(name, session.kvStore, session.logger) if err != nil { diff --git a/pkg/engine/experimentbuilder_test.go b/pkg/engine/experimentbuilder_test.go index 00a22ef62..30ab72bd6 100644 --- a/pkg/engine/experimentbuilder_test.go +++ b/pkg/engine/experimentbuilder_test.go @@ -1 +1,169 @@ package engine + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-engine/pkg/model" +) + +func TestExperimentBuilderEngineWebConnectivity(t *testing.T) { + // create a session for testing that does not use the network at all + sess := newSessionForTestingNoLookups(t) + + // create an experiment builder for Web Connectivity + builder, err := sess.NewExperimentBuilder("WebConnectivity") + if err != nil { + t.Fatal(err) + } + + // create suitable loader config + config := &model.ExperimentTargetLoaderConfig{ + CheckInConfig: &model.OOAPICheckInConfig{ + // nothing + }, + Session: sess, + StaticInputs: nil, + SourceFiles: nil, + } + + // create the loader + loader := builder.NewTargetLoader(config) + + // create cancelled context to interrupt immediately so that we + // don't use the network when running this test + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + // attempt to load targets + targets, err := loader.Load(ctx) + + // make sure we've got the expected error + if !errors.Is(err, context.Canceled) { + t.Fatal("unexpected err", err) + } + + // make sure there are no targets + if len(targets) != 0 { + t.Fatal("expected zero length targets") + } +} + +func TestExperimentBuilderBasicOperations(t *testing.T) { + // create a session for testing that does not use the network at all + sess := newSessionForTestingNoLookups(t) + + // create an experiment builder for example + builder, err := sess.NewExperimentBuilder("example") + if err != nil { + t.Fatal(err) + } + + // example should be interruptible + t.Run("Interruptible", func(t *testing.T) { + if !builder.Interruptible() { + t.Fatal("example should be interruptible") + } + }) + + // we expect to see the InputNone input policy + t.Run("InputPolicy", func(t *testing.T) { + if builder.InputPolicy() != model.InputNone { + t.Fatal("unexpectyed input policy") + } + }) + + // get the options and check whether they are what we expect + t.Run("Options", func(t *testing.T) { + options, err := builder.Options() + if err != nil { + t.Fatal(err) + } + expectOptions := map[string]model.ExperimentOptionInfo{ + "Message": {Doc: "Message to emit at test completion", Type: "string", Value: "Good day from the example experiment!"}, + "ReturnError": {Doc: "Toogle to return a mocked error", Type: "bool", Value: false}, + "SleepTime": {Doc: "Amount of time to sleep for in nanosecond", Type: "int64", Value: int64(1000000000)}, + } + if diff := cmp.Diff(expectOptions, options); diff != "" { + t.Fatal(diff) + } + }) + + // we can set a specific existing option + t.Run("SetOptionAny", func(t *testing.T) { + if err := builder.SetOptionAny("Message", "foobar"); err != nil { + t.Fatal(err) + } + options, err := builder.Options() + if err != nil { + t.Fatal(err) + } + expectOptions := map[string]model.ExperimentOptionInfo{ + "Message": {Doc: "Message to emit at test completion", Type: "string", Value: "foobar"}, + "ReturnError": {Doc: "Toogle to return a mocked error", Type: "bool", Value: false}, + "SleepTime": {Doc: "Amount of time to sleep for in nanosecond", Type: "int64", Value: int64(1000000000)}, + } + if diff := cmp.Diff(expectOptions, options); diff != "" { + t.Fatal(diff) + } + }) + + // we can set all options at the same time + t.Run("SetOptions", func(t *testing.T) { + inputs := map[string]any{ + "Message": "foobar", + "ReturnError": true, + } + if err := builder.SetOptionsAny(inputs); err != nil { + t.Fatal(err) + } + options, err := builder.Options() + if err != nil { + t.Fatal(err) + } + expectOptions := map[string]model.ExperimentOptionInfo{ + "Message": {Doc: "Message to emit at test completion", Type: "string", Value: "foobar"}, + "ReturnError": {Doc: "Toogle to return a mocked error", Type: "bool", Value: true}, + "SleepTime": {Doc: "Amount of time to sleep for in nanosecond", Type: "int64", Value: int64(1000000000)}, + } + if diff := cmp.Diff(expectOptions, options); diff != "" { + t.Fatal(diff) + } + }) + + // we can set all options using JSON + t.Run("SetOptionsJSON", func(t *testing.T) { + inputs := json.RawMessage(`{ + "Message": "foobar", + "ReturnError": true + }`) + if err := builder.SetOptionsJSON(inputs); err != nil { + t.Fatal(err) + } + options, err := builder.Options() + if err != nil { + t.Fatal(err) + } + expectOptions := map[string]model.ExperimentOptionInfo{ + "Message": {Doc: "Message to emit at test completion", Type: "string", Value: "foobar"}, + "ReturnError": {Doc: "Toogle to return a mocked error", Type: "bool", Value: true}, + "SleepTime": {Doc: "Amount of time to sleep for in nanosecond", Type: "int64", Value: int64(1000000000)}, + } + if diff := cmp.Diff(expectOptions, options); diff != "" { + t.Fatal(diff) + } + }) + + // TODO(bassosimone): we could possibly add more checks here. I am not doing this + // right now, because https://github.com/ooni/probe-cli/pull/1629 mostly cares about + // providing input and the rest of the codebase did not change. + // + // Also, it would make sense to eventually merge experimentbuilder.go with the + // ./internal/registry package, which also has coverage. + // + // In conclusion, our main objective for now is to make sure we don't screw the + // pooch when setting options using the experiment builder. +} diff --git a/pkg/engine/inputloader_network_test.go b/pkg/engine/inputloader_integration_test.go similarity index 52% rename from pkg/engine/inputloader_network_test.go rename to pkg/engine/inputloader_integration_test.go index c79356757..aaa81728d 100644 --- a/pkg/engine/inputloader_network_test.go +++ b/pkg/engine/inputloader_integration_test.go @@ -8,15 +8,22 @@ import ( "github.com/ooni/probe-engine/pkg/engine" "github.com/ooni/probe-engine/pkg/kvstore" "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/targetloading" ) -func TestInputLoaderInputOrQueryBackendWithNoInput(t *testing.T) { +// This historical integration test ensures that we're able to fetch URLs from +// the dev infrastructure. We say this test's historical because the targetloading.Loader +// belonged to the engine package before we introduced richer input. It kind of feels +// good to keep this integration test here since we want to use a real session and a real +// Loader and double check whether we can get inputs. In a more distant future it would +// kind of make sense to have a broader package with this kind of integration tests. +func TestTargetLoaderInputOrQueryBackendWithNoInput(t *testing.T) { if testing.Short() { t.Skip("skip test in short mode") } sess, err := engine.NewSession(context.Background(), engine.SessionConfig{ AvailableProbeServices: []model.OOAPIService{{ - Address: "https://ams-pg-test.ooni.org/", + Address: "https://backend-hel.ooni.org/", Type: "https", }}, KVStore: &kvstore.Memory{}, @@ -29,12 +36,14 @@ func TestInputLoaderInputOrQueryBackendWithNoInput(t *testing.T) { t.Fatal(err) } defer sess.Close() - il := &engine.InputLoader{ + il := &targetloading.Loader{ InputPolicy: model.InputOrQueryBackend, Session: sess, } ctx := context.Background() out, err := il.Load(ctx) + // TODO(decfox): it seems `backend-hel.ooni.org` returns a different response + // than intended which is why the test fails. if err != nil { t.Fatal(err) } diff --git a/pkg/engine/session.go b/pkg/engine/session.go index 1f4c14662..449ec1fc9 100644 --- a/pkg/engine/session.go +++ b/pkg/engine/session.go @@ -368,9 +368,44 @@ func (s *Session) FetchTorTargets( if err != nil { return nil, err } + + // TODO(bassosimone,DecFox): here we could also lock the mutex + // or we should consider using the same strategy we used for the + // experiments, where we separated mutable state into dedicated types. return clnt.FetchTorTargets(ctx, cc) } +// FetchOpenVPNConfig fetches openvpn config from the API if it's not found in the +// internal cache. We do this to avoid hitting the API for every input. +func (s *Session) FetchOpenVPNConfig( + ctx context.Context, provider, cc string) (*model.OOAPIVPNProviderConfig, error) { + clnt, err := s.newOrchestraClient(ctx) + if err != nil { + return nil, err + } + + // ensure that we have fetched the location before fetching openvpn configuration. + if err := s.MaybeLookupLocationContext(ctx); err != nil { + return nil, err + } + + // IMPORTANT! + // + // We cannot lock earlier because newOrchestraClient and + // MaybeLookupLocation both lock the mutex. + // + // TODO(bassosimone,DecFox): we should consider using the same strategy we used for the + // experiments, where we separated mutable state into dedicated types. + defer s.mu.Unlock() + s.mu.Lock() + + config, err := clnt.FetchOpenVPNConfig(ctx, provider, cc) + if err != nil { + return nil, err + } + return &config, nil +} + // KeyValueStore returns the configured key-value store. func (s *Session) KeyValueStore() model.KeyValueStore { return s.kvStore @@ -643,8 +678,8 @@ func (s *Session) MaybeLookupBackendsContext(ctx context.Context) error { if selected == nil { return ErrAllProbeServicesFailed } - s.logger.Infof("session: using probe services: %+v", selected.Endpoint) - s.selectedProbeService = &selected.Endpoint + s.logger.Infof("session: using probe services: %+v", selected.Service) + s.selectedProbeService = &selected.Service s.availableTestHelpers = selected.TestHelpers return nil } diff --git a/pkg/engine/session_integration_test.go b/pkg/engine/session_integration_test.go index c25d513bf..d1307ff0b 100644 --- a/pkg/engine/session_integration_test.go +++ b/pkg/engine/session_integration_test.go @@ -94,7 +94,7 @@ func TestSessionTorArgsTorBinary(t *testing.T) { } sess, err := NewSession(context.Background(), SessionConfig{ AvailableProbeServices: []model.OOAPIService{{ - Address: "https://ams-pg-test.ooni.org", + Address: "https://backend-hel.ooni.org", Type: "https", }}, Logger: model.DiscardLogger, @@ -126,7 +126,7 @@ func TestSessionTorArgsTorBinary(t *testing.T) { func newSessionForTestingNoLookupsWithProxyURL(t *testing.T, URL *url.URL) *Session { sess, err := NewSession(context.Background(), SessionConfig{ AvailableProbeServices: []model.OOAPIService{{ - Address: "https://ams-pg-test.ooni.org", + Address: "https://backend-hel.ooni.org", Type: "https", }}, Logger: model.DiscardLogger, @@ -179,7 +179,7 @@ func TestInitOrchestraClientMaybeRegisterError(t *testing.T) { sess := newSessionForTestingNoLookups(t) defer sess.Close() clnt, err := probeservices.NewClient(sess, model.OOAPIService{ - Address: "https://ams-pg-test.ooni.org/", + Address: "https://backend-hel.ooni.org/", Type: "https", }) if err != nil { @@ -204,7 +204,7 @@ func TestInitOrchestraClientMaybeLoginError(t *testing.T) { sess := newSessionForTestingNoLookups(t) defer sess.Close() clnt, err := probeservices.NewClient(sess, model.OOAPIService{ - Address: "https://ams-pg-test.ooni.org/", + Address: "https://backend-hel.ooni.org/", Type: "https", }) if err != nil { @@ -483,7 +483,7 @@ func TestNewOrchestraClientProbeServicesNewClientFailure(t *testing.T) { svc.Type = "antani" // should really not be supported for a long time } client, err := sess.newOrchestraClient(context.Background()) - if !errors.Is(err, probeservices.ErrUnsupportedEndpoint) { + if !errors.Is(err, probeservices.ErrUnsupportedServiceType) { t.Fatal("not the error we expected") } if client != nil { diff --git a/pkg/engine/session_internal_test.go b/pkg/engine/session_internal_test.go index a2ea161ca..56088cc8b 100644 --- a/pkg/engine/session_internal_test.go +++ b/pkg/engine/session_internal_test.go @@ -253,6 +253,19 @@ func TestSessionMaybeLookupLocationContextLookupLocationContextFailure(t *testin } } +func TestSessionFetchOpenVPNConfigWithCancelledContext(t *testing.T) { + sess := &Session{} + ctx, cancel := context.WithCancel(context.Background()) + cancel() // cause failure + resp, err := sess.FetchOpenVPNConfig(ctx, "riseup", "XX") + if !errors.Is(err, context.Canceled) { + t.Fatal("not the error we expected", err) + } + if resp != nil { + t.Fatal("expected nil response here") + } +} + func TestSessionFetchTorTargetsWithCancelledContext(t *testing.T) { sess := &Session{} ctx, cancel := context.WithCancel(context.Background()) diff --git a/pkg/experiment/dnscheck/dnscheck.go b/pkg/experiment/dnscheck/dnscheck.go index 0107094b1..619a03183 100644 --- a/pkg/experiment/dnscheck/dnscheck.go +++ b/pkg/experiment/dnscheck/dnscheck.go @@ -19,6 +19,7 @@ import ( "github.com/ooni/probe-engine/pkg/legacy/tracex" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/runtimex" + "github.com/ooni/probe-engine/pkg/targetloading" ) const ( @@ -96,7 +97,6 @@ type TestKeys struct { // Measurer performs the measurement. type Measurer struct { - Config Endpoints *Endpoints } @@ -114,7 +114,8 @@ func (m *Measurer) ExperimentVersion() string { // errors are in addition to any other errors returned by the low level packages // that are used by this experiment to implement its functionality. var ( - ErrInputRequired = errors.New("this experiment needs input") + ErrInputRequired = targetloading.ErrInputRequired + ErrInvalidInputType = targetloading.ErrInvalidInputType ErrInvalidURL = errors.New("the input URL is invalid") ErrUnsupportedURLScheme = errors.New("unsupported URL scheme") ) @@ -125,6 +126,17 @@ func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { measurement := args.Measurement sess := args.Session + // 0. obtain the richer input target, config, and input or fail + if args.Target == nil { + return ErrInputRequired + } + target, ok := args.Target.(*Target) + if !ok { + return ErrInvalidInputType + } + config, input := target.Config, target.URL + sess.Logger().Infof("dnscheck: using richer input: %+v %+v", config, input) + // 1. fill the measurement with test keys tk := new(TestKeys) tk.Lookups = make(map[string]urlgetter.TestKeys) @@ -133,20 +145,19 @@ func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { // 2. select the domain to resolve or use default and, while there, also // ensure that we register all the other options we're using. - domain := m.Config.Domain + domain := config.Domain if domain == "" { domain = defaultDomain } - tk.DefaultAddrs = m.Config.DefaultAddrs + tk.DefaultAddrs = config.DefaultAddrs tk.Domain = domain - tk.HTTP3Enabled = m.Config.HTTP3Enabled - tk.HTTPHost = m.Config.HTTPHost - tk.TLSServerName = m.Config.TLSServerName - tk.TLSVersion = m.Config.TLSVersion + tk.HTTP3Enabled = config.HTTP3Enabled + tk.HTTPHost = config.HTTPHost + tk.TLSServerName = config.TLSServerName + tk.TLSVersion = config.TLSVersion tk.Residual = m.Endpoints != nil // 3. parse the input URL describing the resolver to use - input := string(measurement.Input) if input == "" { return ErrInputRequired } @@ -191,7 +202,7 @@ func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { for _, addr := range addrs { allAddrs[addr] = true } - for _, addr := range strings.Split(m.Config.DefaultAddrs, " ") { + for _, addr := range strings.Split(config.DefaultAddrs, " ") { if addr != "" { allAddrs[addr] = true } @@ -208,10 +219,10 @@ func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { for addr := range allAddrs { inputs = append(inputs, urlgetter.MultiInput{ Config: urlgetter.Config{ - DNSHTTPHost: m.httpHost(URL.Host), - DNSTLSServerName: m.tlsServerName(URL.Hostname()), - DNSTLSVersion: m.Config.TLSVersion, - HTTP3Enabled: m.Config.HTTP3Enabled, + DNSHTTPHost: config.httpHost(URL.Host), + DNSTLSServerName: config.tlsServerName(URL.Hostname()), + DNSTLSVersion: config.TLSVersion, + HTTP3Enabled: config.HTTP3Enabled, RejectDNSBogons: true, // bogons are errors in this context ResolverURL: makeResolverURL(URL, addr), Timeout: 15 * time.Second, @@ -244,17 +255,17 @@ func (m *Measurer) lookupHost(ctx context.Context, hostname string, r model.Reso // httpHost returns the configured HTTP host, if set, otherwise // it will return the host provide as argument. -func (m *Measurer) httpHost(httpHost string) string { - if m.Config.HTTPHost != "" { - return m.Config.HTTPHost +func (c *Config) httpHost(httpHost string) string { + if c.HTTPHost != "" { + return c.HTTPHost } return httpHost } // tlsServerName is like httpHost for the TLS server name. -func (m *Measurer) tlsServerName(tlsServerName string) string { - if m.Config.TLSServerName != "" { - return m.Config.TLSServerName +func (c *Config) tlsServerName(tlsServerName string) string { + if c.TLSServerName != "" { + return c.TLSServerName } return tlsServerName } @@ -311,9 +322,8 @@ func makeResolverURL(URL *url.URL, addr string) string { } // NewExperimentMeasurer creates a new ExperimentMeasurer. -func NewExperimentMeasurer(config Config) model.ExperimentMeasurer { +func NewExperimentMeasurer() model.ExperimentMeasurer { return &Measurer{ - Config: config, Endpoints: nil, // disabled by default } } diff --git a/pkg/experiment/dnscheck/dnscheck_test.go b/pkg/experiment/dnscheck/dnscheck_test.go index 80d472a9a..1519094b6 100644 --- a/pkg/experiment/dnscheck/dnscheck_test.go +++ b/pkg/experiment/dnscheck/dnscheck_test.go @@ -8,44 +8,40 @@ import ( "time" "github.com/apex/log" - "github.com/ooni/probe-engine/pkg/legacy/mockable" + "github.com/ooni/probe-engine/pkg/mocks" "github.com/ooni/probe-engine/pkg/model" ) func TestHTTPHostWithOverride(t *testing.T) { - m := Measurer{Config: Config{HTTPHost: "antani"}} - result := m.httpHost("mascetti") - if result != "antani" { + c := &Config{HTTPHost: "antani"} + if result := c.httpHost("mascetti"); result != "antani" { t.Fatal("not the result we expected") } } func TestHTTPHostWithoutOverride(t *testing.T) { - m := Measurer{Config: Config{}} - result := m.httpHost("mascetti") - if result != "mascetti" { + c := &Config{} + if result := c.httpHost("mascetti"); result != "mascetti" { t.Fatal("not the result we expected") } } func TestTLSServerNameWithOverride(t *testing.T) { - m := Measurer{Config: Config{TLSServerName: "antani"}} - result := m.tlsServerName("mascetti") - if result != "antani" { + c := &Config{TLSServerName: "antani"} + if result := c.tlsServerName("mascetti"); result != "antani" { t.Fatal("not the result we expected") } } func TestTLSServerNameWithoutOverride(t *testing.T) { - m := Measurer{Config: Config{}} - result := m.tlsServerName("mascetti") - if result != "mascetti" { + c := &Config{} + if result := c.tlsServerName("mascetti"); result != "mascetti" { t.Fatal("not the result we expected") } } func TestExperimentNameAndVersion(t *testing.T) { - measurer := NewExperimentMeasurer(Config{Domain: "example.com"}) + measurer := NewExperimentMeasurer() if measurer.ExperimentName() != "dnscheck" { t.Error("unexpected experiment name") } @@ -54,12 +50,32 @@ func TestExperimentNameAndVersion(t *testing.T) { } } +func TestDNSCheckFailsWithInvalidInputType(t *testing.T) { + measurer := NewExperimentMeasurer() + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: new(model.Measurement), + Session: newsession(), + Target: &model.OOAPIURLInfo{}, // not the expected input type + } + err := measurer.Run(context.Background(), args) + if !errors.Is(err, ErrInvalidInputType) { + t.Fatal("expected invalid-input-type error") + } +} + func TestDNSCheckFailsWithoutInput(t *testing.T) { - measurer := NewExperimentMeasurer(Config{Domain: "example.com"}) + measurer := NewExperimentMeasurer() args := &model.ExperimentArgs{ Callbacks: model.NewPrinterCallbacks(log.Log), Measurement: new(model.Measurement), Session: newsession(), + Target: &Target{ + URL: "", // explicitly empty + Config: &Config{ + Domain: "example.com", + }, + }, } err := measurer.Run(context.Background(), args) if !errors.Is(err, ErrInputRequired) { @@ -68,11 +84,15 @@ func TestDNSCheckFailsWithoutInput(t *testing.T) { } func TestDNSCheckFailsWithInvalidURL(t *testing.T) { - measurer := NewExperimentMeasurer(Config{}) + measurer := NewExperimentMeasurer() args := &model.ExperimentArgs{ Callbacks: model.NewPrinterCallbacks(log.Log), Measurement: &model.Measurement{Input: "Not a valid URL \x7f"}, Session: newsession(), + Target: &Target{ + URL: "Not a valid URL \x7f", + Config: &Config{}, + }, } err := measurer.Run(context.Background(), args) if !errors.Is(err, ErrInvalidURL) { @@ -81,11 +101,15 @@ func TestDNSCheckFailsWithInvalidURL(t *testing.T) { } func TestDNSCheckFailsWithUnsupportedProtocol(t *testing.T) { - measurer := NewExperimentMeasurer(Config{}) + measurer := NewExperimentMeasurer() args := &model.ExperimentArgs{ Callbacks: model.NewPrinterCallbacks(log.Log), Measurement: &model.Measurement{Input: "file://1.1.1.1"}, Session: newsession(), + Target: &Target{ + URL: "file://1.1.1.1", + Config: &Config{}, + }, } err := measurer.Run(context.Background(), args) if !errors.Is(err, ErrUnsupportedURLScheme) { @@ -96,14 +120,18 @@ func TestDNSCheckFailsWithUnsupportedProtocol(t *testing.T) { func TestWithCancelledContext(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() // immediately cancel the context - measurer := NewExperimentMeasurer(Config{ - DefaultAddrs: "1.1.1.1 1.0.0.1", - }) + measurer := NewExperimentMeasurer() measurement := &model.Measurement{Input: "dot://one.one.one.one"} args := &model.ExperimentArgs{ Callbacks: model.NewPrinterCallbacks(log.Log), Measurement: measurement, Session: newsession(), + Target: &Target{ + URL: "dot://one.one.one.one", + Config: &Config{ + DefaultAddrs: "1.1.1.1 1.0.0.1", + }, + }, } err := measurer.Run(ctx, args) if err != nil { @@ -111,6 +139,21 @@ func TestWithCancelledContext(t *testing.T) { } } +func TestDNSCheckFailsWithNilTarget(t *testing.T) { + measurer := NewExperimentMeasurer() + measurement := &model.Measurement{Input: "dot://one.one.one.one"} + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: newsession(), + Target: nil, // explicitly nil + } + err := measurer.Run(context.Background(), args) + if !errors.Is(err, ErrInputRequired) { + t.Fatal("unexpected err", err) + } +} + func TestMakeResolverURL(t *testing.T) { // test address substitution addr := "255.255.255.0" @@ -140,14 +183,18 @@ func TestDNSCheckValid(t *testing.T) { t.Skip("skip test in short mode") } - measurer := NewExperimentMeasurer(Config{ - DefaultAddrs: "1.1.1.1 1.0.0.1", - }) + measurer := NewExperimentMeasurer() measurement := model.Measurement{Input: "dot://one.one.one.one:853"} args := &model.ExperimentArgs{ Callbacks: model.NewPrinterCallbacks(log.Log), Measurement: &measurement, Session: newsession(), + Target: &Target{ + URL: "dot://one.one.one.one:853", + Config: &Config{ + DefaultAddrs: "1.1.1.1 1.0.0.1", + }, + }, } err := measurer.Run(context.Background(), args) if err != nil { @@ -169,7 +216,11 @@ func TestDNSCheckValid(t *testing.T) { } func newsession() model.ExperimentSession { - return &mockable.Session{MockableLogger: log.Log} + return &mocks.Session{ + MockLogger: func() model.Logger { + return log.Log + }, + } } func TestDNSCheckWait(t *testing.T) { @@ -182,11 +233,15 @@ func TestDNSCheckWait(t *testing.T) { } measurer := &Measurer{Endpoints: endpoints} run := func(input string) { - measurement := model.Measurement{Input: model.MeasurementTarget(input)} + measurement := model.Measurement{Input: model.MeasurementInput(input)} args := &model.ExperimentArgs{ Callbacks: model.NewPrinterCallbacks(log.Log), Measurement: &measurement, Session: newsession(), + Target: &Target{ + URL: input, + Config: &Config{}, + }, } err := measurer.Run(context.Background(), args) if err != nil { diff --git a/pkg/experiment/dnscheck/richerinput.go b/pkg/experiment/dnscheck/richerinput.go new file mode 100644 index 000000000..9ab8e8ded --- /dev/null +++ b/pkg/experiment/dnscheck/richerinput.go @@ -0,0 +1,129 @@ +package dnscheck + +import ( + "context" + + "github.com/ooni/probe-engine/pkg/experimentconfig" + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/reflectx" + "github.com/ooni/probe-engine/pkg/targetloading" +) + +// Target is a richer-input target that this experiment should measure. +type Target struct { + // Config contains the configuration. + Config *Config + + // URL is the input URL. + URL string +} + +var _ model.ExperimentTarget = &Target{} + +// Category implements [model.ExperimentTarget]. +func (t *Target) Category() string { + return model.DefaultCategoryCode +} + +// Country implements [model.ExperimentTarget]. +func (t *Target) Country() string { + return model.DefaultCountryCode +} + +// Input implements [model.ExperimentTarget]. +func (t *Target) Input() string { + return t.URL +} + +// Options implements [model.ExperimentTarget]. +func (t *Target) Options() []string { + return experimentconfig.DefaultOptionsSerializer(t.Config) +} + +// String implements [model.ExperimentTarget]. +func (t *Target) String() string { + return t.URL +} + +// NewLoader constructs a new [model.ExperimentTargerLoader] instance. +// +// This function PANICS if options is not an instance of [*dnscheck.Config]. +func NewLoader(loader *targetloading.Loader, gopts any) model.ExperimentTargetLoader { + // Panic if we cannot convert the options to the expected type. + // + // We do not expect a panic here because the type is managed by the registry package. + options := gopts.(*Config) + + // Construct the proper loader instance. + return &targetLoader{ + defaultInput: defaultInput, + loader: loader, + options: options, + } +} + +// targetLoader loads targets for this experiment. +type targetLoader struct { + defaultInput []model.ExperimentTarget + loader *targetloading.Loader + options *Config +} + +// Load implements model.ExperimentTargetLoader. +func (tl *targetLoader) Load(ctx context.Context) ([]model.ExperimentTarget, error) { + // If inputs and files are all empty and there are no options, let's use the backend + if len(tl.loader.StaticInputs) <= 0 && len(tl.loader.SourceFiles) <= 0 && + reflectx.StructOrStructPtrIsZero(tl.options) { + return tl.loadFromBackend(ctx) + } + + // Otherwise, attempt to load the static inputs from CLI and files + inputs, err := targetloading.LoadStatic(tl.loader) + + // Handle the case where we couldn't + if err != nil { + return nil, err + } + + // Build the list of targets that we should measure. + var targets []model.ExperimentTarget + for _, input := range inputs { + targets = append(targets, &Target{ + Config: tl.options, + URL: input, + }) + } + return targets, nil +} + +var defaultInput = []model.ExperimentTarget{ + // + // https://dns.google/dns-query + // + // Measure HTTP/3 first and then HTTP/2 (see https://github.com/ooni/probe/issues/2675). + // + // Make sure we include the typical IP addresses for the domain. + // + &Target{ + URL: "https://dns.google/dns-query", + Config: &Config{ + HTTP3Enabled: true, + DefaultAddrs: "8.8.8.8 8.8.4.4", + }, + }, + &Target{ + URL: "https://dns.google/dns-query", + Config: &Config{ + DefaultAddrs: "8.8.8.8 8.8.4.4", + }, + }, + + // TODO(bassosimone,DecFox): before releasing, we need to either sync up + // this list with ./internal/targetloader or implement a backend API. +} + +func (tl *targetLoader) loadFromBackend(_ context.Context) ([]model.ExperimentTarget, error) { + // TODO(https://github.com/ooni/probe/issues/1390): serve DNSCheck + // inputs using richer input (aka check-in v2). + return defaultInput, nil +} diff --git a/pkg/experiment/dnscheck/richerinput_test.go b/pkg/experiment/dnscheck/richerinput_test.go new file mode 100644 index 000000000..318668bce --- /dev/null +++ b/pkg/experiment/dnscheck/richerinput_test.go @@ -0,0 +1,325 @@ +package dnscheck + +import ( + "context" + "errors" + "io/fs" + "path/filepath" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-engine/pkg/mocks" + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/targetloading" +) + +func TestTarget(t *testing.T) { + target := &Target{ + URL: "https://dns.google/dns-query", + Config: &Config{ + DefaultAddrs: "8.8.8.8 8.8.4.4", + Domain: "example.com", + HTTP3Enabled: false, + HTTPHost: "dns.google", + TLSServerName: "dns.google.com", + TLSVersion: "TLSv1.3", + }, + } + + t.Run("Category", func(t *testing.T) { + if target.Category() != model.DefaultCategoryCode { + t.Fatal("invalid Category") + } + }) + + t.Run("Country", func(t *testing.T) { + if target.Country() != model.DefaultCountryCode { + t.Fatal("invalid Country") + } + }) + + t.Run("Input", func(t *testing.T) { + if target.Input() != "https://dns.google/dns-query" { + t.Fatal("invalid Input") + } + }) + + t.Run("Options", func(t *testing.T) { + expect := []string{ + "DefaultAddrs=8.8.8.8 8.8.4.4", + "Domain=example.com", + "HTTPHost=dns.google", + "TLSServerName=dns.google.com", + "TLSVersion=TLSv1.3", + } + if diff := cmp.Diff(expect, target.Options()); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("String", func(t *testing.T) { + if target.String() != "https://dns.google/dns-query" { + t.Fatal("invalid String") + } + }) +} + +func TestNewLoader(t *testing.T) { + // create the pointers we expect to see + child := &targetloading.Loader{} + options := &Config{} + + // create the loader and cast it to its private type + loader := NewLoader(child, options).(*targetLoader) + + // make sure the default input is okay + if diff := cmp.Diff(defaultInput, loader.defaultInput); diff != "" { + t.Fatal(diff) + } + + // make sure the loader is okay + if child != loader.loader { + t.Fatal("invalid loader pointer") + } + + // make sure the options are okay + if options != loader.options { + t.Fatal("invalid options pointer") + } +} + +// testDefaultInput is the default input used by [TestTargetLoaderLoad]. +var testDefaultInput = []model.ExperimentTarget{ + &Target{ + URL: "https://dns.google/dns-query", + Config: &Config{ + HTTP3Enabled: true, + DefaultAddrs: "8.8.8.8 8.8.4.4", + }, + }, + &Target{ + URL: "https://dns.google/dns-query", + Config: &Config{ + DefaultAddrs: "8.8.8.8 8.8.4.4", + }, + }, +} + +func TestTargetLoaderLoad(t *testing.T) { + // testcase is a test case implemented by this function + type testcase struct { + // name is the test case name + name string + + // options contains the options to use + options *Config + + // loader is the loader to use + loader *targetloading.Loader + + // expectErr is the error we expect + expectErr error + + // expectResults contains the expected results + expectTargets []model.ExperimentTarget + } + + cases := []testcase{ + + { + name: "with options, inputs, and files", + options: &Config{ + DefaultAddrs: "1.1.1.1 1.0.0.1", + }, + loader: &targetloading.Loader{ + CheckInConfig: &model.OOAPICheckInConfig{ /* nothing */ }, + ExperimentName: "dnscheck", + InputPolicy: model.InputOrStaticDefault, + Logger: model.DiscardLogger, + Session: &mocks.Session{}, + StaticInputs: []string{ + "https://dns.cloudflare.com/dns-query", + "https://one.one.one.one/dns-query", + }, + SourceFiles: []string{ + filepath.Join("testdata", "input.txt"), + }, + }, + expectErr: nil, + expectTargets: []model.ExperimentTarget{ + &Target{ + URL: "https://dns.cloudflare.com/dns-query", + Config: &Config{ + DefaultAddrs: "1.1.1.1 1.0.0.1", + }, + }, + &Target{ + URL: "https://one.one.one.one/dns-query", + Config: &Config{ + DefaultAddrs: "1.1.1.1 1.0.0.1", + }, + }, + &Target{ + URL: "https://1dot1dot1dot1dot.com/dns-query", + Config: &Config{ + DefaultAddrs: "1.1.1.1 1.0.0.1", + }, + }, + &Target{ + URL: "https://dns.cloudflare/dns-query", + Config: &Config{ + DefaultAddrs: "1.1.1.1 1.0.0.1", + }, + }, + }, + }, + + { + name: "with an unreadable file", + options: &Config{ + DefaultAddrs: "1.1.1.1 1.0.0.1", + }, + loader: &targetloading.Loader{ + CheckInConfig: &model.OOAPICheckInConfig{ /* nothing */ }, + ExperimentName: "dnscheck", + InputPolicy: model.InputOrStaticDefault, + Logger: model.DiscardLogger, + Session: &mocks.Session{}, + StaticInputs: []string{ + "https://dns.cloudflare.com/dns-query", + "https://one.one.one.one/dns-query", + }, + SourceFiles: []string{ + filepath.Join("testdata", "nonexistent.txt"), + }, + }, + expectErr: fs.ErrNotExist, + expectTargets: nil, + }, + + { + name: "with just inputs", + options: &Config{}, + loader: &targetloading.Loader{ + CheckInConfig: &model.OOAPICheckInConfig{ /* nothing */ }, + ExperimentName: "dnscheck", + InputPolicy: model.InputOrStaticDefault, + Logger: model.DiscardLogger, + Session: &mocks.Session{}, + StaticInputs: []string{ + "https://dns.cloudflare.com/dns-query", + "https://one.one.one.one/dns-query", + }, + SourceFiles: []string{}, + }, + expectErr: nil, + expectTargets: []model.ExperimentTarget{ + &Target{ + URL: "https://dns.cloudflare.com/dns-query", + Config: &Config{}, + }, + &Target{ + URL: "https://one.one.one.one/dns-query", + Config: &Config{}, + }, + }, + }, + + { + name: "with just files", + options: &Config{}, + loader: &targetloading.Loader{ + CheckInConfig: &model.OOAPICheckInConfig{ /* nothing */ }, + ExperimentName: "dnscheck", + InputPolicy: model.InputOrStaticDefault, + Logger: model.DiscardLogger, + Session: &mocks.Session{}, + StaticInputs: []string{}, + SourceFiles: []string{ + filepath.Join("testdata", "input.txt"), + }, + }, + expectErr: nil, + expectTargets: []model.ExperimentTarget{ + &Target{ + URL: "https://1dot1dot1dot1dot.com/dns-query", + Config: &Config{}, + }, + &Target{ + URL: "https://dns.cloudflare/dns-query", + Config: &Config{}, + }, + }, + }, + + { + name: "with just options", + options: &Config{ + DefaultAddrs: "1.1.1.1 1.0.0.1", + }, + loader: &targetloading.Loader{ + CheckInConfig: &model.OOAPICheckInConfig{ /* nothing */ }, + ExperimentName: "dnscheck", + InputPolicy: model.InputOrStaticDefault, + Logger: model.DiscardLogger, + Session: &mocks.Session{}, + StaticInputs: []string{}, + SourceFiles: []string{}, + }, + expectErr: nil, + expectTargets: nil, + }, + + { + name: "with no options, not inputs, no files", + options: &Config{}, + loader: &targetloading.Loader{ + CheckInConfig: &model.OOAPICheckInConfig{ /* nothing */ }, + ExperimentName: "dnscheck", + InputPolicy: model.InputOrStaticDefault, + Logger: model.DiscardLogger, + Session: &mocks.Session{}, + StaticInputs: []string{}, + SourceFiles: []string{}, + }, + expectErr: nil, + expectTargets: testDefaultInput, + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // create a target loader using the given config + // + // note that we use a default test input for results predictability + // since the static list may change over time + tl := &targetLoader{ + defaultInput: testDefaultInput, + loader: tc.loader, + options: tc.options, + } + + // load targets + targets, err := tl.Load(context.Background()) + + // make sure error is consistent + switch { + case err == nil && tc.expectErr == nil: + // fallthrough + + case err != nil && tc.expectErr != nil: + if !errors.Is(err, tc.expectErr) { + t.Fatal("unexpected error", err) + } + // fallthrough + + default: + t.Fatal("expected", tc.expectErr, "got", err) + } + + // make sure the targets are consistent + if diff := cmp.Diff(tc.expectTargets, targets); diff != "" { + t.Fatal(diff) + } + }) + } +} diff --git a/pkg/experiment/dnscheck/testdata/input.txt b/pkg/experiment/dnscheck/testdata/input.txt new file mode 100644 index 000000000..42a1896f1 --- /dev/null +++ b/pkg/experiment/dnscheck/testdata/input.txt @@ -0,0 +1,2 @@ +https://1dot1dot1dot1dot.com/dns-query +https://dns.cloudflare/dns-query diff --git a/pkg/experiment/dnsping/dnsping_test.go b/pkg/experiment/dnsping/dnsping_test.go index 0becfc93a..a404cf698 100644 --- a/pkg/experiment/dnsping/dnsping_test.go +++ b/pkg/experiment/dnsping/dnsping_test.go @@ -56,7 +56,7 @@ func TestMeasurer_run(t *testing.T) { } ctx := context.Background() meas := &model.Measurement{ - Input: model.MeasurementTarget(input), + Input: model.MeasurementInput(input), } sess := &mocks.Session{ MockLogger: func() model.Logger { return model.DiscardLogger }, diff --git a/pkg/experiment/example/example.go b/pkg/experiment/example/example.go index 69cf7be22..bf5487761 100644 --- a/pkg/experiment/example/example.go +++ b/pkg/experiment/example/example.go @@ -14,6 +14,8 @@ import ( const testVersion = "0.1.0" +const testName = "example" + // Config contains the experiment config. // // This contains all the settings that user can set to modify the behaviour @@ -22,7 +24,7 @@ const testVersion = "0.1.0" type Config struct { Message string `ooni:"Message to emit at test completion"` ReturnError bool `ooni:"Toogle to return a mocked error"` - SleepTime int64 `ooni:"Amount of time to sleep for"` + SleepTime int64 `ooni:"Amount of time to sleep for in nanosecond"` } // TestKeys contains the experiment's result. @@ -38,13 +40,12 @@ type TestKeys struct { // Measurer performs the measurement. type Measurer struct { - config Config - testName string + config Config } // ExperimentName implements model.ExperimentMeasurer.ExperimentName. func (m Measurer) ExperimentName() string { - return m.testName + return testName } // ExperimentVersion implements model.ExperimentMeasurer.ExperimentVersion. @@ -81,6 +82,6 @@ func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { } // NewExperimentMeasurer creates a new ExperimentMeasurer. -func NewExperimentMeasurer(config Config, testName string) model.ExperimentMeasurer { - return Measurer{config: config, testName: testName} +func NewExperimentMeasurer(config Config) model.ExperimentMeasurer { + return Measurer{config: config} } diff --git a/pkg/experiment/example/example_test.go b/pkg/experiment/example/example_test.go index 4d64d88d7..4afd3c7fb 100644 --- a/pkg/experiment/example/example_test.go +++ b/pkg/experiment/example/example_test.go @@ -8,14 +8,14 @@ import ( "github.com/apex/log" "github.com/ooni/probe-engine/pkg/experiment/example" - "github.com/ooni/probe-engine/pkg/legacy/mockable" + "github.com/ooni/probe-engine/pkg/mocks" "github.com/ooni/probe-engine/pkg/model" ) func TestSuccess(t *testing.T) { m := example.NewExperimentMeasurer(example.Config{ SleepTime: int64(2 * time.Millisecond), - }, "example") + }) if m.ExperimentName() != "example" { t.Fatal("invalid ExperimentName") } @@ -23,7 +23,11 @@ func TestSuccess(t *testing.T) { t.Fatal("invalid ExperimentVersion") } ctx := context.Background() - sess := &mockable.Session{MockableLogger: log.Log} + sess := &mocks.Session{ + MockLogger: func() model.Logger { + return log.Log + }, + } callbacks := model.NewPrinterCallbacks(sess.Logger()) measurement := new(model.Measurement) args := &model.ExperimentArgs{ @@ -41,9 +45,13 @@ func TestFailure(t *testing.T) { m := example.NewExperimentMeasurer(example.Config{ SleepTime: int64(2 * time.Millisecond), ReturnError: true, - }, "example") + }) ctx := context.Background() - sess := &mockable.Session{MockableLogger: log.Log} + sess := &mocks.Session{ + MockLogger: func() model.Logger { + return log.Log + }, + } callbacks := model.NewPrinterCallbacks(sess.Logger()) args := &model.ExperimentArgs{ Callbacks: callbacks, diff --git a/pkg/experiment/openvpn/endpoint.go b/pkg/experiment/openvpn/endpoint.go new file mode 100644 index 000000000..331f60e3c --- /dev/null +++ b/pkg/experiment/openvpn/endpoint.go @@ -0,0 +1,199 @@ +package openvpn + +import ( + "fmt" + "net" + "net/url" + "slices" + "strings" + + vpnconfig "github.com/ooni/minivpn/pkg/config" + vpntracex "github.com/ooni/minivpn/pkg/tracex" + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/targetloading" +) + +var ( + ErrInputRequired = targetloading.ErrInputRequired + ErrInvalidInput = targetloading.ErrInvalidInput +) + +// endpoint is a single endpoint to be probed. +// The information contained in here is not sufficient to complete a connection: +// we need to augment it with more info, as cipher selection or obfuscating proxy credentials. +type endpoint struct { + // IPAddr is the IP Address for this endpoint. + IPAddr string + + // Obfuscation is any obfuscation method use to connect to this endpoint. + // Valid values are: obfs4, none. + Obfuscation string + + // Port is the Port for this endpoint. + Port string + + // Protocol is the tunneling protocol (openvpn, openvpn+obfs4). + Protocol string + + // Provider is a unique label identifying the provider maintaining this endpoint. + Provider string + + // Transport is the underlying transport used for this endpoint. Valid transports are `tcp` and `udp`. + Transport string +} + +// newEndpointFromInputString constructs an endpoint after parsing an input string. +// +// The input URI is in the form: +// "openvpn://provider.corp/?address=1.2.3.4:1194&transport=udp +// "openvpn+obfs4://provider.corp/address=1.2.3.4:1194?&cert=deadbeef&iat=0" +func newEndpointFromInputString(uri string) (*endpoint, error) { + if uri == "" { + return nil, ErrInputRequired + } + parsedURL, err := url.Parse(uri) + if err != nil { + return nil, fmt.Errorf("%w: %s", ErrInvalidInput, err) + } + var obfuscation string + switch parsedURL.Scheme { + case "openvpn": + obfuscation = "none" + case "openvpn+obfs4": + obfuscation = "obfs4" + default: + return nil, fmt.Errorf("%w: unknown scheme: %s", ErrInvalidInput, parsedURL.Scheme) + } + + provider := strings.TrimSuffix(parsedURL.Hostname(), ".corp") + if provider == "" { + return nil, fmt.Errorf("%w: expected provider as host: %s", ErrInvalidInput, parsedURL.Host) + } + if !isValidProvider(provider) { + return nil, fmt.Errorf("%w: unknown provider: %s", ErrInvalidInput, provider) + } + + params := parsedURL.Query() + + transport := params.Get("transport") + if transport != "tcp" && transport != "udp" { + return nil, fmt.Errorf("%w: invalid transport: %s", ErrInvalidInput, transport) + } + + address := params.Get("address") + if address == "" { + return nil, fmt.Errorf("%w: please specify an address as part of the input", ErrInvalidInput) + } + ip, port, err := net.SplitHostPort(address) + if err != nil { + return nil, fmt.Errorf("%w: cannot split ip:port", ErrInvalidInput) + } + if parsedIP := net.ParseIP(ip); parsedIP == nil { + return nil, fmt.Errorf("%w: bad ip", ErrInvalidInput) + } + + endpoint := &endpoint{ + IPAddr: ip, + Port: port, + Obfuscation: obfuscation, + Protocol: "openvpn", + Provider: provider, + Transport: transport, + } + return endpoint, nil +} + +// String implements [fmt.Stringer]. This is a compact representation of the endpoint, +// which differs from the input URI scheme. This is the canonical representation, that can be used +// to deterministically slice a list of endpoints, sort them lexicographically, etc. +func (e *endpoint) String() string { + var proto string + if e.Obfuscation == "obfs4" { + proto = e.Protocol + "+obfs4" + } else { + proto = e.Protocol + } + url := &url.URL{ + Scheme: proto, + Host: net.JoinHostPort(e.IPAddr, e.Port), + Path: e.Transport, + } + return url.String() +} + +// AsInputURI is a string representation of this endpoint, as used in the experiment input URI format. +func (e *endpoint) AsInputURI() string { + var proto string + if e.Obfuscation == "obfs4" { + proto = e.Protocol + "+obfs4" + } else { + proto = e.Protocol + } + + provider := e.Provider + if provider == "" { + provider = "unknown" + } + + values := map[string][]string{ + "address": {net.JoinHostPort(e.IPAddr, e.Port)}, + "transport": {e.Transport}, + } + + url := &url.URL{ + Scheme: proto, + Host: provider + ".corp", + RawQuery: url.Values(values).Encode(), + } + return url.String() +} + +// APIEnabledProviders is the list of providers that the stable API Endpoint and/or this +// experiment knows about. +var APIEnabledProviders = []string{ + "riseupvpn", + "oonivpn", +} + +// isValidProvider returns true if the provider is found as key in the array of [APIEnabledProviders]. +func isValidProvider(provider string) bool { + return slices.Contains(APIEnabledProviders, provider) +} + +// newOpenVPNConfig returns a properly configured [*vpnconfig.Config] object for the given endpoint. +// To obtain that, we merge the endpoint specific configuration with the options passed as richer input targets. +func newOpenVPNConfig( + tracer *vpntracex.Tracer, + logger model.Logger, + endpoint *endpoint, + config *Config) (*vpnconfig.Config, error) { + + provider := endpoint.Provider + if !isValidProvider(provider) { + return nil, fmt.Errorf("%w: unknown provider: %s", ErrInvalidInput, provider) + } + + cfg := vpnconfig.NewConfig( + vpnconfig.WithLogger(logger), + vpnconfig.WithOpenVPNOptions( + &vpnconfig.OpenVPNOptions{ + // endpoint-specific options. + Remote: endpoint.IPAddr, + Port: endpoint.Port, + Proto: vpnconfig.Proto(endpoint.Transport), + Compress: vpnconfig.Compression(config.Compress), + + // options and credentials come from the experiment + // richer input targets. + Cipher: config.Cipher, + Auth: config.Auth, + CA: []byte(config.SafeCA), + Cert: []byte(config.SafeCert), + Key: []byte(config.SafeKey), + }, + ), + vpnconfig.WithHandshakeTracer(tracer), + ) + + return cfg, nil +} diff --git a/pkg/experiment/openvpn/endpoint_test.go b/pkg/experiment/openvpn/endpoint_test.go new file mode 100644 index 000000000..bfd3eb602 --- /dev/null +++ b/pkg/experiment/openvpn/endpoint_test.go @@ -0,0 +1,351 @@ +package openvpn + +import ( + "errors" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + vpntracex "github.com/ooni/minivpn/pkg/tracex" +) + +func Test_newEndpointFromInputString(t *testing.T) { + type args struct { + uri string + } + tests := []struct { + name string + args args + want *endpoint + wantErr error + }{ + { + name: "empty input returns error", + args: args{""}, + want: nil, + wantErr: ErrInputRequired, + }, + { + name: "invalid protocol returns error", + args: args{"bad://foo.bar"}, + want: nil, + wantErr: ErrInvalidInput, + }, + { + name: "uri with illegal chars returns error", + args: args{"openvpn://\x7f/#"}, + want: nil, + wantErr: ErrInvalidInput, + }, + { + name: "valid input uri returns good endpoint", + args: args{"openvpn://riseupvpn.corp/?address=1.1.1.1:1194&transport=tcp"}, + want: &endpoint{ + IPAddr: "1.1.1.1", + Obfuscation: "none", + Port: "1194", + Protocol: "openvpn", + Provider: "riseupvpn", + Transport: "tcp", + }, + wantErr: nil, + }, + { + name: "bad url fails", + args: args{"://address=1.1.1.1:1194&transport=tcp"}, + want: nil, + wantErr: ErrInvalidInput, + }, + { + name: "openvpn+obfs4 does not fail", + args: args{"openvpn+obfs4://riseupvpn.corp/?address=1.1.1.1:1194&transport=tcp"}, + want: &endpoint{ + IPAddr: "1.1.1.1", + Obfuscation: "obfs4", + Port: "1194", + Protocol: "openvpn", + Provider: "riseupvpn", + Transport: "tcp", + }, + wantErr: nil, + }, + { + name: "unknown proto fails", + args: args{"unknown://riseupvpn.corp/?address=1.1.1.1:1194&transport=tcp"}, + want: nil, + wantErr: ErrInvalidInput, + }, + { + name: "any tld other than .corp fails", + args: args{"openvpn://riseupvpn.org/?address=1.1.1.1:1194&transport=tcp"}, + want: nil, + wantErr: ErrInvalidInput, + }, + { + name: "empty provider fails", + args: args{"openvpn://.corp/?address=1.1.1.1:1194&transport=tcp"}, + want: nil, + wantErr: ErrInvalidInput, + }, + { + name: "non-registered provider fails", + args: args{"openvpn://nsavpn.corp/?address=1.1.1.1:1194&transport=tcp"}, + want: nil, + wantErr: ErrInvalidInput, + }, + { + name: "endpoint with invalid ipv4 fails", + args: args{"openvpn://riseupvpn.corp/?address=example.com:1194&transport=tcp"}, + want: nil, + wantErr: ErrInvalidInput, + }, + { + name: "endpoint with no port fails", + args: args{"openvpn://riseupvpn.corp/?address=1.1.1.1&transport=tcp"}, + want: nil, + wantErr: ErrInvalidInput, + }, + { + name: "endpoint with empty transport fails", + args: args{"openvpn://riseupvpn.corp/?address=1.1.1.1:1194&transport="}, + want: nil, + wantErr: ErrInvalidInput, + }, + { + name: "endpoint with no transport fails", + args: args{"openvpn://riseupvpn.corp/?address=1.1.1.1:1194"}, + want: nil, + wantErr: ErrInvalidInput, + }, + { + name: "endpoint with unknown transport fails", + args: args{"openvpn://riseupvpn.corp/?address=1.1.1.1:1194&transport=uh"}, + want: nil, + wantErr: ErrInvalidInput, + }, + { + name: "endpoint with no address fails", + args: args{"openvpn://riseupvpn.corp/?transport=tcp"}, + want: nil, + wantErr: ErrInvalidInput, + }, + { + name: "endpoint with empty address fails", + args: args{"openvpn://riseupvpn.corp/?address=&transport=tcp"}, + want: nil, + wantErr: ErrInvalidInput, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := newEndpointFromInputString(tt.args.uri) + if !errors.Is(err, tt.wantErr) { + t.Errorf("newEndpointFromInputString() error = %v, wantErr %v", err, tt.wantErr) + return + } + if diff := cmp.Diff(got, tt.want); diff != "" { + t.Errorf(diff) + } + }) + } +} + +func Test_EndpointToInputURI(t *testing.T) { + type args struct { + endpoint endpoint + } + tests := []struct { + name string + args args + want string + }{ + { + name: "good endpoint with plain openvpn", + args: args{ + endpoint{ + IPAddr: "1.1.1.1", + Obfuscation: "none", + Port: "443", + Protocol: "openvpn", + Provider: "shady", + Transport: "udp", + }, + }, + want: "openvpn://shady.corp?address=1.1.1.1%3A443&transport=udp", + }, + { + name: "good endpoint with openvpn+obfs4", + args: args{ + endpoint{ + IPAddr: "1.1.1.1", + Obfuscation: "obfs4", + Port: "443", + Protocol: "openvpn", + Provider: "shady", + Transport: "udp", + }, + }, + want: "openvpn+obfs4://shady.corp?address=1.1.1.1%3A443&transport=udp", + }, + { + name: "empty provider is marked as unknown", + args: args{ + endpoint{ + IPAddr: "1.1.1.1", + Obfuscation: "obfs4", + Port: "443", + Protocol: "openvpn", + Provider: "", + Transport: "udp", + }, + }, + want: "openvpn+obfs4://unknown.corp?address=1.1.1.1%3A443&transport=udp", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.args.endpoint.AsInputURI(); cmp.Diff(got, tt.want) != "" { + fmt.Println("GOT", got) + t.Error(cmp.Diff(got, tt.want)) + } + }) + } +} + +func Test_endpoint_String(t *testing.T) { + type fields struct { + IPAddr string + Obfuscation string + Port string + Protocol string + Provider string + Transport string + } + tests := []struct { + name string + fields fields + want string + }{ + { + name: "well formed endpoint returns a well formed endpoint string", + fields: fields{ + IPAddr: "1.1.1.1", + Obfuscation: "none", + Port: "1194", + Protocol: "openvpn", + Provider: "unknown", + Transport: "tcp", + }, + want: "openvpn://1.1.1.1:1194/tcp", + }, + { + name: "well formed endpoint, openvpn+obfs4", + fields: fields{ + IPAddr: "1.1.1.1", + Obfuscation: "obfs4", + Port: "1194", + Protocol: "openvpn", + Provider: "unknown", + Transport: "tcp", + }, + want: "openvpn+obfs4://1.1.1.1:1194/tcp", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := &endpoint{ + IPAddr: tt.fields.IPAddr, + Obfuscation: tt.fields.Obfuscation, + Port: tt.fields.Port, + Protocol: tt.fields.Protocol, + Provider: tt.fields.Provider, + Transport: tt.fields.Transport, + } + if got := e.String(); got != tt.want { + t.Errorf("endpoint.String() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_isValidProvider(t *testing.T) { + if valid := isValidProvider("riseupvpn"); !valid { + t.Fatal("riseup is the only valid provider now") + } + if valid := isValidProvider("nsa"); valid { + t.Fatal("nsa will never be a provider") + } +} + +func Test_newVPNConfig(t *testing.T) { + tracer := vpntracex.NewTracer(time.Now()) + e := &endpoint{ + Provider: "riseupvpn", + IPAddr: "1.1.1.1", + Port: "443", + Transport: "udp", + } + + config := &Config{ + Auth: "SHA512", + Cipher: "AES-256-GCM", + SafeCA: "ca", + SafeCert: "cert", + SafeKey: "key", + } + + cfg, err := newOpenVPNConfig(tracer, nil, e, config) + if err != nil { + t.Fatalf("did not expect error, got: %v", err) + } + if cfg.Tracer() != tracer { + t.Fatal("config tracer is not what passed") + } + if auth := cfg.OpenVPNOptions().Auth; auth != "SHA512" { + t.Errorf("expected auth %s, got %s", "SHA512", auth) + } + if cipher := cfg.OpenVPNOptions().Cipher; cipher != "AES-256-GCM" { + t.Errorf("expected cipher %s, got %s", "AES-256-GCM", cipher) + } + if remote := cfg.OpenVPNOptions().Remote; remote != e.IPAddr { + t.Errorf("expected remote %s, got %s", e.IPAddr, remote) + } + if port := cfg.OpenVPNOptions().Port; port != e.Port { + t.Errorf("expected port %s, got %s", e.Port, port) + } + if transport := cfg.OpenVPNOptions().Proto; string(transport) != e.Transport { + t.Errorf("expected transport %s, got %s", e.Transport, transport) + } + if transport := cfg.OpenVPNOptions().Proto; string(transport) != e.Transport { + t.Errorf("expected transport %s, got %s", e.Transport, transport) + } + if diff := cmp.Diff(cfg.OpenVPNOptions().CA, []byte(config.SafeCA)); diff != "" { + t.Error(diff) + } + if diff := cmp.Diff(cfg.OpenVPNOptions().Cert, []byte(config.SafeCert)); diff != "" { + t.Error(diff) + } + if diff := cmp.Diff(cfg.OpenVPNOptions().Key, []byte(config.SafeKey)); diff != "" { + t.Error(diff) + } +} + +func Test_mergeOpenVPNConfig_with_unknown_provider(t *testing.T) { + tracer := vpntracex.NewTracer(time.Now()) + e := &endpoint{ + Provider: "nsa", + IPAddr: "1.1.1.1", + Port: "443", + Transport: "udp", + } + cfg := &Config{ + SafeCA: "ca", + SafeCert: "cert", + SafeKey: "key", + } + _, err := newOpenVPNConfig(tracer, nil, e, cfg) + if !errors.Is(err, ErrInvalidInput) { + t.Fatalf("expected invalid input error, got: %v", err) + } +} diff --git a/pkg/experiment/openvpn/openvpn.go b/pkg/experiment/openvpn/openvpn.go new file mode 100644 index 000000000..9dc719386 --- /dev/null +++ b/pkg/experiment/openvpn/openvpn.go @@ -0,0 +1,271 @@ +// Package openvpn contains a generic openvpn experiment. +package openvpn + +import ( + "context" + "strconv" + "time" + + "github.com/ooni/probe-engine/pkg/measurexlite" + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/targetloading" + + vpnconfig "github.com/ooni/minivpn/pkg/config" + vpntracex "github.com/ooni/minivpn/pkg/tracex" + "github.com/ooni/minivpn/pkg/tunnel" +) + +const ( + testName = "openvpn" + testVersion = "0.1.5" + openVPNProtocol = "openvpn" +) + +var ( + ErrInvalidInputType = targetloading.ErrInvalidInputType +) + +// Config contains the experiment config. +// +// This contains all the settings that user can set to modify the behaviour +// of this experiment. By tagging these variables with `ooni:"..."`, we allow +// miniooni's -O flag to find them and set them. +// TODO(ainghazal): do pass Auth, Cipher and Compress to OpenVPN config options. +type Config struct { + Auth string `ooni:"OpenVPN authentication to use"` + Cipher string `ooni:"OpenVPN cipher to use"` + Compress string `ooni:"OpenVPN compression to use"` + Provider string `ooni:"VPN provider"` + Obfuscation string `ooni:"Obfuscation to use (obfs4, none)"` + SafeKey string `ooni:"key to connect to the OpenVPN endpoint"` + SafeCert string `ooni:"cert to connect to the OpenVPN endpoint"` + SafeCA string `ooni:"ca to connect to the OpenVPN endpoint"` +} + +// TestKeys contains the experiment's result. +type TestKeys struct { + Success bool `json:"success"` + NetworkEvents []*vpntracex.Event `json:"network_events"` + TCPConnect []*model.ArchivalTCPConnectResult `json:"tcp_connect,omitempty"` + OpenVPNHandshake []*model.ArchivalOpenVPNHandshakeResult `json:"openvpn_handshake"` + BootstrapTime float64 `json:"bootstrap_time"` + Tunnel string `json:"tunnel"` + Failure *string `json:"failure"` +} + +// NewTestKeys creates new openvpn TestKeys. +func NewTestKeys() *TestKeys { + return &TestKeys{ + Success: false, + NetworkEvents: []*vpntracex.Event{}, + TCPConnect: []*model.ArchivalTCPConnectResult{}, + OpenVPNHandshake: []*model.ArchivalOpenVPNHandshakeResult{}, + BootstrapTime: 0, + Tunnel: "openvpn", + Failure: nil, + } +} + +// SingleConnection contains the results of a single handshake. +type SingleConnection struct { + BootstrapTime float64 + TCPConnect *model.ArchivalTCPConnectResult `json:"tcp_connect,omitempty"` + OpenVPNHandshake *model.ArchivalOpenVPNHandshakeResult `json:"openvpn_handshake"` + NetworkEvents []*vpntracex.Event `json:"network_events"` + // TODO(ainghazal): in the future, we will want to store more operations under this struct for a single connection, + // like pingResults or urlgetter calls. Be sure to modify the spec when that happens. +} + +// AddConnectionTestKeys adds the result of a single OpenVPN connection attempt to the +// corresponding array in the [TestKeys] object. +func (tk *TestKeys) AddConnectionTestKeys(result *SingleConnection) { + // Note that TCPConnect is nil when we're using UDP. + if result.TCPConnect != nil { + tk.TCPConnect = append(tk.TCPConnect, result.TCPConnect) + } + tk.OpenVPNHandshake = append(tk.OpenVPNHandshake, result.OpenVPNHandshake) + tk.NetworkEvents = append(tk.NetworkEvents, result.NetworkEvents...) + + // we assume one measurement has exactly one effective connection + tk.BootstrapTime = result.BootstrapTime + + if result.OpenVPNHandshake.Failure != nil { + tk.Failure = result.OpenVPNHandshake.Failure + tk.BootstrapTime = 0 + } +} + +// AllConnectionsSuccessful returns true if all the registered handshakes have nil failures. +func (tk *TestKeys) AllConnectionsSuccessful() bool { + if len(tk.OpenVPNHandshake) == 0 { + return false + } + for _, c := range tk.OpenVPNHandshake { + if c.Failure != nil { + return false + } + } + return true +} + +// Measurer performs the measurement. +type Measurer struct{} + +// NewExperimentMeasurer creates a new ExperimentMeasurer. +func NewExperimentMeasurer() model.ExperimentMeasurer { + return &Measurer{} +} + +// ExperimentName implements model.ExperimentMeasurer.ExperimentName. +func (m *Measurer) ExperimentName() string { + return testName +} + +// ExperimentVersion implements model.ExperimentMeasurer.ExperimentVersion. +func (m *Measurer) ExperimentVersion() string { + return testVersion +} + +// AuthMethod is the authentication method used by a provider. +type AuthMethod string + +var ( + // AuthCertificate is used for providers that authenticate clients via certificates. + AuthCertificate = AuthMethod("cert") + + // AuthUserPass is used for providers that authenticate clients via username (or token) and password. + AuthUserPass = AuthMethod("userpass") +) + +// connectAndHandshake dials a connection and attempts an OpenVPN handshake using that dialer. +func (m *Measurer) connectAndHandshake( + ctx context.Context, + zeroTime time.Time, + index int64, + logger model.Logger, + endpoint *endpoint, + openvpnConfig *vpnconfig.Config, + handshakeTracer *vpntracex.Tracer) *SingleConnection { + + // create a trace for the network dialer + trace := measurexlite.NewTrace(index, zeroTime) + dialer := trace.NewDialerWithoutResolver(logger) + + // Create a vpn tun Device that attempts to dial and performs the handshake. + // Any error will be returned as a failure in the SingleConnection result. + tun, err := tunnel.Start(ctx, dialer, openvpnConfig) + if tun != nil { + defer tun.Close() + } + + handshakeEvents := handshakeTracer.Trace() + port, _ := strconv.Atoi(endpoint.Port) + + t0, t, handshakeTime := TimestampsFromHandshake(handshakeEvents) + + // the bootstrap time is defined to be zero if there's a handshake failure. + var bootstrapTime float64 + if err == nil { + bootstrapTime = time.Since(zeroTime).Seconds() + } + + return &SingleConnection{ + BootstrapTime: bootstrapTime, + TCPConnect: trace.FirstTCPConnectOrNil(), + OpenVPNHandshake: &model.ArchivalOpenVPNHandshakeResult{ + HandshakeTime: handshakeTime, + Endpoint: endpoint.String(), + Failure: measurexlite.NewFailure(err), + IP: endpoint.IPAddr, + Port: port, + Transport: endpoint.Transport, + Provider: endpoint.Provider, + OpenVPNOptions: model.ArchivalOpenVPNOptions{ + Cipher: openvpnConfig.OpenVPNOptions().Cipher, + Auth: openvpnConfig.OpenVPNOptions().Auth, + Compression: string(openvpnConfig.OpenVPNOptions().Compress), + }, + T0: t0, + T: t, + Tags: []string{}, + TransactionID: index, + }, + NetworkEvents: handshakeEvents, + } +} + +// TimestampsFromHandshake returns the t0, t and duration of the handshake events. +// If the passed events are a zero-len array, all of the results will be zero. +func TimestampsFromHandshake(events []*vpntracex.Event) (float64, float64, float64) { + var ( + t0 float64 + t float64 + duration float64 + ) + if len(events) > 0 { + t0 = events[0].AtTime + t = events[len(events)-1].AtTime + duration = t - t0 + } + return t0, t, duration +} + +// Run implements model.ExperimentMeasurer.Run. +// A single run expects exactly ONE input (endpoint), but we can modify whether +// to test different transports by settings options. +func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + callbacks := args.Callbacks + measurement := args.Measurement + sess := args.Session + + // 0. fail if there's no richer input target + if args.Target == nil { + return targetloading.ErrInputRequired + } + + tk := NewTestKeys() + + zeroTime := time.Now() + idx := int64(1) + handshakeTracer := vpntracex.NewTracerWithTransactionID(zeroTime, idx) + + // 1. build the input + target, ok := args.Target.(*Target) + if !ok { + return targetloading.ErrInvalidInputType + } + config, input := target.Config, target.URL + + // 2. obtain the endpoint representation from the input URL + endpoint, err := newEndpointFromInputString(input) + if err != nil { + return err + } + + // TODO(ainghazal): validate we have valid config for each endpoint. + // TODO(ainghazal): validate hostname is a valid IP (ipv4 or 6) + // TODO(ainghazal): decide what to do if we have expired certs (abort one measurement or abort the whole experiment?) + + // 3. build openvpn config from endpoint and options + openvpnConfig, err := newOpenVPNConfig(handshakeTracer, sess.Logger(), endpoint, config) + if err != nil { + return err + } + sess.Logger().Infof("Probing endpoint %s", endpoint.String()) + + // 4. initiate openvpn handshake against endpoint + connResult := m.connectAndHandshake(ctx, zeroTime, idx, sess.Logger(), endpoint, openvpnConfig, handshakeTracer) + tk.AddConnectionTestKeys(connResult) + tk.Success = tk.AllConnectionsSuccessful() + + callbacks.OnProgress(1.0, "All endpoints probed") + + // 5. assign the testkeys + measurement.TestKeys = tk + + // Note: if here we return an error, the parent code will assume + // something fundamental was wrong and we don't have a measurement + // to submit to the OONI collector. Keep this in mind when you + // are writing new experiments! + return nil +} diff --git a/pkg/experiment/openvpn/openvpn_test.go b/pkg/experiment/openvpn/openvpn_test.go new file mode 100644 index 000000000..af1fb1a71 --- /dev/null +++ b/pkg/experiment/openvpn/openvpn_test.go @@ -0,0 +1,384 @@ +package openvpn_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/apex/log" + "github.com/google/go-cmp/cmp" + vpntracex "github.com/ooni/minivpn/pkg/tracex" + "github.com/ooni/probe-engine/pkg/experiment/openvpn" + "github.com/ooni/probe-engine/pkg/measurexlite" + "github.com/ooni/probe-engine/pkg/mocks" + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/targetloading" +) + +func makeMockSession() *mocks.Session { + return &mocks.Session{ + MockLogger: func() model.Logger { + return model.DiscardLogger + }, + MockFetchOpenVPNConfig: func(context.Context, string, string) (*model.OOAPIVPNProviderConfig, error) { + return &model.OOAPIVPNProviderConfig{ + Provider: "provider", + Config: &model.OOAPIVPNConfig{ + CA: "ca", + Cert: "cert", + Key: "key", + }, + Inputs: []string{}, + DateUpdated: time.Now(), + }, nil + }, + } +} + +func TestNewExperimentMeasurer(t *testing.T) { + m := openvpn.NewExperimentMeasurer() + if m.ExperimentName() != "openvpn" { + t.Fatal("invalid ExperimentName") + } + if m.ExperimentVersion() != "0.1.5" { + t.Fatal("invalid ExperimentVersion") + } +} + +func TestNewTestKeys(t *testing.T) { + tk := openvpn.NewTestKeys() + if tk.Success != false { + t.Fatal("default success should be false") + } + if tk.NetworkEvents == nil { + t.Fatal("NetworkEvents not initialized") + } + if tk.TCPConnect == nil { + t.Fatal("TCPConnect not initialized") + } + if tk.OpenVPNHandshake == nil { + t.Fatal("OpenVPNHandshake not initialized") + } +} + +func TestAddConnectionTestKeys(t *testing.T) { + t.Run("append tcp connection result to empty keys", func(t *testing.T) { + tk := openvpn.NewTestKeys() + sc := &openvpn.SingleConnection{ + TCPConnect: &model.ArchivalTCPConnectResult{ + IP: "1.1.1.1", + Port: 1194, + Status: model.ArchivalTCPConnectStatus{ + Blocked: new(bool), + Failure: new(string), + Success: false, + }, + T0: 0.1, + T: 0.9, + Tags: []string{}, + TransactionID: 1, + }, + OpenVPNHandshake: &model.ArchivalOpenVPNHandshakeResult{ + HandshakeTime: 1, + Endpoint: "aa", + Failure: nil, + IP: "1.1.1.1", + Port: 1194, + Transport: "tcp", + Provider: "unknown", + OpenVPNOptions: model.ArchivalOpenVPNOptions{}, + T0: 0, + T: 0, + Tags: []string{}, + TransactionID: 1, + }, + NetworkEvents: []*vpntracex.Event{}, + } + tk.AddConnectionTestKeys(sc) + if diff := cmp.Diff(tk.TCPConnect[0], sc.TCPConnect); diff != "" { + t.Fatal(diff) + } + if diff := cmp.Diff(tk.OpenVPNHandshake[0], sc.OpenVPNHandshake); diff != "" { + t.Fatal(diff) + } + if diff := cmp.Diff(tk.NetworkEvents, sc.NetworkEvents); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("append udp connection result to empty keys", func(t *testing.T) { + tk := openvpn.NewTestKeys() + sc := &openvpn.SingleConnection{ + TCPConnect: nil, + OpenVPNHandshake: &model.ArchivalOpenVPNHandshakeResult{ + HandshakeTime: 1, + Endpoint: "aa", + Failure: nil, + IP: "1.1.1.1", + Port: 1194, + Transport: "udp", + Provider: "unknown", + OpenVPNOptions: model.ArchivalOpenVPNOptions{}, + T0: 0, + T: 0, + Tags: []string{}, + TransactionID: 1, + }, + NetworkEvents: []*vpntracex.Event{}, + } + tk.AddConnectionTestKeys(sc) + if len(tk.TCPConnect) != 0 { + t.Fatal("expected empty tcpconnect") + } + if diff := cmp.Diff(tk.OpenVPNHandshake[0], sc.OpenVPNHandshake); diff != "" { + t.Fatal(diff) + } + if diff := cmp.Diff(tk.NetworkEvents, sc.NetworkEvents); diff != "" { + t.Fatal(diff) + } + }) +} + +func TestAllConnectionsSuccessful(t *testing.T) { + t.Run("all success", func(t *testing.T) { + tk := openvpn.NewTestKeys() + tk.OpenVPNHandshake = []*model.ArchivalOpenVPNHandshakeResult{ + {Failure: nil}, + {Failure: nil}, + {Failure: nil}, + } + if tk.AllConnectionsSuccessful() != true { + t.Fatal("expected all connections successful") + } + }) + t.Run("one failure", func(t *testing.T) { + fail := "uh" + tk := openvpn.NewTestKeys() + tk.OpenVPNHandshake = []*model.ArchivalOpenVPNHandshakeResult{ + {Failure: &fail}, + {Failure: nil}, + {Failure: nil}, + } + if tk.AllConnectionsSuccessful() != false { + t.Fatal("expected false") + } + }) + t.Run("all failures", func(t *testing.T) { + fail := "uh" + tk := openvpn.NewTestKeys() + tk.OpenVPNHandshake = []*model.ArchivalOpenVPNHandshakeResult{ + {Failure: &fail}, + {Failure: &fail}, + {Failure: &fail}, + } + if tk.AllConnectionsSuccessful() != false { + t.Fatal("expected false") + } + }) +} + +func TestOpenVPNFailsWithInvalidInputType(t *testing.T) { + measurer := openvpn.NewExperimentMeasurer() + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: new(model.Measurement), + Session: makeMockSession(), + Target: &model.OOAPIURLInfo{}, // not the input type we expect + } + err := measurer.Run(context.Background(), args) + if !errors.Is(err, openvpn.ErrInvalidInputType) { + t.Fatal("expected input error") + } +} + +func TestBadTargetURLFailure(t *testing.T) { + m := openvpn.NewExperimentMeasurer() + ctx := context.Background() + sess := makeMockSession() + callbacks := model.NewPrinterCallbacks(sess.Logger()) + measurement := new(model.Measurement) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + Target: &openvpn.Target{ + URL: "openvpn://badprovider/?address=aa", + Config: &openvpn.Config{}, + }, + } + err := m.Run(ctx, args) + if !errors.Is(err, targetloading.ErrInvalidInput) { + t.Fatalf("expected ErrInvalidInput, got %v", err) + } +} + +func TestSuccess(t *testing.T) { + m := openvpn.NewExperimentMeasurer() + ctx := context.Background() + sess := makeMockSession() + callbacks := model.NewPrinterCallbacks(sess.Logger()) + measurement := new(model.Measurement) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + Target: &openvpn.Target{ + URL: "openvpn://riseupvpn.corp/?address=127.0.0.1:9989&transport=tcp", + Config: &openvpn.Config{}, + }, + } + err := m.Run(ctx, args) + if err != nil { + t.Fatal(err) + } +} + +func TestTimestampsFromHandshake(t *testing.T) { + t.Run("with more than a single event (common case)", func(t *testing.T) { + events := []*vpntracex.Event{{AtTime: 0}, {AtTime: 1}, {AtTime: 2}} + t0, tlast, duration := openvpn.TimestampsFromHandshake(events) + if t0 != 0 { + t.Fatal("expected t0 == 0") + } + if tlast != 2.0 { + t.Fatal("expected t == 2") + } + if duration != 2 { + t.Fatal("expected duration == 2") + } + }) + + t.Run("with a single event", func(t *testing.T) { + events := []*vpntracex.Event{{AtTime: 1}} + t0, tlast, duration := openvpn.TimestampsFromHandshake(events) + if t0 != 1.0 { + t.Fatal("expected t0 == 1.0") + } + if tlast != 1.0 { + t.Fatal("expected t == 1.0") + } + if duration != 0 { + t.Fatal("expected duration == 0") + } + }) + + t.Run("with no events", func(t *testing.T) { + events := []*vpntracex.Event{} + t0, tlast, duration := openvpn.TimestampsFromHandshake(events) + if t0 != 0 { + t.Fatal("expected t0 == 0") + } + if tlast != 0 { + t.Fatal("expected t == 0") + } + if duration != 0 { + t.Fatal("expected duration == 0") + } + }) +} + +func TestBootstrapTimeWithNoFailure(t *testing.T) { + bootstrapTime := 1.2305 + tk := openvpn.NewTestKeys() + sc := &openvpn.SingleConnection{ + BootstrapTime: bootstrapTime, + TCPConnect: &model.ArchivalTCPConnectResult{ + IP: "1.1.1.1", + Port: 1194, + Status: model.ArchivalTCPConnectStatus{ + Blocked: new(bool), + Failure: new(string), + Success: false, + }, + T0: 0.1, + T: 0.9, + Tags: []string{}, + TransactionID: 1, + }, + OpenVPNHandshake: &model.ArchivalOpenVPNHandshakeResult{ + HandshakeTime: 1.20, + Endpoint: "aa", + Failure: nil, + IP: "1.1.1.1", + Port: 1194, + Transport: "tcp", + Provider: "unknown", + OpenVPNOptions: model.ArchivalOpenVPNOptions{}, + T0: 0.03, + T: 1.23, + Tags: []string{}, + TransactionID: 1, + }, + NetworkEvents: []*vpntracex.Event{}, + } + tk.AddConnectionTestKeys(sc) + + if tk.Failure != nil { + t.Fatal("expected nil failure") + } + if tk.BootstrapTime != bootstrapTime { + t.Fatal("wrong bootstrap time") + } + if tk.Tunnel != "openvpn" { + t.Fatal("tunnel should be openvpn") + } +} + +func TestBootstrapTimeWithFailure(t *testing.T) { + bootstrapTime := 6.1 + + handshakeError := errors.New("mocked error") + handshakeFailure := measurexlite.NewFailure(handshakeError) + + tk := openvpn.NewTestKeys() + sc := &openvpn.SingleConnection{ + BootstrapTime: bootstrapTime, + TCPConnect: &model.ArchivalTCPConnectResult{ + IP: "1.1.1.1", + Port: 1194, + Status: model.ArchivalTCPConnectStatus{ + Blocked: new(bool), + Failure: new(string), + Success: false, + }, + T0: 0.1, + T: 0.9, + Tags: []string{}, + TransactionID: 1, + }, + OpenVPNHandshake: &model.ArchivalOpenVPNHandshakeResult{ + HandshakeTime: 1.20, + Endpoint: "aa", + Failure: handshakeFailure, + IP: "1.1.1.1", + Port: 1194, + Transport: "tcp", + Provider: "unknown", + OpenVPNOptions: model.ArchivalOpenVPNOptions{}, + T0: 0.03, + T: 1.23, + Tags: []string{}, + TransactionID: 1, + }, + NetworkEvents: []*vpntracex.Event{}, + } + tk.AddConnectionTestKeys(sc) + + if tk.Failure != handshakeFailure { + t.Fatalf("expected handshake failure, got %v", tk.Failure) + } + if tk.BootstrapTime != 0 { + t.Fatalf("wrong bootstrap time: expected 0, got %v", tk.BootstrapTime) + } + if tk.Tunnel != "openvpn" { + t.Fatal("tunnel should be openvpn") + } +} + +func TestVPNInput(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + // TODO(ainghazal): do a real test, get credentials etc. +} diff --git a/pkg/experiment/openvpn/richerinput.go b/pkg/experiment/openvpn/richerinput.go new file mode 100644 index 000000000..bf9b3923f --- /dev/null +++ b/pkg/experiment/openvpn/richerinput.go @@ -0,0 +1,183 @@ +package openvpn + +import ( + "context" + "fmt" + + "github.com/ooni/probe-engine/pkg/experimentconfig" + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/reflectx" + "github.com/ooni/probe-engine/pkg/targetloading" +) + +// defaultProvider is the provider we will request from API in case we got no provider set +// in the CLI options. +var defaultProvider = "riseupvpn" + +// providerAuthentication is a map so that we know which kind of credentials we +// need to fill in the openvpn options for each known provider. +var providerAuthentication = map[string]AuthMethod{ + "riseupvpn": AuthCertificate, + "tunnelbearvpn": AuthUserPass, + "surfsharkvpn": AuthUserPass, +} + +// Target is a richer-input target that this experiment should measure. +type Target struct { + // Config contains the configuration. + Config *Config + + // URL is the input URL. + URL string +} + +var _ model.ExperimentTarget = &Target{} + +// Category implements [model.ExperimentTarget]. +func (t *Target) Category() string { + return model.DefaultCategoryCode +} + +// Country implements [model.ExperimentTarget]. +func (t *Target) Country() string { + return model.DefaultCountryCode +} + +// Input implements [model.ExperimentTarget]. +func (t *Target) Input() string { + return t.URL +} + +// Options implements [model.ExperimentTarget]. +func (t *Target) Options() (options []string) { + return experimentconfig.DefaultOptionsSerializer(t.Config) +} + +// String implements [model.ExperimentTarget]. +func (t *Target) String() string { + return t.URL +} + +// NewLoader constructs a new [model.ExperimentTargerLoader] instance. +// +// This function PANICS if options is not an instance of [*openvpn.Config]. +func NewLoader(loader *targetloading.Loader, gopts any) model.ExperimentTargetLoader { + // Panic if we cannot convert the options to the expected type. + // + // We do not expect a panic here because the type is managed by the registry package. + options := gopts.(*Config) + + // Construct the proper loader instance. + return &targetLoader{ + loader: loader, + options: options, + session: loader.Session, + } +} + +// targetLoader loads targets for this experiment. +type targetLoader struct { + loader *targetloading.Loader + options *Config + session targetloading.Session +} + +// Load implements model.ExperimentTargetLoader. +func (tl *targetLoader) Load(ctx context.Context) ([]model.ExperimentTarget, error) { + // If inputs and files are all empty and there are no options, let's use the backend + if len(tl.loader.StaticInputs) <= 0 && len(tl.loader.SourceFiles) <= 0 && + reflectx.StructOrStructPtrIsZero(tl.options) { + targets, err := tl.loadFromBackend(ctx) + if err == nil { + return targets, nil + } + } + + tl.loader.Logger.Warnf("Error fetching OpenVPN targets from backend") + + // Otherwise, attempt to load the static inputs from CLI and files + inputs, err := targetloading.LoadStatic(tl.loader) + + // Handle the case where we couldn't load from CLI or files: + if err != nil { + return nil, err + } + + // Build the list of targets that we should measure. + var targets []model.ExperimentTarget + for _, input := range inputs { + targets = append(targets, &Target{ + Config: tl.options, + URL: input, + }) + } + if len(targets) > 0 { + return targets, nil + } + + // Return the hardcoded endpoints. + return tl.loadFromDefaultEndpoints() +} + +func (tl *targetLoader) loadFromDefaultEndpoints() ([]model.ExperimentTarget, error) { + tl.loader.Logger.Warnf("Using default OpenVPN endpoints") + targets := []model.ExperimentTarget{} + if udp, err := defaultOONIOpenVPNTargetUDP(); err == nil { + targets = append(targets, + &Target{ + Config: pickFromDefaultOONIOpenVPNConfig(), + URL: udp, + }) + } + if tcp, err := defaultOONIOpenVPNTargetTCP(); err == nil { + targets = append(targets, + &Target{ + Config: pickFromDefaultOONIOpenVPNConfig(), + URL: tcp, + }) + } + return targets, nil +} + +func (tl *targetLoader) loadFromBackend(ctx context.Context) ([]model.ExperimentTarget, error) { + if tl.options.Provider == "" { + tl.options.Provider = defaultProvider + } + + targets := make([]model.ExperimentTarget, 0) + provider := tl.options.Provider + + apiConfig, err := tl.session.FetchOpenVPNConfig(ctx, provider, tl.session.ProbeCC()) + if err != nil { + tl.session.Logger().Warnf("Cannot fetch openvpn config: %v", err) + return nil, err + } + + auth, ok := providerAuthentication[provider] + if !ok { + return nil, fmt.Errorf("%w: unknown authentication for provider %s", targetloading.ErrInvalidInput, provider) + } + + for _, input := range apiConfig.Inputs { + config := &Config{ + // TODO(ainghazal): Auth and Cipher are hardcoded for now. + // Backend should provide them as richer input; and if empty we can use these as defaults. + Auth: "SHA512", + Cipher: "AES-256-GCM", + } + switch auth { + case AuthCertificate: + config.SafeCA = apiConfig.Config.CA + config.SafeCert = apiConfig.Config.Cert + config.SafeKey = apiConfig.Config.Key + case AuthUserPass: + // TODO(ainghazal): implement (surfshark, etc) + } + targets = append(targets, &Target{ + URL: input, + Config: config, + }) + } + + return targets, nil +} diff --git a/pkg/experiment/openvpn/richerinput_test.go b/pkg/experiment/openvpn/richerinput_test.go new file mode 100644 index 000000000..68fc8a22c --- /dev/null +++ b/pkg/experiment/openvpn/richerinput_test.go @@ -0,0 +1,212 @@ +package openvpn + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-engine/pkg/mocks" + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/targetloading" +) + +func TestTarget(t *testing.T) { + target := &Target{ + URL: "openvpn://unknown.corp?address=1.1.1.1%3A443&transport=udp", + Config: &Config{ + Auth: "SHA512", + Cipher: "AES-256-GCM", + Provider: "unknown", + SafeKey: "aa", + SafeCert: "bb", + SafeCA: "cc", + }, + } + + t.Run("Category", func(t *testing.T) { + if target.Category() != model.DefaultCategoryCode { + t.Fatal("invalid Category") + } + }) + + t.Run("Country", func(t *testing.T) { + if target.Country() != model.DefaultCountryCode { + t.Fatal("invalid Country") + } + }) + + t.Run("Input", func(t *testing.T) { + if target.Input() != "openvpn://unknown.corp?address=1.1.1.1%3A443&transport=udp" { + t.Fatal("invalid Input") + } + }) + + t.Run("Options", func(t *testing.T) { + expect := []string{ + "Auth=SHA512", + "Cipher=AES-256-GCM", + "Provider=unknown", + } + if diff := cmp.Diff(expect, target.Options()); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("String", func(t *testing.T) { + if target.String() != "openvpn://unknown.corp?address=1.1.1.1%3A443&transport=udp" { + t.Fatal("invalid String") + } + }) +} + +func TestNewLoader(t *testing.T) { + // create the pointers we expect to see + child := &targetloading.Loader{} + options := &Config{} + + // create the loader and cast it to its private type + loader := NewLoader(child, options).(*targetLoader) + + // make sure the loader is okay + if child != loader.loader { + t.Fatal("invalid loader pointer") + } + + // make sure the options are okay + if options != loader.options { + t.Fatal("invalid options pointer") + } +} + +func TestTargetLoaderLoad(t *testing.T) { + // testcase is a test case implemented by this function + type testcase struct { + // name is the test case name + name string + + // options contains the options to use + options *Config + + // loader is the loader to use + loader *targetloading.Loader + + // expectErr is the error we expect + expectErr error + + // expectResults contains the expected results + expectTargets []model.ExperimentTarget + } + + cases := []testcase{ + + { + name: "with options and inputs", + options: &Config{ + SafeCA: "aa", + SafeCert: "bb", + SafeKey: "cc", + Provider: "unknown", + }, + loader: &targetloading.Loader{ + ExperimentName: "openvpn", + InputPolicy: model.InputOrQueryBackend, + Logger: model.DiscardLogger, + Session: &mocks.Session{}, + StaticInputs: []string{ + "openvpn://unknown.corp/1.1.1.1", + }, + }, + expectErr: nil, + expectTargets: []model.ExperimentTarget{ + &Target{ + URL: "openvpn://unknown.corp/1.1.1.1", + Config: &Config{ + Provider: "unknown", + SafeCA: "aa", + SafeCert: "bb", + SafeKey: "cc", + }, + }, + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // create a target loader using the given config + tl := &targetLoader{ + loader: tc.loader, + options: tc.options, + } + + // load targets + targets, err := tl.Load(context.Background()) + + // make sure error is consistent + switch { + case err == nil && tc.expectErr == nil: + // fallthrough + + case err != nil && tc.expectErr != nil: + if !errors.Is(err, tc.expectErr) { + t.Fatal("unexpected error", err) + } + // fallthrough + + default: + t.Fatal("expected", tc.expectErr, "got", err) + } + + // make sure the targets are consistent + if diff := cmp.Diff(tc.expectTargets, targets); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestTargetLoaderLoadFromBackend(t *testing.T) { + loader := &targetloading.Loader{ + ExperimentName: "openvpn", + InputPolicy: model.InputOrQueryBackend, + Logger: model.DiscardLogger, + Session: &mocks.Session{}, + } + sess := &mocks.Session{} + sess.MockFetchOpenVPNConfig = func(context.Context, string, string) (*model.OOAPIVPNProviderConfig, error) { + return &model.OOAPIVPNProviderConfig{ + Provider: "riseupvpn", + Config: &model.OOAPIVPNConfig{}, + Inputs: []string{ + "openvpn://target0", + "openvpn://target1", + }, + DateUpdated: time.Now(), + }, nil + } + sess.MockProbeCC = func() string { + return "IT" + } + tl := &targetLoader{ + loader: loader, + options: &Config{}, + session: sess, + } + targets, err := tl.Load(context.Background()) + if err != nil { + t.Fatal("expected no error") + } + fmt.Println("targets", targets) + if len(targets) != 2 { + t.Fatal("expected 2 targets") + } + if targets[0].String() != "openvpn://target0" { + t.Fatal("expected openvpn://target0") + } + if targets[1].String() != "openvpn://target1" { + t.Fatal("expected openvpn://target1") + } +} diff --git a/pkg/experiment/openvpn/targets.go b/pkg/experiment/openvpn/targets.go new file mode 100644 index 000000000..6a85b48ca --- /dev/null +++ b/pkg/experiment/openvpn/targets.go @@ -0,0 +1,78 @@ +package openvpn + +import ( + "fmt" + "math/rand" + "net" +) + +const defaultOpenVPNEndpoint = "openvpn-server1.ooni.io" + +// this is a safety toggle: it's on purpose that the experiment will receive no +// input if the resolution fails. This also implies that we have no way of knowing if this +// target has been blocked at the level of DNS. +// TODO(ain,mehul): we might want to try resolving with other techniques (DoT etc), +// and perhaps also transform DNS failure into a specific failure of the experiment, not +// a skip. +// TODO(ain): update the openvpn spec to reflect the CURRENT state of delivering the targets. +func resolveTarget(domain string) (string, error) { + ips, err := net.LookupIP(domain) + if err != nil { + return "", err + } + if len(ips) > 0 { + return ips[0].String(), nil + } + return "", fmt.Errorf("cannot resolve %v", defaultOpenVPNEndpoint) +} + +func defaultOONITargetURL(ip string) string { + return "openvpn://oonivpn.corp/?address=" + ip + ":1194" +} + +func defaultOONIOpenVPNTargetUDP() (string, error) { + ip, err := resolveTarget(defaultOpenVPNEndpoint) + if err != nil { + return "", err + } + return defaultOONITargetURL(ip) + "&transport=udp", nil +} + +func defaultOONIOpenVPNTargetTCP() (string, error) { + ip, err := resolveTarget(defaultOpenVPNEndpoint) + if err != nil { + return "", err + } + return defaultOONITargetURL(ip) + "&transport=tcp", nil +} + +func pickFromDefaultOONIOpenVPNConfig() *Config { + idx := rand.Intn(len(defaultOONIOpenVPNConfig)) + return defaultOONIOpenVPNConfig[idx] +} + +var defaultCA = "-----BEGIN CERTIFICATE-----\nMIIDSzCCAjOgAwIBAgIUOPlwhp2s96qqGF5zgLOp0noN2uwwDQYJKoZIhvcNAQEL\nBQAwFjEUMBIGA1UEAwwLRWFzeS1SU0EgQ0EwHhcNMjQwNzMxMTc1MDI3WhcNMzQw\nNzI5MTc1MDI3WjAWMRQwEgYDVQQDDAtFYXN5LVJTQSBDQTCCASIwDQYJKoZIhvcN\nAQEBBQADggEPADCCAQoCggEBALfhmQ6YndIaq9K2ya1HNv9e3DiwKO8X7Ferh8KV\n/Yobs1jPJYfK/l1SZTO97FnIptqxPzGAWuxhS/+4n4ZB2RpszJKdu3sHYNY6lZCR\nw8dtxKYDIS5v/1by6AJk052wV3NWizw1QiawCOJl5cNN5Vb4OpLPvBzrx3IN7jvO\n0HxaaRYIiPdQy++cJ/wqQazTvPYpws0rIAF0A9jxzgsJZoWshg8MhQm9OYIMyZ2C\n4WeuBKU5bR7vqjAQnVH6ZsZ8ZX1UILq++PcuLeDYbg7M5YmT0v0SO+3ealgg48SO\nxqStAawEAXI2sOZqWTvFfXiq9l6Uw2uxPwXnzSO8hjjVqc0CAwEAAaOBkDCBjTAM\nBgNVHRMEBTADAQH/MB0GA1UdDgQWBBRyvhkgys8dIIzvcH7+TlcATT6bGTBRBgNV\nHSMESjBIgBRyvhkgys8dIIzvcH7+TlcATT6bGaEapBgwFjEUMBIGA1UEAwwLRWFz\neS1SU0EgQ0GCFDj5cIadrPeqqhhec4CzqdJ6DdrsMAsGA1UdDwQEAwIBBjANBgkq\nhkiG9w0BAQsFAAOCAQEAPpb2z/wBj9tULuzBQ1j6qkIUCkyH6e+QATHcCcJGWQsU\naeEc1w/qBXaJcRS0ahALXC3d/Tz8R2dAj1sO1HEsfjEs5fv1dKGgeVb1rNuZuUW8\n9xEtUdp3jL3xumcqfxKIwOv8Y1fz+AKGJbbPC3yoHptwMDW9zyaRTQ+McKE7Y497\nFZDF2RWQjgpxwCi7P3cScNBLNtt42TPnj6Up3D6Sj57YVDK9dXbrDj94bwmkQa8s\nl8Mp/PFaFeLNXXuGGVEbIlFuw9RY32vbJ1CrS9rrWlVq9Q17NrAmSYSBi9T19mDh\nMFslRMPBN4Jfd/45V26iW2XMpWCONY5aqAfx+2Oz2g==\n-----END CERTIFICATE-----" +var defaultOONIOpenVPNConfig = []*Config{ + { + Cipher: "AES-256-GCM", + Auth: "SHA512", + Compress: "stub", + Provider: "oonivpn", + Obfuscation: "none", + // yes, github, I know this looks like a leaked certificate. That is exactly what it is. + SafeCA: defaultCA, + SafeCert: "-----BEGIN CERTIFICATE-----\nMIIDXjCCAkagAwIBAgIRALM/5njrVcneGXfmqnIX278wDQYJKoZIhvcNAQELBQAw\nFjEUMBIGA1UEAwwLRWFzeS1SU0EgQ0EwHhcNMjQwNzMxMTc1MjAxWhcNMjYxMTAz\nMTc1MjAxWjAaMRgwFgYDVQQDDA9jbGllbnQtY2VydC0wMDIwggEiMA0GCSqGSIb3\nDQEBAQUAA4IBDwAwggEKAoIBAQDfVfY+RK1Wl4Dw+KPJMOu7UT4g8VoWS0r5B3z8\nqzL/RAL9xEMaeJbeJPCkOCMaPiS5Xyuj2X/idSlejINmC+XhAx0+ANbxD7oilhBt\nLO43u8QRE5N2HBt045dJdFiN/lt2OwQOrYAL4p7hEn91zObT35wzK6jfNFMON9HQ\n3JZzEqcs/5SfnCvyAtAnV+Qfr4TolX2lRhu74Yl88OzjNFiGADniK/jJGJWfPEzn\nhqfzbcpXCVKUD38kFje3wBN+DrWQabuXTlJhOfhHANMgUnqoS91ea/TbfdiQ4kni\n1sE9RG/X+v8/Xm1BmJO2db1t1K/Px4wqE5Ku7XvdyVU4U4YHAgMBAAGjgaIwgZ8w\nCQYDVR0TBAIwADAdBgNVHQ4EFgQUAT/5mtUeUGCeFKrgBb/6i0B+2ycwUQYDVR0j\nBEowSIAUcr4ZIMrPHSCM73B+/k5XAE0+mxmhGqQYMBYxFDASBgNVBAMMC0Vhc3kt\nUlNBIENBghQ4+XCGnaz3qqoYXnOAs6nSeg3a7DATBgNVHSUEDDAKBggrBgEFBQcD\nAjALBgNVHQ8EBAMCB4AwDQYJKoZIhvcNAQELBQADggEBAA1VNnz0jz+1uLqQBdH2\nc5D97BdANVHjE6NptELekeoYni4IrqhJ8sjx60tq459nhaHZc4XaCMpuSb/rdxhF\nxh/D+PJlpQQxQkrIFLGTwDGVz0J6OI/PCLgjRwHqWIp7Y1DYtGEUtojhrRYCq6Dt\nHT3tG6Osd08tZTKeW1kOf35JZqu5JFOz52uIO7qmk5DZoR3O4Oxk4mCyA6kdu9tp\nk3n9OnrhQFVWy98N6cQ+k5UIyN1HgdWfhwIjxFJXVt4JfsF3jRyUyUDpuGXPQs6Y\nywVyfOE5EYqUfGDBqgUBEChaQTY2aTHQ9S9QVrIXHE3Gjj6pqjZg4TDUPv+h2fom\nu4c=\n-----END CERTIFICATE-----", + SafeKey: "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDfVfY+RK1Wl4Dw\n+KPJMOu7UT4g8VoWS0r5B3z8qzL/RAL9xEMaeJbeJPCkOCMaPiS5Xyuj2X/idSle\njINmC+XhAx0+ANbxD7oilhBtLO43u8QRE5N2HBt045dJdFiN/lt2OwQOrYAL4p7h\nEn91zObT35wzK6jfNFMON9HQ3JZzEqcs/5SfnCvyAtAnV+Qfr4TolX2lRhu74Yl8\n8OzjNFiGADniK/jJGJWfPEznhqfzbcpXCVKUD38kFje3wBN+DrWQabuXTlJhOfhH\nANMgUnqoS91ea/TbfdiQ4kni1sE9RG/X+v8/Xm1BmJO2db1t1K/Px4wqE5Ku7Xvd\nyVU4U4YHAgMBAAECggEAFXyT4h4XCSeD+xWzeEXxyrCVyB3q8Lh8tY8atMUUdR+f\neWJaTF/Wr/eg2UPMs20leQTRfNmOPATRSUHpMGEA/vrJjkS5aTF9/cQvP2RdD3/2\nlz1UE2m/2yk8TbpY/LbrKL75Hx+3xoJJOSvflxSdY+agWgH0z3NIHFboI5kytk5N\nOQZ+9zlMPx5FOhl38IizbaKG5xXbIFfZqz9TeShtycY1Uj4c1ghveFh/EwEcj51+\nFHoUo6IGVsODpuiSGIj23vlY/D6H4J2/wkwlhUHKg4zPLHEHL1PWlijQt80mYIx/\n+yUKl9QuRCLsEQTRX4h1iUuM8w2jV6FulpljPAZCNQKBgQD53+PjfXnrw+Ft0FoV\nJfTIHScS7U5jS25lRS0p4QSB+GcME0FY/sRPP4L0cloNNK6Ml5rKi90KHnoGGBTt\nIuIqZpChXbeSFLswiel1ye67GYgELXhdZZPI2xAgL2804nzMpoYx7+VHLJCVsskL\nkWHbO3VHmkiiyng5Fib/Bg9ANQKBgQDkz4Z2BaClDu8jn1JyoIPkmrz+Noako0wo\nGtfPW93FV+OqXoitJHJvXQZ962DZbnty6PsjCJixchqHep4HT9m4WUuzwZqJJG4J\ngS4+hlWS9pcvLpobeBeiJ1OnOQXk5FRgAVJ2suwtErPYCmtNsH9RwCkrRrTMxk6V\n7/yBkbisywKBgQCxJpELVIceplXpI+Dpw2oigcCVA5cigHT46S1W1of6mSB9iB05\nOg31XUK7iWLcn+/sDwOX+8avCOJb9bDIWoXbp7F8JdQihf4cMHpKnupYzYYH6DDA\njmZS7TQmjVqbNMNj19+mAb0cU7UB3Kn6QI0O/71rES/T8hV/63ukLCidzQKBgGPJ\nEUXFPILfWXE6mTU+RWbcCNIAq4V/ZcYTtsxKrxPSOYpiQc7olzNz6VHe5dTNNu8t\nJeDobdbtAR7WXbaonzWjU71oEGIAzjA88xL3eLhn7BT6iOCz5fKknfnOh4CEBzv7\nN6BmdVNO1bnBCXzPHSdk209xPYYUcc834vIKv/QzAoGAaciaMwabaecjjZPgUtCq\n+0hE6yWg8YQ7t60+jA8IanU29vpXVFJKr/yTgnExde96sD3POxBln8F2R2tHZK0Z\nR+BFhw5TUzDuSgQJRmhskbcR7u70I39fgm5G0ed9Qt5tx7bl32r9OGxMiWY6yGb+\nabwHOfqbU+03upE8+Of17sg=\n-----END PRIVATE KEY-----", + }, + { + Cipher: "AES-256-GCM", + Auth: "SHA512", + Compress: "stub", + Provider: "oonivpn", + Obfuscation: "none", + // yes, github, I know this looks like a leaked certificate. That is exactly what it is. + SafeCA: defaultCA, + SafeCert: "-----BEGIN CERTIFICATE-----\nMIIDXTCCAkWgAwIBAgIQFiJOrUbahl4vlYa6xv7SmzANBgkqhkiG9w0BAQsFADAW\nMRQwEgYDVQQDDAtFYXN5LVJTQSBDQTAeFw0yNDA3MzExNzUyMDBaFw0yNjExMDMx\nNzUyMDBaMBoxGDAWBgNVBAMMD2NsaWVudC1jZXJ0LTAwMTCCASIwDQYJKoZIhvcN\nAQEBBQADggEPADCCAQoCggEBANXyxnIb9tLqe6di6xIJaCDm0Ue4D8Cy0XYQKnbB\n8Ko9xJiglUm4BXAkjkOHLfSB38hOx9exXTW4whMuYOEJoo26JcdbmLJhaxiVAwTQ\nzMROgCbpJi1lu5cQ8F0U4Sq1/+IZKIGfmiWtxa2YP4Kc4qgEESk+AZ6rtxuKUvQU\nY0rLO1J1FuH8CgYnPG/dkwekVn47v7VnLzIM6XgPdezFNqwGYDAINrxutvnh8dI1\n9hoUZ5sTS9+747kXBy8049xfZqd7rUst9aC47Bt2BOXPUaKCeu1S4v6yEwQcQuSm\nHHoKHJsmlI1DQRS9ZRMq4e0ugFxwIMWz2Wwf2uC1VTOFbCUCAwEAAaOBojCBnzAJ\nBgNVHRMEAjAAMB0GA1UdDgQWBBR4NMh/CIuWF1L7MGM81Hg54ChATzBRBgNVHSME\nSjBIgBRyvhkgys8dIIzvcH7+TlcATT6bGaEapBgwFjEUMBIGA1UEAwwLRWFzeS1S\nU0EgQ0GCFDj5cIadrPeqqhhec4CzqdJ6DdrsMBMGA1UdJQQMMAoGCCsGAQUFBwMC\nMAsGA1UdDwQEAwIHgDANBgkqhkiG9w0BAQsFAAOCAQEARhgY3kmrJ5QP2cz9OcFB\nTjFQaQlnEts7Z4xcl/DNz3WNmqP2HVe1jzcHvZgkkcFNoP3BR/45rW2UiAAw3gx5\nupjcxceJ1GtStmZHM2ReO8mSumtkMZ60Qwo8z+xmbY8art28U2exXRCijtD2BYku\nVV6jaZGrWNk5JgSf4Eaj8oB5SDhuO18flogDAY4Y0iQDScYc8JYLXP9cgYJDLICM\n7wKanE8g3IL/Ruy5/nqNPRIPc28YP2U4sUDSNgIDQJKwXCHQmdUjXhMRNPy/I00I\nccK1qapMxfZHy+zQbUM2OPEsdr4oeqe2GRJowoI4Chb5w0s+GfTGopV5J2o8QfYz\nvw==\n-----END CERTIFICATE-----", + SafeKey: "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDV8sZyG/bS6nun\nYusSCWgg5tFHuA/AstF2ECp2wfCqPcSYoJVJuAVwJI5Dhy30gd/ITsfXsV01uMIT\nLmDhCaKNuiXHW5iyYWsYlQME0MzEToAm6SYtZbuXEPBdFOEqtf/iGSiBn5olrcWt\nmD+CnOKoBBEpPgGeq7cbilL0FGNKyztSdRbh/AoGJzxv3ZMHpFZ+O7+1Zy8yDOl4\nD3XsxTasBmAwCDa8brb54fHSNfYaFGebE0vfu+O5FwcvNOPcX2ane61LLfWguOwb\ndgTlz1GignrtUuL+shMEHELkphx6ChybJpSNQ0EUvWUTKuHtLoBccCDFs9lsH9rg\ntVUzhWwlAgMBAAECggEAGhllZXatV8H0hzEdOYRNwFO//HaC73Aw9qzWrUmP3Qiv\ncqoGRCmveiRveWPkcoMkZDheDx0rIHpTLIlYFqO5AX6PjMALAtm4+ZT1+xCophro\nbba76kZSicVueQBqzm3I0xFcHGH2qTmHV5uuxbVzPelPGZ+fjXZnnjOz1mQlT7J9\nf1846ICm2U3RtD6HtD0SD5SK/r6qsOawxdq2aVXn5Kgb6zwqvrSfzGgKHLwmFh60\nGXFCVko/ohsvesvpQd/ONbMXQEjq99vLMwLa/vjXtU2mSaHKvpR2uokTO9f5LJFq\n1VNzU7x2bCnUs7O3wk6TTYaV0zXsWo5OjMgWayB1MQKBgQDxtSFf1vfq+jCjhMMg\nmE2asnS/rnyH4pog1k8TiJZncubKdZU5QuaozDJhsGkSL16EAa7X2pvi17BhDJhe\ns0fprsziWqjp9isTQjMFMdrZhnL5uEVkK/JiSUGQdvlXxUpd0quhlVWkh8ZmZ8ce\nOQArEUJHWWgtvu5gS78izPnEUQKBgQDimW/7CE7zfdczUlj6Sp4v1+Tbdz/Iket3\nIQDj4f3DD6f1v5Hd9k43oPFPP6jbOwSW9dFMuHzsQW4G7WNSJmA+x0fe6qeKRLoV\nDe89WVMcCVlNX0kvwF68ojyFQ+/V+NjF2A6yeOAiM8tQf4YDHuNokzjSP7TVZIAc\nbeRERwZZlQKBgCqCVppKblOvKLq5cK/c2VkppYrInzIu0jiQOFwRG5KaDKjywQnP\nEE4Di6DOq8v89Lx2p09jLSNaF7UZx/pvwWgBzBrLIwXyu2SpsdtqBzlWggYVOG8D\no59RjuxfYD7lfcy+blz+rI9BKc181vIjyDnK0UNHICFbgQUCjV0Le6nhAoGBAM4M\nKehBuNDuZ+YSBjip60ej8EWkHMq77TnpN87/62kY7minJvOHib5JycN/JoMbGmRO\n6F/0DhwirvL7n2nO3YuYWAEarPgs4GxOvHGzrL/8vEh/0aPrL/olKBUiHo8Z9buJ\naGvfQCe5ozHyk6B40N6BqJR+O2gjN98iCgQP9XU1AoGBAJF9+aKNz5eirXnCT60n\nDjZ+o4jJ1c0C/dpuGBU9Sm0Q0qqLsZKIBLSrfMccmgpxhnsQ8a/9yXAkFkL+E0yX\nSOlGzbO1VhnsreBQg7oIe5PMsie+zulHwZ9gqvwH5T3xYGJJc6AzB1V3CS00jPsf\ncAnOQHux3yai6ZZkdXAAWJ8w\n-----END PRIVATE KEY-----", + }, +} diff --git a/pkg/experiment/openvpn/targets_test.go b/pkg/experiment/openvpn/targets_test.go new file mode 100644 index 000000000..480be65f5 --- /dev/null +++ b/pkg/experiment/openvpn/targets_test.go @@ -0,0 +1,81 @@ +package openvpn + +import ( + "testing" + + "github.com/google/go-cmp/cmp" +) + +func Test_resolveTarget(t *testing.T) { + // TODO: mustHaveExternalNetwork() equivalent. + if testing.Short() { + t.Skip("skip test in short mode") + } + + _, err := resolveTarget("google.com") + + if err != nil { + if err.Error() == "connection_refused" { + // connection_refused is raised when running this test + // on the restricted network for coverage tests. + // so we bail out + return + } + t.Fatal("should be able to resolve the target") + } + + _, err = resolveTarget("nothing.corp") + if err == nil { + t.Fatal("should not be able to resolve the target") + } + + _, err = resolveTarget("asfasfasfasfasfafs.ooni.io") + if err == nil { + t.Fatal("should not be able to resolve the target") + } +} + +func Test_defaultOONIOpenVPNTargetUDP(t *testing.T) { + url, err := defaultOONIOpenVPNTargetUDP() + if err != nil { + if err.Error() == "connection_refused" { + // connection_refused is raised when running this test + // on the restricted network for coverage tests. + // so we bail out + return + } + t.Fatal("unexpected error") + } + expected := "openvpn://oonivpn.corp/?address=37.218.243.98:1194&transport=udp" + if diff := cmp.Diff(url, expected); diff != "" { + t.Fatal(diff) + } +} + +func Test_defaultOONIOpenVPNTargetTCP(t *testing.T) { + url, err := defaultOONIOpenVPNTargetTCP() + if err != nil { + if err.Error() == "connection_refused" { + // connection_refused is raised when running this test + // on the restricted network for coverage tests. + // so we bail out + return + } + t.Fatal("unexpected error") + } + expected := "openvpn://oonivpn.corp/?address=37.218.243.98:1194&transport=tcp" + if diff := cmp.Diff(url, expected); diff != "" { + t.Fatal(diff) + } +} + +func Test_pickFromDefaultOONIOpenVPNConfig(t *testing.T) { + pick := pickFromDefaultOONIOpenVPNConfig() + + if pick.Cipher != "AES-256-GCM" { + t.Fatal("cipher unexpected") + } + if pick.SafeCA != defaultCA { + t.Fatal("ca unexpected") + } +} diff --git a/pkg/experiment/psiphon/psiphon_test.go b/pkg/experiment/psiphon/psiphon_test.go index 19fa1ac97..bbccf4343 100644 --- a/pkg/experiment/psiphon/psiphon_test.go +++ b/pkg/experiment/psiphon/psiphon_test.go @@ -51,7 +51,7 @@ func TestRunWithCancelledContext(t *testing.T) { func TestRunWithCustomInputAndCancelledContext(t *testing.T) { expected := "http://x.org" measurement := &model.Measurement{ - Input: model.MeasurementTarget(expected), + Input: model.MeasurementInput(expected), } measurer := psiphon.NewExperimentMeasurer(psiphon.Config{}) measurer.(*psiphon.Measurer).BeforeGetHook = func(g urlgetter.Getter) { diff --git a/pkg/experiment/quicping/quicping_test.go b/pkg/experiment/quicping/quicping_test.go index b1dac4c35..2d243a1a1 100644 --- a/pkg/experiment/quicping/quicping_test.go +++ b/pkg/experiment/quicping/quicping_test.go @@ -31,7 +31,7 @@ func TestInvalidHost(t *testing.T) { Repetitions: 1, }) measurement := new(model.Measurement) - measurement.Input = model.MeasurementTarget("a.a.a.a") + measurement.Input = model.MeasurementInput("a.a.a.a") sess := &mockable.Session{MockableLogger: log.Log} args := &model.ExperimentArgs{ Callbacks: model.NewPrinterCallbacks(log.Log), @@ -55,7 +55,7 @@ func TestURLInput(t *testing.T) { Repetitions: 1, }) measurement := new(model.Measurement) - measurement.Input = model.MeasurementTarget("https://google.com/") + measurement.Input = model.MeasurementInput("https://google.com/") sess := &mockable.Session{MockableLogger: log.Log} args := &model.ExperimentArgs{ Callbacks: model.NewPrinterCallbacks(log.Log), @@ -79,7 +79,7 @@ func TestSuccess(t *testing.T) { } measurer := NewExperimentMeasurer(Config{}) measurement := new(model.Measurement) - measurement.Input = model.MeasurementTarget("google.com") + measurement.Input = model.MeasurementInput("google.com") sess := &mockable.Session{MockableLogger: log.Log} args := &model.ExperimentArgs{ Callbacks: model.NewPrinterCallbacks(log.Log), @@ -122,7 +122,7 @@ func TestWithCancelledContext(t *testing.T) { measurer := NewExperimentMeasurer(Config{}) measurement := new(model.Measurement) - measurement.Input = model.MeasurementTarget("google.com") + measurement.Input = model.MeasurementInput("google.com") sess := &mockable.Session{MockableLogger: log.Log} ctx, cancel := context.WithCancel(context.Background()) cancel() @@ -153,7 +153,7 @@ func TestListenFails(t *testing.T) { }, }) measurement := new(model.Measurement) - measurement.Input = model.MeasurementTarget("google.com") + measurement.Input = model.MeasurementInput("google.com") sess := &mockable.Session{MockableLogger: log.Log} args := &model.ExperimentArgs{ Callbacks: model.NewPrinterCallbacks(log.Log), @@ -201,7 +201,7 @@ func TestWriteFails(t *testing.T) { Repetitions: 1, }) measurement := new(model.Measurement) - measurement.Input = model.MeasurementTarget("google.com") + measurement.Input = model.MeasurementInput("google.com") sess := &mockable.Session{MockableLogger: log.Log} args := &model.ExperimentArgs{ Callbacks: model.NewPrinterCallbacks(log.Log), @@ -262,7 +262,7 @@ func TestReadFails(t *testing.T) { Repetitions: 1, }) measurement := new(model.Measurement) - measurement.Input = model.MeasurementTarget("google.com") + measurement.Input = model.MeasurementInput("google.com") sess := &mockable.Session{MockableLogger: log.Log} args := &model.ExperimentArgs{ Callbacks: model.NewPrinterCallbacks(log.Log), @@ -298,7 +298,7 @@ func TestNoResponse(t *testing.T) { Repetitions: 1, }) measurement := new(model.Measurement) - measurement.Input = model.MeasurementTarget("ooni.org") + measurement.Input = model.MeasurementInput("ooni.org") sess := &mockable.Session{MockableLogger: log.Log} args := &model.ExperimentArgs{ Callbacks: model.NewPrinterCallbacks(log.Log), diff --git a/pkg/experiment/simplequicping/simplequicping_test.go b/pkg/experiment/simplequicping/simplequicping_test.go index 0ca8993ed..a63cdc735 100644 --- a/pkg/experiment/simplequicping/simplequicping_test.go +++ b/pkg/experiment/simplequicping/simplequicping_test.go @@ -58,7 +58,7 @@ func TestMeasurerRun(t *testing.T) { } meas := &model.Measurement{ - Input: model.MeasurementTarget(input), + Input: model.MeasurementInput(input), } sess := &mocks.Session{ MockLogger: func() model.Logger { return model.DiscardLogger }, diff --git a/pkg/experiment/sniblocking/sniblocking.go b/pkg/experiment/sniblocking/sniblocking.go index b5217b93b..2b1a66ee8 100644 --- a/pkg/experiment/sniblocking/sniblocking.go +++ b/pkg/experiment/sniblocking/sniblocking.go @@ -221,7 +221,7 @@ func processall( // maybeURLToSNI handles the case where the input is from the test-lists // and hence every input is a URL rather than a domain. -func maybeURLToSNI(input model.MeasurementTarget) (model.MeasurementTarget, error) { +func maybeURLToSNI(input model.MeasurementInput) (model.MeasurementInput, error) { parsed, err := url.Parse(string(input)) if err != nil { return "", err @@ -229,7 +229,7 @@ func maybeURLToSNI(input model.MeasurementTarget) (model.MeasurementTarget, erro if parsed.Path == string(input) { return input, nil } - return model.MeasurementTarget(parsed.Hostname()), nil + return model.MeasurementInput(parsed.Hostname()), nil } // Run implements ExperimentMeasurer.Run. diff --git a/pkg/experiment/sniblocking/sniblocking_test.go b/pkg/experiment/sniblocking/sniblocking_test.go index 8faf9ea12..9c2c479ac 100644 --- a/pkg/experiment/sniblocking/sniblocking_test.go +++ b/pkg/experiment/sniblocking/sniblocking_test.go @@ -445,7 +445,7 @@ func TestMeasurerRun(t *testing.T) { }) measurer.(*Measurer).cache = cache measurement := &model.Measurement{ - Input: model.MeasurementTarget(testsni), + Input: model.MeasurementInput(testsni), } args := &model.ExperimentArgs{ Callbacks: model.NewPrinterCallbacks(model.DiscardLogger), diff --git a/pkg/experiment/stunreachability/stunreachability_test.go b/pkg/experiment/stunreachability/stunreachability_test.go index c1658f8e5..192ae3ff4 100644 --- a/pkg/experiment/stunreachability/stunreachability_test.go +++ b/pkg/experiment/stunreachability/stunreachability_test.go @@ -46,7 +46,7 @@ func TestRunWithoutInput(t *testing.T) { func TestRunWithInvalidURL(t *testing.T) { measurer := NewExperimentMeasurer(Config{}) measurement := new(model.Measurement) - measurement.Input = model.MeasurementTarget("\t") // <- invalid URL + measurement.Input = model.MeasurementInput("\t") // <- invalid URL args := &model.ExperimentArgs{ Callbacks: model.NewPrinterCallbacks(log.Log), Measurement: measurement, @@ -61,7 +61,7 @@ func TestRunWithInvalidURL(t *testing.T) { func TestRunWithNoPort(t *testing.T) { measurer := NewExperimentMeasurer(Config{}) measurement := new(model.Measurement) - measurement.Input = model.MeasurementTarget("stun://stun.ekiga.net") + measurement.Input = model.MeasurementInput("stun://stun.ekiga.net") args := &model.ExperimentArgs{ Callbacks: model.NewPrinterCallbacks(log.Log), Measurement: measurement, @@ -76,7 +76,7 @@ func TestRunWithNoPort(t *testing.T) { func TestRunWithUnsupportedURLScheme(t *testing.T) { measurer := NewExperimentMeasurer(Config{}) measurement := new(model.Measurement) - measurement.Input = model.MeasurementTarget("https://stun.ekiga.net:3478") + measurement.Input = model.MeasurementInput("https://stun.ekiga.net:3478") args := &model.ExperimentArgs{ Callbacks: model.NewPrinterCallbacks(log.Log), Measurement: measurement, @@ -95,7 +95,7 @@ func TestRunWithInput(t *testing.T) { measurer := NewExperimentMeasurer(Config{}) measurement := new(model.Measurement) - measurement.Input = model.MeasurementTarget(defaultInput) + measurement.Input = model.MeasurementInput(defaultInput) args := &model.ExperimentArgs{ Callbacks: model.NewPrinterCallbacks(log.Log), Measurement: measurement, @@ -127,7 +127,7 @@ func TestCancelledContext(t *testing.T) { cancel() // immediately fail everything measurer := NewExperimentMeasurer(Config{}) measurement := new(model.Measurement) - measurement.Input = model.MeasurementTarget(defaultInput) + measurement.Input = model.MeasurementInput(defaultInput) args := &model.ExperimentArgs{ Callbacks: model.NewPrinterCallbacks(log.Log), Measurement: measurement, @@ -166,7 +166,7 @@ func TestNewClientFailure(t *testing.T) { } measurer := NewExperimentMeasurer(*config) measurement := new(model.Measurement) - measurement.Input = model.MeasurementTarget(defaultInput) + measurement.Input = model.MeasurementInput(defaultInput) args := &model.ExperimentArgs{ Callbacks: model.NewPrinterCallbacks(log.Log), Measurement: measurement, @@ -202,7 +202,7 @@ func TestStartFailure(t *testing.T) { } measurer := NewExperimentMeasurer(*config) measurement := new(model.Measurement) - measurement.Input = model.MeasurementTarget(defaultInput) + measurement.Input = model.MeasurementInput(defaultInput) args := &model.ExperimentArgs{ Callbacks: model.NewPrinterCallbacks(log.Log), Measurement: measurement, @@ -242,7 +242,7 @@ func TestReadFailure(t *testing.T) { } measurer := NewExperimentMeasurer(*config) measurement := new(model.Measurement) - measurement.Input = model.MeasurementTarget(defaultInput) + measurement.Input = model.MeasurementInput(defaultInput) args := &model.ExperimentArgs{ Callbacks: model.NewPrinterCallbacks(log.Log), Measurement: measurement, diff --git a/pkg/experiment/tcpping/tcpping_test.go b/pkg/experiment/tcpping/tcpping_test.go index 2007a9a27..13667a93c 100644 --- a/pkg/experiment/tcpping/tcpping_test.go +++ b/pkg/experiment/tcpping/tcpping_test.go @@ -46,7 +46,7 @@ func TestMeasurer_run(t *testing.T) { } ctx := context.Background() meas := &model.Measurement{ - Input: model.MeasurementTarget(input), + Input: model.MeasurementInput(input), } sess := &mocks.Session{ MockLogger: func() model.Logger { return model.DiscardLogger }, diff --git a/pkg/experiment/tlsmiddlebox/measurer_test.go b/pkg/experiment/tlsmiddlebox/measurer_test.go index 2216d115d..ddc652395 100644 --- a/pkg/experiment/tlsmiddlebox/measurer_test.go +++ b/pkg/experiment/tlsmiddlebox/measurer_test.go @@ -30,7 +30,7 @@ func TestMeasurer_input_failure(t *testing.T) { SNIControl: sniControl, }) meas := &model.Measurement{ - Input: model.MeasurementTarget(input), + Input: model.MeasurementInput(input), } sess := &mocks.Session{ MockLogger: func() model.Logger { diff --git a/pkg/experiment/tlsping/tlsping_test.go b/pkg/experiment/tlsping/tlsping_test.go index 98da13cd1..bf90e51b9 100644 --- a/pkg/experiment/tlsping/tlsping_test.go +++ b/pkg/experiment/tlsping/tlsping_test.go @@ -58,7 +58,7 @@ func TestMeasurerRun(t *testing.T) { } meas := &model.Measurement{ - Input: model.MeasurementTarget(input), + Input: model.MeasurementInput(input), } sess := &mocks.Session{ MockLogger: func() model.Logger { return model.DiscardLogger }, diff --git a/pkg/experiment/webconnectivity/webconnectivity_test.go b/pkg/experiment/webconnectivity/webconnectivity_test.go index 3694b75e4..d3a3889e2 100644 --- a/pkg/experiment/webconnectivity/webconnectivity_test.go +++ b/pkg/experiment/webconnectivity/webconnectivity_test.go @@ -225,7 +225,7 @@ func TestMeasureWithNoAvailableTestHelpers(t *testing.T) { func newsession(t *testing.T, lookupBackends bool) model.ExperimentSession { sess, err := engine.NewSession(context.Background(), engine.SessionConfig{ AvailableProbeServices: []model.OOAPIService{{ - Address: "https://ams-pg-test.ooni.org", + Address: "https://backend-hel.ooni.org", Type: "https", }}, Logger: log.Log, diff --git a/pkg/experimentconfig/experimentconfig.go b/pkg/experimentconfig/experimentconfig.go new file mode 100644 index 000000000..58932b0e9 --- /dev/null +++ b/pkg/experimentconfig/experimentconfig.go @@ -0,0 +1,84 @@ +// Package experimentconfig contains code to manage experiments configuration. +package experimentconfig + +import ( + "fmt" + "reflect" + "strings" +) + +// TODO(bassosimone): we should probably move here all the code inside +// of registry used to serialize existing options and to set values from +// generic map[string]any types. + +// DefaultOptionsSerializer serializes options for [model.ExperimentTarget] +// honouring its Options method contract: +// +// 1. we do not serialize options whose name starts with "Safe"; +// +// 2. we only serialize scalar values; +// +// 3. we never serialize any zero values. +// +// This method MUST be passed a pointer to a struct. Otherwise, the return +// value will be a zero-length list (either nil or empty). +func DefaultOptionsSerializer(config any) (options []string) { + // as documented, this method MUST be passed a struct pointer + // + // Implementation note: the .Elem method converts a nil + // pointer to a zero-value pointee type. + stval := reflect.ValueOf(config) + if stval.Kind() != reflect.Pointer { + return + } + stval = stval.Elem() + if stval.Kind() != reflect.Struct { + return + } + + // obtain the structure type + stt := stval.Type() + + // cycle through the struct fields + for idx := 0; idx < stval.NumField(); idx++ { + // obtain the field type and value + fieldval, fieldtype := stval.Field(idx), stt.Field(idx) + + // make sure the field is public + if !fieldtype.IsExported() { + continue + } + + // make sure the field name does not start with "Safe" + if strings.HasPrefix(fieldtype.Name, "Safe") { + continue + } + + // add the field iff it's a nonzero scalar + switch fieldval.Kind() { + case reflect.Bool, + reflect.Int, + reflect.Int8, + reflect.Int16, + reflect.Int32, + reflect.Int64, + reflect.Uint, + reflect.Uint8, + reflect.Uint16, + reflect.Uint32, + reflect.Uint64, + reflect.Float32, + reflect.Float64, + reflect.String: + if fieldval.IsZero() { + continue + } + options = append(options, fmt.Sprintf("%s=%v", fieldtype.Name, fieldval.Interface())) + + default: + // nothing + } + } + + return +} diff --git a/pkg/experimentconfig/experimentconfig_test.go b/pkg/experimentconfig/experimentconfig_test.go new file mode 100644 index 000000000..ad5090bcb --- /dev/null +++ b/pkg/experimentconfig/experimentconfig_test.go @@ -0,0 +1,175 @@ +package experimentconfig + +import ( + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestDefaultOptionsSerializer(t *testing.T) { + // configuration is the configuration we're testing the serialization of. + // + // Note that there's no `ooni:"..."` annotation here because we have changed + // our model in https://github.com/ooni/probe-cli/pull/1629, and now this kind + // of annotations are only command-line related. + type configuration struct { + // booleans + ValBool bool + + // integers + ValInt int + ValInt8 int8 + ValInt16 int16 + ValInt32 int32 + ValInt64 int64 + + // unsigned integers + ValUint uint + ValUint8 uint8 + ValUint16 uint16 + ValUint32 uint32 + ValUint64 uint64 + + // floats + ValFloat32 float32 + ValFloat64 float64 + + // strings + ValString string + + // unexported fields we should ignore + privateInt int + privateString string + privateList []int16 + + // safe fields we should ignore + SafeBool bool + SafeInt int + SafeString string + + // non-scalar fields we should ignore + NSList []int64 + NSMap map[string]string + } + + // testcase is a test case implemented by this function + type testcase struct { + // name is the test case name + name string + + // config is the config to transform into a list of options + config any + + // expectConfigType is an extra check to make sure we're actually + // passing the correct type for the config, which is here to ensure + // that, with a nil pointer to struct, we're not crashing. We need + // some extra case here because of how the Go type system work, + // and specifically we want to be sure we're passing an any containing + // a tuple like (type=*configuration,value=nil). + // + // See https://codefibershq.com/blog/golang-why-nil-is-not-always-nil + expectConfigType string + + // expect is the expected result + expect []string + } + + cases := []testcase{ + { + name: "we return a nil list for zero values", + expectConfigType: "*experimentconfig.configuration", + config: &configuration{}, + expect: nil, + }, + + { + name: "we return a nil list for non-pointers", + expectConfigType: "experimentconfig.configuration", + config: configuration{}, + expect: nil, + }, + + { + name: "we return a nil list for non-struct pointers", + expectConfigType: "*int64", + config: func() *int64 { + v := int64(12345) + return &v + }(), + expect: nil, + }, + + { + name: "we return a nil list for a nil struct pointer", + expectConfigType: "*experimentconfig.configuration", + config: func() *configuration { + return (*configuration)(nil) + }(), + expect: nil, + }, + + { + name: "we only serialize the fields that should be exported", + expectConfigType: "*experimentconfig.configuration", + config: &configuration{ + ValBool: true, + ValInt: 1, + ValInt8: 2, + ValInt16: 3, + ValInt32: 4, + ValInt64: 5, + ValUint: 6, + ValUint8: 7, + ValUint16: 8, + ValUint32: 9, + ValUint64: 10, + ValFloat32: 11, + ValFloat64: 12, + ValString: "tredici", + privateInt: 14, + privateString: "quindici", + privateList: []int16{16}, + SafeBool: true, + SafeInt: 18, + SafeString: "diciannove", + NSList: []int64{20}, + NSMap: map[string]string{"21": "22"}, + }, + expect: []string{ + "ValBool=true", + "ValInt=1", + "ValInt8=2", + "ValInt16=3", + "ValInt32=4", + "ValInt64=5", + "ValUint=6", + "ValUint8=7", + "ValUint16=8", + "ValUint32=9", + "ValUint64=10", + "ValFloat32=11", + "ValFloat64=12", + "ValString=tredici", + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // first make sure that tc.config has really the expected + // type for the reason explained in its docstring + if actual := fmt.Sprintf("%T", tc.config); actual != tc.expectConfigType { + t.Fatal("expected", tc.expectConfigType, "got", actual) + } + + // then serialize the content of the config to a list of strings + got := DefaultOptionsSerializer(tc.config) + + // finally, make sure that the result matches expectations + if diff := cmp.Diff(tc.expect, got); diff != "" { + t.Fatal(diff) + } + }) + } +} diff --git a/pkg/experimentname/experimentname.go b/pkg/experimentname/experimentname.go new file mode 100644 index 000000000..18cd37741 --- /dev/null +++ b/pkg/experimentname/experimentname.go @@ -0,0 +1,25 @@ +// Package experimentname contains code to manipulate experiment names. +package experimentname + +import "github.com/ooni/probe-engine/pkg/strcasex" + +// Canonicalize allows code to provide experiment names +// in a more flexible way, where we have aliases. +// +// Because we allow for uppercase experiment names for backwards +// compatibility with MK, we need to add some exceptions here when +// mapping (e.g., DNSCheck => dnscheck). +func Canonicalize(name string) string { + switch name = strcasex.ToSnake(name); name { + case "ndt_7": + name = "ndt" // since 2020-03-18, we use ndt7 to implement ndt by default + case "dns_check": + name = "dnscheck" + case "stun_reachability": + name = "stunreachability" + case "web_connectivity@v_0_5": + name = "web_connectivity@v0.5" + default: + } + return name +} diff --git a/pkg/experimentname/experimentname_test.go b/pkg/experimentname/experimentname_test.go new file mode 100644 index 000000000..df191d768 --- /dev/null +++ b/pkg/experimentname/experimentname_test.go @@ -0,0 +1,55 @@ +// Package experimentname contains code to manipulate experiment names. +package experimentname + +import "testing" + +func TestCanonicalize(t *testing.T) { + tests := []struct { + input string + expect string + }{ + { + input: "example", + expect: "example", + }, + { + input: "Example", + expect: "example", + }, + { + input: "ndt7", + expect: "ndt", + }, + { + input: "Ndt7", + expect: "ndt", + }, + { + input: "DNSCheck", + expect: "dnscheck", + }, + { + input: "dns_check", + expect: "dnscheck", + }, + { + input: "STUNReachability", + expect: "stunreachability", + }, + { + input: "stun_reachability", + expect: "stunreachability", + }, + { + input: "WebConnectivity@v0.5", + expect: "web_connectivity@v0.5", + }, + } + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + if got := Canonicalize(tt.input); got != tt.expect { + t.Errorf("Canonicalize() = %v, want %v", got, tt.expect) + } + }) + } +} diff --git a/pkg/inputparser/inputparser.go b/pkg/inputparser/inputparser.go index f048a31d6..b92161638 100644 --- a/pkg/inputparser/inputparser.go +++ b/pkg/inputparser/inputparser.go @@ -50,7 +50,7 @@ var ErrUnsupportedScheme = errors.New("inputparser: unsupported URL.Scheme") // Parse parses the experiment input using the given config and returns // to the caller either the resulting URL or an error. -func Parse(config *Config, input model.MeasurementTarget) (*url.URL, error) { +func Parse(config *Config, input model.MeasurementInput) (*url.URL, error) { runtimex.Assert(config != nil, "passed nil config") runtimex.Assert(input != "", "passed empty input") diff --git a/pkg/inputparser/inputparser_test.go b/pkg/inputparser/inputparser_test.go index 4df5c5445..4130825f0 100644 --- a/pkg/inputparser/inputparser_test.go +++ b/pkg/inputparser/inputparser_test.go @@ -20,7 +20,7 @@ func TestParse(t *testing.T) { config *Config // input is the MANDATORY string-format input-URL to parse. - input model.MeasurementTarget + input model.MeasurementInput // expectURL is the OPTIONAL URL we expect in output. expectURL *url.URL diff --git a/pkg/legacy/mockable/mockable.go b/pkg/legacy/mockable/mockable.go index 0f1c7edd7..e057fb628 100644 --- a/pkg/legacy/mockable/mockable.go +++ b/pkg/legacy/mockable/mockable.go @@ -26,6 +26,7 @@ type Session struct { MockableFetchPsiphonConfigErr error MockableFetchTorTargetsResult map[string]model.OOAPITorTarget MockableFetchTorTargetsErr error + MockableFetchOpenVPNConfigErr error MockableCheckInInfo *model.OOAPICheckInResultNettests MockableCheckInErr error MockableResolverIP string @@ -34,6 +35,7 @@ type Session struct { MockableTempDir string MockableTorArgs []string MockableTorBinary string + MockableOpenVPNConfig *model.OOAPIVPNProviderConfig MockableTunnelDir string MockableUserAgent string } @@ -60,6 +62,12 @@ func (sess *Session) FetchTorTargets( return sess.MockableFetchTorTargetsResult, sess.MockableFetchTorTargetsErr } +// FetchOpenVPNConfig implements ExperimentSession.FetchOpenVPNConfig +func (sess *Session) FetchOpenVPNConfig( + ctx context.Context, provider, cc string) (*model.OOAPIVPNProviderConfig, error) { + return sess.MockableOpenVPNConfig, sess.MockableFetchOpenVPNConfigErr +} + // KeyValueStore returns the configured key-value store. func (sess *Session) KeyValueStore() model.KeyValueStore { return &kvstore.Memory{} diff --git a/pkg/mocks/experiment.go b/pkg/mocks/experiment.go index 84508b7f5..8d796871b 100644 --- a/pkg/mocks/experiment.go +++ b/pkg/mocks/experiment.go @@ -16,10 +16,8 @@ type Experiment struct { MockReportID func() string - MockMeasureAsync func(ctx context.Context, input string) (<-chan *model.Measurement, error) - MockMeasureWithContext func( - ctx context.Context, input string) (measurement *model.Measurement, err error) + ctx context.Context, target model.ExperimentTarget) (measurement *model.Measurement, err error) MockSaveMeasurement func(measurement *model.Measurement, filePath string) error @@ -45,14 +43,9 @@ func (e *Experiment) ReportID() string { return e.MockReportID() } -func (e *Experiment) MeasureAsync( - ctx context.Context, input string) (<-chan *model.Measurement, error) { - return e.MockMeasureAsync(ctx, input) -} - func (e *Experiment) MeasureWithContext( - ctx context.Context, input string) (measurement *model.Measurement, err error) { - return e.MockMeasureWithContext(ctx, input) + ctx context.Context, target model.ExperimentTarget) (measurement *model.Measurement, err error) { + return e.MockMeasureWithContext(ctx, target) } func (e *Experiment) SaveMeasurement(measurement *model.Measurement, filePath string) error { diff --git a/pkg/mocks/experiment_test.go b/pkg/mocks/experiment_test.go index da4058cef..389cf53d8 100644 --- a/pkg/mocks/experiment_test.go +++ b/pkg/mocks/experiment_test.go @@ -57,30 +57,18 @@ func TestExperiment(t *testing.T) { } }) - t.Run("MeasureAsync", func(t *testing.T) { - expected := errors.New("mocked err") - e := &Experiment{ - MockMeasureAsync: func(ctx context.Context, input string) (<-chan *model.Measurement, error) { - return nil, expected - }, - } - out, err := e.MeasureAsync(context.Background(), "xo") - if !errors.Is(err, expected) { - t.Fatal("unexpected err", err) - } - if out != nil { - t.Fatal("expected nil") - } - }) - t.Run("MeasureWithContext", func(t *testing.T) { expected := errors.New("mocked err") e := &Experiment{ - MockMeasureWithContext: func(ctx context.Context, input string) (measurement *model.Measurement, err error) { + MockMeasureWithContext: func( + ctx context.Context, target model.ExperimentTarget) (measurement *model.Measurement, err error) { return nil, expected }, } - out, err := e.MeasureWithContext(context.Background(), "xo") + out, err := e.MeasureWithContext( + context.Background(), + model.NewOOAPIURLInfoWithDefaultCategoryAndCountry("https://www.example.com/"), + ) if !errors.Is(err, expected) { t.Fatal("unexpected err", err) } diff --git a/pkg/mocks/experimentbuilder.go b/pkg/mocks/experimentbuilder.go index 9221514af..1dc363633 100644 --- a/pkg/mocks/experimentbuilder.go +++ b/pkg/mocks/experimentbuilder.go @@ -1,6 +1,10 @@ package mocks -import "github.com/ooni/probe-engine/pkg/model" +import ( + "encoding/json" + + "github.com/ooni/probe-engine/pkg/model" +) // ExperimentBuilder mocks model.ExperimentBuilder. type ExperimentBuilder struct { @@ -14,11 +18,17 @@ type ExperimentBuilder struct { MockSetOptionsAny func(options map[string]any) error + MockSetOptionsJSON func(value json.RawMessage) error + MockSetCallbacks func(callbacks model.ExperimentCallbacks) MockNewExperiment func() model.Experiment + + MockNewTargetLoader func(config *model.ExperimentTargetLoaderConfig) model.ExperimentTargetLoader } +var _ model.ExperimentBuilder = &ExperimentBuilder{} + func (eb *ExperimentBuilder) Interruptible() bool { return eb.MockInterruptible() } @@ -39,6 +49,10 @@ func (eb *ExperimentBuilder) SetOptionsAny(options map[string]any) error { return eb.MockSetOptionsAny(options) } +func (eb *ExperimentBuilder) SetOptionsJSON(value json.RawMessage) error { + return eb.MockSetOptionsJSON(value) +} + func (eb *ExperimentBuilder) SetCallbacks(callbacks model.ExperimentCallbacks) { eb.MockSetCallbacks(callbacks) } @@ -46,3 +60,7 @@ func (eb *ExperimentBuilder) SetCallbacks(callbacks model.ExperimentCallbacks) { func (eb *ExperimentBuilder) NewExperiment() model.Experiment { return eb.MockNewExperiment() } + +func (eb *ExperimentBuilder) NewTargetLoader(config *model.ExperimentTargetLoaderConfig) model.ExperimentTargetLoader { + return eb.MockNewTargetLoader(config) +} diff --git a/pkg/mocks/experimentbuilder_test.go b/pkg/mocks/experimentbuilder_test.go index 03acf713f..91ddaa882 100644 --- a/pkg/mocks/experimentbuilder_test.go +++ b/pkg/mocks/experimentbuilder_test.go @@ -1,6 +1,7 @@ package mocks import ( + "encoding/json" "errors" "testing" @@ -72,6 +73,19 @@ func TestExperimentBuilder(t *testing.T) { } }) + t.Run("SetOptionsJSON", func(t *testing.T) { + expected := errors.New("mocked error") + eb := &ExperimentBuilder{ + MockSetOptionsJSON: func(value json.RawMessage) error { + return expected + }, + } + err := eb.SetOptionsJSON([]byte(`{}`)) + if !errors.Is(err, expected) { + t.Fatal("unexpected value") + } + }) + t.Run("SetCallbacks", func(t *testing.T) { var called bool eb := &ExperimentBuilder{ @@ -96,4 +110,16 @@ func TestExperimentBuilder(t *testing.T) { t.Fatal("invalid result") } }) + + t.Run("NewTargetLoader", func(t *testing.T) { + tloader := &ExperimentTargetLoader{} + eb := &ExperimentBuilder{ + MockNewTargetLoader: func(*model.ExperimentTargetLoaderConfig) model.ExperimentTargetLoader { + return tloader + }, + } + if out := eb.NewTargetLoader(&model.ExperimentTargetLoaderConfig{}); out != tloader { + t.Fatal("invalid result") + } + }) } diff --git a/pkg/mocks/experimentinputloader.go b/pkg/mocks/experimentinputloader.go deleted file mode 100644 index b7ed9824f..000000000 --- a/pkg/mocks/experimentinputloader.go +++ /dev/null @@ -1,19 +0,0 @@ -package mocks - -import ( - "context" - - "github.com/ooni/probe-engine/pkg/model" -) - -// ExperimentInputLoader mocks model.ExperimentInputLoader -type ExperimentInputLoader struct { - MockLoad func(ctx context.Context) ([]model.OOAPIURLInfo, error) -} - -var _ model.ExperimentInputLoader = &ExperimentInputLoader{} - -// Load calls MockLoad -func (eil *ExperimentInputLoader) Load(ctx context.Context) ([]model.OOAPIURLInfo, error) { - return eil.MockLoad(ctx) -} diff --git a/pkg/mocks/experimenttargetloader.go b/pkg/mocks/experimenttargetloader.go new file mode 100644 index 000000000..3822a489a --- /dev/null +++ b/pkg/mocks/experimenttargetloader.go @@ -0,0 +1,19 @@ +package mocks + +import ( + "context" + + "github.com/ooni/probe-engine/pkg/model" +) + +// ExperimentTargetLoader mocks model.ExperimentTargetLoader +type ExperimentTargetLoader struct { + MockLoad func(ctx context.Context) ([]model.ExperimentTarget, error) +} + +var _ model.ExperimentTargetLoader = &ExperimentTargetLoader{} + +// Load calls MockLoad +func (eil *ExperimentTargetLoader) Load(ctx context.Context) ([]model.ExperimentTarget, error) { + return eil.MockLoad(ctx) +} diff --git a/pkg/mocks/experimentinputloader_test.go b/pkg/mocks/experimenttargetloader_test.go similarity index 80% rename from pkg/mocks/experimentinputloader_test.go rename to pkg/mocks/experimenttargetloader_test.go index af8d2d63f..8a7d0496a 100644 --- a/pkg/mocks/experimentinputloader_test.go +++ b/pkg/mocks/experimenttargetloader_test.go @@ -11,8 +11,8 @@ import ( func TestExperimentInputLoader(t *testing.T) { t.Run("Load", func(t *testing.T) { expected := errors.New("mocked error") - eil := &ExperimentInputLoader{ - MockLoad: func(ctx context.Context) ([]model.OOAPIURLInfo, error) { + eil := &ExperimentTargetLoader{ + MockLoad: func(ctx context.Context) ([]model.ExperimentTarget, error) { return nil, expected }, } diff --git a/pkg/mocks/session.go b/pkg/mocks/session.go index e18f4030c..c7e3044ca 100644 --- a/pkg/mocks/session.go +++ b/pkg/mocks/session.go @@ -18,6 +18,9 @@ type Session struct { MockFetchTorTargets func( ctx context.Context, cc string) (map[string]model.OOAPITorTarget, error) + MockFetchOpenVPNConfig func( + ctx context.Context, provider, cc string) (*model.OOAPIVPNProviderConfig, error) + MockKeyValueStore func() model.KeyValueStore MockLogger func() model.Logger @@ -56,6 +59,16 @@ type Session struct { MockCheckIn func(ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResult, error) + + MockClose func() error + + MockMaybeLookupBackendsContext func(ctx context.Context) error + + MockMaybeLookupLocationContext func(ctx context.Context) error + + MockResolverASNString func() string + + MockResolverNetworkName func() string } func (sess *Session) GetTestHelpersByName(name string) ([]model.OOAPIService, bool) { @@ -70,6 +83,11 @@ func (sess *Session) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { return sess.MockFetchPsiphonConfig(ctx) } +func (sess *Session) FetchOpenVPNConfig( + ctx context.Context, provider, cc string) (*model.OOAPIVPNProviderConfig, error) { + return sess.MockFetchOpenVPNConfig(ctx, provider, cc) +} + func (sess *Session) FetchTorTargets( ctx context.Context, cc string) (map[string]model.OOAPITorTarget, error) { return sess.MockFetchTorTargets(ctx, cc) @@ -151,3 +169,23 @@ func (sess *Session) CheckIn(ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResult, error) { return sess.MockCheckIn(ctx, config) } + +func (sess *Session) Close() error { + return sess.MockClose() +} + +func (sess *Session) MaybeLookupBackendsContext(ctx context.Context) error { + return sess.MockMaybeLookupBackendsContext(ctx) +} + +func (sess *Session) MaybeLookupLocationContext(ctx context.Context) error { + return sess.MockMaybeLookupLocationContext(ctx) +} + +func (sess *Session) ResolverASNString() string { + return sess.MockResolverASNString() +} + +func (sess *Session) ResolverNetworkName() string { + return sess.MockResolverNetworkName() +} diff --git a/pkg/mocks/session_test.go b/pkg/mocks/session_test.go index cb326c3e8..aa808edc7 100644 --- a/pkg/mocks/session_test.go +++ b/pkg/mocks/session_test.go @@ -80,6 +80,22 @@ func TestSession(t *testing.T) { } }) + t.Run("FetchOpenVPNConfig", func(t *testing.T) { + expected := errors.New("mocked err") + s := &Session{ + MockFetchOpenVPNConfig: func(ctx context.Context, provider, cc string) (*model.OOAPIVPNProviderConfig, error) { + return nil, expected + }, + } + cfg, err := s.FetchOpenVPNConfig(context.Background(), "riseup", "XX") + if !errors.Is(err, expected) { + t.Fatal("unexpected err", err) + } + if cfg != nil { + t.Fatal("expected nil cfg") + } + }) + t.Run("KeyValueStore", func(t *testing.T) { expect := &KeyValueStore{} s := &Session{ @@ -338,4 +354,65 @@ func TestSession(t *testing.T) { t.Fatal("unexpected out") } }) + + t.Run("Close", func(t *testing.T) { + expected := errors.New("mocked err") + s := &Session{ + MockClose: func() error { + return expected + }, + } + err := s.Close() + if !errors.Is(err, expected) { + t.Fatal("unexpected err") + } + }) + + t.Run("MaybeLookupBackendsContext", func(t *testing.T) { + expected := errors.New("mocked err") + s := &Session{ + MockMaybeLookupBackendsContext: func(ctx context.Context) error { + return expected + }, + } + err := s.MaybeLookupBackendsContext(context.Background()) + if !errors.Is(err, expected) { + t.Fatal("unexpected err") + } + }) + + t.Run("MaybeLookupLocationContext", func(t *testing.T) { + expected := errors.New("mocked err") + s := &Session{ + MockMaybeLookupLocationContext: func(ctx context.Context) error { + return expected + }, + } + err := s.MaybeLookupLocationContext(context.Background()) + if !errors.Is(err, expected) { + t.Fatal("unexpected err") + } + }) + + t.Run("ResolverASNString", func(t *testing.T) { + s := &Session{ + MockResolverASNString: func() string { + return "xx" + }, + } + if s.ResolverASNString() != "xx" { + t.Fatal("unexpected result") + } + }) + + t.Run("ResolverNetworkName", func(t *testing.T) { + s := &Session{ + MockResolverNetworkName: func() string { + return "xx" + }, + } + if s.ResolverNetworkName() != "xx" { + t.Fatal("unexpected result") + } + }) } diff --git a/pkg/model/archival.go b/pkg/model/archival.go index eca6ed1d6..7aada4f7e 100644 --- a/pkg/model/archival.go +++ b/pkg/model/archival.go @@ -392,3 +392,31 @@ type ArchivalNetworkEvent struct { TransactionID int64 `json:"transaction_id,omitempty"` Tags []string `json:"tags,omitempty"` } + +// +// OpenVPN +// + +// ArchivalOpenVPNHandshakeResult contains the result of a OpenVPN handshake. +type ArchivalOpenVPNHandshakeResult struct { + Endpoint string `json:"endpoint"` + Failure *string `json:"failure"` + HandshakeTime float64 `json:"handshake_time,omitempty"` + IP string `json:"ip"` + Port int `json:"port"` + Transport string `json:"transport"` + Provider string `json:"provider"` + OpenVPNOptions ArchivalOpenVPNOptions `json:"openvpn_options"` + T0 float64 `json:"t0,omitempty"` + T float64 `json:"t"` + Tags []string `json:"tags"` + TransactionID int64 `json:"transaction_id,omitempty"` +} + +// ArchivalOpenVPNOptions is a subset of [vpnconfig.OpenVPNOptions] that we want to include +// in the archived result. +type ArchivalOpenVPNOptions struct { + Auth string `json:"auth,omitempty"` + Cipher string `json:"cipher,omitempty"` + Compression string `json:"compression,omitempty"` +} diff --git a/pkg/model/experiment.go b/pkg/model/experiment.go index 410233f6f..31b80f01f 100644 --- a/pkg/model/experiment.go +++ b/pkg/model/experiment.go @@ -7,7 +7,9 @@ package model import ( "context" + "encoding/json" "errors" + "fmt" ) // ErrNoAvailableTestHelpers is emitted when there are no available test helpers. @@ -21,6 +23,9 @@ type ExperimentSession interface { // DefaultHTTPClient returns the default HTTPClient used by the session. DefaultHTTPClient() HTTPClient + // FetchOpenVPNConfig returns vpn config as a serialized JSON or an error. + FetchOpenVPNConfig(ctx context.Context, provider, cc string) (*OOAPIVPNProviderConfig, error) + // FetchPsiphonConfig returns psiphon's config as a serialized JSON or an error. FetchPsiphonConfig(ctx context.Context) ([]byte, error) @@ -53,57 +58,13 @@ type ExperimentSession interface { UserAgent() string } -// ExperimentAsyncTestKeys is the type of test keys returned by an experiment -// when running in async fashion rather than in sync fashion. -type ExperimentAsyncTestKeys struct { - // Extensions contains the extensions used by this experiment. - Extensions map[string]int64 - - // Input is the input this measurement refers to. - Input MeasurementTarget - - // MeasurementRuntime is the total measurement runtime. - MeasurementRuntime float64 - - // TestHelpers contains the test helpers used in the experiment - TestHelpers map[string]interface{} - - // TestKeys contains the actual test keys. - TestKeys interface{} -} - -// ExperimentMeasurerAsync is a measurer that can run in async fashion. -// -// Currently this functionality is optional, but we will likely -// migrate all experiments to use this functionality in 2022. -type ExperimentMeasurerAsync interface { - // RunAsync runs the experiment in async fashion. - // - // Arguments: - // - // - ctx is the context for deadline/timeout/cancellation - // - // - sess is the measurement session - // - // - input is the input URL to measure - // - // - callbacks contains the experiment callbacks - // - // Returns either a channel where TestKeys are posted or an error. - // - // An error indicates that specific preconditions for running the experiment - // are not met (e.g., the input URL is invalid). - // - // On success, the experiment will post on the channel each new - // measurement until it is done and closes the channel. - RunAsync(ctx context.Context, sess ExperimentSession, input string, - callbacks ExperimentCallbacks) (<-chan *ExperimentAsyncTestKeys, error) -} - -// ExperimentCallbacks contains experiment event-handling callbacks +// ExperimentCallbacks contains experiment event-handling callbacks. type ExperimentCallbacks interface { - // OnProgress provides information about an experiment progress. - OnProgress(percentage float64, message string) + // OnProgress provides information about the experiment's progress. + // + // The prog field is a number between 0.0 and 1.0 representing progress, where + // 0.0 corresponds to 0% and 1.0 corresponds to 100%. + OnProgress(prog float64, message string) } // PrinterCallbacks is the default event handler @@ -121,6 +82,48 @@ func (d PrinterCallbacks) OnProgress(percentage float64, message string) { d.Logger.Infof("[%5.1f%%] %s", percentage*100, message) } +// ExperimentTarget contains a target for the experiment to measure. +type ExperimentTarget interface { + // Category returns the github.com/citizenlab/test-lists category + // code for this piece of richer input. + // + // Return [DefaultCategoryCode] if there's no applicable category code. + Category() string + + // Country returns the country code for this + // piece of richer input. + // + // Return [DefaultCountryCode] if there's not applicable country code. + Country() string + + // Input returns the experiment input, which is typically a URL. + Input() string + + // Options transforms the options contained by this target + // into a []string containing options as they were provided + // using the command line `-O option=value` syntax. + // + // This method MUST NOT serialize all the options whose name + // starts with the "Safe" prefix. This method MUST skip serializing + // sensitive options, non-scalar options, and zero value options. + // + // Consider using the [experimentconfig] package to serialize. + Options() []string + + // String MUST return the experiment input. + // + // Implementation note: previously existing code often times treated + // the input as a string and, crucially, printed it using %s. To be + // robust with respect to introducing richer input, we would like the + // code to print in output the same value as before, which possibly + // is processed by the desktop app. This is the reason why we are + // introducing an explicit String() method and why we say that this + // method MUST return the experiment input. + String() string +} + +var _ fmt.Stringer = ExperimentTarget(nil) + // ExperimentArgs contains the arguments passed to an experiment. type ExperimentArgs struct { // Callbacks contains MANDATORY experiment callbacks. @@ -132,6 +135,12 @@ type ExperimentArgs struct { // Session is the MANDATORY session the experiment can use. Session ExperimentSession + + // Target is the OPTIONAL target we're measuring. + // + // Only richer-input-aware experiments use this field. These experiments + // SHOULD be defensive and handle the case where this field is nil. + Target ExperimentTarget } // ExperimentMeasurer is the interface that allows to run a @@ -166,51 +175,21 @@ type Experiment interface { // ReportID returns the open report's ID, if we have opened a report // successfully before, or an empty string, otherwise. - // - // Deprecated: new code should use a Submitter. ReportID() string - // MeasureAsync runs an async measurement. This operation could post - // one or more measurements onto the returned channel. We'll close the - // channel when we've emitted all the measurements. - // - // Arguments: - // - // - ctx is the context for deadline/cancellation/timeout; - // - // - input is the input (typically a URL but it could also be - // just an endpoint or an empty string for input-less experiments - // such as, e.g., ndt7 and dash). - // - // Return value: - // - // - on success, channel where to post measurements (the channel - // will be closed when done) and nil error; - // - // - on failure, nil channel and non-nil error. - MeasureAsync(ctx context.Context, input string) (<-chan *Measurement, error) - - // MeasureWithContext performs a synchronous measurement. - // - // Return value: strictly either a non-nil measurement and - // a nil error or a nil measurement and a non-nil error. + // MeasureWithContext measures the given experiment target. // - // CAVEAT: while this API is perfectly fine for experiments that - // return a single measurement, it will only return the first measurement - // when used with an asynchronous experiment. - MeasureWithContext(ctx context.Context, input string) (measurement *Measurement, err error) + // Return value: either a non-nil measurement and a nil error + // or a nil measurement and a non-nil error. + MeasureWithContext(ctx context.Context, target ExperimentTarget) (measurement *Measurement, err error) // SubmitAndUpdateMeasurementContext submits a measurement and updates the // fields whose value has changed as part of the submission. - // - // Deprecated: new code should use a Submitter. SubmitAndUpdateMeasurementContext( ctx context.Context, measurement *Measurement) error // OpenReportContext will open a report using the given context // to possibly limit the lifetime of this operation. - // - // Deprecated: new code should use a Submitter. OpenReportContext(ctx context.Context) error } @@ -268,11 +247,58 @@ type ExperimentBuilder interface { // the SetOptionAny method for more information. SetOptionsAny(options map[string]any) error + // SetOptionsJSON uses the given [json.RawMessage] to initialize fields + // of the configuration for running the experiment. The [json.RawMessage], if + // not empty, MUST contain a serialization of the experiment config's + // type. An empty [json.RawMessage] will silently be ignored. + SetOptionsJSON(value json.RawMessage) error + // SetCallbacks sets the experiment's interactive callbacks. SetCallbacks(callbacks ExperimentCallbacks) - // NewExperiment creates the experiment instance. + // NewExperiment creates the [Experiment] instance. NewExperiment() Experiment + + // NewTargetLoader creates the [ExperimentTargetLoader] instance. + NewTargetLoader(config *ExperimentTargetLoaderConfig) ExperimentTargetLoader +} + +// ExperimentTargetLoaderConfig is the configuration to create a new [ExperimentTargetLoader]. +// +// The zero value is not ready to use; please, init the MANDATORY fields. +type ExperimentTargetLoaderConfig struct { + // CheckInConfig contains OPTIONAL options for the CheckIn API. If not set, then we'll create a + // default config. If set but there are fields inside it that are not set, then we will set them + // to a default value. + CheckInConfig *OOAPICheckInConfig + + // Session is the MANDATORY current measurement session. + Session ExperimentTargetLoaderSession + + // StaticInputs contains OPTIONAL input to be added + // to the resulting input list if possible. + StaticInputs []string + + // SourceFiles contains OPTIONAL files to read input + // from. Each file should contain a single input string + // per line. We will fail if any file is unreadable + // as well as if any file is empty. + SourceFiles []string +} + +// ExperimentTargetLoaderSession is the session according to [ExperimentTargetLoader]. +type ExperimentTargetLoaderSession interface { + // CheckIn invokes the check-in API. + CheckIn(ctx context.Context, config *OOAPICheckInConfig) (*OOAPICheckInResult, error) + + // FetchOpenVPNConfig fetches the OpenVPN experiment configuration. + FetchOpenVPNConfig(ctx context.Context, provider, cc string) (*OOAPIVPNProviderConfig, error) + + // Logger returns the logger to use. + Logger() Logger + + // ProbeCC returns the probe country code. + ProbeCC() string } // ExperimentOptionInfo contains info about an experiment option. @@ -282,11 +308,14 @@ type ExperimentOptionInfo struct { // Type contains the type. Type string + + // Value contains the current option value. + Value any } -// ExperimentInputLoader loads inputs from local or remote sources. -type ExperimentInputLoader interface { - Load(ctx context.Context) ([]OOAPIURLInfo, error) +// ExperimentTargetLoader loads targets from local or remote sources. +type ExperimentTargetLoader interface { + Load(ctx context.Context) ([]ExperimentTarget, error) } // Submitter submits a measurement to the OONI collector. diff --git a/pkg/model/http.go b/pkg/model/http.go index 9e04df769..2b2d92d72 100644 --- a/pkg/model/http.go +++ b/pkg/model/http.go @@ -15,7 +15,7 @@ const ( // HTTPHeaderUserAgent is the User-Agent header used for measuring. The current header // is 36.86% of the browser population as of 2024-05-13 according to the // https://www.useragents.me/ webpage. - HTTPHeaderUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.3" + HTTPHeaderUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.3" ) // Additional strings used to report HTTP errors. They're currently only used by diff --git a/pkg/model/measurement.go b/pkg/model/measurement.go index 0d29e2822..f59db128c 100644 --- a/pkg/model/measurement.go +++ b/pkg/model/measurement.go @@ -54,11 +54,11 @@ func MeasurementFormatTimeNowUTC() string { return time.Now().UTC().Format(MeasurementDateFormat) } -// MeasurementTarget is the target of a OONI measurement. -type MeasurementTarget string +// MeasurementInput is the input of an OONI measurement. +type MeasurementInput string -// MarshalJSON serializes the MeasurementTarget. -func (t MeasurementTarget) MarshalJSON() ([]byte, error) { +// MarshalJSON serializes the [MeasurementInput]. +func (t MeasurementInput) MarshalJSON() ([]byte, error) { if t == "" { return json.Marshal(nil) } @@ -84,7 +84,7 @@ type Measurement struct { ID string `json:"id,omitempty"` // Input is the measurement input - Input MeasurementTarget `json:"input"` + Input MeasurementInput `json:"input"` // InputHashes contains input hashes InputHashes []string `json:"input_hashes,omitempty"` diff --git a/pkg/model/measurement_test.go b/pkg/model/measurement_test.go index 7b425adbb..848597c64 100644 --- a/pkg/model/measurement_test.go +++ b/pkg/model/measurement_test.go @@ -20,8 +20,8 @@ func TestMeasurementFormatTimeNowUTC(t *testing.T) { }) } -func TestMeasurementTargetMarshalJSON(t *testing.T) { - var mt MeasurementTarget +func TestMeasurementInputMarshalJSON(t *testing.T) { + var mt MeasurementInput data, err := json.Marshal(mt) if err != nil { t.Fatal(err) @@ -83,7 +83,7 @@ func makeMeasurement(config makeMeasurementConfig) *Measurement { return &Measurement{ DataFormatVersion: "0.3.0", ID: "bdd20d7a-bba5-40dd-a111-9863d7908572", - Input: MeasurementTarget(config.Input), + Input: MeasurementInput(config.Input), MeasurementStartTime: "2018-11-01 15:33:20", ProbeIP: config.ProbeIP, ProbeASN: config.ProbeASN, diff --git a/pkg/model/ooapi.go b/pkg/model/ooapi.go index 940bc4ce5..8a0449568 100644 --- a/pkg/model/ooapi.go +++ b/pkg/model/ooapi.go @@ -99,6 +99,41 @@ type OOAPICheckReportIDResponse struct { V int64 `json:"v"` } +// OOAPIVPNConfig contains the configuration needed to start an OpenVPN connection, returned as part of +// [OOAPIVPNProviderConfig]. +type OOAPIVPNConfig struct { + // CA is the Certificate Authority for the endpoints by this provider. + CA string `json:"ca"` + + // Cert is a valid certificate, for providers that use x509 certificate authentication. + Cert string `json:"cert,omitempty"` + + // Key is a valid key, for providers that use x509 certificate authentication. + Key string `json:"key,omitempty"` + + // Username is a valid username, for providers that use password authentication. + Username string `json:"username,omitempty"` + + // Password is a valid password, for providers that use password authentication. + Password string `json:"password,omitempty"` +} + +// OOAPIVPNProviderConfig is a minimal valid configuration subset for the openvpn experiment; at the moment it provides +// credentials valid for endpoints in a provider, and a list of inputs to be tested on this provider. +type OOAPIVPNProviderConfig struct { + // Provider is the label for this provider. + Provider string `json:"provider,omitempty"` + + // Config is the provider-specific VPN Config. + Config *OOAPIVPNConfig `json:"config"` + + // Inputs is an array of valid endpoints for this provider. + Inputs []string `json:"endpoints"` + + // DateUpdated is when the credential set was last updated in the server database. + DateUpdated time.Time `json:"date_updated"` +} + // OOAPIService describes a backend service. // // The fields of this struct have the meaning described in v2.0.0 of the OONI @@ -147,6 +182,68 @@ type OOAPIURLInfo struct { URL string `json:"url"` } +const ( + // DefaultCategoryCode is the default category code to use + // when a URL's category code is unknown. + DefaultCategoryCode = "MISC" + + // DefaultCountryCode is the default country code to use + // when a URL's country code is unknown. + // + // We use XX because it is the same string that the URL + // prioritization code would return. + // + // See https://github.com/ooni/backend/blob/f7a93f477111c7278424996815b91e6300d66b83/api/ooniapi/prio.py#L182 + DefaultCountryCode = "XX" +) + +// NewOOAPIURLInfoWithDefaultCategoryAndCountry constructs a new instance +// of [*OOAPIURLInfo] with default category and country code. +func NewOOAPIURLInfoWithDefaultCategoryAndCountry(URL string) *OOAPIURLInfo { + return &OOAPIURLInfo{ + CategoryCode: DefaultCategoryCode, + CountryCode: DefaultCountryCode, + URL: URL, + } +} + +var _ ExperimentTarget = &OOAPIURLInfo{} + +// Category implements [ExperimentTarget]. +func (o *OOAPIURLInfo) Category() string { + return o.CategoryCode +} + +// Country implements [ExperimentTarget]. +func (o *OOAPIURLInfo) Country() string { + return o.CountryCode +} + +// Input implements [ExperimentTarget]. +func (o *OOAPIURLInfo) Input() string { + return o.URL +} + +// Options implements ExperimentTarget. +func (o *OOAPIURLInfo) Options() []string { + // Implementation note: we're not serializing any options for now. If/when + // we do that, remember the Options contract: + // + // 1. skip options whose name begins with "Safe"; + // + // 2. skip options that are not scalars; + // + // 3. avoid serializing zero values. + // + // Consider using the [experimentconfig] package to serialize. + return nil +} + +// String implements [ExperimentTarget]. +func (o *OOAPIURLInfo) String() string { + return o.URL +} + const ( // OOAPIReportDefaultDataFormatVersion is the default data format version. // diff --git a/pkg/model/ooapi_test.go b/pkg/model/ooapi_test.go index 65cf1a623..a02958183 100644 --- a/pkg/model/ooapi_test.go +++ b/pkg/model/ooapi_test.go @@ -108,3 +108,31 @@ func TestOOAPIProbeMetadataValid(t *testing.T) { } }) } + +func TestOOAPIURLInfo(t *testing.T) { + info := &OOAPIURLInfo{ + CategoryCode: "SOCIAL", + CountryCode: "IT", + URL: "https://www.facebook.com/", + } + + if info.Category() != "SOCIAL" { + t.Fatal("invalid Category") + } + + if info.Country() != "IT" { + t.Fatal("invalid Country") + } + + if info.Input() != "https://www.facebook.com/" { + t.Fatal("invalid Input") + } + + if info.Options() != nil { + t.Fatal("invalid Options") + } + + if info.String() != "https://www.facebook.com/" { + t.Fatal("invalid String") + } +} diff --git a/pkg/netxlite/certifi.go b/pkg/netxlite/certifi.go index 41f51b51b..06ff8fa8d 100644 --- a/pkg/netxlite/certifi.go +++ b/pkg/netxlite/certifi.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2024-05-13 18:43:22.647694 +0200 CEST m=+0.938119584 +// 2024-08-07 18:58:47.007704 +0530 IST m=+1.550272418 // https://curl.haxx.se/ca/cacert.pem package netxlite @@ -10,7 +10,7 @@ const pemcerts string = ` ## ## Bundle of CA Root Certificates ## -## Certificate data from Mozilla as of: Mon Mar 11 15:25:27 2024 GMT +## Certificate data from Mozilla as of: Tue Jul 2 03:12:04 2024 GMT ## ## This is a bundle of X.509 certificates of public Certificate Authorities ## (CA). These were automatically extracted from Mozilla's root certificates @@ -23,7 +23,7 @@ const pemcerts string = ` ## Just configure this file as the SSLCACertificateFile. ## ## Conversion done with mk-ca-bundle.pl version 1.29. -## SHA256: 4d96bd539f4719e9ace493757afbe4a23ee8579de1c97fbebc50bba3c12e8c1e +## SHA256: 456ff095dde6dd73354c5c28c73d9c06f53b61a803963414cb91a1d92945cdd3 ## @@ -2609,36 +2609,6 @@ vLtoURMMA/cVi4RguYv/Uo7njLwcAjA8+RHUjE7AwWHCFUyqqx0LMV87HOIAl0Qx5v5zli/altP+ CAezNIm8BZ/3Hobui3A= -----END CERTIFICATE----- -GLOBALTRUST 2020 -================ ------BEGIN CERTIFICATE----- -MIIFgjCCA2qgAwIBAgILWku9WvtPilv6ZeUwDQYJKoZIhvcNAQELBQAwTTELMAkGA1UEBhMCQVQx -IzAhBgNVBAoTGmUtY29tbWVyY2UgbW9uaXRvcmluZyBHbWJIMRkwFwYDVQQDExBHTE9CQUxUUlVT -VCAyMDIwMB4XDTIwMDIxMDAwMDAwMFoXDTQwMDYxMDAwMDAwMFowTTELMAkGA1UEBhMCQVQxIzAh -BgNVBAoTGmUtY29tbWVyY2UgbW9uaXRvcmluZyBHbWJIMRkwFwYDVQQDExBHTE9CQUxUUlVTVCAy -MDIwMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAri5WrRsc7/aVj6B3GyvTY4+ETUWi -D59bRatZe1E0+eyLinjF3WuvvcTfk0Uev5E4C64OFudBc/jbu9G4UeDLgztzOG53ig9ZYybNpyrO -VPu44sB8R85gfD+yc/LAGbaKkoc1DZAoouQVBGM+uq/ufF7MpotQsjj3QWPKzv9pj2gOlTblzLmM -CcpL3TGQlsjMH/1WljTbjhzqLL6FLmPdqqmV0/0plRPwyJiT2S0WR5ARg6I6IqIoV6Lr/sCMKKCm -fecqQjuCgGOlYx8ZzHyyZqjC0203b+J+BlHZRYQfEs4kUmSFC0iAToexIiIwquuuvuAC4EDosEKA -A1GqtH6qRNdDYfOiaxaJSaSjpCuKAsR49GiKweR6NrFvG5Ybd0mN1MkGco/PU+PcF4UgStyYJ9OR -JitHHmkHr96i5OTUawuzXnzUJIBHKWk7buis/UDr2O1xcSvy6Fgd60GXIsUf1DnQJ4+H4xj04KlG -DfV0OoIu0G4skaMxXDtG6nsEEFZegB31pWXogvziB4xiRfUg3kZwhqG8k9MedKZssCz3AwyIDMvU -clOGvGBG85hqwvG/Q/lwIHfKN0F5VVJjjVsSn8VoxIidrPIwq7ejMZdnrY8XD2zHc+0klGvIg5rQ -mjdJBKuxFshsSUktq6HQjJLyQUp5ISXbY9e2nKd+Qmn7OmMCAwEAAaNjMGEwDwYDVR0TAQH/BAUw -AwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYEFNwuH9FhN3nkq9XVsxJxaD1qaJwiMB8GA1Ud -IwQYMBaAFNwuH9FhN3nkq9XVsxJxaD1qaJwiMA0GCSqGSIb3DQEBCwUAA4ICAQCR8EICaEDuw2jA -VC/f7GLDw56KoDEoqoOOpFaWEhCGVrqXctJUMHytGdUdaG/7FELYjQ7ztdGl4wJCXtzoRlgHNQIw -4Lx0SsFDKv/bGtCwr2zD/cuz9X9tAy5ZVp0tLTWMstZDFyySCstd6IwPS3BD0IL/qMy/pJTAvoe9 -iuOTe8aPmxadJ2W8esVCgmxcB9CpwYhgROmYhRZf+I/KARDOJcP5YBugxZfD0yyIMaK9MOzQ0MAS -8cE54+X1+NZK3TTN+2/BT+MAi1bikvcoskJ3ciNnxz8RFbLEAwW+uxF7Cr+obuf/WEPPm2eggAe2 -HcqtbepBEX4tdJP7wry+UUTF72glJ4DjyKDUEuzZpTcdN3y0kcra1LGWge9oXHYQSa9+pTeAsRxS -vTOBTI/53WXZFM2KJVj04sWDpQmQ1GwUY7VA3+vA/MRYfg0UFodUJ25W5HCEuGwyEn6CMUO+1918 -oa2u1qsgEu8KwxCMSZY13At1XrFP1U80DhEgB3VDRemjEdqso5nCtnkn4rnvyOL2NSl6dPrFf4IF -YqYK6miyeUcGbvJXqBUzxvd4Sj1Ce2t+/vdG6tHrju+IaFvowdlxfv1k7/9nR4hYJS8+hge9+6jl -gqispdNpQ80xiEmEU5LAsTkbOYMBMMTyqfrQA71yN2BWHzZ8vTmR9W0Nv3vXkg== ------END CERTIFICATE----- - ANF Secure Server Root CA ========================= -----BEGIN CERTIFICATE----- @@ -3589,4 +3559,21 @@ HVlNjM7IDiPCtyaaEBRx/pOyiriA8A4QntOoUAw3gi/q4Iqd4Sw5/7W0cwDk90imc6y/st53BIe0 o82bNSQ3+pCTE4FCxpgmdTdmQRCsu/WU48IxK63nI1bMNSWSs1A= -----END CERTIFICATE----- +FIRMAPROFESIONAL CA ROOT-A WEB +============================== +-----BEGIN CERTIFICATE----- +MIICejCCAgCgAwIBAgIQMZch7a+JQn81QYehZ1ZMbTAKBggqhkjOPQQDAzBuMQswCQYDVQQGEwJF +UzEcMBoGA1UECgwTRmlybWFwcm9mZXNpb25hbCBTQTEYMBYGA1UEYQwPVkFURVMtQTYyNjM0MDY4 +MScwJQYDVQQDDB5GSVJNQVBST0ZFU0lPTkFMIENBIFJPT1QtQSBXRUIwHhcNMjIwNDA2MDkwMTM2 +WhcNNDcwMzMxMDkwMTM2WjBuMQswCQYDVQQGEwJFUzEcMBoGA1UECgwTRmlybWFwcm9mZXNpb25h +bCBTQTEYMBYGA1UEYQwPVkFURVMtQTYyNjM0MDY4MScwJQYDVQQDDB5GSVJNQVBST0ZFU0lPTkFM +IENBIFJPT1QtQSBXRUIwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAARHU+osEaR3xyrq89Zfe9MEkVz6 +iMYiuYMQYneEMy3pA4jU4DP37XcsSmDq5G+tbbT4TIqk5B/K6k84Si6CcyvHZpsKjECcfIr28jlg +st7L7Ljkb+qbXbdTkBgyVcUgt5SjYzBhMA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUk+FD +Y1w8ndYn81LsF7Kpryz3dvgwHQYDVR0OBBYEFJPhQ2NcPJ3WJ/NS7Beyqa8s93b4MA4GA1UdDwEB +/wQEAwIBBjAKBggqhkjOPQQDAwNoADBlAjAdfKR7w4l1M+E7qUW/Runpod3JIha3RxEL2Jq68cgL +cFBTApFwhVmpHqTm6iMxoAACMQD94vizrxa5HnPEluPBMBnYfubDl94cT7iJLzPrSA8Z94dGXSaQ +pYXFuXqUPoeovQA= +-----END CERTIFICATE----- + ` diff --git a/pkg/netxlite/errno.go b/pkg/netxlite/errno.go index 0dadf82b5..5c1cc2897 100644 --- a/pkg/netxlite/errno.go +++ b/pkg/netxlite/errno.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2024-05-13 18:43:23.514043 +0200 CEST m=+0.480330667 +// Generated: 2024-08-07 18:58:47.809137 +0530 IST m=+0.327216084 package netxlite diff --git a/pkg/netxlite/errno_darwin.go b/pkg/netxlite/errno_darwin.go index 4df9980ce..cbf6a1878 100644 --- a/pkg/netxlite/errno_darwin.go +++ b/pkg/netxlite/errno_darwin.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2024-05-13 18:43:23.034437 +0200 CEST m=+0.000713334 +// Generated: 2024-08-07 18:58:47.483524 +0530 IST m=+0.001600709 package netxlite diff --git a/pkg/netxlite/errno_darwin_test.go b/pkg/netxlite/errno_darwin_test.go index cc990de9e..08e9682b3 100644 --- a/pkg/netxlite/errno_darwin_test.go +++ b/pkg/netxlite/errno_darwin_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2024-05-13 18:43:23.30085 +0200 CEST m=+0.267132417 +// Generated: 2024-08-07 18:58:47.697871 +0530 IST m=+0.215949876 package netxlite diff --git a/pkg/netxlite/errno_freebsd.go b/pkg/netxlite/errno_freebsd.go index 0fc15fa20..33a1350dc 100644 --- a/pkg/netxlite/errno_freebsd.go +++ b/pkg/netxlite/errno_freebsd.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2024-05-13 18:43:23.327874 +0200 CEST m=+0.294157167 +// Generated: 2024-08-07 18:58:47.714639 +0530 IST m=+0.232717459 package netxlite diff --git a/pkg/netxlite/errno_freebsd_test.go b/pkg/netxlite/errno_freebsd_test.go index 6972334e4..67e595c88 100644 --- a/pkg/netxlite/errno_freebsd_test.go +++ b/pkg/netxlite/errno_freebsd_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2024-05-13 18:43:23.354839 +0200 CEST m=+0.321123042 +// Generated: 2024-08-07 18:58:47.729768 +0530 IST m=+0.247846751 package netxlite diff --git a/pkg/netxlite/errno_linux.go b/pkg/netxlite/errno_linux.go index 7a4a2cf62..3e5e27d03 100644 --- a/pkg/netxlite/errno_linux.go +++ b/pkg/netxlite/errno_linux.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2024-05-13 18:43:23.42483 +0200 CEST m=+0.391115126 +// Generated: 2024-08-07 18:58:47.765674 +0530 IST m=+0.283752917 package netxlite diff --git a/pkg/netxlite/errno_linux_test.go b/pkg/netxlite/errno_linux_test.go index b7409b697..cf86cf07e 100644 --- a/pkg/netxlite/errno_linux_test.go +++ b/pkg/netxlite/errno_linux_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2024-05-13 18:43:23.448805 +0200 CEST m=+0.415090876 +// Generated: 2024-08-07 18:58:47.777235 +0530 IST m=+0.295313709 package netxlite diff --git a/pkg/netxlite/errno_openbsd.go b/pkg/netxlite/errno_openbsd.go index 85f89d009..903fcc00b 100644 --- a/pkg/netxlite/errno_openbsd.go +++ b/pkg/netxlite/errno_openbsd.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2024-05-13 18:43:23.377383 +0200 CEST m=+0.343667459 +// Generated: 2024-08-07 18:58:47.741885 +0530 IST m=+0.259963959 package netxlite diff --git a/pkg/netxlite/errno_openbsd_test.go b/pkg/netxlite/errno_openbsd_test.go index a9283453b..dbb608527 100644 --- a/pkg/netxlite/errno_openbsd_test.go +++ b/pkg/netxlite/errno_openbsd_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2024-05-13 18:43:23.402976 +0200 CEST m=+0.369261209 +// Generated: 2024-08-07 18:58:47.75471 +0530 IST m=+0.272788667 package netxlite diff --git a/pkg/netxlite/errno_windows.go b/pkg/netxlite/errno_windows.go index 33d27476c..c52106889 100644 --- a/pkg/netxlite/errno_windows.go +++ b/pkg/netxlite/errno_windows.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2024-05-13 18:43:23.47073 +0200 CEST m=+0.437016626 +// Generated: 2024-08-07 18:58:47.788017 +0530 IST m=+0.306096167 package netxlite diff --git a/pkg/netxlite/errno_windows_test.go b/pkg/netxlite/errno_windows_test.go index 5a95d8aa0..24c9f39e2 100644 --- a/pkg/netxlite/errno_windows_test.go +++ b/pkg/netxlite/errno_windows_test.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// Generated: 2024-05-13 18:43:23.492462 +0200 CEST m=+0.458748876 +// Generated: 2024-08-07 18:58:47.798286 +0530 IST m=+0.316365417 package netxlite diff --git a/pkg/oonirun/experiment.go b/pkg/oonirun/experiment.go index 2232a8d05..07bba0381 100644 --- a/pkg/oonirun/experiment.go +++ b/pkg/oonirun/experiment.go @@ -6,13 +6,11 @@ package oonirun import ( "context" - "fmt" + "encoding/json" "math/rand" - "strings" "sync/atomic" "time" - "github.com/ooni/probe-engine/pkg/engine" "github.com/ooni/probe-engine/pkg/humanize" "github.com/ooni/probe-engine/pkg/model" ) @@ -26,9 +24,18 @@ type Experiment struct { // Annotations contains OPTIONAL Annotations for the experiment. Annotations map[string]string - // ExtraOptions contains OPTIONAL extra options for the experiment. + // ExtraOptions contains OPTIONAL extra options that modify the + // default experiment-specific configuration. We apply + // the changes described by this field after using the InitialOptions + // field to initialize the experiment-specific configuration. ExtraOptions map[string]any + // InitialOptions contains an OPTIONAL [json.RawMessage] object + // used to initialize the default experiment-specific + // configuration. After we have initialized the configuration + // as such, we then apply the changes described by the ExtraOptions. + InitialOptions json.RawMessage + // Inputs contains the OPTIONAL experiment Inputs Inputs []string @@ -60,17 +67,17 @@ type Experiment struct { // newExperimentBuilderFn is OPTIONAL and used for testing. newExperimentBuilderFn func(experimentName string) (model.ExperimentBuilder, error) - // newInputLoaderFn is OPTIONAL and used for testing. - newInputLoaderFn func(inputPolicy model.InputPolicy) inputLoader + // newTargetLoaderFn is OPTIONAL and used for testing. + newTargetLoaderFn func(builder model.ExperimentBuilder) targetLoader // newSubmitterFn is OPTIONAL and used for testing. newSubmitterFn func(ctx context.Context) (model.Submitter, error) // newSaverFn is OPTIONAL and used for testing. - newSaverFn func(experiment model.Experiment) (model.Saver, error) + newSaverFn func() (model.Saver, error) // newInputProcessorFn is OPTIONAL and used for testing. - newInputProcessorFn func(experiment model.Experiment, inputList []model.OOAPIURLInfo, + newInputProcessorFn func(experiment model.Experiment, inputList []model.ExperimentTarget, saver model.Saver, submitter model.Submitter) inputProcessor } @@ -83,27 +90,32 @@ func (ed *Experiment) Run(ctx context.Context) error { return err } - // 2. create input loader and load input for this experiment - inputLoader := ed.newInputLoader(builder.InputPolicy()) - inputList, err := inputLoader.Load(ctx) + // 2. configure experiment's options + // + // This MUST happen before loading targets because the options will + // possibly be used to produce richer input targets. + if err := ed.setOptions(builder); err != nil { + return err + } + + // 3. create target loader and load targets for this experiment + targetLoader := ed.newTargetLoader(builder) + targetList, err := targetLoader.Load(ctx) if err != nil { return err } - // 3. randomize input, if needed + // 4. randomize input, if needed if ed.Random { - rnd := rand.New(rand.NewSource(time.Now().UnixNano())) // #nosec G404 -- not really important - rnd.Shuffle(len(inputList), func(i, j int) { - inputList[i], inputList[j] = inputList[j], inputList[i] + // Note: since go1.20 the default random generator is randomly seeded + // + // See https://tip.golang.org/doc/go1.20 + rand.Shuffle(len(targetList), func(i, j int) { + targetList[i], targetList[j] = targetList[j], targetList[i] }) experimentShuffledInputs.Add(1) } - // 4. configure experiment's options - if err := builder.SetOptionsAny(ed.ExtraOptions); err != nil { - return err - } - // 5. construct the experiment instance experiment := builder.NewExperiment() logger := ed.Session.Logger() @@ -121,24 +133,34 @@ func (ed *Experiment) Run(ctx context.Context) error { } // 7. create the saver - saver, err := ed.newSaver(experiment) + saver, err := ed.newSaver() if err != nil { return err } // 8. create an input processor - inputProcessor := ed.newInputProcessor(experiment, inputList, saver, submitter) + inputProcessor := ed.newInputProcessor(experiment, targetList, saver, submitter) // 9. process input and generate measurements return inputProcessor.Run(ctx) } +func (ed *Experiment) setOptions(builder model.ExperimentBuilder) error { + // We first unmarshal the InitialOptions into the experiment + // configuration and afterwards we modify the configuration using + // the values contained inside the ExtraOptions field. + if err := builder.SetOptionsJSON(ed.InitialOptions); err != nil { + return err + } + return builder.SetOptionsAny(ed.ExtraOptions) +} + // inputProcessor is an alias for model.ExperimentInputProcessor type inputProcessor = model.ExperimentInputProcessor // newInputProcessor creates a new inputProcessor instance. func (ed *Experiment) newInputProcessor(experiment model.Experiment, - inputList []model.OOAPIURLInfo, saver model.Saver, submitter model.Submitter) inputProcessor { + inputList []model.ExperimentTarget, saver model.Saver, submitter model.Submitter) inputProcessor { if ed.newInputProcessorFn != nil { return ed.newInputProcessorFn(experiment, inputList, saver, submitter) } @@ -151,7 +173,6 @@ func (ed *Experiment) newInputProcessor(experiment model.Experiment, }, Inputs: inputList, MaxRuntime: time.Duration(ed.MaxRuntime) * time.Second, - Options: experimentOptionsToStringList(ed.ExtraOptions), Saver: NewInputProcessorSaverWrapper(saver), Submitter: &experimentSubmitterWrapper{ child: NewInputProcessorSubmitterWrapper(submitter), @@ -161,9 +182,9 @@ func (ed *Experiment) newInputProcessor(experiment model.Experiment, } // newSaver creates a new engine.Saver instance. -func (ed *Experiment) newSaver(experiment model.Experiment) (model.Saver, error) { +func (ed *Experiment) newSaver() (model.Saver, error) { if ed.newSaverFn != nil { - return ed.newSaverFn(experiment) + return ed.newSaverFn() } return NewSaver(SaverConfig{ Enabled: !ed.NoJSON, @@ -192,42 +213,24 @@ func (ed *Experiment) newExperimentBuilder(experimentName string) (model.Experim return ed.Session.NewExperimentBuilder(ed.Name) } -// inputLoader is an alias for model.ExperimentInputLoader -type inputLoader = model.ExperimentInputLoader +// targetLoader is an alias for [model.ExperimentTargetLoader]. +type targetLoader = model.ExperimentTargetLoader -// newInputLoader creates a new inputLoader. -func (ed *Experiment) newInputLoader(inputPolicy model.InputPolicy) inputLoader { - if ed.newInputLoaderFn != nil { - return ed.newInputLoaderFn(inputPolicy) +// newTargetLoader creates a new [model.ExperimentTargetLoader]. +func (ed *Experiment) newTargetLoader(builder model.ExperimentBuilder) targetLoader { + if ed.newTargetLoaderFn != nil { + return ed.newTargetLoaderFn(builder) } - return &engine.InputLoader{ + return builder.NewTargetLoader(&model.ExperimentTargetLoaderConfig{ CheckInConfig: &model.OOAPICheckInConfig{ RunType: model.RunTypeManual, OnWiFi: true, // meaning: not on 4G Charging: true, }, - ExperimentName: ed.Name, - InputPolicy: inputPolicy, - StaticInputs: ed.Inputs, - SourceFiles: ed.InputFilePaths, - Session: ed.Session, - } -} - -// experimentOptionsToStringList convers the options to []string, which is -// the format with which we include them into a OONI Measurement. The resulting -// []string will skip any option that is named with a `Safe` prefix (case -// sensitive). -func experimentOptionsToStringList(options map[string]any) (out []string) { - // the prefix to skip inclusion in the string list - safeOptionPrefix := "Safe" - for key, value := range options { - if strings.HasPrefix(key, safeOptionPrefix) { - continue - } - out = append(out, fmt.Sprintf("%s=%v", key, value)) - } - return + StaticInputs: ed.Inputs, + SourceFiles: ed.InputFilePaths, + Session: ed.Session, + }) } // experimentWrapper wraps an experiment and logs progress @@ -242,12 +245,12 @@ type experimentWrapper struct { total int } -func (ew *experimentWrapper) MeasureAsync( - ctx context.Context, input string, idx int) (<-chan *model.Measurement, error) { - if input != "" { - ew.logger.Infof("[%d/%d] running with input: %s", idx+1, ew.total, input) +func (ew *experimentWrapper) MeasureWithContext( + ctx context.Context, target model.ExperimentTarget, idx int) (*model.Measurement, error) { + if target.Input() != "" { + ew.logger.Infof("[%d/%d] running with input: %s", idx+1, ew.total, target) } - return ew.child.MeasureAsync(ctx, input, idx) + return ew.child.MeasureWithContext(ctx, target, idx) } // experimentSubmitterWrapper implements a submission policy where we don't diff --git a/pkg/oonirun/experiment_test.go b/pkg/oonirun/experiment_test.go index 7a068d661..2fea2cfa4 100644 --- a/pkg/oonirun/experiment_test.go +++ b/pkg/oonirun/experiment_test.go @@ -2,12 +2,13 @@ package oonirun import ( "context" + "encoding/json" "errors" - "reflect" - "sort" "testing" "time" + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-engine/pkg/engine" "github.com/ooni/probe-engine/pkg/mocks" "github.com/ooni/probe-engine/pkg/model" "github.com/ooni/probe-engine/pkg/testingx" @@ -16,6 +17,7 @@ import ( func TestExperimentRunWithFailureToSubmitAndShuffle(t *testing.T) { shuffledInputsPrev := experimentShuffledInputs.Load() var calledSetOptionsAny int + var calledSetOptionsJSON int var failedToSubmit int var calledKibiBytesReceived int var calledKibiBytesSent int @@ -27,9 +29,6 @@ func TestExperimentRunWithFailureToSubmitAndShuffle(t *testing.T) { ExtraOptions: map[string]any{ "SleepTime": int64(10 * time.Millisecond), }, - Inputs: []string{ - "a", "b", "c", - }, InputFilePaths: []string{}, MaxRuntime: 0, Name: "example", @@ -43,22 +42,22 @@ func TestExperimentRunWithFailureToSubmitAndShuffle(t *testing.T) { MockInputPolicy: func() model.InputPolicy { return model.InputOptional }, + MockSetOptionsJSON: func(value json.RawMessage) error { + calledSetOptionsJSON++ + return nil + }, MockSetOptionsAny: func(options map[string]any) error { calledSetOptionsAny++ return nil }, MockNewExperiment: func() model.Experiment { exp := &mocks.Experiment{ - MockMeasureAsync: func(ctx context.Context, input string) (<-chan *model.Measurement, error) { - out := make(chan *model.Measurement) - go func() { - defer close(out) - ff := &testingx.FakeFiller{} - var meas model.Measurement - ff.Fill(&meas) - out <- &meas - }() - return out, nil + MockMeasureWithContext: func( + ctx context.Context, target model.ExperimentTarget) (*model.Measurement, error) { + ff := &testingx.FakeFiller{} + var meas model.Measurement + ff.Fill(&meas) + return &meas, nil }, MockKibiBytesReceived: func() float64 { calledKibiBytesReceived++ @@ -71,6 +70,18 @@ func TestExperimentRunWithFailureToSubmitAndShuffle(t *testing.T) { } return exp }, + MockNewTargetLoader: func(config *model.ExperimentTargetLoaderConfig) model.ExperimentTargetLoader { + return &mocks.ExperimentTargetLoader{ + MockLoad: func(ctx context.Context) ([]model.ExperimentTarget, error) { + results := []model.ExperimentTarget{ + model.NewOOAPIURLInfoWithDefaultCategoryAndCountry("a"), + model.NewOOAPIURLInfoWithDefaultCategoryAndCountry("b"), + model.NewOOAPIURLInfoWithDefaultCategoryAndCountry("c"), + } + return results, nil + }, + } + }, } return eb, nil }, @@ -79,7 +90,7 @@ func TestExperimentRunWithFailureToSubmitAndShuffle(t *testing.T) { }, }, newExperimentBuilderFn: nil, - newInputLoaderFn: nil, + newTargetLoaderFn: nil, newSubmitterFn: func(ctx context.Context) (model.Submitter, error) { subm := &mocks.Submitter{ MockSubmit: func(ctx context.Context, m *model.Measurement) error { @@ -104,6 +115,9 @@ func TestExperimentRunWithFailureToSubmitAndShuffle(t *testing.T) { if calledSetOptionsAny < 1 { t.Fatal("should have called SetOptionsAny") } + if calledSetOptionsJSON < 1 { + t.Fatal("should have called SetOptionsJSON") + } if calledKibiBytesReceived < 1 { t.Fatal("did not call KibiBytesReceived") } @@ -112,45 +126,64 @@ func TestExperimentRunWithFailureToSubmitAndShuffle(t *testing.T) { } } -func Test_experimentOptionsToStringList(t *testing.T) { - type args struct { - options map[string]any +// This test ensures that we honour InitialOptions then ExtraOptions. +func TestExperimentSetOptions(t *testing.T) { + + // create the Experiment we're using for this test + exp := &Experiment{ + ExtraOptions: map[string]any{ + "Message": "jarjarbinks", + }, + InitialOptions: []byte(`{"Message": "foobar", "ReturnError": true}`), + Name: "example", + + // TODO(bassosimone): A zero-value session works here. The proper change + // however would be to write a engine.NewExperimentBuilder factory that takes + // as input an interface for the session. This would help testing. + Session: &engine.Session{}, } - tests := []struct { - name string - args args - wantOut []string - }{ - { - name: "happy path: a map with three entries returns three items", - args: args{ - map[string]any{ - "foo": 1, - "bar": 2, - "baaz": 3, - }, - }, - wantOut: []string{"baaz=3", "bar=2", "foo=1"}, + + // create the experiment builder manually + builder, err := exp.newExperimentBuilder(exp.Name) + if err != nil { + t.Fatal(err) + } + + // invoke the method we're testing + if err := exp.setOptions(builder); err != nil { + t.Fatal(err) + } + + // obtain the options + options, err := builder.Options() + if err != nil { + t.Fatal(err) + } + + // describe what we expect to happen + // + // we basically want ExtraOptions to override InitialOptions + expect := map[string]model.ExperimentOptionInfo{ + "Message": { + Doc: "Message to emit at test completion", + Type: "string", + Value: string("jarjarbinks"), // set by ExtraOptions }, - { - name: "an option beginning with `Safe` is skipped from the output", - args: args{ - map[string]any{ - "foo": 1, - "Safefoo": 42, - }, - }, - wantOut: []string{"foo=1"}, + "ReturnError": { + Doc: "Toogle to return a mocked error", + Type: "bool", + Value: bool(true), // set by InitialOptions + }, + "SleepTime": { + Doc: "Amount of time to sleep for in nanosecond", + Type: "int64", + Value: int64(1000000000), // still the default nonzero value }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - gotOut := experimentOptionsToStringList(tt.args.options) - sort.Strings(gotOut) - if !reflect.DeepEqual(gotOut, tt.wantOut) { - t.Errorf("experimentOptionsToStringList() = %v, want %v", gotOut, tt.wantOut) - } - }) + + // make sure the result equals expectation + if diff := cmp.Diff(expect, options); diff != "" { + t.Fatal(diff) } } @@ -169,10 +202,11 @@ func TestExperimentRun(t *testing.T) { ReportFile string Session Session newExperimentBuilderFn func(experimentName string) (model.ExperimentBuilder, error) - newInputLoaderFn func(inputPolicy model.InputPolicy) inputLoader + newTargetLoaderFn func(builder model.ExperimentBuilder) targetLoader newSubmitterFn func(ctx context.Context) (model.Submitter, error) - newSaverFn func(experiment model.Experiment) (model.Saver, error) - newInputProcessorFn func(experiment model.Experiment, inputList []model.OOAPIURLInfo, saver model.Saver, submitter model.Submitter) inputProcessor + newSaverFn func() (model.Saver, error) + newInputProcessorFn func(experiment model.Experiment, + inputList []model.ExperimentTarget, saver model.Saver, submitter model.Submitter) inputProcessor } type args struct { ctx context.Context @@ -192,20 +226,44 @@ func TestExperimentRun(t *testing.T) { args: args{}, expectErr: errMocked, }, { - name: "cannot load input", + name: "cannot set InitialOptions", fields: fields{ newExperimentBuilderFn: func(experimentName string) (model.ExperimentBuilder, error) { eb := &mocks.ExperimentBuilder{ - MockInputPolicy: func() model.InputPolicy { - return model.InputOptional + MockSetOptionsJSON: func(value json.RawMessage) error { + return errMocked }, } return eb, nil }, - newInputLoaderFn: func(inputPolicy model.InputPolicy) inputLoader { - return &mocks.ExperimentInputLoader{ - MockLoad: func(ctx context.Context) ([]model.OOAPIURLInfo, error) { - return nil, errMocked + newTargetLoaderFn: func(builder model.ExperimentBuilder) targetLoader { + return &mocks.ExperimentTargetLoader{ + MockLoad: func(ctx context.Context) ([]model.ExperimentTarget, error) { + return []model.ExperimentTarget{}, nil + }, + } + }, + }, + args: args{}, + expectErr: errMocked, + }, { + name: "cannot set ExtraOptions", + fields: fields{ + newExperimentBuilderFn: func(experimentName string) (model.ExperimentBuilder, error) { + eb := &mocks.ExperimentBuilder{ + MockSetOptionsJSON: func(value json.RawMessage) error { + return nil + }, + MockSetOptionsAny: func(options map[string]any) error { + return errMocked + }, + } + return eb, nil + }, + newTargetLoaderFn: func(builder model.ExperimentBuilder) targetLoader { + return &mocks.ExperimentTargetLoader{ + MockLoad: func(ctx context.Context) ([]model.ExperimentTarget, error) { + return []model.ExperimentTarget{}, nil }, } }, @@ -213,23 +271,26 @@ func TestExperimentRun(t *testing.T) { args: args{}, expectErr: errMocked, }, { - name: "cannot set options", + name: "cannot load input", fields: fields{ newExperimentBuilderFn: func(experimentName string) (model.ExperimentBuilder, error) { eb := &mocks.ExperimentBuilder{ MockInputPolicy: func() model.InputPolicy { return model.InputOptional }, + MockSetOptionsJSON: func(value json.RawMessage) error { + return nil + }, MockSetOptionsAny: func(options map[string]any) error { - return errMocked + return nil }, } return eb, nil }, - newInputLoaderFn: func(inputPolicy model.InputPolicy) inputLoader { - return &mocks.ExperimentInputLoader{ - MockLoad: func(ctx context.Context) ([]model.OOAPIURLInfo, error) { - return []model.OOAPIURLInfo{}, nil + newTargetLoaderFn: func(builder model.ExperimentBuilder) targetLoader { + return &mocks.ExperimentTargetLoader{ + MockLoad: func(ctx context.Context) ([]model.ExperimentTarget, error) { + return nil, errMocked }, } }, @@ -249,6 +310,9 @@ func TestExperimentRun(t *testing.T) { MockInputPolicy: func() model.InputPolicy { return model.InputOptional }, + MockSetOptionsJSON: func(value json.RawMessage) error { + return nil + }, MockSetOptionsAny: func(options map[string]any) error { return nil }, @@ -266,10 +330,10 @@ func TestExperimentRun(t *testing.T) { } return eb, nil }, - newInputLoaderFn: func(inputPolicy model.InputPolicy) inputLoader { - return &mocks.ExperimentInputLoader{ - MockLoad: func(ctx context.Context) ([]model.OOAPIURLInfo, error) { - return []model.OOAPIURLInfo{}, nil + newTargetLoaderFn: func(builder model.ExperimentBuilder) targetLoader { + return &mocks.ExperimentTargetLoader{ + MockLoad: func(ctx context.Context) ([]model.ExperimentTarget, error) { + return []model.ExperimentTarget{}, nil }, } }, @@ -292,6 +356,9 @@ func TestExperimentRun(t *testing.T) { MockInputPolicy: func() model.InputPolicy { return model.InputOptional }, + MockSetOptionsJSON: func(value json.RawMessage) error { + return nil + }, MockSetOptionsAny: func(options map[string]any) error { return nil }, @@ -309,17 +376,17 @@ func TestExperimentRun(t *testing.T) { } return eb, nil }, - newInputLoaderFn: func(inputPolicy model.InputPolicy) inputLoader { - return &mocks.ExperimentInputLoader{ - MockLoad: func(ctx context.Context) ([]model.OOAPIURLInfo, error) { - return []model.OOAPIURLInfo{}, nil + newTargetLoaderFn: func(builder model.ExperimentBuilder) targetLoader { + return &mocks.ExperimentTargetLoader{ + MockLoad: func(ctx context.Context) ([]model.ExperimentTarget, error) { + return []model.ExperimentTarget{}, nil }, } }, newSubmitterFn: func(ctx context.Context) (model.Submitter, error) { return &mocks.Submitter{}, nil }, - newSaverFn: func(experiment model.Experiment) (model.Saver, error) { + newSaverFn: func() (model.Saver, error) { return nil, errMocked }, }, @@ -338,6 +405,9 @@ func TestExperimentRun(t *testing.T) { MockInputPolicy: func() model.InputPolicy { return model.InputOptional }, + MockSetOptionsJSON: func(value json.RawMessage) error { + return nil + }, MockSetOptionsAny: func(options map[string]any) error { return nil }, @@ -355,20 +425,20 @@ func TestExperimentRun(t *testing.T) { } return eb, nil }, - newInputLoaderFn: func(inputPolicy model.InputPolicy) inputLoader { - return &mocks.ExperimentInputLoader{ - MockLoad: func(ctx context.Context) ([]model.OOAPIURLInfo, error) { - return []model.OOAPIURLInfo{}, nil + newTargetLoaderFn: func(builder model.ExperimentBuilder) targetLoader { + return &mocks.ExperimentTargetLoader{ + MockLoad: func(ctx context.Context) ([]model.ExperimentTarget, error) { + return []model.ExperimentTarget{}, nil }, } }, newSubmitterFn: func(ctx context.Context) (model.Submitter, error) { return &mocks.Submitter{}, nil }, - newSaverFn: func(experiment model.Experiment) (model.Saver, error) { + newSaverFn: func() (model.Saver, error) { return &mocks.Saver{}, nil }, - newInputProcessorFn: func(experiment model.Experiment, inputList []model.OOAPIURLInfo, + newInputProcessorFn: func(experiment model.Experiment, inputList []model.ExperimentTarget, saver model.Saver, submitter model.Submitter) inputProcessor { return &mocks.ExperimentInputProcessor{ MockRun: func(ctx context.Context) error { @@ -395,7 +465,7 @@ func TestExperimentRun(t *testing.T) { ReportFile: tt.fields.ReportFile, Session: tt.fields.Session, newExperimentBuilderFn: tt.fields.newExperimentBuilderFn, - newInputLoaderFn: tt.fields.newInputLoaderFn, + newTargetLoaderFn: tt.fields.newTargetLoaderFn, newSubmitterFn: tt.fields.newSubmitterFn, newSaverFn: tt.fields.newSaverFn, newInputProcessorFn: tt.fields.newInputProcessorFn, diff --git a/pkg/oonirun/inputprocessor.go b/pkg/oonirun/inputprocessor.go index 5fcf6707e..e38b049c6 100644 --- a/pkg/oonirun/inputprocessor.go +++ b/pkg/oonirun/inputprocessor.go @@ -10,15 +10,13 @@ import ( // InputProcessorExperiment is the Experiment // according to InputProcessor. type InputProcessorExperiment interface { - MeasureAsync( - ctx context.Context, input string) (<-chan *model.Measurement, error) + MeasureWithContext(ctx context.Context, target model.ExperimentTarget) (*model.Measurement, error) } // InputProcessorExperimentWrapper is a wrapper for an // Experiment that also allow to pass around the input index. type InputProcessorExperimentWrapper interface { - MeasureAsync( - ctx context.Context, input string, idx int) (<-chan *model.Measurement, error) + MeasureWithContext(ctx context.Context, target model.ExperimentTarget, idx int) (*model.Measurement, error) } // NewInputProcessorExperimentWrapper creates a new @@ -32,9 +30,9 @@ type inputProcessorExperimentWrapper struct { exp InputProcessorExperiment } -func (ipew inputProcessorExperimentWrapper) MeasureAsync( - ctx context.Context, input string, idx int) (<-chan *model.Measurement, error) { - return ipew.exp.MeasureAsync(ctx, input) +func (ipew inputProcessorExperimentWrapper) MeasureWithContext( + ctx context.Context, target model.ExperimentTarget, idx int) (*model.Measurement, error) { + return ipew.exp.MeasureWithContext(ctx, target) } var _ InputProcessorExperimentWrapper = inputProcessorExperimentWrapper{} @@ -49,7 +47,7 @@ type InputProcessor struct { Experiment InputProcessorExperimentWrapper // Inputs is the list of inputs to measure. - Inputs []model.OOAPIURLInfo + Inputs []model.ExperimentTarget // MaxRuntime is the optional maximum runtime // when looping over a list of inputs (e.g. when @@ -57,9 +55,6 @@ type InputProcessor struct { // there will be no MaxRuntime limit. MaxRuntime time.Duration - // Options contains command line options for this experiment. - Options []string - // Saver is the code that will save measurement results // on persistent storage (e.g. the file system). Saver InputProcessorSaverWrapper @@ -137,34 +132,29 @@ const ( // also returns the reason why we stopped. func (ip *InputProcessor) run(ctx context.Context) (int, error) { start := time.Now() - for idx, url := range ip.Inputs { + for idx, target := range ip.Inputs { if ip.MaxRuntime > 0 && time.Since(start) > ip.MaxRuntime { return stopMaxRuntime, nil } - input := url.URL - var measurements []*model.Measurement - source, err := ip.Experiment.MeasureAsync(ctx, input, idx) + meas, err := ip.Experiment.MeasureWithContext(ctx, target, idx) if err != nil { return 0, err } - // NOTE: we don't want to intermix measuring with submitting - // therefore we collect all measurements first - for meas := range source { - measurements = append(measurements, meas) + meas.AddAnnotations(ip.Annotations) + err = ip.Submitter.Submit(ctx, idx, meas) + if err != nil { + // TODO(bassosimone): when re-reading this code, I find it confusing that + // we return on error because I am always like "wait, this is not the right + // thing to do here". Then, I remember that the experimentSubmitterWrapper{} + // ignores this error and so it's like it does not exist. Maybe we should + // rewrite the code to do the right thing here 😬😬😬. + return 0, err } - for _, meas := range measurements { - meas.AddAnnotations(ip.Annotations) - meas.Options = ip.Options - err = ip.Submitter.Submit(ctx, idx, meas) - if err != nil { - return 0, err - } - // Note: must be after submission because submission modifies - // the measurement to include the report ID. - err = ip.Saver.SaveMeasurement(idx, meas) - if err != nil { - return 0, err - } + // Note: must be after submission because submission modifies + // the measurement to include the report ID. + err = ip.Saver.SaveMeasurement(idx, meas) + if err != nil { + return 0, err } } return stopNormal, nil diff --git a/pkg/oonirun/inputprocessor_test.go b/pkg/oonirun/inputprocessor_test.go index a4e875121..5159e1b34 100644 --- a/pkg/oonirun/inputprocessor_test.go +++ b/pkg/oonirun/inputprocessor_test.go @@ -15,8 +15,8 @@ type FakeInputProcessorExperiment struct { M []*model.Measurement } -func (fipe *FakeInputProcessorExperiment) MeasureAsync( - ctx context.Context, input string) (<-chan *model.Measurement, error) { +func (fipe *FakeInputProcessorExperiment) MeasureWithContext( + ctx context.Context, target model.ExperimentTarget) (*model.Measurement, error) { if fipe.Err != nil { return nil, fipe.Err } @@ -28,14 +28,9 @@ func (fipe *FakeInputProcessorExperiment) MeasureAsync( // is MERGING annotations as opposed to overwriting them. m.AddAnnotation("antani", "antani") m.AddAnnotation("foo", "baz") // would be bar below - m.Input = model.MeasurementTarget(input) + m.Input = model.MeasurementInput(target.Input()) fipe.M = append(fipe.M, m) - out := make(chan *model.Measurement) - go func() { - defer close(out) - out <- m - }() - return out, nil + return m, nil } func TestInputProcessorMeasurementFailed(t *testing.T) { @@ -44,9 +39,9 @@ func TestInputProcessorMeasurementFailed(t *testing.T) { Experiment: NewInputProcessorExperimentWrapper( &FakeInputProcessorExperiment{Err: expected}, ), - Inputs: []model.OOAPIURLInfo{{ - URL: "https://www.kernel.org/", - }}, + Inputs: []model.ExperimentTarget{ + model.NewOOAPIURLInfoWithDefaultCategoryAndCountry("https://www.kernel.org/"), + }, } ctx := context.Background() if err := ip.Run(ctx); !errors.Is(err, expected) { @@ -73,10 +68,9 @@ func TestInputProcessorSubmissionFailed(t *testing.T) { "foo": "bar", }, Experiment: NewInputProcessorExperimentWrapper(fipe), - Inputs: []model.OOAPIURLInfo{{ - URL: "https://www.kernel.org/", - }}, - Options: []string{"fake=true"}, + Inputs: []model.ExperimentTarget{ + model.NewOOAPIURLInfoWithDefaultCategoryAndCountry("https://www.kernel.org/"), + }, Submitter: NewInputProcessorSubmitterWrapper( &FakeInputProcessorSubmitter{Err: expected}, ), @@ -101,9 +95,6 @@ func TestInputProcessorSubmissionFailed(t *testing.T) { if m.Annotations["antani"] != "antani" { t.Fatal("invalid annotation: antani") } - if len(m.Options) != 1 || m.Options[0] != "fake=true" { - t.Fatal("options not set") - } } type FakeInputProcessorSaver struct { @@ -122,10 +113,9 @@ func TestInputProcessorSaveOnDiskFailed(t *testing.T) { Experiment: NewInputProcessorExperimentWrapper( &FakeInputProcessorExperiment{}, ), - Inputs: []model.OOAPIURLInfo{{ - URL: "https://www.kernel.org/", - }}, - Options: []string{"fake=true"}, + Inputs: []model.ExperimentTarget{ + model.NewOOAPIURLInfoWithDefaultCategoryAndCountry("https://www.kernel.org/"), + }, Saver: NewInputProcessorSaverWrapper( &FakeInputProcessorSaver{Err: expected}, ), @@ -145,12 +135,10 @@ func TestInputProcessorGood(t *testing.T) { submitter := &FakeInputProcessorSubmitter{Err: nil} ip := &InputProcessor{ Experiment: NewInputProcessorExperimentWrapper(fipe), - Inputs: []model.OOAPIURLInfo{{ - URL: "https://www.kernel.org/", - }, { - URL: "https://www.slashdot.org/", - }}, - Options: []string{"fake=true"}, + Inputs: []model.ExperimentTarget{ + model.NewOOAPIURLInfoWithDefaultCategoryAndCountry("https://www.kernel.org/"), + model.NewOOAPIURLInfoWithDefaultCategoryAndCountry("https://www.slashdot.org/"), + }, Saver: NewInputProcessorSaverWrapper(saver), Submitter: NewInputProcessorSubmitterWrapper(submitter), } @@ -187,13 +175,11 @@ func TestInputProcessorMaxRuntime(t *testing.T) { submitter := &FakeInputProcessorSubmitter{Err: nil} ip := &InputProcessor{ Experiment: NewInputProcessorExperimentWrapper(fipe), - Inputs: []model.OOAPIURLInfo{{ - URL: "https://www.kernel.org/", - }, { - URL: "https://www.slashdot.org/", - }}, + Inputs: []model.ExperimentTarget{ + model.NewOOAPIURLInfoWithDefaultCategoryAndCountry("https://www.kernel.org/"), + model.NewOOAPIURLInfoWithDefaultCategoryAndCountry("https://www.slashdot.org/"), + }, MaxRuntime: 1 * time.Nanosecond, - Options: []string{"fake=true"}, Saver: NewInputProcessorSaverWrapper(saver), Submitter: NewInputProcessorSubmitterWrapper(submitter), } diff --git a/pkg/oonirun/link.go b/pkg/oonirun/link.go index da2d1ec76..6f0f423ca 100644 --- a/pkg/oonirun/link.go +++ b/pkg/oonirun/link.go @@ -19,6 +19,10 @@ type LinkConfig struct { // reviewing what it contains or what has changed. AcceptChanges bool + // AuthFile is OPTIONAL and will add an authentication header to the + // request used for fetching this OONI Run link. + AuthFile string + // Annotations contains OPTIONAL Annotations for the experiment. Annotations map[string]string diff --git a/pkg/oonirun/session.go b/pkg/oonirun/session.go index 89710d522..0427d5fb7 100644 --- a/pkg/oonirun/session.go +++ b/pkg/oonirun/session.go @@ -10,14 +10,14 @@ package oonirun // import ( - "github.com/ooni/probe-engine/pkg/engine" "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/targetloading" ) // Session is the definition of Session used by this package. type Session interface { - // A Session is also an InputLoaderSession. - engine.InputLoaderSession + // A Session is also an [targetloading.Session]. + targetloading.Session // A Session is also a SubmitterSession. SubmitterSession diff --git a/pkg/oonirun/v1.go b/pkg/oonirun/v1.go index 48bf89470..8a5edeaec 100644 --- a/pkg/oonirun/v1.go +++ b/pkg/oonirun/v1.go @@ -84,7 +84,7 @@ func v1Measure(ctx context.Context, config *LinkConfig, URL string) error { ReportFile: config.ReportFile, Session: config.Session, newExperimentBuilderFn: nil, - newInputLoaderFn: nil, + newTargetLoaderFn: nil, newSubmitterFn: nil, newSaverFn: nil, newInputProcessorFn: nil, diff --git a/pkg/oonirun/v1_test.go b/pkg/oonirun/v1_test.go index 645176b58..39f3026c5 100644 --- a/pkg/oonirun/v1_test.go +++ b/pkg/oonirun/v1_test.go @@ -2,6 +2,7 @@ package oonirun import ( "context" + "encoding/json" "errors" "net/http" "strings" @@ -23,21 +24,20 @@ func newMinimalFakeSession() *mocks.Session { MockInputPolicy: func() model.InputPolicy { return model.InputNone }, + MockSetOptionsJSON: func(value json.RawMessage) error { + return nil + }, MockSetOptionsAny: func(options map[string]any) error { return nil }, MockNewExperiment: func() model.Experiment { exp := &mocks.Experiment{ - MockMeasureAsync: func(ctx context.Context, input string) (<-chan *model.Measurement, error) { - out := make(chan *model.Measurement) - go func() { - defer close(out) - ff := &testingx.FakeFiller{} - var meas model.Measurement - ff.Fill(&meas) - out <- &meas - }() - return out, nil + MockMeasureWithContext: func( + ctx context.Context, target model.ExperimentTarget) (*model.Measurement, error) { + ff := &testingx.FakeFiller{} + var meas model.Measurement + ff.Fill(&meas) + return &meas, nil }, MockKibiBytesReceived: func() float64 { return 1.1 @@ -48,6 +48,16 @@ func newMinimalFakeSession() *mocks.Session { } return exp }, + MockNewTargetLoader: func(config *model.ExperimentTargetLoaderConfig) model.ExperimentTargetLoader { + return &mocks.ExperimentTargetLoader{ + MockLoad: func(ctx context.Context) ([]model.ExperimentTarget, error) { + // Implementation note: the convention for input-less experiments is that + // they require a single entry containing an empty input. + entry := model.NewOOAPIURLInfoWithDefaultCategoryAndCountry("") + return []model.ExperimentTarget{entry}, nil + }, + } + }, } return eb, nil }, diff --git a/pkg/oonirun/v2.go b/pkg/oonirun/v2.go index 1dd4f53d9..635e43dd8 100644 --- a/pkg/oonirun/v2.go +++ b/pkg/oonirun/v2.go @@ -5,15 +5,19 @@ package oonirun // import ( + "bufio" "context" + "encoding/base64" "encoding/json" "errors" "fmt" + "strings" "sync/atomic" "github.com/hexops/gotextdiff" "github.com/hexops/gotextdiff/myers" "github.com/hexops/gotextdiff/span" + "github.com/ooni/probe-engine/pkg/fsx" "github.com/ooni/probe-engine/pkg/httpclientx" "github.com/ooni/probe-engine/pkg/kvstore" "github.com/ooni/probe-engine/pkg/model" @@ -54,7 +58,7 @@ type V2Nettest struct { // `Safe` will be available for the experiment run, but omitted from // the serialized Measurement that the experiment builder will submit // to the OONI backend. - Options map[string]any `json:"options"` + Options json.RawMessage `json:"options"` // TestName contains the nettest name. TestName string `json:"test_name"` @@ -63,12 +67,16 @@ type V2Nettest struct { // getV2DescriptorFromHTTPSURL GETs a v2Descriptor instance from // a static URL (e.g., from a GitHub repo or from a Gist). func getV2DescriptorFromHTTPSURL(ctx context.Context, client model.HTTPClient, - logger model.Logger, URL string) (*V2Descriptor, error) { + logger model.Logger, URL, auth string) (*V2Descriptor, error) { + if auth != "" { + // we assume a bearer token + auth = fmt.Sprintf("Bearer %s", auth) + } return httpclientx.GetJSON[*V2Descriptor]( ctx, httpclientx.NewEndpoint(URL), &httpclientx.Config{ - Authorization: "", // not needed + Authorization: auth, Client: client, Logger: logger, UserAgent: model.HTTPHeaderUserAgent, @@ -140,9 +148,9 @@ func v2DescriptorCacheLoad(fsstore model.KeyValueStore) (*v2DescriptorCache, err // - err is the error that occurred, or nil in case of success. func (cache *v2DescriptorCache) PullChangesWithoutSideEffects( ctx context.Context, client model.HTTPClient, logger model.Logger, - URL string) (oldValue, newValue *V2Descriptor, err error) { + URL, auth string) (oldValue, newValue *V2Descriptor, err error) { oldValue = cache.Entries[URL] - newValue, err = getV2DescriptorFromHTTPSURL(ctx, client, logger, URL) + newValue, err = getV2DescriptorFromHTTPSURL(ctx, client, logger, URL, auth) return } @@ -183,7 +191,8 @@ func V2MeasureDescriptor(ctx context.Context, config *LinkConfig, desc *V2Descri // construct an experiment from the current nettest exp := &Experiment{ Annotations: config.Annotations, - ExtraOptions: nettest.Options, + ExtraOptions: make(map[string]any), + InitialOptions: nettest.Options, Inputs: nettest.Inputs, InputFilePaths: nil, MaxRuntime: config.MaxRuntime, @@ -194,7 +203,7 @@ func V2MeasureDescriptor(ctx context.Context, config *LinkConfig, desc *V2Descri ReportFile: config.ReportFile, Session: config.Session, newExperimentBuilderFn: nil, - newInputLoaderFn: nil, + newTargetLoaderFn: nil, newSubmitterFn: nil, newSaverFn: nil, newInputProcessorFn: nil, @@ -262,7 +271,11 @@ func v2MeasureHTTPS(ctx context.Context, config *LinkConfig, URL string) error { // pull a possibly new descriptor without updating the old descriptor clnt := config.Session.DefaultHTTPClient() - oldValue, newValue, err := cache.PullChangesWithoutSideEffects(ctx, clnt, logger, URL) + auth, err := v2MaybeGetAuthenticationTokenFromFile(config.AuthFile) + if err != nil { + logger.Warnf("oonirun: failed to retrieve auth token: %v", err) + } + oldValue, newValue, err := cache.PullChangesWithoutSideEffects(ctx, clnt, logger, URL, auth) if err != nil { return err } @@ -289,3 +302,46 @@ func v2MeasureHTTPS(ctx context.Context, config *LinkConfig, URL string) error { // note: this function gracefully handles nil values return V2MeasureDescriptor(ctx, config, newValue) } + +func v2MaybeGetAuthenticationTokenFromFile(path string) (string, error) { + if path != "" { + return v2ReadBearerTokenFromFile(path) + } + return "", nil +} + +// v2ReadBearerTokenFromFile tries to extract a valid (base64) bearer token from +// the first line of the passed text file. +// If there is an error while reading from the file, the error will be returned. +// If we can read from the file but there's no valid token found, an empty string will be returned. +func v2ReadBearerTokenFromFile(fileName string) (string, error) { + filep, err := fsx.OpenFile(fileName) + if err != nil { + return "", err + } + defer filep.Close() + + scanner := bufio.NewScanner(filep) + + // Scan the first line + if scanner.Scan() { + line := scanner.Text() + + token := strings.TrimSpace(line) + + // if this is not a valid base64 token, return empty string + if _, err := base64.StdEncoding.DecodeString(token); err != nil { + return "", nil + } + + return token, nil + } + + // Check for any scanning error + if err := scanner.Err(); err != nil { + return "", err + } + + // Return empty string if file is empty + return "", nil +} diff --git a/pkg/oonirun/v2_test.go b/pkg/oonirun/v2_test.go index 5d5520922..c1fe2264a 100644 --- a/pkg/oonirun/v2_test.go +++ b/pkg/oonirun/v2_test.go @@ -6,8 +6,9 @@ import ( "errors" "net/http" "net/http/httptest" + "os" + "path/filepath" "testing" - "time" "github.com/ooni/probe-engine/pkg/httpclientx" "github.com/ooni/probe-engine/pkg/kvstore" @@ -27,9 +28,9 @@ func TestOONIRunV2LinkCommonCase(t *testing.T) { Author: "", Nettests: []V2Nettest{{ Inputs: []string{}, - Options: map[string]any{ - "SleepTime": int64(10 * time.Millisecond), - }, + Options: json.RawMessage(`{ + "SleepTime": 10000000 + }`), TestName: "example", }}, } @@ -73,9 +74,9 @@ func TestOONIRunV2LinkCannotUpdateCache(t *testing.T) { Author: "", Nettests: []V2Nettest{{ Inputs: []string{}, - Options: map[string]any{ - "SleepTime": int64(10 * time.Millisecond), - }, + Options: json.RawMessage(`{ + "SleepTime": 10000000 + }`), TestName: "example", }}, } @@ -132,9 +133,9 @@ func TestOONIRunV2LinkWithoutAcceptChanges(t *testing.T) { Author: "", Nettests: []V2Nettest{{ Inputs: []string{}, - Options: map[string]any{ - "SleepTime": int64(10 * time.Millisecond), - }, + Options: json.RawMessage(`{ + "SleepTime": 10000000 + }`), TestName: "example", }}, } @@ -220,9 +221,9 @@ func TestOONIRunV2LinkEmptyTestName(t *testing.T) { Author: "", Nettests: []V2Nettest{{ Inputs: []string{}, - Options: map[string]any{ - "SleepTime": int64(10 * time.Millisecond), - }, + Options: json.RawMessage(`{ + "SleepTime": 10000000 + }`), TestName: "", // empty! }}, } @@ -263,6 +264,131 @@ func TestOONIRunV2LinkEmptyTestName(t *testing.T) { } } +func TestOONIRunV2LinkWithAuthentication(t *testing.T) { + + t.Run("authentication raises error if no token is passed", func(t *testing.T) { + token := "c2VjcmV0" + bearerToken := "Bearer " + token + + // make a local server that returns a reasonable descriptor for the example experiment + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + authHeader := r.Header.Get("Authorization") + if authHeader != bearerToken { + // If the header is not what expected, return a 401 Unauthorized status + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + descriptor := &V2Descriptor{ + Name: "", + Description: "", + Author: "", + Nettests: []V2Nettest{{ + Inputs: []string{}, + Options: json.RawMessage(`{ + "SleepTime": 10000000 + }`), + TestName: "example", + }}, + } + data, err := json.Marshal(descriptor) + runtimex.PanicOnError(err, "json.Marshal failed") + w.Write(data) + })) + + defer server.Close() + ctx := context.Background() + + // create a minimal link configuration + config := &LinkConfig{ + AcceptChanges: true, // avoid "oonirun: need to accept changes" error + Annotations: map[string]string{ + "platform": "linux", + }, + KVStore: &kvstore.Memory{}, + MaxRuntime: 0, + NoCollector: true, + NoJSON: true, + Random: false, + ReportFile: "", + Session: newMinimalFakeSession(), + } + + // construct a link runner relative to the local server URL + r := NewLinkRunner(config, server.URL) + + if err := r.Run(ctx); err != nil { + if err.Error() != "httpx: request failed" { + t.Fatal("expected error") + } + } + }) + + t.Run("authentication does not fail the auth token is passed", func(t *testing.T) { + token := "c2VjcmV0" + bearerToken := "Bearer " + token + + // make a local server that returns a reasonable descriptor for the example experiment + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + authHeader := r.Header.Get("Authorization") + if authHeader != bearerToken { + // If the header is not what expected, return a 401 Unauthorized status + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + descriptor := &V2Descriptor{ + Name: "", + Description: "", + Author: "", + Nettests: []V2Nettest{{ + Inputs: []string{}, + Options: json.RawMessage(`{ + "SleepTime": 10000000 + }`), + TestName: "example", + }}, + } + data, err := json.Marshal(descriptor) + runtimex.PanicOnError(err, "json.Marshal failed") + w.Write(data) + })) + + defer server.Close() + ctx := context.Background() + + authFile, err := os.CreateTemp(t.TempDir(), "token-") + if err != nil { + t.Fatal(err) + } + defer authFile.Close() + defer os.Remove(authFile.Name()) + + authFile.Write([]byte(token)) + + // create a minimal link configuration + config := &LinkConfig{ + AcceptChanges: true, // avoid "oonirun: need to accept changes" error + Annotations: map[string]string{ + "platform": "linux", + }, + AuthFile: authFile.Name(), + KVStore: &kvstore.Memory{}, + MaxRuntime: 0, + NoCollector: true, + NoJSON: true, + Random: false, + ReportFile: "", + Session: newMinimalFakeSession(), + } + + // construct a link runner relative to the local server URL + r := NewLinkRunner(config, server.URL) + + if err := r.Run(ctx); err != nil { + t.Fatal(err) + } + }) +} + func TestOONIRunV2LinkConnectionResetByPeer(t *testing.T) { // create a local server that will reset the connection immediately. // actually contains an empty test name, which is what we want to test @@ -374,12 +500,16 @@ func TestV2MeasureDescriptor(t *testing.T) { MockInputPolicy: func() model.InputPolicy { return model.InputNone }, + MockSetOptionsJSON: func(value json.RawMessage) error { + return nil + }, MockSetOptionsAny: func(options map[string]any) error { return nil }, MockNewExperiment: func() model.Experiment { exp := &mocks.Experiment{ - MockMeasureAsync: func(ctx context.Context, input string) (<-chan *model.Measurement, error) { + MockMeasureWithContext: func( + ctx context.Context, target model.ExperimentTarget) (*model.Measurement, error) { return nil, expected }, MockKibiBytesReceived: func() float64 { @@ -391,6 +521,16 @@ func TestV2MeasureDescriptor(t *testing.T) { } return exp }, + MockNewTargetLoader: func(config *model.ExperimentTargetLoaderConfig) model.ExperimentTargetLoader { + return &mocks.ExperimentTargetLoader{ + MockLoad: func(ctx context.Context) ([]model.ExperimentTarget, error) { + // Implementation note: the convention for input-less experiments is that + // they require a single entry containing an empty input. + entry := model.NewOOAPIURLInfoWithDefaultCategoryAndCountry("") + return []model.ExperimentTarget{entry}, nil + }, + } + }, } return eb, nil } @@ -415,7 +555,7 @@ func TestV2MeasureDescriptor(t *testing.T) { Author: "", Nettests: []V2Nettest{{ Inputs: []string{}, - Options: map[string]any{}, + Options: json.RawMessage(`{}`), TestName: "example", }}, } @@ -496,7 +636,6 @@ func TestV2MeasureHTTPS(t *testing.T) { t.Fatal("unexpected err", err) } }) - } func TestV2DescriptorCacheLoad(t *testing.T) { @@ -521,5 +660,139 @@ func TestV2DescriptorCacheLoad(t *testing.T) { t.Fatal("expected nil cache") } }) +} + +func Test_readFirstLineFromFile(t *testing.T) { + + t.Run("return empty string if file is empty", func(t *testing.T) { + f, err := os.CreateTemp(t.TempDir(), "auth-") + if err != nil { + t.Fatal(err) + } + f.Write([]byte("")) + defer f.Close() + defer os.Remove(f.Name()) + + line, err := v2ReadBearerTokenFromFile(f.Name()) + if line != "" { + t.Fatal("expected empty string") + } + if err != nil { + t.Fatal("expected err==nil") + } + }) + + t.Run("return empty string if first line is just whitespace", func(t *testing.T) { + f, err := os.CreateTemp(t.TempDir(), "auth-") + if err != nil { + t.Fatal(err) + } + f.Write([]byte(" \n")) + defer f.Close() + defer os.Remove(f.Name()) + + line, err := v2ReadBearerTokenFromFile(f.Name()) + if line != "" { + t.Fatal("expected empty string") + } + if err != nil { + t.Fatal("expected err==nil") + } + }) + t.Run("return error if file does not exist", func(t *testing.T) { + line, err := v2ReadBearerTokenFromFile(filepath.Join(t.TempDir(), "non-existent")) + if line != "" { + t.Fatal("expected empty string") + } + if !errors.Is(err, os.ErrNotExist) { + t.Fatal("expected ErrNotExist") + } + }) + + t.Run("return first line with a file of one line without EOL", func(t *testing.T) { + f, err := os.CreateTemp(t.TempDir(), "auth-") + if err != nil { + t.Fatal(err) + } + + token := "c2VjcmV0" // b64("secret") + f.Write([]byte(token)) + defer f.Close() + defer os.Remove(f.Name()) + + line, err := v2ReadBearerTokenFromFile(f.Name()) + if line != token { + t.Fatalf("expected %s, got %s", token, line) + } + if err != nil { + t.Fatal("expected err==nil") + } + }) + + t.Run("return first line with a file of one line with EOL", func(t *testing.T) { + f, err := os.CreateTemp(t.TempDir(), "auth-") + if err != nil { + t.Fatal(err) + } + + token := "c2VjcmV0" // b64("secret") + f.Write(append([]byte(token), '\n')) + defer f.Close() + defer os.Remove(f.Name()) + + line, err := v2ReadBearerTokenFromFile(f.Name()) + if line != token { + t.Fatalf("expected %s, got %s", token, line) + } + if err != nil { + t.Fatal("expected err==nil") + } + }) + + t.Run("return first line with a file of >1 line", func(t *testing.T) { + f, err := os.CreateTemp(t.TempDir(), "auth-") + if err != nil { + t.Fatal(err) + } + + token := "c2VjcmV0" // b64("secret") + f.Write([]byte(token)) + f.Write([]byte("\n")) + f.Write([]byte("something\nelse\nand\nsomething\nmore")) + defer f.Close() + defer os.Remove(f.Name()) + + line, err := v2ReadBearerTokenFromFile(f.Name()) + if line != token { + t.Fatalf("expected %s, got %s", token, line) + } + if err != nil { + t.Fatal("expected err==nil") + } + }) + + t.Run("return empty string if not a valid b64 token", func(t *testing.T) { + f, err := os.CreateTemp(t.TempDir(), "auth-") + if err != nil { + t.Fatal(err) + } + + token := "secret!" + f.Write([]byte(token)) + f.Write([]byte("\n")) + f.Write([]byte(" antani\n")) + defer f.Close() + defer os.Remove(f.Name()) + + expected := "" + + line, err := v2ReadBearerTokenFromFile(f.Name()) + if line != expected { + t.Fatalf("expected empty string, got %s", line) + } + if err != nil { + t.Fatal("expected err==nil") + } + }) } diff --git a/pkg/probeservices/benchselect.go b/pkg/probeservices/benchselect.go index c486a0d88..825131fb0 100644 --- a/pkg/probeservices/benchselect.go +++ b/pkg/probeservices/benchselect.go @@ -19,8 +19,8 @@ func Default() []model.OOAPIService { }} } -// SortEndpoints gives priority to https, then cloudfronted, then onion. -func SortEndpoints(in []model.OOAPIService) (out []model.OOAPIService) { +// SortServices gives priority to https, then cloudfronted, then onion. +func SortServices(in []model.OOAPIService) (out []model.OOAPIService) { for _, entry := range in { if entry.Type == "https" { out = append(out, entry) @@ -39,7 +39,7 @@ func SortEndpoints(in []model.OOAPIService) (out []model.OOAPIService) { return } -// OnlyHTTPS returns the HTTPS endpoints only. +// OnlyHTTPS returns the HTTPS services only. func OnlyHTTPS(in []model.OOAPIService) (out []model.OOAPIService) { for _, entry := range in { if entry.Type == "https" { @@ -49,9 +49,9 @@ func OnlyHTTPS(in []model.OOAPIService) (out []model.OOAPIService) { return } -// OnlyFallbacks returns the fallback endpoints only. +// OnlyFallbacks returns the fallback services only. func OnlyFallbacks(in []model.OOAPIService) (out []model.OOAPIService) { - for _, entry := range SortEndpoints(in) { + for _, entry := range SortServices(in) { if entry.Type != "https" { out = append(out, entry) } @@ -67,15 +67,16 @@ type Candidate struct { // Err indicates whether the service works. Err error - // Endpoint is the service endpoint. - Endpoint model.OOAPIService + // Service is the service to use. + Service model.OOAPIService - // TestHelpers contains the data returned by the endpoint. + // TestHelpers contains the data returned by the service when + // querying the /api/v1/test-helpers API endpoint. TestHelpers map[string][]model.OOAPIService } func (c *Candidate) try(ctx context.Context, sess Session) { - client, err := NewClient(sess, c.Endpoint) + client, err := NewClient(sess, c.Service) if err != nil { c.Err = err return @@ -85,11 +86,11 @@ func (c *Candidate) try(ctx context.Context, sess Session) { c.Duration = time.Since(start) c.Err = err c.TestHelpers = testhelpers - sess.Logger().Debugf("probe services: %+v: %+v %s", c.Endpoint, err, c.Duration) + sess.Logger().Debugf("probe services: %+v: %+v %s", c.Service, err, c.Duration) } func try(ctx context.Context, sess Session, svc model.OOAPIService) *Candidate { - candidate := &Candidate{Endpoint: svc} + candidate := &Candidate{Service: svc} candidate.try(ctx, sess) return candidate } diff --git a/pkg/probeservices/checkin_test.go b/pkg/probeservices/checkin_test.go index 3cb6623ce..bf354c3b9 100644 --- a/pkg/probeservices/checkin_test.go +++ b/pkg/probeservices/checkin_test.go @@ -42,7 +42,7 @@ func TestCheckIn(t *testing.T) { } client := newclient() - client.BaseURL = "https://ams-pg-test.ooni.org" // use the test infra + client.BaseURL = "https://backend-hel.ooni.org" // use the test infra ctx := context.Background() diff --git a/pkg/probeservices/openvpn.go b/pkg/probeservices/openvpn.go new file mode 100644 index 000000000..ef8ece0e6 --- /dev/null +++ b/pkg/probeservices/openvpn.go @@ -0,0 +1,45 @@ +package probeservices + +import ( + "context" + "fmt" + "net/url" + "strings" + + "github.com/ooni/probe-engine/pkg/httpclientx" + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/urlx" +) + +// FetchOpenVPNConfig returns valid configuration for the openvpn experiment. +// It accepts the provider label, and the country code for the probe, in case the API wants to +// return different targets to us depending on where we are located. +func (c Client) FetchOpenVPNConfig(ctx context.Context, provider, cc string) (result model.OOAPIVPNProviderConfig, err error) { + // create query string + query := url.Values{} + query.Add("country_code", cc) + + // TODO(ainghazal): remove temporary fix + if !strings.HasSuffix(provider, "vpn") { + provider = provider + "vpn" + } + + URL, err := urlx.ResolveReference(c.BaseURL, + fmt.Sprintf("/api/v2/ooniprobe/vpn-config/%s", provider), + query.Encode()) + if err != nil { + return + } + + // get response + // + // use a model.DiscardLogger to avoid logging config + return httpclientx.GetJSON[model.OOAPIVPNProviderConfig]( + ctx, + httpclientx.NewEndpoint(URL).WithHostOverride(c.Host), + &httpclientx.Config{ + Client: c.HTTPClient, + Logger: model.DiscardLogger, + UserAgent: c.UserAgent, + }) +} diff --git a/pkg/probeservices/openvpn_test.go b/pkg/probeservices/openvpn_test.go new file mode 100644 index 000000000..c7ccdaf2f --- /dev/null +++ b/pkg/probeservices/openvpn_test.go @@ -0,0 +1,105 @@ +package probeservices + +import ( + "context" + "fmt" + "net/http" + "net/url" + "testing" + + "github.com/apex/log" + "github.com/ooni/probe-engine/pkg/legacy/mockable" + "github.com/ooni/probe-engine/pkg/mocks" + "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/runtimex" + "github.com/ooni/probe-engine/pkg/testingx" +) + +func newclientWithStagingEnv() *Client { + client := runtimex.Try1(NewClient( + &mockable.Session{ + MockableHTTPClient: http.DefaultClient, + MockableLogger: log.Log, + }, + model.OOAPIService{ + Address: "https://api.dev.ooni.io/", + Type: "https", + }, + )) + return client +} + +func TestFetchOpenVPNConfig(t *testing.T) { + // First, let's check whether we can get a response from the real OONI backend. + t.Run("is working as intended with the real backend", func(t *testing.T) { + if testing.Short() { + t.Skip("skip test in short mode") + } + + // TODO(ain): switch to newclient() when backend in all environments + // deploys the vpn-config endpoint. + clnt := newclientWithStagingEnv() + + // run the tor flow + config, err := clnt.FetchOpenVPNConfig(context.Background(), "riseup", "ZZ") + + // we do not expect an error here + if err != nil { + t.Fatal(err) + } + + // we expect non-zero length targets + if len(config.Inputs) <= 0 { + fmt.Println(config) + t.Fatal("expected non-zero-length inputs") + } + }) + + t.Run("is working as intended with a local test server", func(t *testing.T) { + // create state for emulating the OONI backend + state := &testingx.OONIBackendWithLoginFlow{} + + // return something that matches thes expected data + state.SetOpenVPNConfig([]byte(`{ +"provider": "demovpn", +"protocol": "openvpn", +"config": { + "ca": "deadbeef", + "cert": "deadbeef", + "key": "deadbeef" + }, + "date_updated": "2024-05-06T15:22:13.152242Z", + "endpoints": [ + "openvpn://demovpn.corp/?address=1.1.1.1:53&transport=udp" + ] +} +`)) + + // expose the state via HTTP + srv := testingx.MustNewHTTPServer(state.NewMux()) + defer srv.Close() + + client := newclient() + + // override the HTTP client so we speak with our local server rather than the true backend + client.HTTPClient = &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + URL := runtimex.Try1(url.Parse(srv.URL)) + req.URL.Scheme = URL.Scheme + req.URL.Host = URL.Host + return http.DefaultClient.Do(req) + }, + MockCloseIdleConnections: func() { + http.DefaultClient.CloseIdleConnections() + }, + } + + // then we can try to fetch the config + _, err := client.FetchOpenVPNConfig(context.Background(), "demo", "ZZ") + + // we do not expect an error here + if err != nil { + t.Fatal(err) + } + }) +} diff --git a/pkg/probeservices/probeservices.go b/pkg/probeservices/probeservices.go index 18d711f9f..b719bf3e9 100644 --- a/pkg/probeservices/probeservices.go +++ b/pkg/probeservices/probeservices.go @@ -1,6 +1,6 @@ // Package probeservices contains code to contact OONI probe services. // -// The probe services are HTTPS endpoints distributed across a bunch of data +// The probe services are HTTPS services distributed across a bunch of data // centres implementing a bunch of OONI APIs. When started, OONI will benchmark // the available probe services and select the fastest one. Eventually all the // possible OONI APIs will run as probe services. @@ -32,8 +32,8 @@ import ( ) var ( - // ErrUnsupportedEndpoint indicates that we don't support this endpoint type. - ErrUnsupportedEndpoint = errors.New("probe services: unsupported endpoint type") + // ErrUnsupportedServiceType indicates that we don't support this service type. + ErrUnsupportedServiceType = errors.New("probe services: unsupported service type") // ErrUnsupportedCloudFrontAddress indicates that we don't support this // cloudfront address (e.g. wrong scheme, presence of port). @@ -90,11 +90,11 @@ func (c Client) GetCredsAndAuth() (*model.OOAPILoginCredentials, *model.OOAPILog return creds, auth, nil } -// NewClient creates a new client for the specified probe services endpoint. This -// function fails, e.g., we don't support the specified endpoint. -func NewClient(sess Session, endpoint model.OOAPIService) (*Client, error) { +// NewClient creates a new client for the specified probe services service. This +// function fails, e.g., we don't support the specified service. +func NewClient(sess Session, service model.OOAPIService) (*Client, error) { client := &Client{ - BaseURL: endpoint.Address, + BaseURL: service.Address, HTTPClient: sess.DefaultHTTPClient(), Host: "", KVStore: sess.KeyValueStore(), @@ -104,7 +104,7 @@ func NewClient(sess Session, endpoint model.OOAPIService) (*Client, error) { StateFile: NewStateFile(sess.KeyValueStore()), UserAgent: sess.UserAgent(), } - switch endpoint.Type { + switch service.Type { case "https": return client, nil case "cloudfront": @@ -119,13 +119,13 @@ func NewClient(sess Session, endpoint model.OOAPIService) (*Client, error) { return nil, ErrUnsupportedCloudFrontAddress } client.Host = URL.Hostname() - URL.Host = endpoint.Front + URL.Host = service.Front client.BaseURL = URL.String() if _, err := url.Parse(client.BaseURL); err != nil { return nil, err } return client, nil default: - return nil, ErrUnsupportedEndpoint + return nil, ErrUnsupportedServiceType } } diff --git a/pkg/probeservices/probeservices_test.go b/pkg/probeservices/probeservices_test.go index 8ee959f16..89c7934c0 100644 --- a/pkg/probeservices/probeservices_test.go +++ b/pkg/probeservices/probeservices_test.go @@ -23,7 +23,7 @@ func newclient() *Client { MockableLogger: log.Log, }, model.OOAPIService{ - Address: "https://ams-pg-test.ooni.org/", + Address: "https://backend-hel.ooni.org/", Type: "https", }, ) @@ -47,13 +47,13 @@ func TestNewClientHTTPS(t *testing.T) { } } -func TestNewClientUnsupportedEndpoint(t *testing.T) { +func TestNewClientUnsupportedService(t *testing.T) { client, err := NewClient( &mockable.Session{}, model.OOAPIService{ Address: "https://x.org", Type: "onion", }) - if !errors.Is(err, ErrUnsupportedEndpoint) { + if !errors.Is(err, ErrUnsupportedServiceType) { t.Fatal("not the error we expected") } if client != nil { @@ -193,7 +193,7 @@ func TestDefaultProbeServicesWorkAsIntended(t *testing.T) { } } -func TestSortEndpoints(t *testing.T) { +func TestSortServices(t *testing.T) { in := []model.OOAPIService{{ Type: "onion", Address: "httpo://jehhrikjjqrlpufu.onion", @@ -216,7 +216,7 @@ func TestSortEndpoints(t *testing.T) { Type: "onion", Address: "httpo://jehhrikjjqrlpufu.onion", }} - out := SortEndpoints(in) + out := SortServices(in) diff := cmp.Diff(out, expect) if diff != "" { t.Fatal(diff) @@ -259,7 +259,7 @@ func TestOnlyHTTPS(t *testing.T) { } func TestOnlyFallbacks(t *testing.T) { - // put onion first so we also verify that we sort the endpoints + // put onion first so we also verify that we sort the services in := []model.OOAPIService{{ Type: "onion", Address: "httpo://jehhrikjjqrlpufu.onion", @@ -293,7 +293,7 @@ func TestOnlyFallbacks(t *testing.T) { } func TestTryAllCanceledContext(t *testing.T) { - // put onion first so we also verify that we sort the endpoints + // put onion first so we also verify that we sort the services in := []model.OOAPIService{{ Type: "onion", Address: "httpo://jehhrikjjqrlpufu.onion", @@ -328,11 +328,11 @@ func TestTryAllCanceledContext(t *testing.T) { if !errors.Is(out[0].Err, context.Canceled) { t.Fatal("invalid error") } - if out[0].Endpoint.Type != "https" { - t.Fatal("invalid endpoint type") + if out[0].Service.Type != "https" { + t.Fatal("invalid service type") } - if out[0].Endpoint.Address != "https://ams-ps-nonexistent.ooni.io" { - t.Fatal("invalid endpoint type") + if out[0].Service.Address != "https://ams-ps-nonexistent.ooni.io" { + t.Fatal("invalid service type") } // if out[1].Duration <= 0 { @@ -341,11 +341,11 @@ func TestTryAllCanceledContext(t *testing.T) { if !errors.Is(out[1].Err, context.Canceled) { t.Fatal("invalid error") } - if out[1].Endpoint.Type != "https" { - t.Fatal("invalid endpoint type") + if out[1].Service.Type != "https" { + t.Fatal("invalid service type") } - if out[1].Endpoint.Address != "https://hkg-ps-nonexistent.ooni.io" { - t.Fatal("invalid endpoint type") + if out[1].Service.Address != "https://hkg-ps-nonexistent.ooni.io" { + t.Fatal("invalid service type") } // if out[2].Duration <= 0 { @@ -354,11 +354,11 @@ func TestTryAllCanceledContext(t *testing.T) { if !errors.Is(out[2].Err, context.Canceled) { t.Fatal("invalid error") } - if out[2].Endpoint.Type != "https" { - t.Fatal("invalid endpoint type") + if out[2].Service.Type != "https" { + t.Fatal("invalid service type") } - if out[2].Endpoint.Address != "https://mia-ps-nonexistent.ooni.io" { - t.Fatal("invalid endpoint type") + if out[2].Service.Address != "https://mia-ps-nonexistent.ooni.io" { + t.Fatal("invalid service type") } // if out[3].Duration <= 0 { @@ -367,28 +367,28 @@ func TestTryAllCanceledContext(t *testing.T) { if !errors.Is(out[3].Err, context.Canceled) { t.Fatal("invalid error") } - if out[3].Endpoint.Type != "cloudfront" { - t.Fatal("invalid endpoint type") + if out[3].Service.Type != "cloudfront" { + t.Fatal("invalid service type") } - if out[3].Endpoint.Front != "dkyhjv0wpi2dk.cloudfront.net" { - t.Fatal("invalid endpoint type") + if out[3].Service.Front != "dkyhjv0wpi2dk.cloudfront.net" { + t.Fatal("invalid service type") } - if out[3].Endpoint.Address != "https://dkyhjv0wpi2dk.cloudfront.net" { - t.Fatal("invalid endpoint type") + if out[3].Service.Address != "https://dkyhjv0wpi2dk.cloudfront.net" { + t.Fatal("invalid service type") } // - // Note: here duration may be zero because the endpoint is not supported + // Note: here duration may be zero because the service is not supported // and so we don't basically do anything. But it also may be nonzero since // we also run tests in the cloud, which is slower than my desktop. So, I // have not written a specific test concerning out[4].Duration. - if !errors.Is(out[4].Err, ErrUnsupportedEndpoint) { + if !errors.Is(out[4].Err, ErrUnsupportedServiceType) { t.Fatal("invalid error") } - if out[4].Endpoint.Type != "onion" { - t.Fatal("invalid endpoint type") + if out[4].Service.Type != "onion" { + t.Fatal("invalid service type") } - if out[4].Endpoint.Address != "httpo://jehhrikjjqrlpufu.onion" { - t.Fatal("invalid endpoint type") + if out[4].Service.Address != "httpo://jehhrikjjqrlpufu.onion" { + t.Fatal("invalid service type") } } @@ -396,7 +396,7 @@ func TestTryAllIntegrationWeRaceForFastestHTTPS(t *testing.T) { if testing.Short() { t.Skip("skip test in short mode") } - // put onion first so we also verify that we sort the endpoints + // put onion first so we also verify that we sort the services in := []model.OOAPIService{{ Type: "onion", Address: "httpo://jehhrikjjqrlpufu.onion", @@ -423,11 +423,11 @@ func TestTryAllIntegrationWeRaceForFastestHTTPS(t *testing.T) { if out[0].Err != nil { t.Fatal("invalid error") } - if out[0].Endpoint.Type != "https" { - t.Fatal("invalid endpoint type") + if out[0].Service.Type != "https" { + t.Fatal("invalid service type") } - if out[0].Endpoint.Address != "https://api.ooni.io" { - t.Fatal("invalid endpoint address") + if out[0].Service.Address != "https://api.ooni.io" { + t.Fatal("invalid service address") } } @@ -435,7 +435,7 @@ func TestTryAllIntegrationWeFallback(t *testing.T) { if testing.Short() { t.Skip("skip test in short mode") } - // put onion first so we also verify that we sort the endpoints + // put onion first so we also verify that we sort the services in := []model.OOAPIService{{ Type: "onion", Address: "httpo://jehhrikjjqrlpufu.onion", @@ -468,11 +468,11 @@ func TestTryAllIntegrationWeFallback(t *testing.T) { if !strings.HasSuffix(out[0].Err.Error(), "no such host") { t.Fatal("invalid error") } - if out[0].Endpoint.Type != "https" { - t.Fatal("invalid endpoint type") + if out[0].Service.Type != "https" { + t.Fatal("invalid service type") } - if out[0].Endpoint.Address != "https://ps-nonexistent.ooni.io" { - t.Fatal("invalid endpoint type") + if out[0].Service.Address != "https://ps-nonexistent.ooni.io" { + t.Fatal("invalid service type") } // if out[1].Duration <= 0 { @@ -481,11 +481,11 @@ func TestTryAllIntegrationWeFallback(t *testing.T) { if !strings.HasSuffix(out[1].Err.Error(), "no such host") { t.Fatal("invalid error") } - if out[1].Endpoint.Type != "https" { - t.Fatal("invalid endpoint type") + if out[1].Service.Type != "https" { + t.Fatal("invalid service type") } - if out[1].Endpoint.Address != "https://hkg-ps-nonexistent.ooni.nu" { - t.Fatal("invalid endpoint type") + if out[1].Service.Address != "https://hkg-ps-nonexistent.ooni.nu" { + t.Fatal("invalid service type") } // if out[2].Duration <= 0 { @@ -494,11 +494,11 @@ func TestTryAllIntegrationWeFallback(t *testing.T) { if !strings.HasSuffix(out[2].Err.Error(), "no such host") { t.Fatal("invalid error") } - if out[2].Endpoint.Type != "https" { - t.Fatal("invalid endpoint type") + if out[2].Service.Type != "https" { + t.Fatal("invalid service type") } - if out[2].Endpoint.Address != "https://mia-ps2-nonexistent.ooni.nu" { - t.Fatal("invalid endpoint type") + if out[2].Service.Address != "https://mia-ps2-nonexistent.ooni.nu" { + t.Fatal("invalid service type") } // if out[3].Duration <= 0 { @@ -507,13 +507,13 @@ func TestTryAllIntegrationWeFallback(t *testing.T) { if out[3].Err != nil { t.Fatal("invalid error") } - if out[3].Endpoint.Type != "cloudfront" { - t.Fatal("invalid endpoint type") + if out[3].Service.Type != "cloudfront" { + t.Fatal("invalid service type") } - if out[3].Endpoint.Address != "https://dkyhjv0wpi2dk.cloudfront.net" { - t.Fatal("invalid endpoint type") + if out[3].Service.Address != "https://dkyhjv0wpi2dk.cloudfront.net" { + t.Fatal("invalid service type") } - if out[3].Endpoint.Front != "dkyhjv0wpi2dk.cloudfront.net" { + if out[3].Service.Front != "dkyhjv0wpi2dk.cloudfront.net" { t.Fatal("invalid front") } } @@ -537,32 +537,32 @@ func TestSelectBestOnlyFailures(t *testing.T) { func TestSelectBestSelectsTheFastest(t *testing.T) { in := []*Candidate{{ Duration: 10 * time.Millisecond, - Endpoint: model.OOAPIService{ + Service: model.OOAPIService{ Address: "https://ps1.ooni.nonexistent", Type: "https", }, }, { Duration: 4 * time.Millisecond, - Endpoint: model.OOAPIService{ + Service: model.OOAPIService{ Address: "https://ps2.ooni.nonexistent", Type: "https", }, }, { Duration: 7 * time.Millisecond, - Endpoint: model.OOAPIService{ + Service: model.OOAPIService{ Address: "https://ps3.ooni.nonexistent", Type: "https", }, }, { Duration: 11 * time.Millisecond, - Endpoint: model.OOAPIService{ + Service: model.OOAPIService{ Address: "https://ps4.ooni.nonexistent", Type: "https", }, }} expected := &Candidate{ Duration: 4 * time.Millisecond, - Endpoint: model.OOAPIService{ + Service: model.OOAPIService{ Address: "https://ps2.ooni.nonexistent", Type: "https", }, diff --git a/pkg/probeservices/tor_test.go b/pkg/probeservices/tor_test.go index 149fb8f85..7cffee65e 100644 --- a/pkg/probeservices/tor_test.go +++ b/pkg/probeservices/tor_test.go @@ -242,7 +242,7 @@ func TestFetchTorTargets(t *testing.T) { t.Run("when we're not registered", func(t *testing.T) { clnt := newclient() - // With explicitly empty state so it's pretty obvioust there's no state + // With explicitly empty state so it's pretty obvious there's no state state := State{} // force the state to be empty diff --git a/pkg/reflectx/reflectx.go b/pkg/reflectx/reflectx.go new file mode 100644 index 000000000..d2ade978c --- /dev/null +++ b/pkg/reflectx/reflectx.go @@ -0,0 +1,31 @@ +// Package reflectx contains [reflect] extensions. +package reflectx + +import ( + "reflect" + + "github.com/ooni/probe-engine/pkg/runtimex" +) + +// StructOrStructPtrIsZero returns whether a given struct or struct pointer +// only contains zero-value public fields. This function panic if passed a value +// that is neither a pointer to struct nor a struct. This function panics if +// passed a nil struct pointer. +func StructOrStructPtrIsZero(vop any) bool { + vx := reflect.ValueOf(vop) + if vx.Kind() == reflect.Pointer { + vx = vx.Elem() + } + runtimex.Assert(vx.Kind() == reflect.Struct, "not a struct") + tx := vx.Type() + for idx := 0; idx < tx.NumField(); idx++ { + fvx, ftx := vx.Field(idx), tx.Field(idx) + if !ftx.IsExported() { + continue + } + if !fvx.IsZero() { + return false + } + } + return true +} diff --git a/pkg/reflectx/reflectx_test.go b/pkg/reflectx/reflectx_test.go new file mode 100644 index 000000000..e0046f407 --- /dev/null +++ b/pkg/reflectx/reflectx_test.go @@ -0,0 +1,87 @@ +package reflectx + +import ( + "testing" + + "github.com/ooni/probe-engine/pkg/testingx" +) + +type example struct { + Age int64 + Ch chan int64 + F bool + Fmp map[string]*bool + Fp *bool + Fv []bool + Fvp []*bool + Name string + Ptr *int64 + V []int64 + namex string + num int64 +} + +var nonzero example + +func init() { + ff := &testingx.FakeFiller{} + ff.Fill(&nonzero) + nonzero.namex = "foo" + nonzero.num = 123 +} + +func TestStructOrStructPtrIsZero(t *testing.T) { + + // testcase is a test case implemented by this function + type testcase struct { + // name is the name of the test case + name string + + // input is the input + input any + + // expect is the expected result + expect bool + } + + cases := []testcase{{ + name: "[struct] with zero value", + input: example{}, + expect: true, + }, { + name: "[ptr] with zero value", + input: &example{}, + expect: true, + }, { + name: "[struct] with nonzero value", + input: nonzero, + expect: false, + }, { + name: "[ptr] with nonzero value", + input: &nonzero, + expect: false, + }, { + name: "[struct] with only private fields being nonzero", + input: example{ + namex: "abc", + num: 128, + }, + expect: true, + }, { + name: "[ptr] with only private fields being nonzero", + input: &example{ + namex: "abc", + num: 128, + }, + expect: true, + }} + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Logf("input: %#v", tc.input) + if got := StructOrStructPtrIsZero(tc.input); got != tc.expect { + t.Fatal("expected", tc.expect, "got", got) + } + }) + } +} diff --git a/pkg/registry/allexperiments.go b/pkg/registry/allexperiments.go index 39ce61226..8fe9edd33 100644 --- a/pkg/registry/allexperiments.go +++ b/pkg/registry/allexperiments.go @@ -3,7 +3,7 @@ package registry import "sort" // Where we register all the available experiments. -var AllExperiments = map[string]*Factory{} +var AllExperiments = map[string]func() *Factory{} // ExperimentNames returns the name of all experiments func ExperimentNames() (names []string) { diff --git a/pkg/registry/dash.go b/pkg/registry/dash.go index d55656b2a..245e5f9ce 100644 --- a/pkg/registry/dash.go +++ b/pkg/registry/dash.go @@ -10,15 +10,19 @@ import ( ) func init() { - AllExperiments["dash"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return dash.NewExperimentMeasurer( - *config.(*dash.Config), - ) - }, - config: &dash.Config{}, - enabledByDefault: true, - interruptible: true, - inputPolicy: model.InputNone, + const canonicalName = "dash" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return dash.NewExperimentMeasurer( + *config.(*dash.Config), + ) + }, + canonicalName: canonicalName, + config: &dash.Config{}, + enabledByDefault: true, + interruptible: true, + inputPolicy: model.InputNone, + } } } diff --git a/pkg/registry/dnscheck.go b/pkg/registry/dnscheck.go index 5f95fc4ef..a88cf57a2 100644 --- a/pkg/registry/dnscheck.go +++ b/pkg/registry/dnscheck.go @@ -10,14 +10,18 @@ import ( ) func init() { - AllExperiments["dnscheck"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return dnscheck.NewExperimentMeasurer( - *config.(*dnscheck.Config), - ) - }, - config: &dnscheck.Config{}, - enabledByDefault: true, - inputPolicy: model.InputOrStaticDefault, + const canonicalName = "dnscheck" + AllExperiments[canonicalName] = func() *Factory { + // TODO(bassosimone,DecFox): for now, we MUST keep the InputOrStaticDefault + // policy because otherwise ./pkg/oonimkall should break. + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return dnscheck.NewExperimentMeasurer() + }, + canonicalName: canonicalName, + config: &dnscheck.Config{}, + enabledByDefault: true, + inputPolicy: model.InputOrStaticDefault, + } } } diff --git a/pkg/registry/dnsping.go b/pkg/registry/dnsping.go index 1c98ff176..7d6d932b5 100644 --- a/pkg/registry/dnsping.go +++ b/pkg/registry/dnsping.go @@ -10,14 +10,18 @@ import ( ) func init() { - AllExperiments["dnsping"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return dnsping.NewExperimentMeasurer( - *config.(*dnsping.Config), - ) - }, - config: &dnsping.Config{}, - enabledByDefault: true, - inputPolicy: model.InputOrStaticDefault, + const canonicalName = "dnsping" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return dnsping.NewExperimentMeasurer( + *config.(*dnsping.Config), + ) + }, + canonicalName: canonicalName, + config: &dnsping.Config{}, + enabledByDefault: true, + inputPolicy: model.InputOrStaticDefault, + } } } diff --git a/pkg/registry/dslxtutorial.go b/pkg/registry/dslxtutorial.go index eb4c91bac..fa250401d 100644 --- a/pkg/registry/dslxtutorial.go +++ b/pkg/registry/dslxtutorial.go @@ -10,13 +10,17 @@ import ( ) func init() { - AllExperiments["simple_sni"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return chapter02.NewExperimentMeasurer( - *config.(*chapter02.Config), - ) - }, - config: &chapter02.Config{}, - inputPolicy: model.InputOrQueryBackend, + const canonicalName = "simple_sni" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return chapter02.NewExperimentMeasurer( + *config.(*chapter02.Config), + ) + }, + canonicalName: canonicalName, + config: &chapter02.Config{}, + inputPolicy: model.InputOrQueryBackend, + } } } diff --git a/pkg/registry/echcheck.go b/pkg/registry/echcheck.go index a97ac0768..435126c23 100644 --- a/pkg/registry/echcheck.go +++ b/pkg/registry/echcheck.go @@ -10,13 +10,17 @@ import ( ) func init() { - AllExperiments["echcheck"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return echcheck.NewExperimentMeasurer( - *config.(*echcheck.Config), - ) - }, - config: &echcheck.Config{}, - inputPolicy: model.InputOptional, + const canonicalName = "echcheck" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return echcheck.NewExperimentMeasurer( + *config.(*echcheck.Config), + ) + }, + canonicalName: canonicalName, + config: &echcheck.Config{}, + inputPolicy: model.InputOptional, + } } } diff --git a/pkg/registry/example.go b/pkg/registry/example.go index c3ee38ca2..2cd5b3ca7 100644 --- a/pkg/registry/example.go +++ b/pkg/registry/example.go @@ -12,18 +12,27 @@ import ( ) func init() { - AllExperiments["example"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return example.NewExperimentMeasurer( - *config.(*example.Config), "example", - ) - }, - config: &example.Config{ - Message: "Good day from the example experiment!", - SleepTime: int64(time.Second), - }, - enabledByDefault: true, - interruptible: true, - inputPolicy: model.InputNone, + const canonicalName = "example" + AllExperiments[canonicalName] = func() *Factory { + // TODO(bassosimone,DecFox): as pointed out by @ainghazal, this experiment + // should be the one that people modify to start out new experiments, so it's + // kind of suboptimal that it has a constructor with explicit experiment + // name to ease writing some tests that ./pkg/oonimkall needs given that no + // other experiment ever sets the experiment name externally! + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return example.NewExperimentMeasurer( + *config.(*example.Config), + ) + }, + canonicalName: canonicalName, + config: &example.Config{ + Message: "Good day from the example experiment!", + SleepTime: int64(time.Second), + }, + enabledByDefault: true, + interruptible: true, + inputPolicy: model.InputNone, + } } } diff --git a/pkg/registry/factory.go b/pkg/registry/factory.go index 29e4bf58d..a63a60c34 100644 --- a/pkg/registry/factory.go +++ b/pkg/registry/factory.go @@ -5,6 +5,7 @@ package registry // import ( + "encoding/json" "errors" "fmt" "math" @@ -13,8 +14,9 @@ import ( "strconv" "github.com/ooni/probe-engine/pkg/checkincache" + "github.com/ooni/probe-engine/pkg/experimentname" "github.com/ooni/probe-engine/pkg/model" - "github.com/ooni/probe-engine/pkg/strcasex" + "github.com/ooni/probe-engine/pkg/targetloading" ) // Factory allows to construct an experiment measurer. @@ -22,6 +24,9 @@ type Factory struct { // build is the constructor that build an experiment with the given config. build func(config interface{}) model.ExperimentMeasurer + // canonicalName is the canonical name of the experiment. + canonicalName string + // config contains the experiment's config. config any @@ -33,6 +38,35 @@ type Factory struct { // interruptible indicates whether the experiment is interruptible. interruptible bool + + // newLoader is the OPTIONAL function to create a new loader. + newLoader func(config *targetloading.Loader, options any) model.ExperimentTargetLoader +} + +// Session is the session definition according to this package. +type Session = model.ExperimentTargetLoaderSession + +// NewTargetLoader creates a new [model.ExperimentTargetLoader] instance. +func (b *Factory) NewTargetLoader(config *model.ExperimentTargetLoaderConfig) model.ExperimentTargetLoader { + // Construct the default loader used in the non-richer input case. + loader := &targetloading.Loader{ + CheckInConfig: config.CheckInConfig, // OPTIONAL + ExperimentName: b.canonicalName, + InputPolicy: b.inputPolicy, + Logger: config.Session.Logger(), + Session: config.Session, + StaticInputs: config.StaticInputs, + SourceFiles: config.SourceFiles, + } + + // If an experiment implements richer input, it will use its custom loader + // that will use experiment specific policy for loading targets. + if b.newLoader != nil { + return b.newLoader(loader, b.config) + } + + // Otherwise just return the default loader. + return loader } // Interruptible returns whether the experiment is interruptible. @@ -72,22 +106,47 @@ var ( // Options returns the options exposed by this experiment. func (b *Factory) Options() (map[string]model.ExperimentOptionInfo, error) { + // create the result value result := make(map[string]model.ExperimentOptionInfo) + + // make sure we're dealing with a pointer ptrinfo := reflect.ValueOf(b.config) if ptrinfo.Kind() != reflect.Ptr { return nil, ErrConfigIsNotAStructPointer } - structinfo := ptrinfo.Elem().Type() - if structinfo.Kind() != reflect.Struct { + + // obtain information about the value and its type + valueinfo := ptrinfo.Elem() + typeinfo := valueinfo.Type() + + // make sure we're dealing with a struct + if typeinfo.Kind() != reflect.Struct { return nil, ErrConfigIsNotAStructPointer } - for i := 0; i < structinfo.NumField(); i++ { - field := structinfo.Field(i) - result[field.Name] = model.ExperimentOptionInfo{ - Doc: field.Tag.Get("ooni"), - Type: field.Type.String(), + + // cycle through the fields + for i := 0; i < typeinfo.NumField(); i++ { + fieldType, fieldValue := typeinfo.Field(i), valueinfo.Field(i) + + // do not include private fields into our list of fields + if !fieldType.IsExported() { + continue + } + + // skip fields that are missing an `ooni` tag + docs := fieldType.Tag.Get("ooni") + if docs == "" { + continue + } + + // create a description of this field + result[fieldType.Name] = model.ExperimentOptionInfo{ + Doc: docs, + Type: fieldType.Type.String(), + Value: fieldValue.Interface(), } } + return result, nil } @@ -195,6 +254,19 @@ func (b *Factory) SetOptionsAny(options map[string]any) error { return nil } +// SetOptionsJSON unmarshals the given [json.RawMessage] inside +// the experiment specific configuration. +func (b *Factory) SetOptionsJSON(value json.RawMessage) error { + // handle the case where the options are empty + if len(value) <= 0 { + return nil + } + + // otherwise unmarshal into the configuration, which we assume + // to be a pointer to a structure. + return json.Unmarshal(value, b.config) +} + // fieldbyname return v's field whose name is equal to the given key. func (b *Factory) fieldbyname(v interface{}, key string) (reflect.Value, error) { // See https://stackoverflow.com/a/6396678/4354461 @@ -213,32 +285,11 @@ func (b *Factory) fieldbyname(v interface{}, key string) (reflect.Value, error) return field, nil } -// NewExperimentMeasurer creates the experiment +// NewExperimentMeasurer creates a new [model.ExperimentMeasurer] instance. func (b *Factory) NewExperimentMeasurer() model.ExperimentMeasurer { return b.build(b.config) } -// CanonicalizeExperimentName allows code to provide experiment names -// in a more flexible way, where we have aliases. -// -// Because we allow for uppercase experiment names for backwards -// compatibility with MK, we need to add some exceptions here when -// mapping (e.g., DNSCheck => dnscheck). -func CanonicalizeExperimentName(name string) string { - switch name = strcasex.ToSnake(name); name { - case "ndt_7": - name = "ndt" // since 2020-03-18, we use ndt7 to implement ndt by default - case "dns_check": - name = "dnscheck" - case "stun_reachability": - name = "stunreachability" - case "web_connectivity@v_0_5": - name = "web_connectivity@v0.5" - default: - } - return name -} - // ErrNoSuchExperiment indicates a given experiment does not exist. var ErrNoSuchExperiment = errors.New("no such experiment") @@ -285,7 +336,7 @@ const OONI_FORCE_ENABLE_EXPERIMENT = "OONI_FORCE_ENABLE_EXPERIMENT" func NewFactory(name string, kvStore model.KeyValueStore, logger model.Logger) (*Factory, error) { // Make sure we are deadling with the canonical experiment name. Historically MK used // names such as WebConnectivity and we want to continue supporting this use case. - name = CanonicalizeExperimentName(name) + name = experimentname.Canonicalize(name) // Handle A/B testing where we dynamically choose LTE for some users. The current policy // only relates to a few users to collect data. @@ -305,10 +356,11 @@ func NewFactory(name string, kvStore model.KeyValueStore, logger model.Logger) ( } // Obtain the factory for the canonical name. - factory := AllExperiments[name] - if factory == nil { + ff := AllExperiments[name] + if ff == nil { return nil, fmt.Errorf("%w: %s", ErrNoSuchExperiment, name) } + factory := ff() // Some experiments are not enabled by default. To enable them we use // the cached check-in response or an environment variable. diff --git a/pkg/registry/factory_test.go b/pkg/registry/factory_test.go index 04d45faab..00b01087f 100644 --- a/pkg/registry/factory_test.go +++ b/pkg/registry/factory_test.go @@ -1,27 +1,43 @@ package registry import ( + "context" + "encoding/json" "errors" "fmt" "math" "os" + "reflect" "testing" + "github.com/apex/log" "github.com/google/go-cmp/cmp" "github.com/ooni/probe-engine/pkg/checkincache" "github.com/ooni/probe-engine/pkg/experiment/webconnectivitylte" + "github.com/ooni/probe-engine/pkg/experimentname" "github.com/ooni/probe-engine/pkg/kvstore" + "github.com/ooni/probe-engine/pkg/mocks" "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/targetloading" ) -type fakeExperimentConfig struct { - Chan chan any `ooni:"we cannot set this"` - String string `ooni:"a string"` - Truth bool `ooni:"something that no-one knows"` - Value int64 `ooni:"a number"` -} +func TestFactoryOptions(t *testing.T) { + + // the fake configuration we're using in this test + type fakeExperimentConfig struct { + // values that should be included into the Options return value + Chan chan any `ooni:"we cannot set this"` + String string `ooni:"a string"` + Truth bool `ooni:"something that no-one knows"` + Value int64 `ooni:"a number"` + + // values that should not be included because they're private + private int64 `ooni:"a private number"` + + // values that should not be included because they lack "ooni"'s tag + Invisible int64 + } -func TestExperimentBuilderOptions(t *testing.T) { t.Run("when config is not a pointer", func(t *testing.T) { b := &Factory{ config: 17, @@ -50,7 +66,14 @@ func TestExperimentBuilderOptions(t *testing.T) { }) t.Run("when config is a pointer to struct", func(t *testing.T) { - config := &fakeExperimentConfig{} + config := &fakeExperimentConfig{ + Chan: make(chan any), + String: "foobar", + Truth: true, + Value: 177114, + private: 55, + Invisible: 9876, + } b := &Factory{ config: config, } @@ -58,6 +81,7 @@ func TestExperimentBuilderOptions(t *testing.T) { if err != nil { t.Fatal(err) } + for name, value := range options { switch name { case "Chan": @@ -67,6 +91,10 @@ func TestExperimentBuilderOptions(t *testing.T) { if value.Type != "chan interface {}" { t.Fatal("invalid type", value.Type) } + if value.Value.(chan any) == nil { + t.Fatal("expected non-nil channel here") + } + case "String": if value.Doc != "a string" { t.Fatal("invalid doc") @@ -74,6 +102,10 @@ func TestExperimentBuilderOptions(t *testing.T) { if value.Type != "string" { t.Fatal("invalid type", value.Type) } + if v := value.Value.(string); v != "foobar" { + t.Fatal("unexpected string value", v) + } + case "Truth": if value.Doc != "something that no-one knows" { t.Fatal("invalid doc") @@ -81,6 +113,10 @@ func TestExperimentBuilderOptions(t *testing.T) { if value.Type != "bool" { t.Fatal("invalid type", value.Type) } + if v := value.Value.(bool); !v { + t.Fatal("unexpected bool value", v) + } + case "Value": if value.Doc != "a number" { t.Fatal("invalid doc") @@ -88,14 +124,27 @@ func TestExperimentBuilderOptions(t *testing.T) { if value.Type != "int64" { t.Fatal("invalid type", value.Type) } + if v := value.Value.(int64); v != 177114 { + t.Fatal("unexpected int64 value", v) + } + default: - t.Fatal("unknown name", name) + t.Fatal("unexpected option name", name) } } }) } -func TestExperimentBuilderSetOptionAny(t *testing.T) { +func TestFactorySetOptionAny(t *testing.T) { + + // the fake configuration we're using in this test + type fakeExperimentConfig struct { + Chan chan any `ooni:"we cannot set this"` + String string `ooni:"a string"` + Truth bool `ooni:"something that no-one knows"` + Value int64 `ooni:"a number"` + } + var inputs = []struct { TestCaseName string InitialConfig any @@ -294,6 +343,15 @@ func TestExperimentBuilderSetOptionAny(t *testing.T) { FieldValue: 1.11, ExpectErr: ErrCannotSetIntegerOption, ExpectConfig: &fakeExperimentConfig{}, + }, { + TestCaseName: "[int] for float64 with zero fractional value", + InitialConfig: &fakeExperimentConfig{}, + FieldName: "Value", + FieldValue: float64(16.0), + ExpectErr: nil, + ExpectConfig: &fakeExperimentConfig{ + Value: 16, + }, }, { TestCaseName: "[string] for serialized bool value while setting a string value", InitialConfig: &fakeExperimentConfig{}, @@ -352,7 +410,17 @@ func TestExperimentBuilderSetOptionAny(t *testing.T) { } } -func TestExperimentBuilderSetOptionsAny(t *testing.T) { +func TestFactorySetOptionsAny(t *testing.T) { + + // the fake configuration we're using in this test + type fakeExperimentConfig struct { + // values that should be included into the Options return value + Chan chan any `ooni:"we cannot set this"` + String string `ooni:"a string"` + Truth bool `ooni:"something that no-one knows"` + Value int64 `ooni:"a number"` + } + b := &Factory{config: &fakeExperimentConfig{}} t.Run("we correctly handle an empty map", func(t *testing.T) { @@ -395,6 +463,107 @@ func TestExperimentBuilderSetOptionsAny(t *testing.T) { }) } +func TestFactorySetOptionsJSON(t *testing.T) { + + // PersonRecord is a fake experiment configuration. + // + // Note how the `ooni` tag here is missing because we don't care + // about whether such a tag is present when using JSON. + type PersonRecord struct { + Name string + Age int64 + Friends []string + } + + // testcase is a test case for this function. + type testcase struct { + // name is the name of the test case + name string + + // mutableConfig is the config in which we should unmarshal the JSON + mutableConfig *PersonRecord + + // rawJSON contains the raw JSON to unmarshal into mutableConfig + rawJSON json.RawMessage + + // expectErr is the error we expect + expectErr error + + // expectRecord is what we expectRecord to see in the end + expectRecord *PersonRecord + } + + cases := []testcase{ + { + name: "we correctly accept zero-length options", + mutableConfig: &PersonRecord{ + Name: "foo", + Age: 55, + Friends: []string{"bar", "baz"}, + }, + rawJSON: []byte{}, + expectRecord: &PersonRecord{ + Name: "foo", + Age: 55, + Friends: []string{"bar", "baz"}, + }, + }, + + { + name: "we return an error on JSON parsing error", + mutableConfig: &PersonRecord{}, + rawJSON: []byte(`{`), + expectErr: errors.New("unexpected end of JSON input"), + expectRecord: &PersonRecord{}, + }, + + { + name: "we correctly unmarshal into the existing config", + mutableConfig: &PersonRecord{ + Name: "foo", + Age: 55, + Friends: []string{"bar", "baz"}, + }, + rawJSON: []byte(`{"Friends":["foo","oof"]}`), + expectErr: nil, + expectRecord: &PersonRecord{ + Name: "foo", + Age: 55, + Friends: []string{"foo", "oof"}, + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + + // create the factory to use + factory := &Factory{config: tc.mutableConfig} + + // unmarshal into the mutableConfig + err := factory.SetOptionsJSON(tc.rawJSON) + + // make sure the error is the one we actually expect + switch { + case err == nil && tc.expectErr == nil: + if diff := cmp.Diff(tc.expectRecord, tc.mutableConfig); diff != "" { + t.Fatal(diff) + } + return + + case err != nil && tc.expectErr != nil: + if err.Error() != tc.expectErr.Error() { + t.Fatal("expected", tc.expectErr, "got", err) + } + return + + default: + t.Fatal("expected", tc.expectErr, "got", err) + } + }) + } +} + func TestNewFactory(t *testing.T) { // experimentSpecificExpectations contains expectations for an experiment type experimentSpecificExpectations struct { @@ -455,6 +624,11 @@ func TestNewFactory(t *testing.T) { inputPolicy: model.InputNone, interruptible: true, }, + "openvpn": { + enabledByDefault: true, + inputPolicy: model.InputOrQueryBackend, + interruptible: true, + }, "portfiltering": { enabledByDefault: true, inputPolicy: model.InputNone, @@ -686,7 +860,7 @@ func TestNewFactory(t *testing.T) { // get experiment expectations -- note that here we must canonicalize the // experiment name otherwise we won't find it into the map when testing non-canonical names - expectations := expectationsMap[CanonicalizeExperimentName(tc.experimentName)] + expectations := expectationsMap[experimentname.Canonicalize(tc.experimentName)] if expectations == nil { t.Fatal("no expectations for", tc.experimentName) } @@ -796,3 +970,149 @@ func TestNewFactory(t *testing.T) { } }) } + +// Make sure the target loader for web connectivity is WAI when using no static inputs. +func TestFactoryNewTargetLoaderWebConnectivity(t *testing.T) { + // construct the proper factory instance + store := &kvstore.Memory{} + factory, err := NewFactory("web_connectivity", store, log.Log) + if err != nil { + t.Fatal(err) + } + + // define the expected error. + expected := errors.New("antani") + + // create suitable loader config. + config := &model.ExperimentTargetLoaderConfig{ + CheckInConfig: &model.OOAPICheckInConfig{ + // nothing + }, + Session: &mocks.Session{ + MockCheckIn: func(ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResult, error) { + return nil, expected + }, + MockLogger: func() model.Logger { + return log.Log + }, + }, + StaticInputs: nil, + SourceFiles: nil, + } + + // obtain the loader + loader := factory.NewTargetLoader(config) + + // attempt to load targets + targets, err := loader.Load(context.Background()) + + // make sure we've got the expected error + if !errors.Is(err, expected) { + t.Fatal("unexpected error", err) + } + + // make sure there are no targets + if len(targets) != 0 { + t.Fatal("expected zero length targets") + } +} + +// customConfig is a custom config for [TestFactoryCustomTargetLoaderForRicherInput]. +type customConfig struct{} + +// customTargetLoader is a custom target loader for [TestFactoryCustomTargetLoaderForRicherInput]. +type customTargetLoader struct{} + +// Load implements [model.ExperimentTargetLoader]. +func (c *customTargetLoader) Load(ctx context.Context) ([]model.ExperimentTarget, error) { + panic("should not be called") +} + +func TestFactoryNewTargetLoader(t *testing.T) { + t.Run("with custom target loader", func(t *testing.T) { + // create factory creating a custom target loader + factory := &Factory{ + build: nil, + canonicalName: "", + config: &customConfig{}, + enabledByDefault: false, + inputPolicy: "", + interruptible: false, + newLoader: func(config *targetloading.Loader, options any) model.ExperimentTargetLoader { + return &customTargetLoader{} + }, + } + + // create config for creating a new target loader + config := &model.ExperimentTargetLoaderConfig{ + CheckInConfig: &model.OOAPICheckInConfig{ /* nothing */ }, + Session: &mocks.Session{ + MockLogger: func() model.Logger { + return model.DiscardLogger + }, + }, + StaticInputs: []string{}, + SourceFiles: []string{}, + } + + // create the loader + loader := factory.NewTargetLoader(config) + + // make sure the type is the one we expected + if _, good := loader.(*customTargetLoader); !good { + t.Fatalf("expected a *customTargetLoader, got %T", loader) + } + }) + + t.Run("with default target loader", func(t *testing.T) { + // create factory creating a default target loader + factory := &Factory{ + build: nil, + canonicalName: "", + config: &customConfig{}, + enabledByDefault: false, + inputPolicy: "", + interruptible: false, + newLoader: nil, // explicitly nil + } + + // create config for creating a new target loader + config := &model.ExperimentTargetLoaderConfig{ + CheckInConfig: &model.OOAPICheckInConfig{ /* nothing */ }, + Session: &mocks.Session{ + MockLogger: func() model.Logger { + return model.DiscardLogger + }, + }, + StaticInputs: []string{}, + SourceFiles: []string{}, + } + + // create the loader + loader := factory.NewTargetLoader(config) + + // make sure the type is the one we expected + if _, good := loader.(*targetloading.Loader); !good { + t.Fatalf("expected a *targetloading.Loader, got %T", loader) + } + }) +} + +// This test is important because SetOptionsJSON assumes that the experiment +// config is a struct pointer into which it is possible to write +func TestExperimentConfigIsAlwaysAPointerToStruct(t *testing.T) { + for name, ffunc := range AllExperiments { + t.Run(name, func(t *testing.T) { + factory := ffunc() + config := factory.config + ctype := reflect.TypeOf(config) + if ctype.Kind() != reflect.Pointer { + t.Fatal("expected a pointer") + } + ctype = ctype.Elem() + if ctype.Kind() != reflect.Struct { + t.Fatal("expected a struct") + } + }) + } +} diff --git a/pkg/registry/fbmessenger.go b/pkg/registry/fbmessenger.go index a89456076..f01197b3f 100644 --- a/pkg/registry/fbmessenger.go +++ b/pkg/registry/fbmessenger.go @@ -10,14 +10,18 @@ import ( ) func init() { - AllExperiments["facebook_messenger"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return fbmessenger.NewExperimentMeasurer( - *config.(*fbmessenger.Config), - ) - }, - config: &fbmessenger.Config{}, - enabledByDefault: true, - inputPolicy: model.InputNone, + const canonicalName = "facebook_messenger" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return fbmessenger.NewExperimentMeasurer( + *config.(*fbmessenger.Config), + ) + }, + canonicalName: canonicalName, + config: &fbmessenger.Config{}, + enabledByDefault: true, + inputPolicy: model.InputNone, + } } } diff --git a/pkg/registry/hhfm.go b/pkg/registry/hhfm.go index 1adfaecd0..67a6a0fea 100644 --- a/pkg/registry/hhfm.go +++ b/pkg/registry/hhfm.go @@ -10,14 +10,18 @@ import ( ) func init() { - AllExperiments["http_header_field_manipulation"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return hhfm.NewExperimentMeasurer( - *config.(*hhfm.Config), - ) - }, - config: &hhfm.Config{}, - enabledByDefault: true, - inputPolicy: model.InputNone, + const canonicalName = "http_header_field_manipulation" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return hhfm.NewExperimentMeasurer( + *config.(*hhfm.Config), + ) + }, + canonicalName: canonicalName, + config: &hhfm.Config{}, + enabledByDefault: true, + inputPolicy: model.InputNone, + } } } diff --git a/pkg/registry/hirl.go b/pkg/registry/hirl.go index 6ffa57f25..311a0b1a3 100644 --- a/pkg/registry/hirl.go +++ b/pkg/registry/hirl.go @@ -10,14 +10,18 @@ import ( ) func init() { - AllExperiments["http_invalid_request_line"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return hirl.NewExperimentMeasurer( - *config.(*hirl.Config), - ) - }, - config: &hirl.Config{}, - enabledByDefault: true, - inputPolicy: model.InputNone, + const canonicalName = "http_invalid_request_line" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return hirl.NewExperimentMeasurer( + *config.(*hirl.Config), + ) + }, + canonicalName: canonicalName, + config: &hirl.Config{}, + enabledByDefault: true, + inputPolicy: model.InputNone, + } } } diff --git a/pkg/registry/httphostheader.go b/pkg/registry/httphostheader.go index b9503e0da..d61e7719b 100644 --- a/pkg/registry/httphostheader.go +++ b/pkg/registry/httphostheader.go @@ -10,14 +10,18 @@ import ( ) func init() { - AllExperiments["http_host_header"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return httphostheader.NewExperimentMeasurer( - *config.(*httphostheader.Config), - ) - }, - config: &httphostheader.Config{}, - enabledByDefault: true, - inputPolicy: model.InputOrQueryBackend, + const canonicalName = "http_host_header" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return httphostheader.NewExperimentMeasurer( + *config.(*httphostheader.Config), + ) + }, + canonicalName: canonicalName, + config: &httphostheader.Config{}, + enabledByDefault: true, + inputPolicy: model.InputOrQueryBackend, + } } } diff --git a/pkg/registry/ndt.go b/pkg/registry/ndt.go index 49407b284..81822bffd 100644 --- a/pkg/registry/ndt.go +++ b/pkg/registry/ndt.go @@ -10,15 +10,19 @@ import ( ) func init() { - AllExperiments["ndt"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return ndt7.NewExperimentMeasurer( - *config.(*ndt7.Config), - ) - }, - config: &ndt7.Config{}, - enabledByDefault: true, - interruptible: true, - inputPolicy: model.InputNone, + const canonicalName = "ndt" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return ndt7.NewExperimentMeasurer( + *config.(*ndt7.Config), + ) + }, + canonicalName: canonicalName, + config: &ndt7.Config{}, + enabledByDefault: true, + interruptible: true, + inputPolicy: model.InputNone, + } } } diff --git a/pkg/registry/openvpn.go b/pkg/registry/openvpn.go new file mode 100644 index 000000000..2752db741 --- /dev/null +++ b/pkg/registry/openvpn.go @@ -0,0 +1,27 @@ +package registry + +// +// Registers the `openvpn' experiment. +// + +import ( + "github.com/ooni/probe-engine/pkg/experiment/openvpn" + "github.com/ooni/probe-engine/pkg/model" +) + +func init() { + const canonicalName = "openvpn" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return openvpn.NewExperimentMeasurer() + }, + canonicalName: canonicalName, + config: &openvpn.Config{}, + enabledByDefault: true, + interruptible: true, + inputPolicy: model.InputOrQueryBackend, + newLoader: openvpn.NewLoader, + } + } +} diff --git a/pkg/registry/portfiltering.go b/pkg/registry/portfiltering.go index 5921e56e9..b1c0e1033 100644 --- a/pkg/registry/portfiltering.go +++ b/pkg/registry/portfiltering.go @@ -10,15 +10,19 @@ import ( ) func init() { - AllExperiments["portfiltering"] = &Factory{ - build: func(config any) model.ExperimentMeasurer { - return portfiltering.NewExperimentMeasurer( - config.(portfiltering.Config), - ) - }, - config: portfiltering.Config{}, - enabledByDefault: true, - interruptible: false, - inputPolicy: model.InputNone, + const canonicalName = "portfiltering" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config any) model.ExperimentMeasurer { + return portfiltering.NewExperimentMeasurer( + *config.(*portfiltering.Config), + ) + }, + canonicalName: canonicalName, + config: &portfiltering.Config{}, + enabledByDefault: true, + interruptible: false, + inputPolicy: model.InputNone, + } } } diff --git a/pkg/registry/psiphon.go b/pkg/registry/psiphon.go index 202f7e04f..0e09bb6f3 100644 --- a/pkg/registry/psiphon.go +++ b/pkg/registry/psiphon.go @@ -10,14 +10,18 @@ import ( ) func init() { - AllExperiments["psiphon"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return psiphon.NewExperimentMeasurer( - *config.(*psiphon.Config), - ) - }, - config: &psiphon.Config{}, - enabledByDefault: true, - inputPolicy: model.InputOptional, + const canonicalName = "psiphon" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return psiphon.NewExperimentMeasurer( + *config.(*psiphon.Config), + ) + }, + canonicalName: canonicalName, + config: &psiphon.Config{}, + enabledByDefault: true, + inputPolicy: model.InputOptional, + } } } diff --git a/pkg/registry/quicping.go b/pkg/registry/quicping.go index 2229523cf..c3fa8b468 100644 --- a/pkg/registry/quicping.go +++ b/pkg/registry/quicping.go @@ -10,14 +10,18 @@ import ( ) func init() { - AllExperiments["quicping"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return quicping.NewExperimentMeasurer( - *config.(*quicping.Config), - ) - }, - config: &quicping.Config{}, - enabledByDefault: true, - inputPolicy: model.InputStrictlyRequired, + const canonicalName = "quicping" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return quicping.NewExperimentMeasurer( + *config.(*quicping.Config), + ) + }, + canonicalName: canonicalName, + config: &quicping.Config{}, + enabledByDefault: true, + inputPolicy: model.InputStrictlyRequired, + } } } diff --git a/pkg/registry/riseupvpn.go b/pkg/registry/riseupvpn.go index 20cee9a02..1ee21df83 100644 --- a/pkg/registry/riseupvpn.go +++ b/pkg/registry/riseupvpn.go @@ -10,13 +10,17 @@ import ( ) func init() { - AllExperiments["riseupvpn"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return riseupvpn.NewExperimentMeasurer( - *config.(*riseupvpn.Config), - ) - }, - config: &riseupvpn.Config{}, - inputPolicy: model.InputNone, + const canonicalName = "riseupvpn" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return riseupvpn.NewExperimentMeasurer( + *config.(*riseupvpn.Config), + ) + }, + canonicalName: canonicalName, + config: &riseupvpn.Config{}, + inputPolicy: model.InputNone, + } } } diff --git a/pkg/registry/signal.go b/pkg/registry/signal.go index 4d7000185..1ff83562d 100644 --- a/pkg/registry/signal.go +++ b/pkg/registry/signal.go @@ -10,14 +10,18 @@ import ( ) func init() { - AllExperiments["signal"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return signal.NewExperimentMeasurer( - *config.(*signal.Config), - ) - }, - config: &signal.Config{}, - enabledByDefault: true, - inputPolicy: model.InputNone, + const canonicalName = "signal" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return signal.NewExperimentMeasurer( + *config.(*signal.Config), + ) + }, + canonicalName: canonicalName, + config: &signal.Config{}, + enabledByDefault: true, + inputPolicy: model.InputNone, + } } } diff --git a/pkg/registry/simplequicping.go b/pkg/registry/simplequicping.go index 1f1ded8f2..f9961c740 100644 --- a/pkg/registry/simplequicping.go +++ b/pkg/registry/simplequicping.go @@ -10,14 +10,18 @@ import ( ) func init() { - AllExperiments["simplequicping"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return simplequicping.NewExperimentMeasurer( - *config.(*simplequicping.Config), - ) - }, - config: &simplequicping.Config{}, - enabledByDefault: true, - inputPolicy: model.InputStrictlyRequired, + const canonicalName = "simplequicping" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return simplequicping.NewExperimentMeasurer( + *config.(*simplequicping.Config), + ) + }, + canonicalName: canonicalName, + config: &simplequicping.Config{}, + enabledByDefault: true, + inputPolicy: model.InputStrictlyRequired, + } } } diff --git a/pkg/registry/sniblocking.go b/pkg/registry/sniblocking.go index db8521359..24bd0ab73 100644 --- a/pkg/registry/sniblocking.go +++ b/pkg/registry/sniblocking.go @@ -10,14 +10,18 @@ import ( ) func init() { - AllExperiments["sni_blocking"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return sniblocking.NewExperimentMeasurer( - *config.(*sniblocking.Config), - ) - }, - config: &sniblocking.Config{}, - enabledByDefault: true, - inputPolicy: model.InputOrQueryBackend, + const canonicalName = "sni_blocking" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return sniblocking.NewExperimentMeasurer( + *config.(*sniblocking.Config), + ) + }, + canonicalName: canonicalName, + config: &sniblocking.Config{}, + enabledByDefault: true, + inputPolicy: model.InputOrQueryBackend, + } } } diff --git a/pkg/registry/stunreachability.go b/pkg/registry/stunreachability.go index 865789f4f..24a3cd99d 100644 --- a/pkg/registry/stunreachability.go +++ b/pkg/registry/stunreachability.go @@ -10,14 +10,18 @@ import ( ) func init() { - AllExperiments["stunreachability"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return stunreachability.NewExperimentMeasurer( - *config.(*stunreachability.Config), - ) - }, - config: &stunreachability.Config{}, - enabledByDefault: true, - inputPolicy: model.InputOrStaticDefault, + const canonicalName = "stunreachability" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return stunreachability.NewExperimentMeasurer( + *config.(*stunreachability.Config), + ) + }, + canonicalName: canonicalName, + config: &stunreachability.Config{}, + enabledByDefault: true, + inputPolicy: model.InputOrStaticDefault, + } } } diff --git a/pkg/registry/tcpping.go b/pkg/registry/tcpping.go index 5786af757..f76f5a761 100644 --- a/pkg/registry/tcpping.go +++ b/pkg/registry/tcpping.go @@ -10,14 +10,18 @@ import ( ) func init() { - AllExperiments["tcpping"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return tcpping.NewExperimentMeasurer( - *config.(*tcpping.Config), - ) - }, - config: &tcpping.Config{}, - enabledByDefault: true, - inputPolicy: model.InputStrictlyRequired, + const canonicalName = "tcpping" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return tcpping.NewExperimentMeasurer( + *config.(*tcpping.Config), + ) + }, + canonicalName: canonicalName, + config: &tcpping.Config{}, + enabledByDefault: true, + inputPolicy: model.InputStrictlyRequired, + } } } diff --git a/pkg/registry/telegram.go b/pkg/registry/telegram.go index cef5786ff..c42159fb3 100644 --- a/pkg/registry/telegram.go +++ b/pkg/registry/telegram.go @@ -10,15 +10,19 @@ import ( ) func init() { - AllExperiments["telegram"] = &Factory{ - build: func(config any) model.ExperimentMeasurer { - return telegram.NewExperimentMeasurer( - config.(telegram.Config), - ) - }, - config: telegram.Config{}, - enabledByDefault: true, - interruptible: false, - inputPolicy: model.InputNone, + const canonicalName = "telegram" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config any) model.ExperimentMeasurer { + return telegram.NewExperimentMeasurer( + *config.(*telegram.Config), + ) + }, + canonicalName: canonicalName, + config: &telegram.Config{}, + enabledByDefault: true, + interruptible: false, + inputPolicy: model.InputNone, + } } } diff --git a/pkg/registry/tlsmiddlebox.go b/pkg/registry/tlsmiddlebox.go index 1f956787e..f99a95aae 100644 --- a/pkg/registry/tlsmiddlebox.go +++ b/pkg/registry/tlsmiddlebox.go @@ -10,14 +10,18 @@ import ( ) func init() { - AllExperiments["tlsmiddlebox"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return tlsmiddlebox.NewExperimentMeasurer( - *config.(*tlsmiddlebox.Config), - ) - }, - config: &tlsmiddlebox.Config{}, - enabledByDefault: true, - inputPolicy: model.InputStrictlyRequired, + const canonicalName = "tlsmiddlebox" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return tlsmiddlebox.NewExperimentMeasurer( + *config.(*tlsmiddlebox.Config), + ) + }, + canonicalName: canonicalName, + config: &tlsmiddlebox.Config{}, + enabledByDefault: true, + inputPolicy: model.InputStrictlyRequired, + } } } diff --git a/pkg/registry/tlsping.go b/pkg/registry/tlsping.go index 529bbb2a6..6e8e4a206 100644 --- a/pkg/registry/tlsping.go +++ b/pkg/registry/tlsping.go @@ -10,14 +10,18 @@ import ( ) func init() { - AllExperiments["tlsping"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return tlsping.NewExperimentMeasurer( - *config.(*tlsping.Config), - ) - }, - config: &tlsping.Config{}, - enabledByDefault: true, - inputPolicy: model.InputStrictlyRequired, + const canonicalName = "tlsping" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return tlsping.NewExperimentMeasurer( + *config.(*tlsping.Config), + ) + }, + canonicalName: canonicalName, + config: &tlsping.Config{}, + enabledByDefault: true, + inputPolicy: model.InputStrictlyRequired, + } } } diff --git a/pkg/registry/tlstool.go b/pkg/registry/tlstool.go index 1cc9d6a47..f41be1d9a 100644 --- a/pkg/registry/tlstool.go +++ b/pkg/registry/tlstool.go @@ -10,14 +10,18 @@ import ( ) func init() { - AllExperiments["tlstool"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return tlstool.NewExperimentMeasurer( - *config.(*tlstool.Config), - ) - }, - config: &tlstool.Config{}, - enabledByDefault: true, - inputPolicy: model.InputOrQueryBackend, + const canonicalName = "tlstool" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return tlstool.NewExperimentMeasurer( + *config.(*tlstool.Config), + ) + }, + canonicalName: canonicalName, + config: &tlstool.Config{}, + enabledByDefault: true, + inputPolicy: model.InputOrQueryBackend, + } } } diff --git a/pkg/registry/tor.go b/pkg/registry/tor.go index 2248eb77e..dc9adfdc1 100644 --- a/pkg/registry/tor.go +++ b/pkg/registry/tor.go @@ -10,14 +10,18 @@ import ( ) func init() { - AllExperiments["tor"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return tor.NewExperimentMeasurer( - *config.(*tor.Config), - ) - }, - config: &tor.Config{}, - enabledByDefault: true, - inputPolicy: model.InputNone, + const canonicalName = "tor" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return tor.NewExperimentMeasurer( + *config.(*tor.Config), + ) + }, + canonicalName: canonicalName, + config: &tor.Config{}, + enabledByDefault: true, + inputPolicy: model.InputNone, + } } } diff --git a/pkg/registry/torsf.go b/pkg/registry/torsf.go index f07267340..59057d6b7 100644 --- a/pkg/registry/torsf.go +++ b/pkg/registry/torsf.go @@ -10,14 +10,18 @@ import ( ) func init() { - AllExperiments["torsf"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return torsf.NewExperimentMeasurer( - *config.(*torsf.Config), - ) - }, - config: &torsf.Config{}, - enabledByDefault: false, - inputPolicy: model.InputNone, + const canonicalName = "torsf" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return torsf.NewExperimentMeasurer( + *config.(*torsf.Config), + ) + }, + canonicalName: canonicalName, + config: &torsf.Config{}, + enabledByDefault: false, + inputPolicy: model.InputNone, + } } } diff --git a/pkg/registry/urlgetter.go b/pkg/registry/urlgetter.go index 6f29a8d81..0a34962a7 100644 --- a/pkg/registry/urlgetter.go +++ b/pkg/registry/urlgetter.go @@ -10,14 +10,18 @@ import ( ) func init() { - AllExperiments["urlgetter"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return urlgetter.NewExperimentMeasurer( - *config.(*urlgetter.Config), - ) - }, - config: &urlgetter.Config{}, - enabledByDefault: true, - inputPolicy: model.InputStrictlyRequired, + const canonicalName = "urlgetter" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return urlgetter.NewExperimentMeasurer( + *config.(*urlgetter.Config), + ) + }, + canonicalName: canonicalName, + config: &urlgetter.Config{}, + enabledByDefault: true, + inputPolicy: model.InputStrictlyRequired, + } } } diff --git a/pkg/registry/vanillator.go b/pkg/registry/vanillator.go index e9865e4bd..0b897029d 100644 --- a/pkg/registry/vanillator.go +++ b/pkg/registry/vanillator.go @@ -10,17 +10,21 @@ import ( ) func init() { - AllExperiments["vanilla_tor"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return vanillator.NewExperimentMeasurer( - *config.(*vanillator.Config), - ) - }, - config: &vanillator.Config{}, - // We discussed this topic with @aanorbel. On Android this experiment crashes - // frequently because of https://github.com/ooni/probe/issues/2406. So, it seems - // more cautious to disable it by default and let the check-in API decide. - enabledByDefault: false, - inputPolicy: model.InputNone, + const canonicalName = "vanilla_tor" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return vanillator.NewExperimentMeasurer( + *config.(*vanillator.Config), + ) + }, + canonicalName: canonicalName, + config: &vanillator.Config{}, + // We discussed this topic with @aanorbel. On Android this experiment crashes + // frequently because of https://github.com/ooni/probe/issues/2406. So, it seems + // more cautious to disable it by default and let the check-in API decide. + enabledByDefault: false, + inputPolicy: model.InputNone, + } } } diff --git a/pkg/registry/webconnectivity.go b/pkg/registry/webconnectivity.go index e94ea6e81..562c514fd 100644 --- a/pkg/registry/webconnectivity.go +++ b/pkg/registry/webconnectivity.go @@ -10,15 +10,19 @@ import ( ) func init() { - AllExperiments["web_connectivity"] = &Factory{ - build: func(config any) model.ExperimentMeasurer { - return webconnectivity.NewExperimentMeasurer( - config.(webconnectivity.Config), - ) - }, - config: webconnectivity.Config{}, - enabledByDefault: true, - interruptible: false, - inputPolicy: model.InputOrQueryBackend, + const canonicalName = "web_connectivity" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config any) model.ExperimentMeasurer { + return webconnectivity.NewExperimentMeasurer( + *config.(*webconnectivity.Config), + ) + }, + canonicalName: canonicalName, + config: &webconnectivity.Config{}, + enabledByDefault: true, + interruptible: false, + inputPolicy: model.InputOrQueryBackend, + } } } diff --git a/pkg/registry/webconnectivityv05.go b/pkg/registry/webconnectivityv05.go index 77bd1519d..1f430fb86 100644 --- a/pkg/registry/webconnectivityv05.go +++ b/pkg/registry/webconnectivityv05.go @@ -12,15 +12,19 @@ import ( ) func init() { - AllExperiments["web_connectivity@v0.5"] = &Factory{ - build: func(config any) model.ExperimentMeasurer { - return webconnectivitylte.NewExperimentMeasurer( - config.(*webconnectivitylte.Config), - ) - }, - config: &webconnectivitylte.Config{}, - enabledByDefault: true, - interruptible: false, - inputPolicy: model.InputOrQueryBackend, + const canonicalName = "web_connectivity@v0.5" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config any) model.ExperimentMeasurer { + return webconnectivitylte.NewExperimentMeasurer( + config.(*webconnectivitylte.Config), + ) + }, + canonicalName: canonicalName, + config: &webconnectivitylte.Config{}, + enabledByDefault: true, + interruptible: false, + inputPolicy: model.InputOrQueryBackend, + } } } diff --git a/pkg/registry/whatsapp.go b/pkg/registry/whatsapp.go index bbd252854..720d807db 100644 --- a/pkg/registry/whatsapp.go +++ b/pkg/registry/whatsapp.go @@ -10,14 +10,18 @@ import ( ) func init() { - AllExperiments["whatsapp"] = &Factory{ - build: func(config interface{}) model.ExperimentMeasurer { - return whatsapp.NewExperimentMeasurer( - *config.(*whatsapp.Config), - ) - }, - config: &whatsapp.Config{}, - enabledByDefault: true, - inputPolicy: model.InputNone, + const canonicalName = "whatsapp" + AllExperiments[canonicalName] = func() *Factory { + return &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return whatsapp.NewExperimentMeasurer( + *config.(*whatsapp.Config), + ) + }, + canonicalName: canonicalName, + config: &whatsapp.Config{}, + enabledByDefault: true, + inputPolicy: model.InputNone, + } } } diff --git a/pkg/engine/inputloader.go b/pkg/targetloading/targetloading.go similarity index 68% rename from pkg/engine/inputloader.go rename to pkg/targetloading/targetloading.go index 271ed17ff..5e9fd2714 100644 --- a/pkg/engine/inputloader.go +++ b/pkg/targetloading/targetloading.go @@ -1,4 +1,5 @@ -package engine +// Package targetloading contains common code to load richer-input targets. +package targetloading import ( "bufio" @@ -9,34 +10,33 @@ import ( "net/url" "github.com/apex/log" + "github.com/ooni/probe-engine/pkg/experimentname" "github.com/ooni/probe-engine/pkg/fsx" "github.com/ooni/probe-engine/pkg/model" - "github.com/ooni/probe-engine/pkg/registry" "github.com/ooni/probe-engine/pkg/stuninput" ) -// These errors are returned by the InputLoader. +// These errors are returned by the [*Loader] or the experiment execution. var ( ErrNoURLsReturned = errors.New("no URLs returned") ErrDetectedEmptyFile = errors.New("file did not contain any input") ErrInputRequired = errors.New("no input provided") ErrNoInputExpected = errors.New("we did not expect any input") ErrNoStaticInput = errors.New("no static input for this experiment") + ErrInvalidInputType = errors.New("invalid richer input type") + ErrInvalidInput = errors.New("input does not conform to spec") ) -// InputLoaderSession is the session according to an InputLoader. We -// introduce this abstraction because it helps us with testing. -type InputLoaderSession interface { - CheckIn(ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResult, error) -} +// Session is the session according to a [*Loader] instance. +type Session = model.ExperimentTargetLoaderSession -// InputLoaderLogger is the logger according to an InputLoader. -type InputLoaderLogger interface { +// Logger is the [model.Logger] according to a [*Loader]. +type Logger interface { // Warnf formats and emits a warning message. Warnf(format string, v ...interface{}) } -// InputLoader loads input according to the specified policy +// Loader loads input according to the specified policy // either from command line and input files or from OONI services. The // behaviour depends on the input policy as described below. // @@ -71,7 +71,7 @@ type InputLoaderLogger interface { // // We gather input from StaticInput and SourceFiles. If there is // input, we return it. Otherwise, we return an error. -type InputLoader struct { +type Loader struct { // CheckInConfig contains options for the CheckIn API. If // not set, then we'll create a default config. If set but // there are fields inside it that are not set, then we @@ -88,14 +88,14 @@ type InputLoader struct { // this field. InputPolicy model.InputPolicy - // Logger is the optional logger that the InputLoader + // Logger is the optional logger that the [*Loader] // should be using. If not set, we will use the default // logger of github.com/apex/log. - Logger InputLoaderLogger + Logger Logger // Session is the current measurement session. You // MUST fill in this field. - Session InputLoaderSession + Session Session // StaticInputs contains optional input to be added // to the resulting input list if possible. @@ -110,7 +110,7 @@ type InputLoader struct { // Load attempts to load input using the specified input loader. We will // return a list of URLs because this is the only input we support. -func (il *InputLoader) Load(ctx context.Context) ([]model.OOAPIURLInfo, error) { +func (il *Loader) Load(ctx context.Context) ([]model.ExperimentTarget, error) { switch il.InputPolicy { case model.InputOptional: return il.loadOptional() @@ -126,26 +126,29 @@ func (il *InputLoader) Load(ctx context.Context) ([]model.OOAPIURLInfo, error) { } // loadNone implements the InputNone policy. -func (il *InputLoader) loadNone() ([]model.OOAPIURLInfo, error) { +func (il *Loader) loadNone() ([]model.ExperimentTarget, error) { if len(il.StaticInputs) > 0 || len(il.SourceFiles) > 0 { return nil, ErrNoInputExpected } - // Note that we need to return a single empty entry. - return []model.OOAPIURLInfo{{}}, nil + // Implementation note: the convention for input-less experiments is that + // they require a single entry containing an empty input. + entry := model.NewOOAPIURLInfoWithDefaultCategoryAndCountry("") + return []model.ExperimentTarget{entry}, nil } // loadOptional implements the InputOptional policy. -func (il *InputLoader) loadOptional() ([]model.OOAPIURLInfo, error) { +func (il *Loader) loadOptional() ([]model.ExperimentTarget, error) { inputs, err := il.loadLocal() if err == nil && len(inputs) <= 0 { - // Note that we need to return a single empty entry. - inputs = []model.OOAPIURLInfo{{}} + // Implementation note: the convention for input-less experiments is that + // they require a single entry containing an empty input. + inputs = []model.ExperimentTarget{model.NewOOAPIURLInfoWithDefaultCategoryAndCountry("")} } return inputs, err } // loadStrictlyRequired implements the InputStrictlyRequired policy. -func (il *InputLoader) loadStrictlyRequired(_ context.Context) ([]model.OOAPIURLInfo, error) { +func (il *Loader) loadStrictlyRequired(_ context.Context) ([]model.ExperimentTarget, error) { inputs, err := il.loadLocal() if err != nil || len(inputs) > 0 { return inputs, err @@ -154,12 +157,13 @@ func (il *InputLoader) loadStrictlyRequired(_ context.Context) ([]model.OOAPIURL } // loadOrQueryBackend implements the InputOrQueryBackend policy. -func (il *InputLoader) loadOrQueryBackend(ctx context.Context) ([]model.OOAPIURLInfo, error) { +func (il *Loader) loadOrQueryBackend(ctx context.Context) ([]model.ExperimentTarget, error) { inputs, err := il.loadLocal() if err != nil || len(inputs) > 0 { return inputs, err } - return il.loadRemote(ctx) + // This assumes that the default experiment for loading remote targets is WebConnectivity. + return il.loadRemoteWebConnectivity(ctx) } // TODO(https://github.com/ooni/probe/issues/1390): we need to @@ -211,23 +215,21 @@ var dnsCheckDefaultInput = []string{ var stunReachabilityDefaultInput = stuninput.AsnStunReachabilityInput() -// StaticBareInputForExperiment returns the list of strings an +// staticBareInputForExperiment returns the list of strings an // experiment should use as static input. In case there is no // static input for this experiment, we return an error. -func StaticBareInputForExperiment(name string) ([]string, error) { +func staticBareInputForExperiment(name string) ([]string, error) { // Implementation note: we may be called from pkg/oonimkall // with a non-canonical experiment name, so we need to convert // the experiment name to be canonical before proceeding. - // - // TODO(https://github.com/ooni/probe/issues/1390): serve DNSCheck - // inputs using richer input (aka check-in v2). - // - // TODO(https://github.com/ooni/probe/issues/2557): server STUNReachability - // inputs using richer input (aka check-in v2). - switch registry.CanonicalizeExperimentName(name) { + switch experimentname.Canonicalize(name) { case "dnscheck": + // TODO(https://github.com/ooni/probe/issues/1390): serve DNSCheck + // inputs using richer input (aka check-in v2). return dnsCheckDefaultInput, nil case "stunreachability": + // TODO(https://github.com/ooni/probe/issues/2557): server STUNReachability + // inputs using richer input (aka check-in v2). return stunReachabilityDefaultInput, nil default: return nil, ErrNoStaticInput @@ -236,12 +238,12 @@ func StaticBareInputForExperiment(name string) ([]string, error) { // staticInputForExperiment returns the static input for the given experiment // or an error if there's no static input for the experiment. -func staticInputForExperiment(name string) ([]model.OOAPIURLInfo, error) { - return stringListToModelURLInfo(StaticBareInputForExperiment(name)) +func staticInputForExperiment(name string) ([]model.ExperimentTarget, error) { + return stringListToModelExperimentTarget(staticBareInputForExperiment(name)) } // loadOrStaticDefault implements the InputOrStaticDefault policy. -func (il *InputLoader) loadOrStaticDefault(_ context.Context) ([]model.OOAPIURLInfo, error) { +func (il *Loader) loadOrStaticDefault(_ context.Context) ([]model.ExperimentTarget, error) { inputs, err := il.loadLocal() if err != nil || len(inputs) > 0 { return inputs, err @@ -249,33 +251,26 @@ func (il *InputLoader) loadOrStaticDefault(_ context.Context) ([]model.OOAPIURLI return staticInputForExperiment(il.ExperimentName) } -// loadLocal loads inputs from StaticInputs and SourceFiles. -func (il *InputLoader) loadLocal() ([]model.OOAPIURLInfo, error) { - inputs := []model.OOAPIURLInfo{} - for _, input := range il.StaticInputs { - inputs = append(inputs, model.OOAPIURLInfo{URL: input}) +// loadLocal loads inputs from the [*Loader] StaticInputs and SourceFiles. +func (il *Loader) loadLocal() ([]model.ExperimentTarget, error) { + inputs, err := LoadStatic(il) + if err != nil { + return nil, err } - for _, filepath := range il.SourceFiles { - extra, err := il.readfile(filepath, fsx.OpenFile) - if err != nil { - return nil, err - } - // See https://github.com/ooni/probe-engine/issues/1123. - if len(extra) <= 0 { - return nil, fmt.Errorf("%w: %s", ErrDetectedEmptyFile, filepath) - } - inputs = append(inputs, extra...) + var targets []model.ExperimentTarget + for _, input := range inputs { + targets = append(targets, model.NewOOAPIURLInfoWithDefaultCategoryAndCountry(input)) } - return inputs, nil + return targets, nil } -// inputLoaderOpenFn is the type of the function to open a file. -type inputLoaderOpenFn func(filepath string) (fs.File, error) +// openFunc is the type of the function to open a file. +type openFunc func(filepath string) (fs.File, error) // readfile reads inputs from the specified file. The open argument should be // compatible with stdlib's fs.Open and helps us with unit testing. -func (il *InputLoader) readfile(filepath string, open inputLoaderOpenFn) ([]model.OOAPIURLInfo, error) { - inputs := []model.OOAPIURLInfo{} +func readfile(filepath string, open openFunc) ([]string, error) { + inputs := []string{} filep, err := open(filepath) if err != nil { return nil, err @@ -288,7 +283,7 @@ func (il *InputLoader) readfile(filepath string, open inputLoaderOpenFn) ([]mode for scanner.Scan() { line := scanner.Text() if line != "" { - inputs = append(inputs, model.OOAPIURLInfo{URL: line}) + inputs = append(inputs, line) } } if scanner.Err() != nil { @@ -297,8 +292,25 @@ func (il *InputLoader) readfile(filepath string, open inputLoaderOpenFn) ([]mode return inputs, nil } -// loadRemote loads inputs from a remote source. -func (il *InputLoader) loadRemote(ctx context.Context) ([]model.OOAPIURLInfo, error) { +// LoadStatic loads inputs from the [*Loader] StaticInputs and SourceFiles. +func LoadStatic(config *Loader) ([]string, error) { + inputs := append([]string{}, config.StaticInputs...) + for _, filepath := range config.SourceFiles { + extra, err := readfile(filepath, fsx.OpenFile) + if err != nil { + return nil, err + } + // See https://github.com/ooni/probe-engine/issues/1123. + if len(extra) <= 0 { + return nil, fmt.Errorf("%w: %s", ErrDetectedEmptyFile, filepath) + } + inputs = append(inputs, extra...) + } + return inputs, nil +} + +// loadRemoteWebConnectivity loads webconnectivity inputs from a remote source. +func (il *Loader) loadRemoteWebConnectivity(ctx context.Context) ([]model.ExperimentTarget, error) { config := il.CheckInConfig if config == nil { // Note: Session.CheckIn documentation says it will fill in @@ -314,13 +326,30 @@ func (il *InputLoader) loadRemote(ctx context.Context) ([]model.OOAPIURLInfo, er if reply.WebConnectivity == nil || len(reply.WebConnectivity.URLs) <= 0 { return nil, ErrNoURLsReturned } - return reply.WebConnectivity.URLs, nil + output := modelOOAPIURLInfoToModelExperimentTarget(reply.WebConnectivity.URLs) + return output, nil +} + +func modelOOAPIURLInfoToModelExperimentTarget( + inputs []model.OOAPIURLInfo) (outputs []model.ExperimentTarget) { + for _, input := range inputs { + // Note: Dammit! Before we switch to go1.22 we need to continue to + // stay careful about the variable over which we're looping! + // + // See https://go.dev/blog/loopvar-preview for more information. + outputs = append(outputs, &model.OOAPIURLInfo{ + CategoryCode: input.CategoryCode, + CountryCode: input.CountryCode, + URL: input.URL, + }) + } + return } // checkIn executes the check-in and filters the returned URLs to exclude // the URLs that are not part of the requested categories. This is done for // robustness, just in case we or the API do something wrong. -func (il *InputLoader) checkIn( +func (il *Loader) checkIn( ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResultNettests, error) { reply, err := il.Session.CheckIn(ctx, config) if err != nil { @@ -338,7 +367,7 @@ func (il *InputLoader) checkIn( // preventMistakes makes the code more robust with respect to any possible // integration issue where the backend returns to us URLs that don't // belong to the category codes we requested. -func (il *InputLoader) preventMistakes(input []model.OOAPIURLInfo, categories []string) (output []model.OOAPIURLInfo) { +func (il *Loader) preventMistakes(input []model.OOAPIURLInfo, categories []string) (output []model.OOAPIURLInfo) { if len(categories) <= 0 { return input } @@ -360,32 +389,28 @@ func (il *InputLoader) preventMistakes(input []model.OOAPIURLInfo, categories [] } // logger returns the configured logger or apex/log's default. -func (il *InputLoader) logger() InputLoaderLogger { +func (il *Loader) logger() Logger { if il.Logger != nil { return il.Logger } return log.Log } -// stringListToModelURLInfo is an utility function to convert -// a list of strings containing URLs into a list of model.URLInfo +// stringListToModelExperimentTarget is an utility function to convert +// a list of strings containing URLs into a list of model.ExperimentTarget // which would have been returned by an hypothetical backend // API serving input for a test for which we don't have an API // yet (e.g., stunreachability and dnscheck). -func stringListToModelURLInfo(input []string, err error) ([]model.OOAPIURLInfo, error) { +func stringListToModelExperimentTarget(input []string, err error) ([]model.ExperimentTarget, error) { if err != nil { return nil, err } - var output []model.OOAPIURLInfo + var output []model.ExperimentTarget for _, URL := range input { if _, err := url.Parse(URL); err != nil { return nil, err } - output = append(output, model.OOAPIURLInfo{ - CategoryCode: "MISC", // hard to find a category - CountryCode: "XX", // representing no country - URL: URL, - }) + output = append(output, model.NewOOAPIURLInfoWithDefaultCategoryAndCountry(URL)) } return output, nil } diff --git a/pkg/engine/inputloader_test.go b/pkg/targetloading/targetloading_test.go similarity index 54% rename from pkg/engine/inputloader_test.go rename to pkg/targetloading/targetloading_test.go index 2811b72a1..b4696904a 100644 --- a/pkg/engine/inputloader_test.go +++ b/pkg/targetloading/targetloading_test.go @@ -1,4 +1,4 @@ -package engine +package targetloading import ( "context" @@ -12,12 +12,13 @@ import ( "github.com/apex/log" "github.com/google/go-cmp/cmp" - "github.com/ooni/probe-engine/pkg/kvstore" + "github.com/ooni/probe-engine/pkg/mocks" "github.com/ooni/probe-engine/pkg/model" + "github.com/ooni/probe-engine/pkg/runtimex" ) -func TestInputLoaderInputNoneWithStaticInputs(t *testing.T) { - il := &InputLoader{ +func TestTargetLoaderInputNoneWithStaticInputs(t *testing.T) { + il := &Loader{ StaticInputs: []string{"https://www.google.com/"}, InputPolicy: model.InputNone, } @@ -31,11 +32,11 @@ func TestInputLoaderInputNoneWithStaticInputs(t *testing.T) { } } -func TestInputLoaderInputNoneWithFilesInputs(t *testing.T) { - il := &InputLoader{ +func TestTargetLoaderInputNoneWithFilesInputs(t *testing.T) { + il := &Loader{ SourceFiles: []string{ - "testdata/inputloader1.txt", - "testdata/inputloader2.txt", + "testdata/loader1.txt", + "testdata/loader2.txt", }, InputPolicy: model.InputNone, } @@ -49,12 +50,12 @@ func TestInputLoaderInputNoneWithFilesInputs(t *testing.T) { } } -func TestInputLoaderInputNoneWithBothInputs(t *testing.T) { - il := &InputLoader{ +func TestTargetLoaderInputNoneWithBothInputs(t *testing.T) { + il := &Loader{ StaticInputs: []string{"https://www.google.com/"}, SourceFiles: []string{ - "testdata/inputloader1.txt", - "testdata/inputloader2.txt", + "testdata/loader1.txt", + "testdata/loader2.txt", }, InputPolicy: model.InputNone, } @@ -68,8 +69,8 @@ func TestInputLoaderInputNoneWithBothInputs(t *testing.T) { } } -func TestInputLoaderInputNoneWithNoInput(t *testing.T) { - il := &InputLoader{ +func TestTargetLoaderInputNoneWithNoInput(t *testing.T) { + il := &Loader{ InputPolicy: model.InputNone, } ctx := context.Background() @@ -77,13 +78,13 @@ func TestInputLoaderInputNoneWithNoInput(t *testing.T) { if err != nil { t.Fatal(err) } - if len(out) != 1 || out[0].URL != "" { + if len(out) != 1 || out[0].Input() != "" { t.Fatal("not the output we expected") } } -func TestInputLoaderInputOptionalWithNoInput(t *testing.T) { - il := &InputLoader{ +func TestTargetLoaderInputOptionalWithNoInput(t *testing.T) { + il := &Loader{ InputPolicy: model.InputOptional, } ctx := context.Background() @@ -91,17 +92,17 @@ func TestInputLoaderInputOptionalWithNoInput(t *testing.T) { if err != nil { t.Fatal(err) } - if len(out) != 1 || out[0].URL != "" { + if len(out) != 1 || out[0].Input() != "" { t.Fatal("not the output we expected") } } -func TestInputLoaderInputOptionalWithInput(t *testing.T) { - il := &InputLoader{ +func TestTargetLoaderInputOptionalWithInput(t *testing.T) { + il := &Loader{ StaticInputs: []string{"https://www.google.com/"}, SourceFiles: []string{ - "testdata/inputloader1.txt", - "testdata/inputloader2.txt", + "testdata/loader1.txt", + "testdata/loader2.txt", }, InputPolicy: model.InputOptional, } @@ -113,25 +114,45 @@ func TestInputLoaderInputOptionalWithInput(t *testing.T) { if len(out) != 5 { t.Fatal("not the output length we expected") } - expect := []model.OOAPIURLInfo{ - {URL: "https://www.google.com/"}, - {URL: "https://www.x.org/"}, - {URL: "https://www.slashdot.org/"}, - {URL: "https://abc.xyz/"}, - {URL: "https://run.ooni.io/"}, + expect := []model.ExperimentTarget{ + &model.OOAPIURLInfo{ + CountryCode: model.DefaultCountryCode, + CategoryCode: model.DefaultCategoryCode, + URL: "https://www.google.com/", + }, + &model.OOAPIURLInfo{ + CountryCode: model.DefaultCountryCode, + CategoryCode: model.DefaultCategoryCode, + URL: "https://www.x.org/", + }, + &model.OOAPIURLInfo{ + CountryCode: model.DefaultCountryCode, + CategoryCode: model.DefaultCategoryCode, + URL: "https://www.slashdot.org/", + }, + &model.OOAPIURLInfo{ + CountryCode: model.DefaultCountryCode, + CategoryCode: model.DefaultCategoryCode, + URL: "https://abc.xyz/", + }, + &model.OOAPIURLInfo{ + CountryCode: model.DefaultCountryCode, + CategoryCode: model.DefaultCategoryCode, + URL: "https://run.ooni.io/", + }, } if diff := cmp.Diff(out, expect); diff != "" { t.Fatal(diff) } } -func TestInputLoaderInputOptionalNonexistentFile(t *testing.T) { - il := &InputLoader{ +func TestTargetLoaderInputOptionalNonexistentFile(t *testing.T) { + il := &Loader{ StaticInputs: []string{"https://www.google.com/"}, SourceFiles: []string{ - "testdata/inputloader1.txt", + "testdata/loader1.txt", "/nonexistent", - "testdata/inputloader2.txt", + "testdata/loader2.txt", }, InputPolicy: model.InputOptional, } @@ -145,12 +166,12 @@ func TestInputLoaderInputOptionalNonexistentFile(t *testing.T) { } } -func TestInputLoaderInputStrictlyRequiredWithInput(t *testing.T) { - il := &InputLoader{ +func TestTargetLoaderInputStrictlyRequiredWithInput(t *testing.T) { + il := &Loader{ StaticInputs: []string{"https://www.google.com/"}, SourceFiles: []string{ - "testdata/inputloader1.txt", - "testdata/inputloader2.txt", + "testdata/loader1.txt", + "testdata/loader2.txt", }, InputPolicy: model.InputStrictlyRequired, } @@ -162,20 +183,40 @@ func TestInputLoaderInputStrictlyRequiredWithInput(t *testing.T) { if len(out) != 5 { t.Fatal("not the output length we expected") } - expect := []model.OOAPIURLInfo{ - {URL: "https://www.google.com/"}, - {URL: "https://www.x.org/"}, - {URL: "https://www.slashdot.org/"}, - {URL: "https://abc.xyz/"}, - {URL: "https://run.ooni.io/"}, + expect := []model.ExperimentTarget{ + &model.OOAPIURLInfo{ + CountryCode: model.DefaultCountryCode, + CategoryCode: model.DefaultCategoryCode, + URL: "https://www.google.com/", + }, + &model.OOAPIURLInfo{ + CountryCode: model.DefaultCountryCode, + CategoryCode: model.DefaultCategoryCode, + URL: "https://www.x.org/", + }, + &model.OOAPIURLInfo{ + CountryCode: model.DefaultCountryCode, + CategoryCode: model.DefaultCategoryCode, + URL: "https://www.slashdot.org/", + }, + &model.OOAPIURLInfo{ + CountryCode: model.DefaultCountryCode, + CategoryCode: model.DefaultCategoryCode, + URL: "https://abc.xyz/", + }, + &model.OOAPIURLInfo{ + CountryCode: model.DefaultCountryCode, + CategoryCode: model.DefaultCategoryCode, + URL: "https://run.ooni.io/", + }, } if diff := cmp.Diff(out, expect); diff != "" { t.Fatal(diff) } } -func TestInputLoaderInputStrictlyRequiredWithoutInput(t *testing.T) { - il := &InputLoader{ +func TestTargetLoaderInputStrictlyRequiredWithoutInput(t *testing.T) { + il := &Loader{ InputPolicy: model.InputStrictlyRequired, } ctx := context.Background() @@ -188,13 +229,13 @@ func TestInputLoaderInputStrictlyRequiredWithoutInput(t *testing.T) { } } -func TestInputLoaderInputStrictlyRequiredWithEmptyFile(t *testing.T) { - il := &InputLoader{ +func TestTargetLoaderInputStrictlyRequiredWithEmptyFile(t *testing.T) { + il := &Loader{ InputPolicy: model.InputStrictlyRequired, SourceFiles: []string{ - "testdata/inputloader1.txt", - "testdata/inputloader3.txt", // we want it before inputloader2.txt - "testdata/inputloader2.txt", + "testdata/loader1.txt", + "testdata/loader3.txt", // we want it before loader2.txt + "testdata/loader2.txt", }, } ctx := context.Background() @@ -207,13 +248,13 @@ func TestInputLoaderInputStrictlyRequiredWithEmptyFile(t *testing.T) { } } -func TestInputLoaderInputOrStaticDefaultWithInput(t *testing.T) { - il := &InputLoader{ +func TestTargetLoaderInputOrStaticDefaultWithInput(t *testing.T) { + il := &Loader{ ExperimentName: "dnscheck", StaticInputs: []string{"https://www.google.com/"}, SourceFiles: []string{ - "testdata/inputloader1.txt", - "testdata/inputloader2.txt", + "testdata/loader1.txt", + "testdata/loader2.txt", }, InputPolicy: model.InputOrStaticDefault, } @@ -225,26 +266,46 @@ func TestInputLoaderInputOrStaticDefaultWithInput(t *testing.T) { if len(out) != 5 { t.Fatal("not the output length we expected") } - expect := []model.OOAPIURLInfo{ - {URL: "https://www.google.com/"}, - {URL: "https://www.x.org/"}, - {URL: "https://www.slashdot.org/"}, - {URL: "https://abc.xyz/"}, - {URL: "https://run.ooni.io/"}, + expect := []model.ExperimentTarget{ + &model.OOAPIURLInfo{ + CountryCode: model.DefaultCountryCode, + CategoryCode: model.DefaultCategoryCode, + URL: "https://www.google.com/", + }, + &model.OOAPIURLInfo{ + CountryCode: model.DefaultCountryCode, + CategoryCode: model.DefaultCategoryCode, + URL: "https://www.x.org/", + }, + &model.OOAPIURLInfo{ + CountryCode: model.DefaultCountryCode, + CategoryCode: model.DefaultCategoryCode, + URL: "https://www.slashdot.org/", + }, + &model.OOAPIURLInfo{ + CountryCode: model.DefaultCountryCode, + CategoryCode: model.DefaultCategoryCode, + URL: "https://abc.xyz/", + }, + &model.OOAPIURLInfo{ + CountryCode: model.DefaultCountryCode, + CategoryCode: model.DefaultCategoryCode, + URL: "https://run.ooni.io/", + }, } if diff := cmp.Diff(out, expect); diff != "" { t.Fatal(diff) } } -func TestInputLoaderInputOrStaticDefaultWithEmptyFile(t *testing.T) { - il := &InputLoader{ +func TestTargetLoaderInputOrStaticDefaultWithEmptyFile(t *testing.T) { + il := &Loader{ ExperimentName: "dnscheck", InputPolicy: model.InputOrStaticDefault, SourceFiles: []string{ - "testdata/inputloader1.txt", - "testdata/inputloader3.txt", // we want it before inputloader2.txt - "testdata/inputloader2.txt", + "testdata/loader1.txt", + "testdata/loader3.txt", // we want it before loader2.txt + "testdata/loader2.txt", }, } ctx := context.Background() @@ -257,8 +318,8 @@ func TestInputLoaderInputOrStaticDefaultWithEmptyFile(t *testing.T) { } } -func TestInputLoaderInputOrStaticDefaultWithoutInputDNSCheck(t *testing.T) { - il := &InputLoader{ +func TestTargetLoaderInputOrStaticDefaultWithoutInputDNSCheck(t *testing.T) { + il := &Loader{ ExperimentName: "dnscheck", InputPolicy: model.InputOrStaticDefault, } @@ -272,20 +333,20 @@ func TestInputLoaderInputOrStaticDefaultWithoutInputDNSCheck(t *testing.T) { } for idx := 0; idx < len(dnsCheckDefaultInput); idx++ { e := out[idx] - if e.CategoryCode != "MISC" { + if e.Category() != model.DefaultCategoryCode { t.Fatal("invalid category code") } - if e.CountryCode != "XX" { + if e.Country() != model.DefaultCountryCode { t.Fatal("invalid country code") } - if e.URL != dnsCheckDefaultInput[idx] { + if e.Input() != dnsCheckDefaultInput[idx] { t.Fatal("invalid URL") } } } -func TestInputLoaderInputOrStaticDefaultWithoutInputStunReachability(t *testing.T) { - il := &InputLoader{ +func TestTargetLoaderInputOrStaticDefaultWithoutInputStunReachability(t *testing.T) { + il := &Loader{ ExperimentName: "stunreachability", InputPolicy: model.InputOrStaticDefault, } @@ -299,13 +360,13 @@ func TestInputLoaderInputOrStaticDefaultWithoutInputStunReachability(t *testing. } for idx := 0; idx < len(stunReachabilityDefaultInput); idx++ { e := out[idx] - if e.CategoryCode != "MISC" { + if e.Category() != model.DefaultCategoryCode { t.Fatal("invalid category code") } - if e.CountryCode != "XX" { + if e.Country() != model.DefaultCountryCode { t.Fatal("invalid country code") } - if e.URL != stunReachabilityDefaultInput[idx] { + if e.Input() != stunReachabilityDefaultInput[idx] { t.Fatal("invalid URL") } } @@ -320,8 +381,8 @@ func TestStaticBareInputForExperimentWorksWithNonCanonicalNames(t *testing.T) { } } -func TestInputLoaderInputOrStaticDefaultWithoutInputOtherName(t *testing.T) { - il := &InputLoader{ +func TestTargetLoaderInputOrStaticDefaultWithoutInputOtherName(t *testing.T) { + il := &Loader{ ExperimentName: "xx", InputPolicy: model.InputOrStaticDefault, } @@ -335,12 +396,12 @@ func TestInputLoaderInputOrStaticDefaultWithoutInputOtherName(t *testing.T) { } } -func TestInputLoaderInputOrQueryBackendWithInput(t *testing.T) { - il := &InputLoader{ +func TestTargetLoaderInputOrQueryBackendWithInput(t *testing.T) { + il := &Loader{ StaticInputs: []string{"https://www.google.com/"}, SourceFiles: []string{ - "testdata/inputloader1.txt", - "testdata/inputloader2.txt", + "testdata/loader1.txt", + "testdata/loader2.txt", }, InputPolicy: model.InputOrQueryBackend, } @@ -352,31 +413,48 @@ func TestInputLoaderInputOrQueryBackendWithInput(t *testing.T) { if len(out) != 5 { t.Fatal("not the output length we expected") } - expect := []model.OOAPIURLInfo{ - {URL: "https://www.google.com/"}, - {URL: "https://www.x.org/"}, - {URL: "https://www.slashdot.org/"}, - {URL: "https://abc.xyz/"}, - {URL: "https://run.ooni.io/"}, + expect := []model.ExperimentTarget{ + &model.OOAPIURLInfo{ + CountryCode: model.DefaultCountryCode, + CategoryCode: model.DefaultCategoryCode, + URL: "https://www.google.com/", + }, + &model.OOAPIURLInfo{ + CountryCode: model.DefaultCountryCode, + CategoryCode: model.DefaultCategoryCode, + URL: "https://www.x.org/", + }, + &model.OOAPIURLInfo{ + CountryCode: model.DefaultCountryCode, + CategoryCode: model.DefaultCategoryCode, + URL: "https://www.slashdot.org/", + }, + &model.OOAPIURLInfo{ + CountryCode: model.DefaultCountryCode, + CategoryCode: model.DefaultCategoryCode, + URL: "https://abc.xyz/", + }, + &model.OOAPIURLInfo{ + CountryCode: model.DefaultCountryCode, + CategoryCode: model.DefaultCategoryCode, + URL: "https://run.ooni.io/", + }, } if diff := cmp.Diff(out, expect); diff != "" { t.Fatal(diff) } } -func TestInputLoaderInputOrQueryBackendWithNoInputAndCancelledContext(t *testing.T) { - sess, err := NewSession(context.Background(), SessionConfig{ - KVStore: &kvstore.Memory{}, - Logger: log.Log, - SoftwareName: "miniooni", - SoftwareVersion: "0.1.0-dev", - TempDir: "testdata", - }) - if err != nil { - t.Fatal(err) +func TestTargetLoaderInputOrQueryBackendWithNoInputAndCancelledContext(t *testing.T) { + sess := &mocks.Session{ + MockCheckIn: func(ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResult, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + panic("should not arrive here") + }, } - defer sess.Close() - il := &InputLoader{ + il := &Loader{ InputPolicy: model.InputOrQueryBackend, Session: sess, } @@ -391,13 +469,13 @@ func TestInputLoaderInputOrQueryBackendWithNoInputAndCancelledContext(t *testing } } -func TestInputLoaderInputOrQueryBackendWithEmptyFile(t *testing.T) { - il := &InputLoader{ +func TestTargetLoaderInputOrQueryBackendWithEmptyFile(t *testing.T) { + il := &Loader{ InputPolicy: model.InputOrQueryBackend, SourceFiles: []string{ - "testdata/inputloader1.txt", - "testdata/inputloader3.txt", // we want it before inputloader2.txt - "testdata/inputloader2.txt", + "testdata/loader1.txt", + "testdata/loader3.txt", // we want it before loader2.txt + "testdata/loader2.txt", }, } ctx := context.Background() @@ -410,29 +488,28 @@ func TestInputLoaderInputOrQueryBackendWithEmptyFile(t *testing.T) { } } -type InputLoaderBrokenFS struct{} +type TargetLoaderBrokenFS struct{} -func (InputLoaderBrokenFS) Open(filepath string) (fs.File, error) { - return InputLoaderBrokenFile{}, nil +func (TargetLoaderBrokenFS) Open(filepath string) (fs.File, error) { + return TargetLoaderBrokenFile{}, nil } -type InputLoaderBrokenFile struct{} +type TargetLoaderBrokenFile struct{} -func (InputLoaderBrokenFile) Stat() (os.FileInfo, error) { +func (TargetLoaderBrokenFile) Stat() (os.FileInfo, error) { return nil, nil } -func (InputLoaderBrokenFile) Read([]byte) (int, error) { +func (TargetLoaderBrokenFile) Read([]byte) (int, error) { return 0, syscall.EFAULT } -func (InputLoaderBrokenFile) Close() error { +func (TargetLoaderBrokenFile) Close() error { return nil } -func TestInputLoaderReadfileScannerFailure(t *testing.T) { - il := &InputLoader{} - out, err := il.readfile("", InputLoaderBrokenFS{}.Open) +func TestReadfileScannerFailure(t *testing.T) { + out, err := readfile("", TargetLoaderBrokenFS{}.Open) if !errors.Is(err, syscall.EFAULT) { t.Fatal("not the error we expected") } @@ -441,20 +518,27 @@ func TestInputLoaderReadfileScannerFailure(t *testing.T) { } } -// InputLoaderMockableSession is a mockable session -// used by InputLoader tests. -type InputLoaderMockableSession struct { +// TargetLoaderMockableSession is a mockable session +// used by TargetLoader tests. +type TargetLoaderMockableSession struct { // Output contains the output of CheckIn. It should // be nil when Error is not-nil. Output *model.OOAPICheckInResult + // FetchOpenVPNConfigOutput contains the output of FetchOpenVPNConfig. + // It should be nil when Error is non-nil. + FetchOpenVPNConfigOutput *model.OOAPIVPNProviderConfig + + // ProbeCountryCode is the probe country code + ProbeCountryCode string + // Error is the error to be returned by CheckIn. It // should be nil when Output is not-nil. Error error } -// CheckIn implements InputLoaderSession.CheckIn. -func (sess *InputLoaderMockableSession) CheckIn( +// CheckIn implements [Session]. +func (sess *TargetLoaderMockableSession) CheckIn( ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResult, error) { if sess.Output == nil && sess.Error == nil { return nil, errors.New("both Output and Error are nil") @@ -462,13 +546,30 @@ func (sess *InputLoaderMockableSession) CheckIn( return sess.Output, sess.Error } -func TestInputLoaderCheckInFailure(t *testing.T) { - il := &InputLoader{ - Session: &InputLoaderMockableSession{ +// FetchOpenVPNConfig implements [Session]. +func (sess *TargetLoaderMockableSession) FetchOpenVPNConfig( + ctx context.Context, provider, cc string) (*model.OOAPIVPNProviderConfig, error) { + runtimex.Assert(!(sess.Error == nil && sess.FetchOpenVPNConfigOutput == nil), "both FetchOpenVPNConfig and Error are nil") + return sess.FetchOpenVPNConfigOutput, sess.Error +} + +func (sess *TargetLoaderMockableSession) ProbeCC() string { + return sess.ProbeCountryCode +} + +// Logger implements [Session]. +func (sess *TargetLoaderMockableSession) Logger() model.Logger { + // Such that we see some logs when running tests + return log.Log +} + +func TestTargetLoaderCheckInFailure(t *testing.T) { + il := &Loader{ + Session: &TargetLoaderMockableSession{ Error: io.EOF, }, } - out, err := il.loadRemote(context.Background()) + out, err := il.loadRemoteWebConnectivity(context.Background()) if !errors.Is(err, io.EOF) { t.Fatalf("not the error we expected: %+v", err) } @@ -477,15 +578,15 @@ func TestInputLoaderCheckInFailure(t *testing.T) { } } -func TestInputLoaderCheckInSuccessWithNilWebConnectivity(t *testing.T) { - il := &InputLoader{ - Session: &InputLoaderMockableSession{ +func TestTargetLoaderCheckInSuccessWithNilWebConnectivity(t *testing.T) { + il := &Loader{ + Session: &TargetLoaderMockableSession{ Output: &model.OOAPICheckInResult{ Tests: model.OOAPICheckInResultNettests{}, }, }, } - out, err := il.loadRemote(context.Background()) + out, err := il.loadRemoteWebConnectivity(context.Background()) if !errors.Is(err, ErrNoURLsReturned) { t.Fatalf("not the error we expected: %+v", err) } @@ -494,9 +595,9 @@ func TestInputLoaderCheckInSuccessWithNilWebConnectivity(t *testing.T) { } } -func TestInputLoaderCheckInSuccessWithNoURLs(t *testing.T) { - il := &InputLoader{ - Session: &InputLoaderMockableSession{ +func TestTargetLoaderCheckInSuccessWithNoURLs(t *testing.T) { + il := &Loader{ + Session: &TargetLoaderMockableSession{ Output: &model.OOAPICheckInResult{ Tests: model.OOAPICheckInResultNettests{ WebConnectivity: &model.OOAPICheckInInfoWebConnectivity{}, @@ -504,7 +605,7 @@ func TestInputLoaderCheckInSuccessWithNoURLs(t *testing.T) { }, }, } - out, err := il.loadRemote(context.Background()) + out, err := il.loadRemoteWebConnectivity(context.Background()) if !errors.Is(err, ErrNoURLsReturned) { t.Fatalf("not the error we expected: %+v", err) } @@ -513,28 +614,31 @@ func TestInputLoaderCheckInSuccessWithNoURLs(t *testing.T) { } } -func TestInputLoaderCheckInSuccessWithSomeURLs(t *testing.T) { - expect := []model.OOAPIURLInfo{{ +func TestTargetLoaderCheckInSuccessWithSomeURLs(t *testing.T) { + inputs0 := model.OOAPIURLInfo{ CategoryCode: "NEWS", CountryCode: "IT", URL: "https://repubblica.it", - }, { + } + inputs1 := model.OOAPIURLInfo{ CategoryCode: "NEWS", CountryCode: "IT", URL: "https://corriere.it", - }} - il := &InputLoader{ - Session: &InputLoaderMockableSession{ + } + inputs := []model.OOAPIURLInfo{inputs0, inputs1} + expect := []model.ExperimentTarget{&inputs0, &inputs1} + il := &Loader{ + Session: &TargetLoaderMockableSession{ Output: &model.OOAPICheckInResult{ Tests: model.OOAPICheckInResultNettests{ WebConnectivity: &model.OOAPICheckInInfoWebConnectivity{ - URLs: expect, + URLs: inputs, }, }, }, }, } - out, err := il.loadRemote(context.Background()) + out, err := il.loadRemoteWebConnectivity(context.Background()) if err != nil { t.Fatal(err) } @@ -566,7 +670,7 @@ func TestPreventMistakesWithCategories(t *testing.T) { URL: "https://addons.mozilla.org/", CountryCode: "XX", }} - il := &InputLoader{} + il := &Loader{} output := il.preventMistakes(input, []string{"NEWS", "FILE"}) if diff := cmp.Diff(desired, output); diff != "" { t.Fatal(diff) @@ -587,7 +691,7 @@ func TestPreventMistakesWithoutCategoriesAndNil(t *testing.T) { URL: "https://addons.mozilla.org/", CountryCode: "XX", }} - il := &InputLoader{} + il := &Loader{} output := il.preventMistakes(input, nil) if diff := cmp.Diff(input, output); diff != "" { t.Fatal(diff) @@ -608,23 +712,23 @@ func TestPreventMistakesWithoutCategoriesAndEmpty(t *testing.T) { URL: "https://addons.mozilla.org/", CountryCode: "XX", }} - il := &InputLoader{} + il := &Loader{} output := il.preventMistakes(input, []string{}) if diff := cmp.Diff(input, output); diff != "" { t.Fatal(diff) } } -// InputLoaderFakeLogger is a fake InputLoaderLogger. -type InputLoaderFakeLogger struct{} +// TargetLoaderFakeLogger is a fake TargetLoaderLogger. +type TargetLoaderFakeLogger struct{} -// Warnf implements InputLoaderLogger.Warnf -func (ilfl *InputLoaderFakeLogger) Warnf(format string, v ...interface{}) {} +// Warnf implements TargetLoaderLogger.Warnf +func (ilfl *TargetLoaderFakeLogger) Warnf(format string, v ...interface{}) {} -func TestInputLoaderLoggerWorksAsIntended(t *testing.T) { - logger := &InputLoaderFakeLogger{} - inputLoader := &InputLoader{Logger: logger} - out := inputLoader.logger() +func TestTargetLoaderLoggerWorksAsIntended(t *testing.T) { + logger := &TargetLoaderFakeLogger{} + targetLoader := &Loader{Logger: logger} + out := targetLoader.logger() if out != logger { t.Fatal("logger not working as intended") } @@ -635,7 +739,7 @@ func TestStringListToModelURLInfoWithValidInput(t *testing.T) { "stun://stun.voip.blackberry.com:3478", "stun://stun.altar.com.pl:3478", } - output, err := stringListToModelURLInfo(input, nil) + output, err := stringListToModelExperimentTarget(input, nil) if err != nil { t.Fatal(err) } @@ -643,13 +747,13 @@ func TestStringListToModelURLInfoWithValidInput(t *testing.T) { t.Fatal("unexpected output length") } for idx := 0; idx < len(input); idx++ { - if input[idx] != output[idx].URL { + if input[idx] != output[idx].Input() { t.Fatal("unexpected entry") } - if output[idx].CategoryCode != "MISC" { + if output[idx].Category() != model.DefaultCategoryCode { t.Fatal("unexpected category") } - if output[idx].CountryCode != "XX" { + if output[idx].Country() != model.DefaultCountryCode { t.Fatal("unexpected country") } } @@ -661,7 +765,7 @@ func TestStringListToModelURLInfoWithInvalidInput(t *testing.T) { "\t", // <- not a valid URL "stun://stun.altar.com.pl:3478", } - output, err := stringListToModelURLInfo(input, nil) + output, err := stringListToModelExperimentTarget(input, nil) if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") { t.Fatal("no the error we expected", err) } @@ -677,9 +781,9 @@ func TestStringListToModelURLInfoWithError(t *testing.T) { "stun://stun.altar.com.pl:3478", } expected := errors.New("mocked error") - output, err := stringListToModelURLInfo(input, expected) + output, err := stringListToModelExperimentTarget(input, expected) if !errors.Is(err, expected) { - t.Fatal("no the error we expected", err) + t.Fatal("not the error we expected", err) } if output != nil { t.Fatal("unexpected nil output") diff --git a/pkg/engine/testdata/inputloader1.txt b/pkg/targetloading/testdata/loader1.txt similarity index 100% rename from pkg/engine/testdata/inputloader1.txt rename to pkg/targetloading/testdata/loader1.txt diff --git a/pkg/engine/testdata/inputloader2.txt b/pkg/targetloading/testdata/loader2.txt similarity index 100% rename from pkg/engine/testdata/inputloader2.txt rename to pkg/targetloading/testdata/loader2.txt diff --git a/pkg/engine/testdata/inputloader3.txt b/pkg/targetloading/testdata/loader3.txt similarity index 100% rename from pkg/engine/testdata/inputloader3.txt rename to pkg/targetloading/testdata/loader3.txt diff --git a/pkg/testingx/fakefill.go b/pkg/testingx/fakefill.go index fdd7261d0..d756d7476 100644 --- a/pkg/testingx/fakefill.go +++ b/pkg/testingx/fakefill.go @@ -91,6 +91,13 @@ func (ff *FakeFiller) doFill(v reflect.Value) { // switch to the element v = v.Elem() } + + // make sure we skip initialization of fields we cannot initialize + // anyway because they're private or immutable + if !v.CanSet() { + return + } + switch v.Type().Kind() { case reflect.String: v.SetString(ff.getRandomString()) diff --git a/pkg/testingx/fakefill_test.go b/pkg/testingx/fakefill_test.go index 7e3f7f197..4704deb68 100644 --- a/pkg/testingx/fakefill_test.go +++ b/pkg/testingx/fakefill_test.go @@ -3,6 +3,8 @@ package testingx import ( "testing" "time" + + "github.com/google/go-cmp/cmp" ) // exampleStructure is an example structure we fill. @@ -25,6 +27,7 @@ func TestFakeFillWorksWithCustomTime(t *testing.T) { if req == nil { t.Fatal("we expected non nil here") } + t.Log(req) } func TestFakeFillAllocatesIntoAPointerToPointer(t *testing.T) { @@ -34,6 +37,7 @@ func TestFakeFillAllocatesIntoAPointerToPointer(t *testing.T) { if req == nil { t.Fatal("we expected non nil here") } + t.Log(req) } func TestFakeFillAllocatesIntoAMapLikeWithStringKeys(t *testing.T) { @@ -46,6 +50,7 @@ func TestFakeFillAllocatesIntoAMapLikeWithStringKeys(t *testing.T) { if len(resp) < 1 { t.Fatal("we expected some data here") } + t.Log(resp) for _, value := range resp { if value == nil { t.Fatal("expected non-nil here") @@ -53,7 +58,7 @@ func TestFakeFillAllocatesIntoAMapLikeWithStringKeys(t *testing.T) { } } -func TestFakeFillAllocatesIntoAMapLikeWithNonStringKeys(t *testing.T) { +func TestFakeFillPanicsWithMapsWithNonStringKeys(t *testing.T) { var panicmsg string func() { defer func() { @@ -83,9 +88,57 @@ func TestFakeFillAllocatesIntoASlice(t *testing.T) { if len(*resp) < 1 { t.Fatal("we expected some data here") } + t.Log(resp) for _, entry := range *resp { if entry == nil { t.Fatal("expected non-nil here") } } } + +func TestFakeFillSkipsPrivateTypes(t *testing.T) { + t.Run("with private struct fields", func(t *testing.T) { + // define structure with mixed private and public fields + type employee struct { + ID int64 + age int64 + name string + } + + // create empty employee + var person employee + + // fake-fill the employee + ff := &FakeFiller{} + ff.Fill(&person) + + // define what we expect to see + expect := employee{ + ID: person.ID, + age: 0, + name: "", + } + + // make sure we've got what we expected + // + // Note: we cannot use cmp.Diff directly because it cannot + // access private fields, so we need to write manual comparison + if person != expect { + t.Fatal("expected", expect, "got", person) + } + }) + + t.Run("make sure we cannot initialize a non-addressable type", func(t *testing.T) { + // create a zero struct + shouldRemainZero := exampleStructure{} + + // attempt to fake fill w/o taking the address + ff := &FakeFiller{} + ff.Fill(shouldRemainZero) + + // make sure it's still zero + if diff := cmp.Diff(exampleStructure{}, shouldRemainZero); diff != "" { + t.Fatal(diff) + } + }) +} diff --git a/pkg/testingx/oonibackendwithlogin.go b/pkg/testingx/oonibackendwithlogin.go index ba39d486c..d12388d6e 100644 --- a/pkg/testingx/oonibackendwithlogin.go +++ b/pkg/testingx/oonibackendwithlogin.go @@ -38,6 +38,9 @@ type OONIBackendWithLoginFlow struct { // mu provides mutual exclusion. mu sync.Mutex + // openVPNConfig is the serialized openvpn config to send to clients. + openVPNConfig []byte + // psiphonConfig is the serialized psiphon config to send to authenticated clients. psiphonConfig []byte @@ -48,6 +51,15 @@ type OONIBackendWithLoginFlow struct { torTargets []byte } +// SetOpenVPNConfig sets openvpn configuration to use. +// +// This method is safe to call concurrently with incoming HTTP requests. +func (h *OONIBackendWithLoginFlow) SetOpenVPNConfig(config []byte) { + defer h.mu.Unlock() + h.mu.Lock() + h.openVPNConfig = config +} + // SetPsiphonConfig sets psiphon configuration to use. // // This method is safe to call concurrently with incoming HTTP requests. @@ -86,6 +98,7 @@ func (h *OONIBackendWithLoginFlow) NewMux() *http.ServeMux { mux.Handle("/api/v1/login", h.handleLogin()) mux.Handle("/api/v1/test-list/psiphon-config", h.withAuthentication(h.handlePsiphonConfig())) mux.Handle("/api/v1/test-list/tor-targets", h.withAuthentication(h.handleTorTargets())) + mux.Handle("/api/v2/ooniprobe/vpn-config/demovpn", h.handleOpenVPNConfig()) return mux } @@ -211,6 +224,21 @@ func (h *OONIBackendWithLoginFlow) handleLogin() http.Handler { }) } +func (h *OONIBackendWithLoginFlow) handleOpenVPNConfig() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // make sure the method is OK + if r.Method != http.MethodGet { + w.WriteHeader(501) + return + } + + // we must lock because of SetOpenVPNConfig + h.mu.Lock() + _, _ = w.Write(h.openVPNConfig) + h.mu.Unlock() + }) +} + func (h *OONIBackendWithLoginFlow) handlePsiphonConfig() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // make sure the method is OK diff --git a/pkg/version/version.go b/pkg/version/version.go index a85f02af1..e89ce40aa 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -2,4 +2,4 @@ package version // Version is the ooniprobe version. -const Version = "3.22.0-alpha" +const Version = "3.23.0" diff --git a/pkg/webconnectivityqa/measurement.go b/pkg/webconnectivityqa/measurement.go index 530baf3cd..dcc5f3e01 100644 --- a/pkg/webconnectivityqa/measurement.go +++ b/pkg/webconnectivityqa/measurement.go @@ -15,7 +15,7 @@ func newMeasurement(input string, measurer model.ExperimentMeasurer, t0 time.Tim DataFormatVersion: "0.2.0", Extensions: nil, ID: "", - Input: model.MeasurementTarget(input), + Input: model.MeasurementInput(input), InputHashes: nil, MeasurementStartTime: t0.Format(model.MeasurementDateFormat), MeasurementStartTimeSaved: t0,