From f86f382466a56a4aad7332493832a43924da3b5c Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 15 Jul 2020 12:20:14 +0200 Subject: [PATCH] Remove Measurement Kit dependency (#806) Part of https://github.com/ooni/probe-engine/issues/776 --- .../ISSUE_TEMPLATE/routine-sprint-releases.md | 2 - .github/workflows/golang.yml | 2 +- .github/workflows/mk.sh | 4 - .github/workflows/mk.yml | 8 - README.md | 43 ++-- build-android.bash | 2 +- build-cli.sh | 14 +- build-ios.bash | 2 +- experiment.go | 8 +- experiment/mkevent/mkevent.go | 75 ------ experiment/mkevent/mkevent_test.go | 104 -------- experiment/mkhelper/mkhelper.go | 33 --- experiment/mkhelper/mkhelper_test.go | 64 ----- experiment/mkrunner/mkrunner.go | 64 ----- experiment/mkrunner/mkrunner_test.go | 38 --- experiment/mktesting/mktesting.go | 56 ---- .../web_connectivity/web_connectivity.go | 63 ----- .../web_connectivity/web_connectivity_test.go | 18 -- experiment_test.go | 4 - measurementkit/measurementkit.go | 241 ------------------ measurementkit/measurementkit_test.go | 28 -- measurementkit/mkcgo/force.cpp | 1 - measurementkit/mkcgo/mkcgo.go | 94 ------- measurementkit/task_mk.go | 15 -- measurementkit/task_nomk.go | 13 - netx/httptransport/system.go | 2 - netx/httptransport/system_compat.go | 31 --- 27 files changed, 38 insertions(+), 991 deletions(-) delete mode 100755 .github/workflows/mk.sh delete mode 100644 .github/workflows/mk.yml delete mode 100644 experiment/mkevent/mkevent.go delete mode 100644 experiment/mkevent/mkevent_test.go delete mode 100644 experiment/mkhelper/mkhelper.go delete mode 100644 experiment/mkhelper/mkhelper_test.go delete mode 100644 experiment/mkrunner/mkrunner.go delete mode 100644 experiment/mkrunner/mkrunner_test.go delete mode 100644 experiment/mktesting/mktesting.go delete mode 100644 experiment/web_connectivity/web_connectivity.go delete mode 100644 experiment/web_connectivity/web_connectivity_test.go delete mode 100644 measurementkit/measurementkit.go delete mode 100644 measurementkit/measurementkit_test.go delete mode 100644 measurementkit/mkcgo/force.cpp delete mode 100644 measurementkit/mkcgo/mkcgo.go delete mode 100644 measurementkit/task_mk.go delete mode 100644 measurementkit/task_nomk.go delete mode 100644 netx/httptransport/system_compat.go diff --git a/.github/ISSUE_TEMPLATE/routine-sprint-releases.md b/.github/ISSUE_TEMPLATE/routine-sprint-releases.md index 1e31fee9..4d7ca75f 100644 --- a/.github/ISSUE_TEMPLATE/routine-sprint-releases.md +++ b/.github/ISSUE_TEMPLATE/routine-sprint-releases.md @@ -11,8 +11,6 @@ assignees: bassosimone - [ ] Update internal/httpheader/useragent.go - [ ] Update version/version.go - [ ] Update internal/resources/assets.go -- [ ] Update measurementkit/mkcgo/mkcgo.go -- [ ] Update .github/workflows/mk.yml - [ ] Tag a new version of ooni/probe-engine - [ ] Create release at GitHub - [ ] Update ooni/probe-engine mobile-staging branch diff --git a/.github/workflows/golang.yml b/.github/workflows/golang.yml index aff9d4c4..3f35bad8 100644 --- a/.github/workflows/golang.yml +++ b/.github/workflows/golang.yml @@ -13,7 +13,7 @@ jobs: with: go-version: ${{ matrix.go }} - uses: actions/checkout@v2 - - run: go test -tags nomk,shaping -v -coverprofile=probe-engine.cov -coverpkg=./... ./... + - run: go test -tags shaping -v -coverprofile=probe-engine.cov -coverpkg=./... ./... - uses: shogo82148/actions-goveralls@v1 with: path-to-profile: probe-engine.cov diff --git a/.github/workflows/mk.sh b/.github/workflows/mk.sh deleted file mode 100755 index de8a0fad..00000000 --- a/.github/workflows/mk.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -set -ex -apk add --no-progress git go -go test -tags ooni,shaping ./... diff --git a/.github/workflows/mk.yml b/.github/workflows/mk.yml deleted file mode 100644 index db1a5ec6..00000000 --- a/.github/workflows/mk.yml +++ /dev/null @@ -1,8 +0,0 @@ -name: mk -on: [push, pull_request] -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - run: docker run -v`pwd`:/mk -w/mk openobservatory/mk-alpine:20200513 /mk/.github/workflows/mk.sh diff --git a/README.md b/README.md index 487a9b6c..1363f6e4 100644 --- a/README.md +++ b/README.md @@ -24,35 +24,26 @@ miniooni CLI will be stable over time. ## Integrating ooni/probe-engine This software uses [Go modules](https://github.com/golang/go/wiki/Modules) -and requires Go v1.14+. We also depend on [Measurement Kit]( -https://github.com/measurement-kit/measurement-kit), a C++14 library -implementing many OONI tests, a.k.a. MK. +and requires Go v1.14+. You can pull the latest version as a dependency from +your modules aware project by using -Note that passing the `-tags nomk` flag to Go will disable linking -Measurement Kit into the resulting Go binaries. You may want that in -cases where you only want to use experiments written in Go. - -We plan on gradually rewriting all OONI tests in Go, therefore the -dependency on Measurement Kit will eventually be removed. A future version -of this document will provide platform specific instructions for -installing Measurement Kit and linking to it. +```bash +go get -v github.com/ooni/probe-engine +``` ## Building miniooni -``` -go build -v -tags nomk ./cmd/miniooni/ +```bash +go build -v ./cmd/miniooni/ ``` -Omit `-tags nomk` to link with MK. - ## Building Android bindings -``` +```bash ./build-android.bash ``` -When building Android bindings, we automatically omit linking with MK. We -automatically build Android bindings whenever commits are pushed to the +We automatically build Android bindings whenever commits are pushed to the `mobile-staging` branch. Such builds could be integrated by using: ```Groovy @@ -68,6 +59,22 @@ implementation "org.ooni:oonimkall:VERSION" Where VERSION is like `2020.03.30-231914` corresponding to the moment in time in which the version has been built. +## Building iOS bindings + +```bash +./build-ios.bash +``` + +We automatically build iOS bindings whenever commits are pushed to the +`mobile-staging` branch. Such builds could be integrated by using: + +```ruby +pod 'oonimkall', :podspec => 'https://dl.bintray.com/ooni/ios/oonimkall-VERSION.podspec' +``` + +Where VERSION is like `2020.03.30-231914` corresponding to the moment in +time in which the version has been built. + ## Release procedure 1. make sure that dependencies are up to date diff --git a/build-android.bash b/build-android.bash index 1b64f92d..efc49330 100755 --- a/build-android.bash +++ b/build-android.bash @@ -31,4 +31,4 @@ output=MOBILE/dist/oonimkall.aar go get -u golang.org/x/mobile/cmd/gomobile gomobile init export GO111MODULE=on -gomobile bind -target=android -o $output -tags nomk -ldflags="-s -w" ./oonimkall +gomobile bind -target=android -o $output -ldflags="-s -w" ./oonimkall diff --git a/build-cli.sh b/build-cli.sh index ee0f1411..6a274189 100755 --- a/build-cli.sh +++ b/build-cli.sh @@ -1,17 +1,15 @@ #!/bin/sh set -ex case $1 in - _linux) - apk add gcc musl-dev - go build -tags "netgo nomk" -ldflags='-s -w -extldflags "-static"' \ - -o ./CLI/linux/amd64/miniooni ./cmd/miniooni;; darwin) - go build -tags nomk -o ./CLI/darwin/amd64 ./cmd/miniooni;; + export GOOS=darwin GOARCH=amd64 + go build -o ./CLI/darwin/amd64 -ldflags="-s -w" ./cmd/miniooni;; linux) - docker run -v`pwd`:/ooni -w/ooni golang:alpine ./build-cli.sh _linux;; + export GOOS=linux GOARCH=amd64 + go build -o ./CLI/linux/amd64 -ldflags="-s -w" ./cmd/miniooni;; windows) - export CC=x86_64-w64-mingw32-gcc CGO_ENABLED=1 GOOS=windows GOARCH=amd64 - go build -tags nomk -o ./CLI/windows/amd64 ./cmd/miniooni;; + export GOOS=windows GOARCH=amd64 + go build -o ./CLI/windows/amd64 -ldflags="-s -w" ./cmd/miniooni;; *) echo "usage: $0 darwin|linux|windows" 1>&2 exit 1 diff --git a/build-ios.bash b/build-ios.bash index 97e73e8a..d02aa1ce 100755 --- a/build-ios.bash +++ b/build-ios.bash @@ -9,7 +9,7 @@ output=MOBILE/dist/oonimkall.framework go get -u golang.org/x/mobile/cmd/gomobile gomobile init export GO111MODULE=on -gomobile bind -target=ios -o $output -tags nomk -ldflags="-s -w" ./oonimkall +gomobile bind -target=ios -o $output -ldflags="-s -w" ./oonimkall # See https://github.com/ooni/probe-engine/issues/668 for header in $output/Headers/*.objc.h; do cat $header | sed 's|^@import Foundation;|#import |g' > $header.new diff --git a/experiment.go b/experiment.go index 45d9cae2..c7a3d2f5 100644 --- a/experiment.go +++ b/experiment.go @@ -26,7 +26,7 @@ import ( "github.com/ooni/probe-engine/experiment/telegram" "github.com/ooni/probe-engine/experiment/tor" "github.com/ooni/probe-engine/experiment/urlgetter" - "github.com/ooni/probe-engine/experiment/web_connectivity" + "github.com/ooni/probe-engine/experiment/webconnectivity" "github.com/ooni/probe-engine/experiment/whatsapp" "github.com/ooni/probe-engine/internal/platform" "github.com/ooni/probe-engine/internal/resources" @@ -649,11 +649,11 @@ var experimentsByName = map[string]func(*Session) *ExperimentBuilder{ "web_connectivity": func(session *Session) *ExperimentBuilder { return &ExperimentBuilder{ build: func(config interface{}) *Experiment { - return NewExperiment(session, web_connectivity.NewExperimentMeasurer( - *config.(*web_connectivity.Config), + return NewExperiment(session, webconnectivity.NewExperimentMeasurer( + *config.(*webconnectivity.Config), )) }, - config: &web_connectivity.Config{}, + config: &webconnectivity.Config{}, inputPolicy: InputRequired, } }, diff --git a/experiment/mkevent/mkevent.go b/experiment/mkevent/mkevent.go deleted file mode 100644 index 57ca2f5b..00000000 --- a/experiment/mkevent/mkevent.go +++ /dev/null @@ -1,75 +0,0 @@ -// Package mkevent processes MK events -package mkevent - -import ( - "encoding/json" - "strings" - - "github.com/ooni/probe-engine/measurementkit" - "github.com/ooni/probe-engine/model" -) - -// Handle handles a measurementkit event -func Handle( - sess model.ExperimentSession, - measurement *model.Measurement, - event measurementkit.Event, - callbacks model.ExperimentCallbacks, -) { - if event.Key == "measurement" { - // We reparse the measurement and overwrite it. This is how we manage to - // return the measurement to the caller. Seriously. - // - // We panic if we cannot parse because since MK v0.9.0 the vendored - // nlohmann/json library should throw if passed non UTF-8 input. - err := json.Unmarshal([]byte(event.Value.JSONStr), measurement) - if err != nil { - panic(err) - } - return - } - if event.Key == "log" { - if strings.HasPrefix(event.Value.LogLevel, "DEBUG") { - sess.Logger().Debug(event.Value.Message) - } else if event.Value.LogLevel == "INFO" { - sess.Logger().Info(event.Value.Message) - } else { - sess.Logger().Warn(event.Value.Message) - } - return - } - if event.Key == "status.progress" { - callbacks.OnProgress(event.Value.Percentage, event.Value.Message) - return - } - if event.Key == "status.end" { - callbacks.OnDataUsage(event.Value.DownloadedKB, event.Value.UploadedKB) - return - } - // I've commented out the events we don't handle. For each event I've - // explained why we don't handle it. - /* - if event.Key == "status.update.performance" { - return // Seems unused by OONI - } - if event.Key == "status.update.websites" { - return // Ditto - } - if event.Key == "status.queued" { - return // Ditto - } - if event.Key == "status.started" { - return // Ditto - } - if event.Key == "status.measurement_start" { - return // We know that because we control the lifecycle - } - if event.Key == "status.measurement_done" { - return // We know that because we control the lifecycle - } - if event.Key == "status.geoip_lookup" { - return // We perform the lookup before calling MK - } - */ - sess.Logger().Debugf("mkevent: %s %+v", event.Key, event.Value) -} diff --git a/experiment/mkevent/mkevent_test.go b/experiment/mkevent/mkevent_test.go deleted file mode 100644 index 55d7f34f..00000000 --- a/experiment/mkevent/mkevent_test.go +++ /dev/null @@ -1,104 +0,0 @@ -package mkevent_test - -import ( - "testing" - - "github.com/apex/log" - "github.com/ooni/probe-engine/experiment/handler" - "github.com/ooni/probe-engine/experiment/mkevent" - "github.com/ooni/probe-engine/internal/mockable" - "github.com/ooni/probe-engine/measurementkit" - "github.com/ooni/probe-engine/model" -) - -func TestIntegrationMeasurementSuccess(t *testing.T) { - sess := &mockable.ExperimentSession{MockableLogger: log.Log} - var m model.Measurement - printer := handler.NewPrinterCallbacks(log.Log) - mkevent.Handle(sess, &m, measurementkit.Event{ - Key: "measurement", - Value: measurementkit.EventValue{ - JSONStr: "{}", - }, - }, printer) -} - -func TestIntegrationMeasurementFailure(t *testing.T) { - defer func() { - if recover() == nil { - t.Fatal("expected a panic here") - } - }() - sess := &mockable.ExperimentSession{MockableLogger: log.Log} - var m model.Measurement - printer := handler.NewPrinterCallbacks(log.Log) - mkevent.Handle(sess, &m, measurementkit.Event{ - Key: "measurement", - Value: measurementkit.EventValue{ - JSONStr: "{", - }, - }, printer) -} - -func TestIntegrationLog(t *testing.T) { - sess := &mockable.ExperimentSession{MockableLogger: log.Log} - var m model.Measurement - printer := handler.NewPrinterCallbacks(log.Log) - mkevent.Handle(sess, &m, measurementkit.Event{ - Key: "log", - Value: measurementkit.EventValue{ - LogLevel: "DEBUG", - Message: "message", - }, - }, printer) - mkevent.Handle(sess, &m, measurementkit.Event{ - Key: "log", - Value: measurementkit.EventValue{ - LogLevel: "INFO", - Message: "message", - }, - }, printer) - mkevent.Handle(sess, &m, measurementkit.Event{ - Key: "log", - Value: measurementkit.EventValue{ - LogLevel: "WARNING", - Message: "message", - }, - }, printer) -} - -func TestIntegrationStatusProgress(t *testing.T) { - sess := &mockable.ExperimentSession{MockableLogger: log.Log} - var m model.Measurement - printer := handler.NewPrinterCallbacks(log.Log) - mkevent.Handle(sess, &m, measurementkit.Event{ - Key: "status.progress", - Value: measurementkit.EventValue{ - Percentage: 0.17, - Message: "message", - }, - }, printer) -} - -func TestIntegrationStatusEnd(t *testing.T) { - sess := &mockable.ExperimentSession{MockableLogger: log.Log} - var m model.Measurement - printer := handler.NewPrinterCallbacks(log.Log) - mkevent.Handle(sess, &m, measurementkit.Event{ - Key: "status.end", - Value: measurementkit.EventValue{ - DownloadedKB: 1234, - UploadedKB: 5678, - }, - }, printer) -} - -func TestIntegrationOtherEvent(t *testing.T) { - sess := &mockable.ExperimentSession{MockableLogger: log.Log} - var m model.Measurement - printer := handler.NewPrinterCallbacks(log.Log) - mkevent.Handle(sess, &m, measurementkit.Event{ - Key: "status.queued", - Value: measurementkit.EventValue{}, - }, printer) -} diff --git a/experiment/mkhelper/mkhelper.go b/experiment/mkhelper/mkhelper.go deleted file mode 100644 index a7d66a8a..00000000 --- a/experiment/mkhelper/mkhelper.go +++ /dev/null @@ -1,33 +0,0 @@ -// Package mkhelper contains common code to get the proper -// helper and configure it into settings. -package mkhelper - -import ( - "fmt" - - "github.com/ooni/probe-engine/measurementkit" - "github.com/ooni/probe-engine/model" -) - -// Set copies a specific helper from the session to MK settings. -func Set( - sess model.ExperimentSession, name, kind string, - settings *measurementkit.Settings, -) error { - ths, ok := sess.GetTestHelpersByName(name) - if !ok { - return fmt.Errorf("No available %s test helper", name) - } - address := "" - for _, th := range ths { - if th.Type == kind { - address = th.Address - break - } - } - if address == "" { - return fmt.Errorf("No suitable %s test helper", name) - } - settings.Options.Backend = address - return nil -} diff --git a/experiment/mkhelper/mkhelper_test.go b/experiment/mkhelper/mkhelper_test.go deleted file mode 100644 index 3ed08f69..00000000 --- a/experiment/mkhelper/mkhelper_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package mkhelper_test - -import ( - "testing" - - "github.com/ooni/probe-engine/experiment/mkhelper" - "github.com/ooni/probe-engine/internal/mockable" - "github.com/ooni/probe-engine/measurementkit" - "github.com/ooni/probe-engine/model" -) - -func TestNoHelpers(t *testing.T) { - sess := &mockable.ExperimentSession{} - var settings measurementkit.Settings - err := mkhelper.Set( - sess, "foobar", "https", &settings, - ) - if err == nil { - t.Fatal("expected an error here") - } -} - -func TestNoSuitableHelper(t *testing.T) { - sess := &mockable.ExperimentSession{ - MockableTestHelpers: map[string][]model.Service{ - "foobar": { - { - Address: "mascetti", - Type: "melandri", - }, - }, - }, - } - var settings measurementkit.Settings - err := mkhelper.Set( - sess, "foobar", "https", &settings, - ) - if err == nil { - t.Fatal("expected an error here") - } -} - -func TestGoodHelper(t *testing.T) { - sess := &mockable.ExperimentSession{ - MockableTestHelpers: map[string][]model.Service{ - "foobar": { - { - Address: "mascetti", - Type: "melandri", - }, - }, - }, - } - var settings measurementkit.Settings - err := mkhelper.Set( - sess, "foobar", "melandri", &settings, - ) - if err != nil { - t.Fatal(err) - } - if settings.Options.Backend != "mascetti" { - t.Fatal("unexpected settings.Options.Backend") - } -} diff --git a/experiment/mkrunner/mkrunner.go b/experiment/mkrunner/mkrunner.go deleted file mode 100644 index fbf56ca9..00000000 --- a/experiment/mkrunner/mkrunner.go +++ /dev/null @@ -1,64 +0,0 @@ -// Package mkrunner contains code to run an MK based test -package mkrunner - -import ( - "errors" - - "github.com/ooni/probe-engine/experiment/mkevent" - "github.com/ooni/probe-engine/measurementkit" - "github.com/ooni/probe-engine/model" -) - -// Do runs a specific measurement-kit based experiment. You should -// pass measurementkit.StartEx as startEx in the common case. -func Do( - settings measurementkit.Settings, - sess model.ExperimentSession, - measurement *model.Measurement, - callbacks model.ExperimentCallbacks, - startEx func( - settings measurementkit.Settings, logger model.Logger, - ) (<-chan measurementkit.Event, error), -) error { - out, err := startEx(settings, sess.Logger()) - if err != nil { - return err - } - // Try to increase reliability a bit. MK does not do a good job at - // that, traditionally. Declare an experiment run failed if we didn't - // at least see a `measurement` event. - err = errors.New("did not see any measurement event") - for ev := range out { - if ev.Key == "measurement" { - err = nil - } - mkevent.Handle(sess, measurement, ev, callbacks) - } - return err -} - -// DoNothingStartEx is a replacement for measurementkit.StartEx that -// does nearly nothing apart emitting a fake measurement. -func DoNothingStartEx( - settings measurementkit.Settings, logger model.Logger, -) (<-chan measurementkit.Event, error) { - out := make(chan measurementkit.Event) - go func() { - defer close(out) - out <- measurementkit.Event{ - Key: "measurement", - Value: measurementkit.EventValue{ - JSONStr: "{}", - }, - } - }() - return out, nil -} - -// FailingStartEx is a replacement for measurementkit.StartEx that -// returns an error immediately. -func FailingStartEx( - settings measurementkit.Settings, logger model.Logger, -) (<-chan measurementkit.Event, error) { - return nil, errors.New("fail immediately") -} diff --git a/experiment/mkrunner/mkrunner_test.go b/experiment/mkrunner/mkrunner_test.go deleted file mode 100644 index 56f02140..00000000 --- a/experiment/mkrunner/mkrunner_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package mkrunner_test - -import ( - "testing" - - apexlog "github.com/apex/log" - "github.com/ooni/probe-engine/experiment/handler" - "github.com/ooni/probe-engine/experiment/mkrunner" - "github.com/ooni/probe-engine/internal/mockable" - "github.com/ooni/probe-engine/measurementkit" - "github.com/ooni/probe-engine/model" -) - -func TestIntegrationSuccess(t *testing.T) { - err := mkrunner.Do( - measurementkit.Settings{}, - &mockable.ExperimentSession{MockableLogger: apexlog.Log}, - &model.Measurement{}, - handler.NewPrinterCallbacks(apexlog.Log), - mkrunner.DoNothingStartEx, - ) - if err != nil { - t.Fatal(err) - } -} - -func TestIntegrationFailure(t *testing.T) { - err := mkrunner.Do( - measurementkit.Settings{}, - &mockable.ExperimentSession{MockableLogger: apexlog.Log}, - &model.Measurement{}, - handler.NewPrinterCallbacks(apexlog.Log), - mkrunner.FailingStartEx, - ) - if err == nil { - t.Fatal("expected an error here") - } -} diff --git a/experiment/mktesting/mktesting.go b/experiment/mktesting/mktesting.go deleted file mode 100644 index 0a7f6ff0..00000000 --- a/experiment/mktesting/mktesting.go +++ /dev/null @@ -1,56 +0,0 @@ -// Package mktesting contains common code to run MK based tests. -package mktesting - -import ( - "github.com/apex/log" - engine "github.com/ooni/probe-engine" - "github.com/ooni/probe-engine/measurementkit" - "github.com/ooni/probe-engine/model" -) - -const ( - softwareName = "ooniprobe-example" - softwareVersion = "0.0.1" -) - -// Run runs the specified experiment. -func Run(input string, factory func() model.ExperimentMeasurer) error { - if !measurementkit.Available() { - return nil - } - log.SetLevel(log.DebugLevel) - - config := engine.SessionConfig{ - AssetsDir: "../../testdata", - Logger: log.Log, - SoftwareName: softwareName, - SoftwareVersion: softwareVersion, - } - sess, err := engine.NewSession(config) - if err != nil { - return err - } - defer sess.Close() - if err := sess.MaybeLookupBackends(); err != nil { - return err - } - if err := sess.MaybeLookupLocation(); err != nil { - return err - } - - measurer := factory() - experiment := engine.NewExperiment(sess, measurer) - if err := experiment.OpenReport(); err != nil { - return err - } - defer experiment.CloseReport() - - measurement, err := experiment.Measure(input) - if err != nil { - return err - } - if err := experiment.SubmitAndUpdateMeasurement(measurement); err != nil { - return err - } - return nil -} diff --git a/experiment/web_connectivity/web_connectivity.go b/experiment/web_connectivity/web_connectivity.go deleted file mode 100644 index 4592753a..00000000 --- a/experiment/web_connectivity/web_connectivity.go +++ /dev/null @@ -1,63 +0,0 @@ -// Package web_connectivity contains the Web Connectivity network experiment. -package web_connectivity - -import ( - "context" - "errors" - - "github.com/ooni/probe-engine/experiment/mkhelper" - "github.com/ooni/probe-engine/experiment/mkrunner" - "github.com/ooni/probe-engine/measurementkit" - "github.com/ooni/probe-engine/model" -) - -const ( - testName = "web_connectivity" - testVersion = "0.0.1" -) - -// Config contains the experiment config. -type Config struct { - // LogLevel is the MK log level. Empty implies "WARNING". - LogLevel string -} - -type measurer struct { - config Config -} - -func (m *measurer) ExperimentName() string { - return testName -} - -func (m *measurer) ExperimentVersion() string { - return testVersion -} - -func (m *measurer) Run( - ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks, -) error { - settings := measurementkit.NewSettings( - "WebConnectivity", sess.SoftwareName(), sess.SoftwareVersion(), - sess.CABundlePath(), sess.ProbeASNString(), sess.ProbeCC(), - sess.ProbeIP(), sess.ProbeNetworkName(), m.config.LogLevel, - ) - settings.Options.GeoIPASNPath = sess.ASNDatabasePath() - if measurement.Input == "" { - return errors.New("web_connectivity: passed an empty input") - } - settings.Inputs = []string{string(measurement.Input)} - err := mkhelper.Set(sess, "web-connectivity", "https", &settings) - if err != nil { - return err - } - return mkrunner.Do( - settings, sess, measurement, callbacks, measurementkit.StartEx, - ) -} - -// NewExperimentMeasurer creates a new ExperimentMeasurer. -func NewExperimentMeasurer(config Config) model.ExperimentMeasurer { - return &measurer{config: config} -} diff --git a/experiment/web_connectivity/web_connectivity_test.go b/experiment/web_connectivity/web_connectivity_test.go deleted file mode 100644 index 39d3be83..00000000 --- a/experiment/web_connectivity/web_connectivity_test.go +++ /dev/null @@ -1,18 +0,0 @@ -package web_connectivity_test - -import ( - "testing" - - "github.com/ooni/probe-engine/experiment/mktesting" - "github.com/ooni/probe-engine/experiment/web_connectivity" - "github.com/ooni/probe-engine/model" -) - -func TestIntegration(t *testing.T) { - err := mktesting.Run("http://www.example.com", func() model.ExperimentMeasurer { - return web_connectivity.NewExperimentMeasurer(web_connectivity.Config{}) - }) - if err != nil { - t.Fatal(err) - } -} diff --git a/experiment_test.go b/experiment_test.go index 4c273fa3..b1848703 100644 --- a/experiment_test.go +++ b/experiment_test.go @@ -14,7 +14,6 @@ import ( "testing" "github.com/ooni/probe-engine/experiment/example" - "github.com/ooni/probe-engine/measurementkit" "github.com/ooni/probe-engine/model" ) @@ -268,9 +267,6 @@ func TestUseOptions(t *testing.T) { } func TestRunHHFM(t *testing.T) { - if !measurementkit.Available() { - t.Skip("Measurement Kit not available; skipping") - } sess := newSessionForTesting(t) defer sess.Close() builder, err := sess.NewExperimentBuilder("http_header_field_manipulation") diff --git a/measurementkit/measurementkit.go b/measurementkit/measurementkit.go deleted file mode 100644 index f618e296..00000000 --- a/measurementkit/measurementkit.go +++ /dev/null @@ -1,241 +0,0 @@ -// Package measurementkit allows to use Measurement Kit. -package measurementkit - -import ( - "encoding/json" - - "github.com/ooni/probe-engine/model" -) - -// Settings contains settings -type Settings struct { - // Annotations contains the annotations - Annotations map[string]string `json:"annotations,omitempty"` - - // DisabledEvents contains disabled events - DisabledEvents []string `json:"disabled_events,omitempty"` - - // Inputs contains the inputs - Inputs []string `json:"inputs,omitempty"` - - // InputFilepaths contains the input file paths - InputFilepaths []string `json:"input_filepaths,omitempty"` - - // LogLevel contains the logs level - LogLevel string `json:"log_level,omitempty"` - - // Name contains the task name - Name string `json:"name"` - - // Options contains the options - Options Options `json:"options"` - - // OutputFilepath contains the output filepath - OutputFilePath string `json:"output_filepath,omitempty"` -} - -// Options contains the options -type Options struct { - // Backend is a test helper for a nettest - Backend string `json:"backend,omitempty"` - - // BouncerBaseURL contains the bouncer base URL - BouncerBaseURL string `json:"bouncer_base_url,omitempty"` - - // CaBundlePath contains the CA bundle path - CaBundlePath string `json:"net/ca_bundle_path,omitempty"` - - // CollectorBaseURL contains the collector base URL - CollectorBaseURL string `json:"collector_base_url,omitempty"` - - // GeoIPCountryPath is the country database path - GeoIPCountryPath string `json:"geoip_country_path,omitempty"` - - // GeoIPASNPath is the ASN database path - GeoIPASNPath string `json:"geoip_asn_path,omitempty"` - - // MaxRuntime is the maximum runtime - MaxRuntime float32 `json:"max_runtime,omitempty"` - - // NoBouncer indicates whether to use a bouncer - NoBouncer bool `json:"no_bouncer,omitempty"` - - // NoCollector indicates whether to use a collector - NoCollector bool `json:"no_collector,omitempty"` - - // NoFileReport indicates whether to write a report file - NoFileReport bool `json:"no_file_report,omitempty"` - - // NoGeoIP indicates whether to perform a GeoIP lookup - NoGeoIP bool `json:"no_geoip,omitempty"` - - // NoResolverLookup indicates whether to perform a resolver lookup - NoResolverLookup bool `json:"no_resolver_lookup"` - - // ProbeASN is the AS number - ProbeASN string `json:"probe_asn,omitempty"` - - // ProbeCC is the probe country code - ProbeCC string `json:"probe_cc,omitempty"` - - // ProbeIP is the probe IP - ProbeIP string `json:"probe_ip,omitempty"` - - // ProbeNetworkName is the probe network name - ProbeNetworkName string `json:"probe_network_name,omitempty"` - - // RandomizeInput indicates whether to randomize inputs - RandomizeInput bool `json:"randomize_input,omitempty"` - - // SaveRealProbeIP indicates whether to save the real probe IP - SaveRealProbeIP bool `json:"save_real_probe_ip,omitempty"` - - // SaveRealProbeIP indicates whether to save the real probe ASN - SaveRealProbeASN bool `json:"save_real_probe_asn,omitempty"` - - // SaveRealProbeCC indicates whether to save the real probe CC - SaveRealProbeCC bool `json:"save_real_probe_cc,omitempty"` - - // SoftwareName is the software name - SoftwareName string `json:"software_name,omitempty"` - - // SoftwareVersion is the software version - SoftwareVersion string `json:"software_version,omitempty"` -} - -func makeLogLevel(s string) string { - if s == "" { - s = "WARNING" - } - return s -} - -// NewSettings creates new Settings -func NewSettings( - taskName string, - softwareName string, - softwareVersion string, - caBundlePath string, - probeASN string, - probeCC string, - probeIP string, - probeNetworkName string, - logLevel string, -) Settings { - return Settings{ - LogLevel: makeLogLevel(logLevel), - Name: taskName, - Options: Options{ - CaBundlePath: caBundlePath, - NoBouncer: true, - NoCollector: true, - NoFileReport: true, - NoGeoIP: true, - NoResolverLookup: true, - ProbeASN: probeASN, - ProbeCC: probeCC, - ProbeIP: probeIP, - ProbeNetworkName: probeNetworkName, - SaveRealProbeIP: true, - SoftwareName: softwareName, - SoftwareVersion: softwareVersion, - }, - } -} - -// EventValue are all the possible value keys -type EventValue struct { - // DownloadedKB is the amount of downloaded KibiBytes - DownloadedKB float64 `json:"downloaded_kb,omitempty"` - - // Failure is the failure that occurred - Failure string `json:"failure,omitempty"` - - // Idx is the measurement index - Idx int64 `json:"idx,omitempty"` - - // Input is the input to which this event is related - Input string `json:"input,omitempty"` - - // JSONStr is a serialized measurement - JSONStr string `json:"json_str,omitempty"` - - // LogLevel is the log level - LogLevel string `json:"log_level,omitempty"` - - // Message is the log message - Message string `json:"message,omitempty"` - - // Percentage is the task progress - Percentage float64 `json:"percentage,omitempty"` - - // ProbeASN is the probe ASN - ProbeASN string `json:"probe_asn,omitempty"` - - // ProbeCC is the probe CC - ProbeCC string `json:"probe_cc,omitempty"` - - // ProbeIP is the probe IP - ProbeIP string `json:"probe_ip,omitempty"` - - // ProbeNetworkName is the probe network name - ProbeNetworkName string `json:"probe_network_name,omitempty"` - - // ReportID is the report ID - ReportID string `json:"report_id,omitempty"` - - // UploadedKB is the amount of uploaded KibiBytes - UploadedKB float64 `json:"uploaded_kb,omitempty"` -} - -// Event is a Measurement Kit event -type Event struct { - // Is the key for the event - Key string `json:"key"` - - // Contains the value for the event - Value EventValue `json:"value"` -} - -func loopEx(in <-chan []byte, out chan<- Event, logger model.Logger) { - defer close(out) - for data := range in { - // Uncomment the following line to debug - //logger.Debugf("measurementkit: event: %s", string(data)) - var event Event - err := json.Unmarshal(data, &event) - if err != nil { - logger.Debugf("measurementkit: JSON processing error: %s", err.Error()) - continue - } - out <- event - } -} - -// StartEx is a more advanced Start that takes input settings -// and that emits Event on the returned channel. -func StartEx(settings Settings, logger model.Logger) (<-chan Event, error) { - data, err := json.Marshal(settings) - if err != nil { - return nil, err - } - logger.Debugf("measurementkit: settings: %s", string(data)) - in, err := Start(data) - if err != nil { - return nil, err - } - out := make(chan Event) - go loopEx(in, out, logger) - return out, nil -} - -// Start starts a Measurement Kit task with the provided settings and -// returns a channel where events are emitted or an error. -func Start(settings []byte) (<-chan []byte, error) { - return start(settings) -} - -// Available indicates whether Measurement Kit support is available. -func Available() bool { - return available() -} diff --git a/measurementkit/measurementkit_test.go b/measurementkit/measurementkit_test.go deleted file mode 100644 index 63e0a913..00000000 --- a/measurementkit/measurementkit_test.go +++ /dev/null @@ -1,28 +0,0 @@ -package measurementkit_test - -import ( - "testing" - - "github.com/apex/log" - "github.com/ooni/probe-engine/measurementkit" -) - -func TestTaskIntegrationNDT(t *testing.T) { - if !measurementkit.Available() { - t.Skip("Measurement Kit support not compiled in") - } - log.SetLevel(log.DebugLevel) - settings := measurementkit.NewSettings( - "Ndt", "ooniprobe-example", "0.1.0", - "../testdata/ca-bundle.pem", "AS30722", "IT", - "130.25.149.142", "Vodafone Italia S.p.A.", - "WARNING", - ) - ch, err := measurementkit.StartEx(settings, log.Log) - if err != nil { - t.Fatal(err) - } - for range ch { - // Drain - } -} diff --git a/measurementkit/mkcgo/force.cpp b/measurementkit/mkcgo/force.cpp deleted file mode 100644 index bd57536c..00000000 --- a/measurementkit/mkcgo/force.cpp +++ /dev/null @@ -1 +0,0 @@ -// Force CGO to compile and link C++ sources diff --git a/measurementkit/mkcgo/mkcgo.go b/measurementkit/mkcgo/mkcgo.go deleted file mode 100644 index 64237405..00000000 --- a/measurementkit/mkcgo/mkcgo.go +++ /dev/null @@ -1,94 +0,0 @@ -// +build !nomk - -// Package mkcgo contains CGO bindings to Measurement Kit. -package mkcgo - -import ( - // #include - // #include - // - // #include - // - // #cgo darwin,amd64 LDFLAGS: /usr/local/lib/libmeasurement_kit.a - // #cgo darwin,amd64 LDFLAGS: /usr/local/opt/libevent/lib/libevent_core.a - // #cgo darwin,amd64 LDFLAGS: /usr/local/opt/libevent/lib/libevent_extra.a - // #cgo darwin,amd64 LDFLAGS: /usr/local/opt/libevent/lib/libevent_openssl.a - // #cgo darwin,amd64 LDFLAGS: /usr/local/opt/libevent/lib/libevent_pthreads.a - // #cgo darwin,amd64 LDFLAGS: /usr/local/opt/libmaxminddb/lib/libmaxminddb.a - // #cgo darwin,amd64 LDFLAGS: /usr/local/opt/openssl@1.1/lib/libssl.a - // #cgo darwin,amd64 LDFLAGS: /usr/local/opt/openssl@1.1/lib/libcrypto.a - // #cgo darwin,amd64 LDFLAGS: -lcurl - // - // #cgo windows LDFLAGS: -static - // #cgo windows,amd64 CFLAGS: -I/usr/local/opt/mingw-w64-measurement-kit/include/ - // #cgo windows,amd64 LDFLAGS: /usr/local/opt/mingw-w64-measurement-kit/lib/libmeasurement_kit.a - // #cgo windows,amd64 LDFLAGS: /usr/local/opt/mingw-w64-libmaxminddb/lib/libmaxminddb.a - // #cgo windows,amd64 LDFLAGS: /usr/local/opt/mingw-w64-libevent/lib/libevent_openssl.a - // #cgo windows,amd64 LDFLAGS: /usr/local/opt/mingw-w64-libressl/lib/libssl.a - // #cgo windows,amd64 LDFLAGS: /usr/local/opt/mingw-w64-libressl/lib/libcrypto.a - // #cgo windows,amd64 LDFLAGS: /usr/local/opt/mingw-w64-libevent/lib/libevent_core.a - // #cgo windows,amd64 LDFLAGS: /usr/local/opt/mingw-w64-libevent/lib/libevent_extra.a - // #cgo windows,amd64 LDFLAGS: -lws2_32 - // #cgo windows,amd64 LDFLAGS: -fstack-protector-strong - // - // #cgo linux,amd64,ooni LDFLAGS: -static - // #cgo linux,amd64,ooni LDFLAGS: /usr/local/lib/libmeasurement_kit.a - // #cgo linux,amd64,ooni LDFLAGS: /usr/lib/libmaxminddb.a - // #cgo linux,amd64,ooni LDFLAGS: /usr/lib/libevent_openssl.a - // #cgo linux,amd64,ooni LDFLAGS: /usr/lib/libssl.a - // #cgo linux,amd64,ooni LDFLAGS: /usr/lib/libcrypto.a - // #cgo linux,amd64,ooni LDFLAGS: /usr/lib/libevent_core.a - // #cgo linux,amd64,ooni LDFLAGS: /usr/lib/libevent_extra.a - // #cgo linux,amd64,ooni LDFLAGS: /usr/lib/libevent_pthreads.a - // #cgo linux,amd64,ooni LDFLAGS: /lib/libz.a - // - // #cgo linux,!ooni LDFLAGS: -lmeasurement_kit - // - // #if MK_VERSION_NUMERIC != 0x00000000010000111LL - // #error "Wrong measurement-kit version, please recompile measurement-kit" - // #endif - "C" - "errors" - "unsafe" -) - -func evprocess(taskp *C.mk_task_t, out chan<- []byte) { - eventp := C.mk_task_wait_for_next_event(taskp) - if eventp == nil { - return - } - defer C.mk_event_destroy(eventp) - events := C.mk_event_serialize(eventp) - if events == nil { - return - } - out <- []byte(C.GoString(events)) -} - -func taskloop(taskp *C.mk_task_t, out chan<- []byte) { - defer close(out) - defer C.mk_task_destroy(taskp) - for C.mk_task_is_done(taskp) == 0 { - evprocess(taskp, out) - } -} - -func taskstart(settings []byte) *C.mk_task_t { - settingsp := C.CString(string(settings)) - if settingsp == nil { - return nil - } - defer C.free(unsafe.Pointer(settingsp)) - return C.mk_task_start(settingsp) -} - -// Start starts a Measurement Kit task. -func Start(settings []byte) (<-chan []byte, error) { - taskp := taskstart(settings) - if taskp == nil { - return nil, errors.New("C.mk_task_start failed") - } - out := make(chan []byte) - go taskloop(taskp, out) - return out, nil -} diff --git a/measurementkit/task_mk.go b/measurementkit/task_mk.go deleted file mode 100644 index eb4cf52f..00000000 --- a/measurementkit/task_mk.go +++ /dev/null @@ -1,15 +0,0 @@ -// +build !nomk - -package measurementkit - -import ( - "github.com/ooni/probe-engine/measurementkit/mkcgo" -) - -func start(settings []byte) (<-chan []byte, error) { - return mkcgo.Start(settings) -} - -func available() bool { - return true -} diff --git a/measurementkit/task_nomk.go b/measurementkit/task_nomk.go deleted file mode 100644 index 090ac9e4..00000000 --- a/measurementkit/task_nomk.go +++ /dev/null @@ -1,13 +0,0 @@ -// +build nomk - -package measurementkit - -import "errors" - -func start(settings []byte) (<-chan []byte, error) { - return nil, errors.New("Measurement Kit not available") -} - -func available() bool { - return false -} diff --git a/netx/httptransport/system.go b/netx/httptransport/system.go index 064999a1..d6857f17 100644 --- a/netx/httptransport/system.go +++ b/netx/httptransport/system.go @@ -1,5 +1,3 @@ -// +build go1.14 - package httptransport import ( diff --git a/netx/httptransport/system_compat.go b/netx/httptransport/system_compat.go deleted file mode 100644 index 30a9d943..00000000 --- a/netx/httptransport/system_compat.go +++ /dev/null @@ -1,31 +0,0 @@ -// +build !go1.14 - -package httptransport - -import ( - "context" - "net" - "net/http" -) - -// NewSystemTransport creates a new "system" HTTP transport. That is a transport -// using the Go standard library with custom dialer and TLS dialer. -func NewSystemTransport(dialer Dialer, tlsDialer TLSDialer) *http.Transport { - txp := http.DefaultTransport.(*http.Transport).Clone() - txp.DialContext = dialer.DialContext - txp.DialTLS = func(network, address string) (net.Conn, error) { - // Go < 1.14 does not have http.Transport.DialTLSContext - return tlsDialer.DialTLSContext(context.Background(), network, address) - } - // Better for Cloudflare DNS and also better because we have less - // noisy events and we can better understand what happened. - txp.MaxConnsPerHost = 1 - // The following (1) reduces the number of headers that Go will - // automatically send for us and (2) ensures that we always receive - // back the true headers, such as Content-Length. This change is - // functional to OONI's goal of observing the network. - txp.DisableCompression = true - return txp -} - -var _ RoundTripper = &http.Transport{}