From 1b63a87e711919377fb5b1969501a297a58e52db Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 8 Jul 2022 15:05:00 +0200 Subject: [PATCH 1/4] feat(oonirun): implement OONIRun v1 This diff adds support for running OONIRun v1 links. Run with `miniooni` using: ``` ./miniooni -i LINK oonirun ``` Part of https://github.com/ooni/probe/issues/2184 --- internal/cmd/miniooni/libminiooni.go | 42 ++++++++++++- internal/oonirun/link.go | 92 ++++++++++++++++++++++++++++ internal/oonirun/v1.go | 90 +++++++++++++++++++++++++++ internal/oonirun/v1_test.go | 36 +++++++++++ 4 files changed, 259 insertions(+), 1 deletion(-) create mode 100644 internal/oonirun/link.go create mode 100644 internal/oonirun/v1.go create mode 100644 internal/oonirun/v1_test.go diff --git a/internal/cmd/miniooni/libminiooni.go b/internal/cmd/miniooni/libminiooni.go index 5eccd4d5f3..a76dd37dc0 100644 --- a/internal/cmd/miniooni/libminiooni.go +++ b/internal/cmd/miniooni/libminiooni.go @@ -369,7 +369,14 @@ func MainWithConfiguration(experimentName string, currentOptions Options) { log.Infof("- resolver's network: %s (%s)", sess.ResolverNetworkName(), sess.ResolverASNString()) - // Run OONI experiments as we normally do. + // We handle the oonirun experiment name specially. The user must specify + // `miniooni -i {OONIRunURL} oonirun` to run a OONI Run URL (v1 or v2). + if experimentName == "oonirun" { + ooniRunMain(ctx, sess, currentOptions, annotations) + return + } + + // Otherwise just run OONI experiments as we normally do. desc := &oonirun.Experiment{ Annotations: annotations, ExtraOptions: extraOptions, @@ -386,3 +393,36 @@ func MainWithConfiguration(experimentName string, currentOptions Options) { err = desc.Run(ctx) runtimex.PanicOnError(err, "cannot run experiment") } + +// ooniRunMain runs the experiments described by the given OONI Run URLs. This +// function works with both v1 and v2 OONI Run URLs. +func ooniRunMain(ctx context.Context, + sess *engine.Session, currentOptions Options, annotations map[string]string) { + runtimex.PanicIfTrue( + len(currentOptions.Inputs) <= 0, + "in oonirun mode you need to specify at least one URL using `-i URL`", + ) + runtimex.PanicIfTrue( + len(currentOptions.InputFilePaths) > 0, + "in oonirun mode you cannot specify any `-f FILE` file", + ) + logger := sess.Logger() + cfg := &oonirun.LinkConfig{ + AcceptChanges: currentOptions.Yes, + Annotations: annotations, + KVStore: sess.KeyValueStore(), + MaxRuntime: currentOptions.MaxRuntime, + NoCollector: currentOptions.NoCollector, + NoJSON: currentOptions.NoJSON, + Random: currentOptions.Random, + ReportFile: currentOptions.ReportFile, + Session: sess, + } + for _, URL := range currentOptions.Inputs { + r := oonirun.NewLinkRunner(cfg, URL) + if err := r.Run(ctx); err != nil { + logger.Warnf("oonirun: Measure failed: %s", err.Error()) + continue + } + } +} diff --git a/internal/oonirun/link.go b/internal/oonirun/link.go new file mode 100644 index 0000000000..4bb8bc0ef9 --- /dev/null +++ b/internal/oonirun/link.go @@ -0,0 +1,92 @@ +package oonirun + +// +// OONI Run v1 and v2 links +// + +import ( + "context" + "strings" + + "github.com/ooni/probe-cli/v3/internal/model" +) + +// LinkConfig contains config for an OONI Run link. You MUST fill all the fields that +// are marked as MANDATORY, or the LinkConfig would cause crashes. +type LinkConfig struct { + // AcceptChanges is OPTIONAL and tells this library that the user is + // okay with running a new or modified OONI Run link without previously + // reviewing what it contains or what has changed. + AcceptChanges bool + + // Annotations contains OPTIONAL Annotations for the experiment. + Annotations map[string]string + + // KVStore is the MANDATORY key-value store to use to keep track of + // OONI Run links and know when they are new or modified. + KVStore model.KeyValueStore + + // MaxRuntime is the OPTIONAL maximum runtime in seconds. + MaxRuntime int64 + + // NoCollector OPTIONALLY indicates we should not be using any collector. + NoCollector bool + + // NoJSON OPTIONALLY indicates we don't want to save measurements to a JSON file. + NoJSON bool + + // Random OPTIONALLY indicates we should randomize inputs. + Random bool + + // ReportFile is the MANDATORY file in which to save reports, which is only + // used when noJSON is set to false. + ReportFile string + + // Session is the MANDATORY Session to use. + Session Session +} + +// LinkRunner knows how to run an OONI Run v1 or v2 link. +type LinkRunner interface { + Run(ctx context.Context) error +} + +// linkRunner implements LinkRunner. +type linkRunner struct { + config *LinkConfig + f func(ctx context.Context, config *LinkConfig, URL string) error + url string +} + +// Run implements LinkRunner.Rune. +func (lr *linkRunner) Run(ctx context.Context) error { + return lr.f(ctx, lr.config, lr.url) +} + +// NewLinkRunner creates a suitable link runner for the current config +// and the given URL, which is one of the following: +// +// 1. OONI Run v1 link with https scheme (e.g., https://run.ooni.io/nettest?...) +// +// 2. OONI Run v1 link with ooni scheme (e.g., ooni://nettest?...) +// +// 3. arbitrary URL of the OONI Run v2 descriptor. +func NewLinkRunner(c *LinkConfig, URL string) LinkRunner { + // TODO(bassosimone): add support for v2 deeplinks. + out := &linkRunner{ + config: c, + f: nil, + url: URL, + } + switch { + case strings.HasPrefix(URL, "https://run.ooni.io/nettest"): + out.f = v1Measure + case strings.HasPrefix(URL, "ooni://nettest"): + out.f = v1Measure + default: + // TODO(bassosimone): this panic will go away when we merge + // the next patch which will implement v2. + panic("unsupported OONI Run link") + } + return out +} diff --git a/internal/oonirun/v1.go b/internal/oonirun/v1.go new file mode 100644 index 0000000000..3002c29db1 --- /dev/null +++ b/internal/oonirun/v1.go @@ -0,0 +1,90 @@ +package oonirun + +// +// OONI Run v1 implementation +// + +import ( + "context" + "encoding/json" + "errors" + "net/url" +) + +var ( + // ErrInvalidV1URLScheme indicates a v1 OONI Run URL has an invalid scheme. + ErrInvalidV1URLScheme = errors.New("oonirun: invalid v1 URL scheme") + + // ErrInvalidV1URLHost indicates a v1 OONI Run URL has an invalid host. + ErrInvalidV1URLHost = errors.New("oonirun: invalid v1 URL host") + + // ErrInvalidV1URLPath indicates a v1 OONI Run URL has an invalid path. + ErrInvalidV1URLPath = errors.New("oonirun: invalid v1 URL path") + + // ErrInvalidV1URLQueryArgument indicates a v1 OONI Run URL query argument is invalid. + ErrInvalidV1URLQueryArgument = errors.New("oonirun: invalid v1 URL query argument") +) + +// v1Arguments contains arguments for a v1 OONI Run URL. These arguments are +// always encoded inside of the "ta" field, which is optional. +type v1Arguments struct { + URLs []string `json:"urls"` +} + +// v1Measure performs a measurement using a v1 OONI Run URL. +func v1Measure(ctx context.Context, config *LinkConfig, URL string) error { + config.Session.Logger().Infof("oonirun/v1: running %s", URL) + pu, err := url.Parse(URL) + if err != nil { + return err + } + switch pu.Scheme { + case "https": + if pu.Host != "run.ooni.io" { + return ErrInvalidV1URLHost + } + if pu.Path != "/nettest" { + return ErrInvalidV1URLPath + } + case "ooni": + if pu.Host != "nettest" { + return ErrInvalidV1URLHost + } + if pu.Path != "" { + return ErrInvalidV1URLPath + } + default: + return ErrInvalidV1URLScheme + } + name := pu.Query().Get("tn") + if name == "" { + return ErrInvalidV1URLQueryArgument + } + var inputs []string + if ra := pu.Query().Get("ta"); ra != "" { + pa, err := url.QueryUnescape(ra) + if err != nil { + return err + } + var arguments v1Arguments + if err := json.Unmarshal([]byte(pa), &arguments); err != nil { + return err + } + inputs = arguments.URLs + } + // TODO(bassosimone): reject mv < 1.2.0 + exp := &Experiment{ + Annotations: config.Annotations, + ExtraOptions: nil, // no way to specify with v1 URLs + Inputs: inputs, + InputFilePaths: nil, + MaxRuntime: config.MaxRuntime, + Name: name, + NoCollector: config.NoCollector, + NoJSON: config.NoJSON, + Random: config.Random, + ReportFile: config.ReportFile, + Session: config.Session, + } + return exp.Run(ctx) +} diff --git a/internal/oonirun/v1_test.go b/internal/oonirun/v1_test.go new file mode 100644 index 0000000000..6abc80e01c --- /dev/null +++ b/internal/oonirun/v1_test.go @@ -0,0 +1,36 @@ +package oonirun + +import ( + "context" + "testing" + + "github.com/ooni/probe-cli/v3/internal/kvstore" +) + +// TODO(bassosimone): it would be cool to write unit tests. However, to do that +// we need to ~redesign the engine package for unit-testability. + +func TestOONIRunV1Link(t *testing.T) { + ctx := context.Background() + config := &LinkConfig{ + AcceptChanges: false, + Annotations: map[string]string{ + "platform": "linux", + }, + KVStore: &kvstore.Memory{}, + MaxRuntime: 0, + NoCollector: true, + NoJSON: true, + Random: false, + ReportFile: "", + Session: newSession(ctx, t), + } + r := NewLinkRunner(config, "https://run.ooni.io/nettest?tn=example&mv=1.2.0") + if err := r.Run(ctx); err != nil { + t.Fatal(err) + } + r = NewLinkRunner(config, "ooni://nettest?tn=example&mv=1.2.0") + if err := r.Run(ctx); err != nil { + t.Fatal(err) + } +} From aa1b2f8988a913aa2026d941ebaf0eee18767ef8 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 8 Jul 2022 15:09:19 +0200 Subject: [PATCH 2/4] Update internal/cmd/miniooni/libminiooni.go --- internal/cmd/miniooni/libminiooni.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/cmd/miniooni/libminiooni.go b/internal/cmd/miniooni/libminiooni.go index a76dd37dc0..9419d03a8f 100644 --- a/internal/cmd/miniooni/libminiooni.go +++ b/internal/cmd/miniooni/libminiooni.go @@ -421,7 +421,7 @@ func ooniRunMain(ctx context.Context, for _, URL := range currentOptions.Inputs { r := oonirun.NewLinkRunner(cfg, URL) if err := r.Run(ctx); err != nil { - logger.Warnf("oonirun: Measure failed: %s", err.Error()) + logger.Warnf("oonirun: running link failed: %s", err.Error()) continue } } From 9c280f3cb266f08da772ea52866956088877ebd3 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 8 Jul 2022 15:10:24 +0200 Subject: [PATCH 3/4] Update internal/oonirun/link.go --- internal/oonirun/link.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/oonirun/link.go b/internal/oonirun/link.go index 4bb8bc0ef9..bfaf4f6d6f 100644 --- a/internal/oonirun/link.go +++ b/internal/oonirun/link.go @@ -58,7 +58,7 @@ type linkRunner struct { url string } -// Run implements LinkRunner.Rune. +// Run implements LinkRunner.Run. func (lr *linkRunner) Run(ctx context.Context) error { return lr.f(ctx, lr.config, lr.url) } From 5b0f6363163c1a77e42df7c43010ad857c08d1f1 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 8 Jul 2022 15:11:39 +0200 Subject: [PATCH 4/4] Update internal/oonirun/v1.go --- internal/oonirun/v1.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/oonirun/v1.go b/internal/oonirun/v1.go index 3002c29db1..f82d6e6040 100644 --- a/internal/oonirun/v1.go +++ b/internal/oonirun/v1.go @@ -31,7 +31,7 @@ type v1Arguments struct { URLs []string `json:"urls"` } -// v1Measure performs a measurement using a v1 OONI Run URL. +// v1Measure performs a measurement using the given v1 OONI Run URL. func v1Measure(ctx context.Context, config *LinkConfig, URL string) error { config.Session.Logger().Infof("oonirun/v1: running %s", URL) pu, err := url.Parse(URL)