diff --git a/.gitignore b/.gitignore index b5d83b3cde..8fd9cd3c16 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,5 @@ /ooporthelper /probe-cli.cov /tmp-* +/tinyooni +/*.zip diff --git a/cmd/ooniprobe/internal/log/handlers/cli/results.go b/cmd/ooniprobe/internal/log/handlers/cli/results.go index 9dd1deb2c3..49c99df971 100644 --- a/cmd/ooniprobe/internal/log/handlers/cli/results.go +++ b/cmd/ooniprobe/internal/log/handlers/cli/results.go @@ -86,7 +86,11 @@ var summarizers = map[string]func(uint64, uint64, string) []string{ } func makeSummary(name string, totalCount uint64, anomalyCount uint64, ss string) []string { - return summarizers[name](totalCount, anomalyCount, ss) + summarizer, ok := summarizers[name] + if !ok { + return []string{"", "", ""} + } + return summarizer(totalCount, anomalyCount, ss) } func logResultItem(w io.Writer, f log.Fields) error { diff --git a/internal/cmd/tinyooni/database.go b/internal/cmd/tinyooni/database.go new file mode 100644 index 0000000000..c3744aff3a --- /dev/null +++ b/internal/cmd/tinyooni/database.go @@ -0,0 +1,69 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "runtime" + + "github.com/ooni/probe-cli/v3/internal/database" + "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// initDatabase initializes a database and returns the corresponding database properties. +func initDatabase(ctx context.Context, sess *engine.Session, globalOptions *GlobalOptions) *model.DatabaseProps { + ooniHome := maybeGetOONIDir(globalOptions.HomeDir) + + db, err := database.Open(databasePath(ooniHome)) + runtimex.PanicOnError(err, "database.Open failed") + + networkDB, err := db.CreateNetwork(sess) + runtimex.PanicOnError(err, "db.Create failed") + + dbResult, err := db.CreateResult(ooniHome, "custom", networkDB.ID) + runtimex.PanicOnError(err, "db.CreateResult failed") + + return &model.DatabaseProps{ + Database: db, + DatabaseNetwork: networkDB, + DatabaseResult: dbResult, + } +} + +// getHomeDir returns the $HOME directory. +func getHomeDir() (string, string) { + // See https://gist.github.com/miguelmota/f30a04a6d64bd52d7ab59ea8d95e54da + if runtime.GOOS == "windows" { + home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH") + if home == "" { + home = os.Getenv("USERPROFILE") + } + return home, "ooniprobe" + } + if runtime.GOOS == "linux" { + home := os.Getenv("XDG_CONFIG_HOME") + if home != "" { + return home, "ooniprobe" + } + // fallthrough + } + return os.Getenv("HOME"), ".ooniprobe" +} + +// maybeGetOONIDir returns the $HOME/.ooniprobe equivalent unless optionsHome +// is already set, in which case it just returns optionsHome. +func maybeGetOONIDir(optionsHome string) string { + if optionsHome != "" { + return optionsHome + } + homeDir, dirName := getHomeDir() + runtimex.Assert(homeDir != "", "homeDir is empty") + return filepath.Join(homeDir, dirName) +} + +// databasePath returns the database path given the OONI_HOME. +func databasePath(ooniHome string) string { + return filepath.Join(ooniHome, "db", "main.sqlite3") +} diff --git a/internal/cmd/tinyooni/groups.json b/internal/cmd/tinyooni/groups.json new file mode 100644 index 0000000000..daf21035a4 --- /dev/null +++ b/internal/cmd/tinyooni/groups.json @@ -0,0 +1,68 @@ +{ + "websites": { + "name": "Websites", + "description": "", + "author": "OONI", + "nettests": [{ + "test_name": "web_connectivity" + }] + }, + "performance": { + "name": "Performance", + "description": "", + "author": "OONI", + "nettests": [{ + "test_name": "dash" + }, { + "test_name": "ndt" + }] + }, + "middlebox": { + "name": "Middlebox", + "description": "", + "author": "OONI", + "nettests": [{ + "test_name": "hirl" + }, { + "test_name": "hhfm" + }] + }, + "im": { + "name": "Instant Messaging", + "description": "Runs the instant messaging experiments", + "author": "OONI", + "nettests": [{ + "test_name": "facebook_messenger" + }, { + "test_name": "signal" + }, { + "test_name": "telegram" + }, { + "test_name": "whatsapp" + }] + }, + "circumvention": { + "name": "Circumvention", + "description": "", + "author": "OONI", + "nettests": [{ + "test_name": "psiphon" + }, { + "test_name": "tor" + }] + }, + "experimental": { + "name": "Experimental", + "description": "Experimental nettests", + "author": "OONI", + "nettests": [{ + "test_name": "dnscheck" + }, { + "test_name": "stun_reachability" + }, { + "test_name": "torsf" + }, { + "test_name": "vanilla_tor" + }] + } +} diff --git a/internal/cmd/tinyooni/main.go b/internal/cmd/tinyooni/main.go new file mode 100644 index 0000000000..c34412e0e6 --- /dev/null +++ b/internal/cmd/tinyooni/main.go @@ -0,0 +1,151 @@ +package main + +import ( + "os" + + "github.com/ooni/probe-cli/v3/internal/version" + "github.com/spf13/cobra" +) + +// GlobalOptions contains the global options. +type GlobalOptions struct { + Emoji bool + HomeDir string + NoJSON bool + NoCollector bool + ProbeServicesURL string + Proxy string + RepeatEvery int64 + ReportFile string + SnowflakeRendezvous string + TorArgs []string + TorBinary string + Tunnel string + Verbose bool + Yes bool +} + +func main() { + var globalOptions GlobalOptions + rootCmd := &cobra.Command{ + Use: "tinyooni", + Short: "tinyooni is like miniooni but more experimental", + Args: cobra.NoArgs, + Version: version.Version, + } + rootCmd.SetVersionTemplate("{{ .Version }}\n") + flags := rootCmd.PersistentFlags() + + flags.BoolVar( + &globalOptions.Emoji, + "emoji", + false, + "whether to use emojis when logging", + ) + + flags.StringVar( + &globalOptions.HomeDir, + "home", + "", + "force specific home directory", + ) + + flags.BoolVarP( + &globalOptions.NoJSON, + "no-json", + "N", + false, + "disable writing to disk", + ) + + flags.BoolVarP( + &globalOptions.NoCollector, + "no-collector", + "n", + false, + "do not submit measurements to the OONI collector", + ) + + flags.StringVar( + &globalOptions.ProbeServicesURL, + "probe-services", + "", + "URL of the OONI backend instance you want to use", + ) + + flags.StringVar( + &globalOptions.Proxy, + "proxy", + "", + "set proxy URL to communicate with the OONI backend (mutually exclusive with --tunnel)", + ) + + flags.Int64Var( + &globalOptions.RepeatEvery, + "repeat-every", + 0, + "wait the given number of seconds and then repeat the same measurement", + ) + + flags.StringVarP( + &globalOptions.ReportFile, + "reportfile", + "o", + "", + "set the output report file path (default: \"report.jsonl\")", + ) + + flags.StringVar( + &globalOptions.SnowflakeRendezvous, + "snowflake-rendezvous", + "domain_fronting", + "rendezvous method for --tunnel=torsf (one of: \"domain_fronting\" and \"amp\")", + ) + + flags.StringSliceVar( + &globalOptions.TorArgs, + "tor-args", + []string{}, + "extra arguments for the tor binary (may be specified multiple times)", + ) + + flags.StringVar( + &globalOptions.TorBinary, + "tor-binary", + "", + "execute a specific tor binary", + ) + + flags.StringVar( + &globalOptions.Tunnel, + "tunnel", + "", + "tunnel to use to communicate with the OONI backend (one of: psiphon, tor, torsf)", + ) + + flags.BoolVarP( + &globalOptions.Verbose, + "verbose", + "v", + false, + "increase verbosity level", + ) + + flags.BoolVarP( + &globalOptions.Yes, + "yes", + "y", + false, + "assume yes as the answer to all questions", + ) + + rootCmd.MarkFlagsMutuallyExclusive("proxy", "tunnel") + + registerRunExperiment(rootCmd, &globalOptions) + registerRunGroup(rootCmd, &globalOptions) + registerOoniRun(rootCmd, &globalOptions) + + if err := rootCmd.Execute(); err != nil { + os.Exit(1) + } +} diff --git a/internal/cmd/tinyooni/oonirun.go b/internal/cmd/tinyooni/oonirun.go new file mode 100644 index 0000000000..2f82790b76 --- /dev/null +++ b/internal/cmd/tinyooni/oonirun.go @@ -0,0 +1,101 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "os" + + "github.com/ooni/probe-cli/v3/internal/oonirun" + "github.com/ooni/probe-cli/v3/internal/oonirunx" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/spf13/cobra" +) + +type oonirunOptions struct { + Inputs []string + InputFilePaths []string +} + +func registerOoniRun(rootCmd *cobra.Command, globalOptions *GlobalOptions) { + options := &oonirunOptions{} + + subCmd := &cobra.Command{ + Use: "run", + Short: "Runs a given experiment group", + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + ooniRunMain(options, globalOptions) + }, + } + rootCmd.AddCommand(subCmd) + flags := subCmd.Flags() + + flags.StringSliceVarP( + &options.Inputs, + "input", + "i", + []string{}, + "URL of the OONI Run v2 descriptor to run (may be specified multiple times)", + ) + flags.StringSliceVarP( + &options.InputFilePaths, + "input-file", + "f", + []string{}, + "Path to the OONI Run v2 descriptor to run (may be specified multiple times)", + ) +} + +func ooniRunMain(options *oonirunOptions, globalOptions *GlobalOptions) { + ctx := context.Background() + + // create a new measurement session + sess, err := newSession(ctx, globalOptions) + runtimex.PanicOnError(err, "newSession failed") + + err = sess.MaybeLookupLocationContext(ctx) + runtimex.PanicOnError(err, "sess.MaybeLookupLocation failed") + + // initialize database + dbProps := initDatabase(ctx, sess, globalOptions) + + logger := sess.Logger() + cfg := &oonirunx.LinkConfig{ + AcceptChanges: globalOptions.Yes, + KVStore: sess.KeyValueStore(), + NoCollector: globalOptions.NoCollector, + NoJSON: globalOptions.NoJSON, + ReportFile: globalOptions.ReportFile, + Session: sess, + DatabaseProps: dbProps, + } + for _, URL := range options.Inputs { + r := oonirunx.NewLinkRunner(cfg, URL) + if err := r.Run(ctx); err != nil { + if errors.Is(err, oonirun.ErrNeedToAcceptChanges) { + logger.Warnf("oonirun: to accept these changes, rerun adding `-y` to the command line") + logger.Warnf("oonirun: we'll show this error every time the upstream link changes") + panic("oonirun: need to accept changes using `-y`") + } + logger.Warnf("oonirun: running link failed: %s", err.Error()) + continue + } + } + for _, filename := range options.InputFilePaths { + data, err := os.ReadFile(filename) + if err != nil { + logger.Warnf("oonirun: reading OONI Run v2 descriptor failed: %s", err.Error()) + continue + } + var descr oonirunx.V2Descriptor + if err := json.Unmarshal(data, &descr); err != nil { + logger.Warnf("oonirun: parsing OONI Run v2 descriptor failed: %s", err.Error()) + continue + } + if err := oonirunx.V2MeasureDescriptor(ctx, cfg, &descr); err != nil { + logger.Warnf("oonirun: running link failed: %s", err.Error()) + continue + } + } +} diff --git a/internal/cmd/tinyooni/run.go b/internal/cmd/tinyooni/run.go new file mode 100644 index 0000000000..5025bee607 --- /dev/null +++ b/internal/cmd/tinyooni/run.go @@ -0,0 +1,81 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + + "github.com/ooni/probe-cli/v3/internal/oonirunx" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/spf13/cobra" +) + +var ( + // TODO: we should probably have groups.json as part of the default OONI + // config in $OONIDir + pathToGroups = "./internal/cmd/tinyooni/groups.json" + + // + groups map[string]json.RawMessage +) + +func registerRunGroup(rootCmd *cobra.Command, globalOptions *GlobalOptions) { + subCmd := &cobra.Command{ + Use: "run", + Short: "Runs a given experiment group", + Args: cobra.NoArgs, + } + rootCmd.AddCommand(subCmd) + registerGroups(subCmd, globalOptions) +} + +func registerGroups(rootCmd *cobra.Command, globalOptions *GlobalOptions) { + data, err := os.ReadFile(pathToGroups) + runtimex.PanicOnError(err, "registerGroups failed: could not read groups.json") + + err = json.Unmarshal(data, &groups) + runtimex.PanicOnError(err, "json.Unmarshal failed") + + for name := range groups { + subCmd := &cobra.Command{ + Use: name, + Short: fmt.Sprintf("Runs the %s experiment group", name), + Run: func(cmd *cobra.Command, args []string) { + runGroupMain(cmd.Use, globalOptions) + }, + } + rootCmd.AddCommand(subCmd) + } +} + +func runGroupMain(experimentName string, globalOptions *GlobalOptions) { + ctx := context.Background() + + // create a new measurement session + sess, err := newSession(ctx, globalOptions) + runtimex.PanicOnError(err, "newSession failed") + + err = sess.MaybeLookupLocationContext(ctx) + runtimex.PanicOnError(err, "sess.MaybeLookupLocation failed") + + // initialize database + dbProps := initDatabase(ctx, sess, globalOptions) + + logger := sess.Logger() + cfg := &oonirunx.LinkConfig{ + AcceptChanges: globalOptions.Yes, + KVStore: sess.KeyValueStore(), + NoCollector: globalOptions.NoCollector, + NoJSON: globalOptions.NoJSON, + ReportFile: globalOptions.ReportFile, + Session: sess, + DatabaseProps: dbProps, + } + var descr oonirunx.V2Descriptor + err = json.Unmarshal(groups[experimentName], &descr) + runtimex.PanicOnError(err, "json.Unmarshal failed") + if err := oonirunx.V2MeasureDescriptor(ctx, cfg, &descr); err != nil { + logger.Warnf("oonirun: running link failed: %s", err.Error()) + } +} diff --git a/internal/cmd/tinyooni/runx.go b/internal/cmd/tinyooni/runx.go new file mode 100644 index 0000000000..9f8c06ba2a --- /dev/null +++ b/internal/cmd/tinyooni/runx.go @@ -0,0 +1,58 @@ +package main + +import ( + "context" + "fmt" + + "github.com/ooni/probe-cli/v3/internal/registryx" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/spf13/cobra" +) + +func registerRunExperiment(rootCmd *cobra.Command, globalOptions *GlobalOptions) { + subCmd := &cobra.Command{ + Use: "runx", + Short: "Runs a given experiment", + Args: cobra.NoArgs, + } + rootCmd.AddCommand(subCmd) + registerAllExperiments(subCmd, globalOptions) +} + +func registerAllExperiments(rootCmd *cobra.Command, globalOptions *GlobalOptions) { + for name, factory := range registryx.AllExperiments { + subCmd := &cobra.Command{ + Use: name, + Short: fmt.Sprintf("Runs the %s experiment", name), + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + runExperimentsMain(cmd.Use, globalOptions) + }, + } + rootCmd.AddCommand(subCmd) + + // build experiment specific flags here + options := registryx.AllExperimentOptions[subCmd.Use] + options.BuildFlags(subCmd.Use, subCmd) + factory.SetOptions(options) + } +} + +func runExperimentsMain(experimentName string, currentOptions *GlobalOptions) { + ctx := context.Background() + + // create a new measurement session + sess, err := newSession(ctx, currentOptions) + runtimex.PanicOnError(err, "newSession failed") + + err = sess.MaybeLookupLocationContext(ctx) + runtimex.PanicOnError(err, "sess.MaybeLookupLocation failed") + + // initialize database + dbProps := initDatabase(ctx, sess, currentOptions) + + factory := registryx.AllExperiments[experimentName] + factory.SetArguments(sess, dbProps, nil) + err = factory.Main(ctx) + runtimex.PanicOnError(err, fmt.Sprintf("%s.Main failed", experimentName)) +} diff --git a/internal/cmd/tinyooni/session.go b/internal/cmd/tinyooni/session.go new file mode 100644 index 0000000000..04a3cdf299 --- /dev/null +++ b/internal/cmd/tinyooni/session.go @@ -0,0 +1,85 @@ +package main + +import ( + "context" + "net/url" + "os" + "path" + "path/filepath" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/kvstore" + "github.com/ooni/probe-cli/v3/internal/legacy/assetsdir" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/version" +) + +const ( + softwareName = "tinyooni" + softwareVersion = version.Version +) + +// newSession creates a new measurement session. +func newSession(ctx context.Context, globalOptions *GlobalOptions) (*engine.Session, error) { + ooniDir := maybeGetOONIDir(globalOptions.HomeDir) + if err := os.MkdirAll(ooniDir, 0700); err != nil { + return nil, err + } + + // We cleanup the assets files used by versions of ooniprobe + // older than v3.9.0, where we started embedding the assets + // into the binary and use that directly. This cleanup doesn't + // remove the whole directory but only known files inside it + // and then the directory itself, if empty. We explicitly discard + // the return value as it does not matter to us here. + _, _ = assetsdir.Cleanup(path.Join(ooniDir, "assets")) + + var ( + proxyURL *url.URL + err error + ) + if globalOptions.Proxy != "" { + proxyURL, err = url.Parse(globalOptions.Proxy) + if err != nil { + return nil, err + } + } + + kvstore2dir := filepath.Join(ooniDir, "kvstore2") + kvstore, err := kvstore.NewFS(kvstore2dir) + if err != nil { + return nil, err + } + + tunnelDir := filepath.Join(ooniDir, "tunnel") + if err := os.MkdirAll(tunnelDir, 0700); err != nil { + return nil, err + } + + config := engine.SessionConfig{ + KVStore: kvstore, + Logger: log.Log, + ProxyURL: proxyURL, + SnowflakeRendezvous: globalOptions.SnowflakeRendezvous, + SoftwareName: softwareName, + SoftwareVersion: softwareVersion, + TorArgs: globalOptions.TorArgs, + TorBinary: globalOptions.TorBinary, + TunnelDir: tunnelDir, + } + if globalOptions.ProbeServicesURL != "" { + config.AvailableProbeServices = []model.OOAPIService{{ + Address: globalOptions.ProbeServicesURL, + Type: "https", + }} + } + + sess, err := engine.NewSession(ctx, config) + if err != nil { + return nil, err + } + + log.Debugf("miniooni temporary directory: %s", sess.TempDir()) + return sess, nil +} diff --git a/internal/database/props.go b/internal/database/props.go new file mode 100644 index 0000000000..645a893389 --- /dev/null +++ b/internal/database/props.go @@ -0,0 +1,18 @@ +package database + +// Database properties retreived on initialization + +import ( + "github.com/ooni/probe-cli/v3/internal/model" +) + +type DatabaseProps struct { + // + Database *Database + + // + DatabaseNetwork *model.DatabaseNetwork + + // + DatabaseResult *model.DatabaseResult +} diff --git a/internal/engine/session.go b/internal/engine/session.go index bec7efc956..d8f216b0ff 100644 --- a/internal/engine/session.go +++ b/internal/engine/session.go @@ -572,6 +572,14 @@ func (s *Session) ResolverNetworkName() string { return nn } +func (s *Session) SubmitMeasurementV2(ctx context.Context, meas *model.Measurement) error { + clnt, err := s.NewProbeServicesClient(ctx) + if err != nil { + return err + } + return clnt.SubmitMeasurementV2(ctx, meas) +} + // SoftwareName returns the application name. func (s *Session) SoftwareName() string { return s.softwareName diff --git a/internal/experiment/dash/main.go b/internal/experiment/dash/main.go new file mode 100644 index 0000000000..55854d866f --- /dev/null +++ b/internal/experiment/dash/main.go @@ -0,0 +1,87 @@ +package dash + +import ( + "context" + "errors" + "os" + "time" + + "github.com/ooni/probe-cli/v3/internal/experiment" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" +) + +// ErrNoCheckInInfo indicates check-in returned no suitable info. +var ErrNoCheckInInfo = errors.New("dash: returned no check-in info") + +// ExperimentMain +type ExperimentMain struct { + options model.ExperimentOptions + args *model.ExperimentMainArgs + configArgs *Config +} + +// SetOptions +func (em *ExperimentMain) SetOptions(options model.ExperimentOptions) { + em.options = options +} + +// SetArguments +func (em *ExperimentMain) SetArguments(sess model.ExperimentSession, db *model.DatabaseProps, extraOptions map[string]any) error { + em.args = em.options.SetArguments(sess, db) + // give precedence to options populated by OONIRun + if extraOptions == nil { + extraOptions = em.options.ExtraOptions() + } + err := setter.SetOptionsAny(em.configArgs, extraOptions) + return err +} + +// Main is the main function of the experiment. +func (em *ExperimentMain) Main(ctx context.Context) error { + sess := em.args.Session + logger := sess.Logger() + + // Create the directory where to store results unless it already exists + if err := os.MkdirAll(em.args.MeasurementDir, 0700); err != nil { + return err + } + + // Attempt to remove the results directory when done unless it + // contains files, in which case we should keep it. + defer os.Remove(em.args.MeasurementDir) + + // Call the check-in API to obtain configuration. Note that the value + // returned here MAY have been cached by the engine. + logger.Infof("calling check-in API...") + checkInResp, err := experiment.CallCheckIn(ctx, em.args, sess) + + // Bail if either the check-in API failed or we don't have a reportID + // with which to submit Web Connectivity measurements results. + if err != nil { + return err + } + if checkInResp.Dash == nil { + return ErrNoCheckInInfo + } + + // Obtain and log the report ID. + reportID := checkInResp.Dash.ReportID + logger.Infof("ReportID: %s", reportID) + + // Create an instance of the experiment's measurer. + measurer := &Measurer{config: *em.configArgs} + + // Record when we started running this nettest. + testStartTime := time.Now() + + return experiment.MeasurePossiblyNilInput( + ctx, + em.args, + measurer, + testStartTime, + reportID, + 0, // inputIdx + nil, // input + ) +} diff --git a/internal/experiment/experiment.go b/internal/experiment/experiment.go new file mode 100644 index 0000000000..410199fea4 --- /dev/null +++ b/internal/experiment/experiment.go @@ -0,0 +1,228 @@ +// Package experiment contains common code for implementing experiments. +package experiment + +// +// Common code for implementing experiments +// + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "net/url" + "os" + "time" + + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// CallCheckIn is a convenience function that calls the +// check-in API using the given arguments. +func CallCheckIn( + ctx context.Context, + args *model.ExperimentMainArgs, + sess model.ExperimentSession, +) (*model.OOAPICheckInResultNettests, error) { + return sess.CheckIn(ctx, &model.OOAPICheckInConfig{ + Charging: args.Charging, + OnWiFi: args.OnWiFi, + Platform: sess.Platform(), + ProbeASN: sess.ProbeASNString(), + ProbeCC: sess.ProbeCC(), + RunType: args.RunType, + SoftwareName: sess.SoftwareName(), + SoftwareVersion: sess.SoftwareVersion(), + WebConnectivity: model.OOAPICheckInConfigWebConnectivity{ + CategoryCodes: args.CategoryCodes, + }, + }) +} + +// MeasurePossiblyNilInput measures a possibly nil +// input using the given experiment measurer. +// +// Arguments: +// +// - ctx is the context for deadline/cancellation/timeout; +// +// - args contains the experiment-main's arguments; +// +// - measurer is the measurer; +// +// - testStartTime is when the nettest started; +// +// - reportID is the reportID to use; +// +// - inputIdx is the POSSIBLY-ZERO input's index; +// +// - input is the POSSIBLY-NIL input to measure. +// +// This function only returns an error in case there is a +// serious situation (e.g., cannot write to disk). +func MeasurePossiblyNilInput( + ctx context.Context, + args *model.ExperimentMainArgs, + measurer model.ExperimentMeasurer, + testStartTime time.Time, + reportID string, + inputIdx int, + input *model.OOAPIURLInfo, +) error { + runtimex.Assert(ctx != nil, "passed nil Context") + runtimex.Assert(args != nil, "passed nil ExperimentMainArgs") + runtimex.Assert(measurer != nil, "passed nil ExperimentMeasurer") + runtimex.Assert(reportID != "", "passed empty report ID") + sess := args.Session + + // Make sure we track this possibly-nil input into the database. + var ( + urlIdx sql.NullInt64 + urlInput string + ) + if input != nil { + index, err := args.Database.CreateOrUpdateURL( + input.URL, + input.CategoryCode, + input.CountryCode, + ) + if err != nil { + return err + } + urlIdx.Int64 = index + urlIdx.Valid = true + urlInput = input.URL + } + + // Create a measurement object inside of the database. + dbMeas, err := args.Database.CreateMeasurement( + sql.NullString{ + String: reportID, + Valid: true, + }, + measurer.ExperimentName(), + args.MeasurementDir, + inputIdx, + args.ResultID, + urlIdx, + ) + if err != nil { + return err + } + + // Create the measurement for this URL. + meas := model.NewMeasurement( + sess, + measurer, + reportID, + urlInput, // possibly the empty string + testStartTime, + args.Annotations, + ) + + // Perform the measurement proper. + err = measurer.Run(ctx, &model.ExperimentArgs{ + Callbacks: args.Callbacks, + Measurement: meas, + Session: sess, + }) + + // In case of error the measurement failed because of some + // fundamental issue, so we don't want to submit. + if err != nil { + return args.Database.Failed(dbMeas, err.Error()) + } + + // Extract the measurement summary and store it inside the database. + summary, err := measurer.GetSummaryKeys(meas) + if err != nil { + return err + } + + // Add summary to database. + err = args.Database.AddTestKeys(dbMeas, summary) + if err != nil { + return err + } + + // Attempt to submit the measurement. + err = SubmitOrStoreLocally(ctx, args, sess, meas, dbMeas) + if err != nil { + return err + } + + // Mark measurement as done. + return args.Database.Done(dbMeas) +} + +// SubmitOrStoreLocally submits the measurement or stores it locally. +// +// Arguments: +// +// - ctx is the context for deadline/cancellation/timeout; +// +// - args contains the experiment's main arguments; +// +// - sess is the measurement session; +// +// - dbMeas is the database's view of the measurement. +// +// This function will return error only in case of fatal errors such as +// not being able to write onto the local disk. +func SubmitOrStoreLocally( + ctx context.Context, + args *model.ExperimentMainArgs, + sess model.ExperimentSession, + meas *model.Measurement, + dbMeas *model.DatabaseMeasurement, +) error { + runtimex.Assert(args != nil, "passed nil arguments") + runtimex.Assert(sess != nil, "passed nil Session") + runtimex.Assert(meas != nil, "passed nil measurement") + runtimex.Assert(dbMeas != nil, "passed nil dbMeas") + logger := sess.Logger() + + if !args.NoCollector { + // Submit the measurement to the OONI backend. + err := sess.SubmitMeasurementV2(ctx, meas) + if err == nil { + logger.Infof("Measurement: %s", ExplorerURL(meas)) + return args.Database.UploadSucceeded(dbMeas) + } + + // Handle the case where we could not submit the measurement. + failure := err.Error() + if err := args.Database.UploadFailed(dbMeas, failure); err != nil { + return err + } + + // Fallthrough and attempt to save measurement to disk + } + + // Serialize to JSON. + data, err := json.Marshal(meas) + if err != nil { + return err + } + + // Write the measurement and return result. + return os.WriteFile(dbMeas.MeasurementFilePath.String, data, 0600) +} + +// ExplorerURL returns the explorer URL associated with a measurement. +func ExplorerURL(meas *model.Measurement) string { + runtimex.Assert(meas != nil, "passed nil measurement") + runtimex.Assert(meas.ReportID != "", "passed empty report ID") + URL := &url.URL{ + Scheme: "https", + Host: "explorer.ooni.org", + Path: fmt.Sprintf("/measurement/%s", meas.ReportID), + } + if meas.Input != "" { + query := url.Values{} + query.Add("input", string(meas.Input)) + URL.RawQuery = query.Encode() + } + return URL.String() +} diff --git a/internal/experiment/fbmessenger/main.go b/internal/experiment/fbmessenger/main.go new file mode 100644 index 0000000000..57195528aa --- /dev/null +++ b/internal/experiment/fbmessenger/main.go @@ -0,0 +1,88 @@ +package fbmessenger + +import ( + "context" + "errors" + "os" + "time" + + "github.com/ooni/probe-cli/v3/internal/experiment" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" +) + +// ErrNoCheckInInfo indicates check-in returned no suitable info. +var ErrNoCheckInInfo = errors.New("fbmessenger: returned no check-in info") + +// ExperimentMain +type ExperimentMain struct { + options model.ExperimentOptions + args *model.ExperimentMainArgs + configArgs *Config +} + +// SetOptions +func (em *ExperimentMain) SetOptions(options model.ExperimentOptions) { + em.options = options +} + +// SetArguments +func (em *ExperimentMain) SetArguments(sess model.ExperimentSession, db *model.DatabaseProps, extraOptions map[string]any) error { + em.args = em.options.SetArguments(sess, db) + em.configArgs = &Config{} + // give precedence to options populated by OONIRun + if extraOptions == nil { + extraOptions = em.options.ExtraOptions() + } + err := setter.SetOptionsAny(em.configArgs, extraOptions) + return err +} + +// Main is the main function of the experiment. +func (em *ExperimentMain) Main(ctx context.Context) error { + sess := em.args.Session + logger := sess.Logger() + + // Create the directory where to store results unless it already exists + if err := os.MkdirAll(em.args.MeasurementDir, 0700); err != nil { + return err + } + + // Attempt to remove the results directory when done unless it + // contains files, in which case we should keep it. + defer os.Remove(em.args.MeasurementDir) + + // Call the check-in API to obtain configuration. Note that the value + // returned here MAY have been cached by the engine. + logger.Infof("calling check-in API...") + checkInResp, err := experiment.CallCheckIn(ctx, em.args, sess) + + // Bail if either the check-in API failed or we don't have a reportID + // with which to submit Web Connectivity measurements results. + if err != nil { + return err + } + if checkInResp.FacebookMessenger == nil { + return ErrNoCheckInInfo + } + + // Obtain and log the report ID. + reportID := checkInResp.FacebookMessenger.ReportID + logger.Infof("ReportID: %s", reportID) + + // Create an instance of the experiment's measurer. + measurer := &Measurer{Config: *em.configArgs} + + // Record when we started running this nettest. + testStartTime := time.Now() + + return experiment.MeasurePossiblyNilInput( + ctx, + em.args, + measurer, + testStartTime, + reportID, + 0, // inputIdx + nil, // input + ) +} diff --git a/internal/experiment/hhfm/main.go b/internal/experiment/hhfm/main.go new file mode 100644 index 0000000000..79163371a7 --- /dev/null +++ b/internal/experiment/hhfm/main.go @@ -0,0 +1,174 @@ +package hhfm + +import ( + "context" + "errors" + "os" + "time" + + "github.com/ooni/probe-cli/v3/internal/experiment" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/setter" +) + +// ErrNoCheckInInfo indicates check-in returned no suitable info. +var ErrNoCheckInInfo = errors.New("http_header_field_manipulation: returned no check-in info") + +// ExperimentMain +type ExperimentMain struct { + options model.ExperimentOptions + args *model.ExperimentMainArgs + configArgs *Config +} + +// SetOptions +func (em *ExperimentMain) SetOptions(options model.ExperimentOptions) { + em.options = options +} + +// SetArguments +func (em *ExperimentMain) SetArguments(sess model.ExperimentSession, db *model.DatabaseProps, extraOptions map[string]any) error { + em.args = em.options.SetArguments(sess, db) + em.configArgs = &Config{} + // give precedence to options populated by OONIRun + if extraOptions == nil { + extraOptions = em.options.ExtraOptions() + } + err := setter.SetOptionsAny(em.configArgs, extraOptions) + return err +} + +// Main is the main function of the experiment. +func (em *ExperimentMain) Main(ctx context.Context) error { + sess := em.args.Session + logger := sess.Logger() + + // Create the directory where to store results unless it already exists + if err := os.MkdirAll(em.args.MeasurementDir, 0700); err != nil { + return err + } + + // Attempt to remove the results directory when done unless it + // contains files, in which case we should keep it. + defer os.Remove(em.args.MeasurementDir) + + // Call the check-in API to obtain configuration. Note that the value + // returned here MAY have been cached by the engine. + logger.Infof("calling check-in API...") + checkInResp, err := experiment.CallCheckIn(ctx, em.args, sess) + + // Bail if either the check-in API failed or we don't have a reportID + // with which to submit Web Connectivity measurements results. + if err != nil { + return err + } + if checkInResp.HHFM == nil { + return ErrNoCheckInInfo + } + + // Obtain and log the report ID. + reportID := checkInResp.HHFM.ReportID + logger.Infof("ReportID: %s", reportID) + + // Obtain experiment inputs. + inputs := getInputs(em.args, checkInResp) + + // Create an instance of the experiment's measurer. + measurer := &Measurer{Config: *em.configArgs} + + // Record when we started running this nettest. + testStartTime := time.Now() + + // Create suitable stop policy. + shouldStop := newStopPolicy(em.args, testStartTime) + + // Create suitable progress emitter. + progresser := newProgressEmitter(em.args, inputs, testStartTime) + + // Measure each URL in sequence. + for inputIdx, input := range inputs { + + // Honour max runtime. + if shouldStop() { + break + } + + // Emit progress. + progresser(inputIdx, input.URL) + + // Measure the current URL. + err := experiment.MeasurePossiblyNilInput( + ctx, + em.args, + measurer, + testStartTime, + reportID, + inputIdx, + &input, + ) + + // An error here means stuff like "cannot write to disk". + if err != nil { + return err + } + } + + return nil +} + +// getInputs obtains inputs from either args or checkInResp giving +// priority to user supplied arguments inside args. +func getInputs(args *model.ExperimentMainArgs, checkInResp *model.OOAPICheckInResultNettests) []model.OOAPIURLInfo { + runtimex.Assert(checkInResp.WebConnectivity != nil, "getInputs passed invalid checkInResp") + inputs := args.Inputs + if len(inputs) < 1 { + return checkInResp.WebConnectivity.URLs + } + outputs := []model.OOAPIURLInfo{} + for _, input := range inputs { + outputs = append(outputs, model.OOAPIURLInfo{ + CategoryCode: "MISC", + CountryCode: "ZZ", + URL: input, + }) + } + return outputs +} + +// newStopPolicy creates a new stop policy depending on the +// arguments passed to the experiment in args. +func newStopPolicy(args *model.ExperimentMainArgs, testStartTime time.Time) func() bool { + if args.MaxRuntime <= 0 { + return func() bool { + return false + } + } + maxRuntime := time.Duration(args.MaxRuntime) * time.Second + return func() bool { + return time.Since(testStartTime) > maxRuntime + } +} + +func newProgressEmitter( + args *model.ExperimentMainArgs, + inputs []model.OOAPIURLInfo, + testStartTime time.Time, +) func(idx int, URL string) { + total := len(inputs) + if total <= 0 { + return func(idx int, URL string) {} // just in case + } + if args.MaxRuntime <= 0 { + return func(idx int, URL string) { + percentage := 100.0 * (float64(idx) / float64(total)) + args.Callbacks.OnProgress(percentage, URL) + } + } + maxRuntime := (time.Duration(args.MaxRuntime) * time.Second) + time.Nanosecond // avoid zero division + return func(idx int, URL string) { + elapsed := time.Since(testStartTime) + percentage := 100.0 * (float64(elapsed) / float64(maxRuntime)) + args.Callbacks.OnProgress(percentage, URL) + } +} diff --git a/internal/experiment/hirl/main.go b/internal/experiment/hirl/main.go new file mode 100644 index 0000000000..41a879c898 --- /dev/null +++ b/internal/experiment/hirl/main.go @@ -0,0 +1,97 @@ +package hirl + +import ( + "context" + "errors" + "os" + "time" + + "github.com/ooni/probe-cli/v3/internal/experiment" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" +) + +// ErrNoCheckInInfo indicates check-in returned no suitable info. +var ErrNoCheckInInfo = errors.New("http_invalid_request_line: returned no check-in info") + +// ExperimentMain +type ExperimentMain struct { + options model.ExperimentOptions + args *model.ExperimentMainArgs + configArgs *Config +} + +// SetOptions +func (em *ExperimentMain) SetOptions(options model.ExperimentOptions) { + em.options = options +} + +// SetArguments +func (em *ExperimentMain) SetArguments(sess model.ExperimentSession, db *model.DatabaseProps, extraOptions map[string]any) error { + em.args = em.options.SetArguments(sess, db) + em.configArgs = &Config{} + // give precedence to options populated by OONIRun + if extraOptions == nil { + extraOptions = em.options.ExtraOptions() + } + err := setter.SetOptionsAny(em.configArgs, extraOptions) + return err +} + +// Main is the main function of the experiment. +func (em *ExperimentMain) Main(ctx context.Context) error { + sess := em.args.Session + logger := sess.Logger() + + // Create the directory where to store results unless it already exists + if err := os.MkdirAll(em.args.MeasurementDir, 0700); err != nil { + return err + } + + // Attempt to remove the results directory when done unless it + // contains files, in which case we should keep it. + defer os.Remove(em.args.MeasurementDir) + + // Call the check-in API to obtain configuration. Note that the value + // returned here MAY have been cached by the engine. + logger.Infof("calling check-in API...") + checkInResp, err := experiment.CallCheckIn(ctx, em.args, sess) + + // Bail if either the check-in API failed or we don't have a reportID + // with which to submit Web Connectivity measurements results. + if err != nil { + return err + } + if checkInResp.HIRL == nil { + return ErrNoCheckInInfo + } + + // Obtain and log the report ID. + reportID := checkInResp.HIRL.ReportID + logger.Infof("ReportID: %s", reportID) + + // Create an instance of the experiment's measurer. + measurer := &Measurer{ + Config: *em.configArgs, + Methods: []Method{ + randomInvalidMethod{}, + randomInvalidFieldCount{}, + randomBigRequestMethod{}, + randomInvalidVersionNumber{}, + squidCacheManager{}, + }, + } + + // Record when we started running this nettest. + testStartTime := time.Now() + + return experiment.MeasurePossiblyNilInput( + ctx, + em.args, + measurer, + testStartTime, + reportID, + 0, // inputIdx + nil, // input + ) +} diff --git a/internal/experiment/ndt7/main.go b/internal/experiment/ndt7/main.go new file mode 100644 index 0000000000..025185c378 --- /dev/null +++ b/internal/experiment/ndt7/main.go @@ -0,0 +1,92 @@ +package ndt7 + +import ( + "context" + "encoding/json" + "errors" + "os" + "time" + + "github.com/ooni/probe-cli/v3/internal/experiment" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" +) + +// ErrNoCheckInInfo indicates check-in returned no suitable info. +var ErrNoCheckInInfo = errors.New("ndt7: returned no check-in info") + +// ExperimentMain +type ExperimentMain struct { + options model.ExperimentOptions + args *model.ExperimentMainArgs + configArgs *Config +} + +// SetOptions +func (em *ExperimentMain) SetOptions(options model.ExperimentOptions) { + em.options = options +} + +// SetArguments +func (em *ExperimentMain) SetArguments(sess model.ExperimentSession, db *model.DatabaseProps, extraOptions map[string]any) error { + em.args = em.options.SetArguments(sess, db) + em.configArgs = &Config{} + // give precedence to options populated by OONIRun + if extraOptions == nil { + extraOptions = em.options.ExtraOptions() + } + err := setter.SetOptionsAny(em.configArgs, extraOptions) + return err +} + +// Main is the main function of the experiment. +func (em *ExperimentMain) Main(ctx context.Context) error { + sess := em.args.Session + logger := sess.Logger() + + // Create the directory where to store results unless it already exists + if err := os.MkdirAll(em.args.MeasurementDir, 0700); err != nil { + return err + } + + // Attempt to remove the results directory when done unless it + // contains files, in which case we should keep it. + defer os.Remove(em.args.MeasurementDir) + + // Call the check-in API to obtain configuration. Note that the value + // returned here MAY have been cached by the engine. + logger.Infof("calling check-in API...") + checkInResp, err := experiment.CallCheckIn(ctx, em.args, sess) + + // Bail if either the check-in API failed or we don't have a reportID + // with which to submit Web Connectivity measurements results. + if err != nil { + return err + } + if checkInResp.NDT == nil { + return ErrNoCheckInInfo + } + + // Obtain and log the report ID. + reportID := checkInResp.NDT.ReportID + logger.Infof("ReportID: %s", reportID) + + // Create an instance of the experiment's measurer. + measurer := &Measurer{ + config: *em.configArgs, + jsonUnmarshal: json.Unmarshal, + } + + // Record when we started running this nettest. + testStartTime := time.Now() + + return experiment.MeasurePossiblyNilInput( + ctx, + em.args, + measurer, + testStartTime, + reportID, + 0, // inputIdx + nil, // input + ) +} diff --git a/internal/experiment/psiphon/main.go b/internal/experiment/psiphon/main.go new file mode 100644 index 0000000000..41a90b4d30 --- /dev/null +++ b/internal/experiment/psiphon/main.go @@ -0,0 +1,88 @@ +package psiphon + +import ( + "context" + "errors" + "os" + "time" + + "github.com/ooni/probe-cli/v3/internal/experiment" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" +) + +// ErrNoCheckInInfo indicates check-in returned no suitable info. +var ErrNoCheckInInfo = errors.New("psiphon: returned no check-in info") + +// ExperimentMain +type ExperimentMain struct { + options model.ExperimentOptions + args *model.ExperimentMainArgs + configArgs *Config +} + +// SetOptions +func (em *ExperimentMain) SetOptions(options model.ExperimentOptions) { + em.options = options +} + +// SetArguments +func (em *ExperimentMain) SetArguments(sess model.ExperimentSession, db *model.DatabaseProps, extraOptions map[string]any) error { + em.args = em.options.SetArguments(sess, db) + em.configArgs = &Config{} + // give precedence to options populated by OONIRun + if extraOptions == nil { + extraOptions = em.options.ExtraOptions() + } + err := setter.SetOptionsAny(em.configArgs, extraOptions) + return err +} + +// Main is the main function of the experiment. +func (em *ExperimentMain) Main(ctx context.Context) error { + sess := em.args.Session + logger := sess.Logger() + + // Create the directory where to store results unless it already exists + if err := os.MkdirAll(em.args.MeasurementDir, 0700); err != nil { + return err + } + + // Attempt to remove the results directory when done unless it + // contains files, in which case we should keep it. + defer os.Remove(em.args.MeasurementDir) + + // Call the check-in API to obtain configuration. Note that the value + // returned here MAY have been cached by the engine. + logger.Infof("calling check-in API...") + checkInResp, err := experiment.CallCheckIn(ctx, em.args, sess) + + // Bail if either the check-in API failed or we don't have a reportID + // with which to submit Web Connectivity measurements results. + if err != nil { + return err + } + if checkInResp.Psiphon == nil { + return ErrNoCheckInInfo + } + + // Obtain and log the report ID. + reportID := checkInResp.Psiphon.ReportID + logger.Infof("ReportID: %s", reportID) + + // Create an instance of the experiment's measurer. + measurer := &Measurer{Config: *em.configArgs} + + // Record when we started running this nettest. + testStartTime := time.Now() + + return experiment.MeasurePossiblyNilInput( + ctx, + em.args, + measurer, + testStartTime, + reportID, + 0, // inputIdx + nil, // input + ) +} diff --git a/internal/experiment/riseupvpn/main.go b/internal/experiment/riseupvpn/main.go new file mode 100644 index 0000000000..301af2e250 --- /dev/null +++ b/internal/experiment/riseupvpn/main.go @@ -0,0 +1,88 @@ +package riseupvpn + +import ( + "context" + "errors" + "os" + "time" + + "github.com/ooni/probe-cli/v3/internal/experiment" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" +) + +// ErrNoCheckInInfo indicates check-in returned no suitable info. +var ErrNoCheckInInfo = errors.New("riseupvpn: returned no check-in info") + +// ExperimentMain +type ExperimentMain struct { + options model.ExperimentOptions + args *model.ExperimentMainArgs + configArgs *Config +} + +// SetOptions +func (em *ExperimentMain) SetOptions(options model.ExperimentOptions) { + em.options = options +} + +// SetArguments +func (em *ExperimentMain) SetArguments(sess model.ExperimentSession, db *model.DatabaseProps, extraOptions map[string]any) error { + em.args = em.options.SetArguments(sess, db) + em.configArgs = &Config{} + // give precedence to options populated by OONIRun + if extraOptions == nil { + extraOptions = em.options.ExtraOptions() + } + err := setter.SetOptionsAny(em.configArgs, extraOptions) + return err +} + +// Main is the main function of the experiment. +func (em *ExperimentMain) Main(ctx context.Context) error { + sess := em.args.Session + logger := sess.Logger() + + // Create the directory where to store results unless it already exists + if err := os.MkdirAll(em.args.MeasurementDir, 0700); err != nil { + return err + } + + // Attempt to remove the results directory when done unless it + // contains files, in which case we should keep it. + defer os.Remove(em.args.MeasurementDir) + + // Call the check-in API to obtain configuration. Note that the value + // returned here MAY have been cached by the engine. + logger.Infof("calling check-in API...") + checkInResp, err := experiment.CallCheckIn(ctx, em.args, sess) + + // Bail if either the check-in API failed or we don't have a reportID + // with which to submit Web Connectivity measurements results. + if err != nil { + return err + } + if checkInResp.RiseupVPN == nil { + return ErrNoCheckInInfo + } + + // Obtain and log the report ID. + reportID := checkInResp.RiseupVPN.ReportID + logger.Infof("ReportID: %s", reportID) + + // Create an instance of the experiment's measurer. + measurer := &Measurer{Config: *em.configArgs} + + // Record when we started running this nettest. + testStartTime := time.Now() + + return experiment.MeasurePossiblyNilInput( + ctx, + em.args, + measurer, + testStartTime, + reportID, + 0, // inputIdx + nil, // input + ) +} diff --git a/internal/experiment/signal/main.go b/internal/experiment/signal/main.go new file mode 100644 index 0000000000..9656e05885 --- /dev/null +++ b/internal/experiment/signal/main.go @@ -0,0 +1,88 @@ +package signal + +import ( + "context" + "errors" + "os" + "time" + + "github.com/ooni/probe-cli/v3/internal/experiment" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" +) + +// ErrNoCheckInInfo indicates check-in returned no suitable info. +var ErrNoCheckInInfo = errors.New("signal: returned no check-in info") + +// ExperimentMain +type ExperimentMain struct { + options model.ExperimentOptions + args *model.ExperimentMainArgs + configArgs *Config +} + +// SetOptions +func (em *ExperimentMain) SetOptions(options model.ExperimentOptions) { + em.options = options +} + +// SetArguments +func (em *ExperimentMain) SetArguments(sess model.ExperimentSession, db *model.DatabaseProps, extraOptions map[string]any) error { + em.args = em.options.SetArguments(sess, db) + em.configArgs = &Config{} + // give precedence to options populated by OONIRun + if extraOptions == nil { + extraOptions = em.options.ExtraOptions() + } + err := setter.SetOptionsAny(em.configArgs, extraOptions) + return err +} + +// Main is the main function of the experiment. +func (em *ExperimentMain) Main(ctx context.Context) error { + sess := em.args.Session + logger := sess.Logger() + + // Create the directory where to store results unless it already exists + if err := os.MkdirAll(em.args.MeasurementDir, 0700); err != nil { + return err + } + + // Attempt to remove the results directory when done unless it + // contains files, in which case we should keep it. + defer os.Remove(em.args.MeasurementDir) + + // Call the check-in API to obtain configuration. Note that the value + // returned here MAY have been cached by the engine. + logger.Infof("calling check-in API...") + checkInResp, err := experiment.CallCheckIn(ctx, em.args, sess) + + // Bail if either the check-in API failed or we don't have a reportID + // with which to submit Web Connectivity measurements results. + if err != nil { + return err + } + if checkInResp.Signal == nil { + return ErrNoCheckInInfo + } + + // Obtain and log the report ID. + reportID := checkInResp.Signal.ReportID + logger.Infof("ReportID: %s", reportID) + + // Create an instance of the experiment's measurer. + measurer := &Measurer{Config: *em.configArgs} + + // Record when we started running this nettest. + testStartTime := time.Now() + + return experiment.MeasurePossiblyNilInput( + ctx, + em.args, + measurer, + testStartTime, + reportID, + 0, // inputIdx + nil, // input + ) +} diff --git a/internal/experiment/telegram/main.go b/internal/experiment/telegram/main.go new file mode 100644 index 0000000000..0649c808e5 --- /dev/null +++ b/internal/experiment/telegram/main.go @@ -0,0 +1,88 @@ +package telegram + +import ( + "context" + "errors" + "os" + "time" + + "github.com/ooni/probe-cli/v3/internal/experiment" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" +) + +// ErrNoCheckInInfo indicates check-in returned no suitable info. +var ErrNoCheckInInfo = errors.New("telegram: returned no check-in info") + +// ExperimentMain +type ExperimentMain struct { + options model.ExperimentOptions + args *model.ExperimentMainArgs + configArgs *Config +} + +// SetOptions +func (em *ExperimentMain) SetOptions(options model.ExperimentOptions) { + em.options = options +} + +// SetArguments +func (em *ExperimentMain) SetArguments(sess model.ExperimentSession, db *model.DatabaseProps, extraOptions map[string]any) error { + em.args = em.options.SetArguments(sess, db) + em.configArgs = &Config{} + // give precedence to options populated by OONIRun + if extraOptions == nil { + extraOptions = em.options.ExtraOptions() + } + err := setter.SetOptionsAny(em.configArgs, extraOptions) + return err +} + +// Main is the main function of the experiment. +func (em *ExperimentMain) Main(ctx context.Context) error { + sess := em.args.Session + logger := sess.Logger() + + // Create the directory where to store results unless it already exists + if err := os.MkdirAll(em.args.MeasurementDir, 0700); err != nil { + return err + } + + // Attempt to remove the results directory when done unless it + // contains files, in which case we should keep it. + defer os.Remove(em.args.MeasurementDir) + + // Call the check-in API to obtain configuration. Note that the value + // returned here MAY have been cached by the engine. + logger.Infof("calling check-in API...") + checkInResp, err := experiment.CallCheckIn(ctx, em.args, sess) + + // Bail if either the check-in API failed or we don't have a reportID + // with which to submit Web Connectivity measurements results. + if err != nil { + return err + } + if checkInResp.Telegram == nil { + return ErrNoCheckInInfo + } + + // Obtain and log the report ID. + reportID := checkInResp.Telegram.ReportID + logger.Infof("ReportID: %s", reportID) + + // Create an instance of the experiment's measurer. + measurer := &Measurer{Config: *em.configArgs} + + // Record when we started running this nettest. + testStartTime := time.Now() + + return experiment.MeasurePossiblyNilInput( + ctx, + em.args, + measurer, + testStartTime, + reportID, + 0, // inputIdx + nil, // input + ) +} diff --git a/internal/experiment/tor/main.go b/internal/experiment/tor/main.go new file mode 100644 index 0000000000..48b587b42a --- /dev/null +++ b/internal/experiment/tor/main.go @@ -0,0 +1,94 @@ +package tor + +import ( + "context" + "errors" + "os" + "time" + + "github.com/ooni/probe-cli/v3/internal/experiment" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" +) + +// ErrNoCheckInInfo indicates check-in returned no suitable info. +var ErrNoCheckInInfo = errors.New("tor: returned no check-in info") + +// ExperimentMain +type ExperimentMain struct { + options model.ExperimentOptions + args *model.ExperimentMainArgs + configArgs *Config +} + +// SetOptions +func (em *ExperimentMain) SetOptions(options model.ExperimentOptions) { + em.options = options +} + +// SetArguments +func (em *ExperimentMain) SetArguments(sess model.ExperimentSession, db *model.DatabaseProps, extraOptions map[string]any) error { + em.args = em.options.SetArguments(sess, db) + em.configArgs = &Config{} + // give precedence to options populated by OONIRun + if extraOptions == nil { + extraOptions = em.options.ExtraOptions() + } + err := setter.SetOptionsAny(em.configArgs, extraOptions) + return err +} + +// Main is the main function of the experiment. +func (em *ExperimentMain) Main(ctx context.Context) error { + sess := em.args.Session + logger := sess.Logger() + + // Create the directory where to store results unless it already exists + if err := os.MkdirAll(em.args.MeasurementDir, 0700); err != nil { + return err + } + + // Attempt to remove the results directory when done unless it + // contains files, in which case we should keep it. + defer os.Remove(em.args.MeasurementDir) + + // Call the check-in API to obtain configuration. Note that the value + // returned here MAY have been cached by the engine. + logger.Infof("calling check-in API...") + checkInResp, err := experiment.CallCheckIn(ctx, em.args, sess) + + // Bail if either the check-in API failed or we don't have a reportID + // with which to submit Web Connectivity measurements results. + if err != nil { + return err + } + if checkInResp.Tor == nil { + return ErrNoCheckInInfo + } + + // Obtain and log the report ID. + reportID := checkInResp.Tor.ReportID + logger.Infof("ReportID: %s", reportID) + + // Create an instance of the experiment's measurer. + measurer := &Measurer{ + config: *em.configArgs, + fetchTorTargets: func(ctx context.Context, sess model.ExperimentSession, + cc string) (map[string]model.OOAPITorTarget, error) { + return sess.FetchTorTargets(ctx, cc) + }, + } + + // Record when we started running this nettest. + testStartTime := time.Now() + + return experiment.MeasurePossiblyNilInput( + ctx, + em.args, + measurer, + testStartTime, + reportID, + 0, // inputIdx + nil, // input + ) +} diff --git a/internal/experiment/webconnectivity/main.go b/internal/experiment/webconnectivity/main.go new file mode 100644 index 0000000000..3c6baa3940 --- /dev/null +++ b/internal/experiment/webconnectivity/main.go @@ -0,0 +1,174 @@ +package webconnectivity + +import ( + "context" + "errors" + "os" + "time" + + "github.com/ooni/probe-cli/v3/internal/experiment" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/setter" +) + +// ErrNoCheckInInfo indicates check-in returned no suitable info. +var ErrNoCheckInInfo = errors.New("webconnectivity: returned no check-in info") + +// ExperimentMain +type ExperimentMain struct { + options model.ExperimentOptions + args *model.ExperimentMainArgs + configArgs *Config +} + +// SetOptions +func (em *ExperimentMain) SetOptions(options model.ExperimentOptions) { + em.options = options +} + +// SetArguments +func (em *ExperimentMain) SetArguments(sess model.ExperimentSession, db *model.DatabaseProps, extraOptions map[string]any) error { + em.args = em.options.SetArguments(sess, db) + em.configArgs = &Config{} + // give precedence to options populated by OONIRun + if extraOptions == nil { + extraOptions = em.options.ExtraOptions() + } + err := setter.SetOptionsAny(em.configArgs, extraOptions) + return err +} + +// Main is the main function of the experiment. +func (em *ExperimentMain) Main(ctx context.Context) error { + sess := em.args.Session + logger := sess.Logger() + + // Create the directory where to store results unless it already exists + if err := os.MkdirAll(em.args.MeasurementDir, 0700); err != nil { + return err + } + + // Attempt to remove the results directory when done unless it + // contains files, in which case we should keep it. + defer os.Remove(em.args.MeasurementDir) + + // Call the check-in API to obtain configuration. Note that the value + // returned here MAY have been cached by the engine. + logger.Infof("calling check-in API...") + checkInResp, err := experiment.CallCheckIn(ctx, em.args, sess) + + // Bail if either the check-in API failed or we don't have a reportID + // with which to submit Web Connectivity measurements results. + if err != nil { + return err + } + if checkInResp.WebConnectivity == nil { + return ErrNoCheckInInfo + } + + // Obtain and log the report ID. + reportID := checkInResp.WebConnectivity.ReportID + logger.Infof("ReportID: %s", reportID) + + // Obtain experiment inputs. + inputs := getInputs(em.args, checkInResp) + + // Create an instance of the experiment's measurer. + measurer := &Measurer{Config: *em.configArgs} + + // Record when we started running this nettest. + testStartTime := time.Now() + + // Create suitable stop policy. + shouldStop := newStopPolicy(em.args, testStartTime) + + // Create suitable progress emitter. + progresser := newProgressEmitter(em.args, inputs, testStartTime) + + // Measure each URL in sequence. + for inputIdx, input := range inputs { + + // Honour max runtime. + if shouldStop() { + break + } + + // Emit progress. + progresser(inputIdx, input.URL) + + // Measure the current URL. + err := experiment.MeasurePossiblyNilInput( + ctx, + em.args, + measurer, + testStartTime, + reportID, + inputIdx, + &input, + ) + + // An error here means stuff like "cannot write to disk". + if err != nil { + return err + } + } + + return nil +} + +// getInputs obtains inputs from either args or checkInResp giving +// priority to user supplied arguments inside args. +func getInputs(args *model.ExperimentMainArgs, checkInResp *model.OOAPICheckInResultNettests) []model.OOAPIURLInfo { + runtimex.Assert(checkInResp.WebConnectivity != nil, "getInputs passed invalid checkInResp") + inputs := args.Inputs + if len(inputs) < 1 { + return checkInResp.WebConnectivity.URLs + } + outputs := []model.OOAPIURLInfo{} + for _, input := range inputs { + outputs = append(outputs, model.OOAPIURLInfo{ + CategoryCode: "MISC", + CountryCode: "ZZ", + URL: input, + }) + } + return outputs +} + +// newStopPolicy creates a new stop policy depending on the +// arguments passed to the experiment in args. +func newStopPolicy(args *model.ExperimentMainArgs, testStartTime time.Time) func() bool { + if args.MaxRuntime <= 0 { + return func() bool { + return false + } + } + maxRuntime := time.Duration(args.MaxRuntime) * time.Second + return func() bool { + return time.Since(testStartTime) > maxRuntime + } +} + +func newProgressEmitter( + args *model.ExperimentMainArgs, + inputs []model.OOAPIURLInfo, + testStartTime time.Time, +) func(idx int, URL string) { + total := len(inputs) + if total <= 0 { + return func(idx int, URL string) {} // just in case + } + if args.MaxRuntime <= 0 { + return func(idx int, URL string) { + percentage := 100.0 * (float64(idx) / float64(total)) + args.Callbacks.OnProgress(percentage, URL) + } + } + maxRuntime := (time.Duration(args.MaxRuntime) * time.Second) + time.Nanosecond // avoid zero division + return func(idx int, URL string) { + elapsed := time.Since(testStartTime) + percentage := 100.0 * (float64(elapsed) / float64(maxRuntime)) + args.Callbacks.OnProgress(percentage, URL) + } +} diff --git a/internal/experiment/whatsapp/main.go b/internal/experiment/whatsapp/main.go new file mode 100644 index 0000000000..77c5e4738d --- /dev/null +++ b/internal/experiment/whatsapp/main.go @@ -0,0 +1,88 @@ +package whatsapp + +import ( + "context" + "errors" + "os" + "time" + + "github.com/ooni/probe-cli/v3/internal/experiment" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" +) + +// ErrNoCheckInInfo indicates check-in returned no suitable info. +var ErrNoCheckInInfo = errors.New("whatsapp: returned no check-in info") + +// ExperimentMain +type ExperimentMain struct { + options model.ExperimentOptions + args *model.ExperimentMainArgs + configArgs *Config +} + +// SetOptions +func (em *ExperimentMain) SetOptions(options model.ExperimentOptions) { + em.options = options +} + +// SetArguments +func (em *ExperimentMain) SetArguments(sess model.ExperimentSession, db *model.DatabaseProps, extraOptions map[string]any) error { + em.args = em.options.SetArguments(sess, db) + em.configArgs = &Config{} + // give precedence to options populated by OONIRun + if extraOptions == nil { + extraOptions = em.options.ExtraOptions() + } + err := setter.SetOptionsAny(em.configArgs, extraOptions) + return err +} + +// Main is the main function of the experiment. +func (em *ExperimentMain) Main(ctx context.Context) error { + sess := em.args.Session + logger := sess.Logger() + + // Create the directory where to store results unless it already exists + if err := os.MkdirAll(em.args.MeasurementDir, 0700); err != nil { + return err + } + + // Attempt to remove the results directory when done unless it + // contains files, in which case we should keep it. + defer os.Remove(em.args.MeasurementDir) + + // Call the check-in API to obtain configuration. Note that the value + // returned here MAY have been cached by the engine. + logger.Infof("calling check-in API...") + checkInResp, err := experiment.CallCheckIn(ctx, em.args, sess) + + // Bail if either the check-in API failed or we don't have a reportID + // with which to submit Web Connectivity measurements results. + if err != nil { + return err + } + if checkInResp.Whatsapp == nil { + return ErrNoCheckInInfo + } + + // Obtain and log the report ID. + reportID := checkInResp.Whatsapp.ReportID + logger.Infof("ReportID: %s", reportID) + + // Create an instance of the experiment's measurer. + measurer := &Measurer{Config: *em.configArgs} + + // Record when we started running this nettest. + testStartTime := time.Now() + + return experiment.MeasurePossiblyNilInput( + ctx, + em.args, + measurer, + testStartTime, + reportID, + 0, // inputIdx + nil, // input + ) +} diff --git a/internal/legacy/mockable/mockable.go b/internal/legacy/mockable/mockable.go index 2f39a254c0..004b7e667f 100644 --- a/internal/legacy/mockable/mockable.go +++ b/internal/legacy/mockable/mockable.go @@ -38,6 +38,11 @@ type Session struct { MockableUserAgent string } +// SubmitMeasurementV2 implements model.ExperimentSession +func (*Session) SubmitMeasurementV2(ctx context.Context, measurement *model.Measurement) error { + panic("unimplemented") +} + // GetTestHelpersByName implements ExperimentSession.GetTestHelpersByName func (sess *Session) GetTestHelpersByName(name string) ([]model.OOAPIService, bool) { services, okay := sess.MockableTestHelpers[name] @@ -140,4 +145,20 @@ func (sess *Session) UserAgent() string { return sess.MockableUserAgent } +func (sess *Session) CheckIn(ctx context.Context, config *model.OOAPICheckInConfig) (*model.OOAPICheckInResultNettests, error) { + panic("not implemented") +} + +func (sess *Session) Platform() string { + panic("not implemented") +} + +func (sess *Session) ResolverASNString() string { + panic("not implemented") +} + +func (sess *Session) ResolverNetworkName() string { + panic("not implemented") +} + var _ model.ExperimentSession = &Session{} diff --git a/internal/model/database.go b/internal/model/database.go index e28a48cf95..374e6cb4da 100644 --- a/internal/model/database.go +++ b/internal/model/database.go @@ -265,3 +265,15 @@ type PerformanceTestKeys struct { Ping float64 `json:"ping"` Bitrate float64 `json:"median_bitrate"` } + +// DatabaseProps contains the database properties for a database instance +type DatabaseProps struct { + // + Database WritableDatabase + + // + DatabaseNetwork *DatabaseNetwork + + // + DatabaseResult *DatabaseResult +} diff --git a/internal/model/experiment.go b/internal/model/experiment.go index 1cba6c1b84..b7026c9926 100644 --- a/internal/model/experiment.go +++ b/internal/model/experiment.go @@ -7,12 +7,14 @@ package model import ( "context" + + "github.com/spf13/cobra" ) // ExperimentSession is the experiment's view of a session. type ExperimentSession interface { - // GetTestHelpersByName returns a list of test helpers with the given name. - GetTestHelpersByName(name string) ([]OOAPIService, bool) + // CheckIn invokes the check-in API. + CheckIn(ctx context.Context, config *OOAPICheckInConfig) (*OOAPICheckInResultNettests, error) // DefaultHTTPClient returns the default HTTPClient used by the session. DefaultHTTPClient() HTTPClient @@ -23,15 +25,42 @@ type ExperimentSession interface { // FetchTorTargets returns the targets for the Tor experiment or an error. FetchTorTargets(ctx context.Context, cc string) (map[string]OOAPITorTarget, error) + // GetTestHelpersByName returns a list of test helpers with the given name. + GetTestHelpersByName(name string) ([]OOAPIService, bool) + // Logger returns the logger used by the session. Logger() Logger + // Platform returns the operating system's platform name. + Platform() string + + // ProbeASNString returns the probe's ASN as a string. + ProbeASNString() string + // ProbeCC returns the country code. ProbeCC() string + // ProbeNetworkName is the name of the probes' ASN. + ProbeNetworkName() string + + // ResolverASNString is the resolver ASN as a string. + ResolverASNString() string + // ResolverIP returns the resolver's IP. ResolverIP() string + // ResolverNetworkName is the name of the resolver's ASN. + ResolverNetworkName() string + + // SoftwareName returns the name of the client software. + SoftwareName() string + + // SoftwareVersion returns the version of the client software. + SoftwareVersion() string + + // SubmitMeasurementV2 submits the given measurement. + SubmitMeasurementV2(ctx context.Context, measurement *Measurement) error + // TempDir returns the session's temporary directory. TempDir() string @@ -130,6 +159,48 @@ type ExperimentArgs struct { Session ExperimentSession } +// ExperimentMainArgs contains the args passed to the experiment's main. +type ExperimentMainArgs struct { + // Annotations contains OPTIONAL annotations. + Annotations map[string]string + + // CategoryCodes OPTIONALLY contains the enabled category codes. + CategoryCodes []string + + // Charging OPTIONALLY indicates whether the phone is charging. + Charging bool + + // Callbacks contains MANDATORY experiment callbacks. + Callbacks ExperimentCallbacks + + // Database is the MANDATORY database to use. + Database WritableDatabase + + // Inputs contains OPTIONAL experiment inputs. + Inputs []string + + // MaxRuntime is the OPTIONAL maximum runtime in seconds. + MaxRuntime int64 + + // MeasurementDir is the MANDATORY directory where to save measurements. + MeasurementDir string + + // NoCollector OPTIONALLY disables submitting the measurements. + NoCollector bool + + // OnWiFi OPTIONALLY indicates whether the phone is using Wi-Fi. + OnWiFi bool + + // ResultID contains the MANDATORY result ID. + ResultID int64 + + // RunType OPTIONALLY indicates in which mode we are running. + RunType RunType + + // Session is the MANDATORY session the experiment can use. + Session ExperimentSession +} + // ExperimentMeasurer is the interface that allows to run a // measurement for a specific experiment. type ExperimentMeasurer interface { @@ -313,3 +384,18 @@ type Saver interface { type ExperimentInputProcessor interface { Run(ctx context.Context) error } + +// ExperimentOptions +type ExperimentOptions interface { + // SetArguments + SetArguments(sess ExperimentSession, db *DatabaseProps) *ExperimentMainArgs + + // ExtraOptions + ExtraOptions() map[string]any + + // BuildWithOONIRun + BuildWithOONIRun(inputs []string, args map[string]any) error + + // BuildFlags + BuildFlags(experimentName string, rootCmd *cobra.Command) +} diff --git a/internal/model/measurement.go b/internal/model/measurement.go index 7cd2a8e13e..668f574c2d 100644 --- a/internal/model/measurement.go +++ b/internal/model/measurement.go @@ -10,9 +10,12 @@ import ( "errors" "fmt" "net" + "runtime" "time" + "github.com/ooni/probe-cli/v3/internal/platform" "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/ooni/probe-cli/v3/internal/version" ) const ( @@ -215,3 +218,55 @@ func scrubTestKeys(m *Measurement, currentIP string) error { data = bytes.ReplaceAll(data, []byte(currentIP), []byte(Scrubbed)) return scrubJSONUnmarshalTestKeys(data, &m.TestKeys) } + +// NewMeasurement creates a new measurement instance. +// +// Arguments: +// +// - sess is the measurement session; +// +// - measurer is the experiment's measurer; +// +// - reportID is the report ID; +// +// - input is the OPTIONAL measurement's input; +// +// - testStartTime is when this nettest started executing; +// +// - annotations contains the annotations. +func NewMeasurement( + sess ExperimentSession, + measurer ExperimentMeasurer, + reportID string, + input string, + testStartTime time.Time, + annotations map[string]string, +) *Measurement { + const dateFormat = "2006-01-02 15:04:05" + utctimenow := time.Now().UTC() + m := &Measurement{ + DataFormatVersion: OOAPIReportDefaultDataFormatVersion, + Input: MeasurementTarget(input), + MeasurementStartTime: utctimenow.Format(dateFormat), + MeasurementStartTimeSaved: utctimenow, + ProbeIP: DefaultProbeIP, + ProbeASN: sess.ProbeASNString(), + ProbeCC: sess.ProbeCC(), + ProbeNetworkName: sess.ProbeNetworkName(), + ReportID: reportID, + ResolverASN: sess.ResolverASNString(), + ResolverIP: sess.ResolverIP(), + ResolverNetworkName: sess.ResolverNetworkName(), + SoftwareName: sess.SoftwareName(), + SoftwareVersion: sess.SoftwareVersion(), + TestName: measurer.ExperimentName(), + TestStartTime: testStartTime.Format(dateFormat), + TestVersion: measurer.ExperimentVersion(), + } + m.AddAnnotations(annotations) // must be before MANDATORY engine annotations + m.AddAnnotation("engine_name", "ooniprobe-engine") + m.AddAnnotation("engine_version", version.Version) + m.AddAnnotation("platform", platform.Name()) + m.AddAnnotation("architecture", runtime.GOARCH) + return m +} diff --git a/internal/model/mocks/session.go b/internal/model/mocks/session.go index d5270f8fd0..c5d11c5d01 100644 --- a/internal/model/mocks/session.go +++ b/internal/model/mocks/session.go @@ -58,6 +58,26 @@ type Session struct { config *model.OOAPICheckInConfig) (*model.OOAPICheckInResultNettests, error) } +// Platform implements model.ExperimentSession +func (*Session) Platform() string { + panic("unimplemented") +} + +// ResolverASNString implements model.ExperimentSession +func (*Session) ResolverASNString() string { + panic("unimplemented") +} + +// ResolverNetworkName implements model.ExperimentSession +func (*Session) ResolverNetworkName() string { + panic("unimplemented") +} + +// SubmitMeasurementV2 implements model.ExperimentSession +func (*Session) SubmitMeasurementV2(ctx context.Context, measurement *model.Measurement) error { + panic("unimplemented") +} + func (sess *Session) GetTestHelpersByName(name string) ([]model.OOAPIService, bool) { return sess.MockGetTestHelpersByName(name) } diff --git a/internal/model/ooapi.go b/internal/model/ooapi.go index 925b112741..5a248c2624 100644 --- a/internal/model/ooapi.go +++ b/internal/model/ooapi.go @@ -55,11 +55,162 @@ type OOAPICheckInInfoWebConnectivity struct { URLs []OOAPIURLInfo `json:"urls"` } +// OOAPICheckInInfoNDT contains the NDT +// part of OOAPICheckInInfo. +type OOAPICheckInInfoNDT struct { + // ReportID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + +// OOAPICheckInInfoDash contains the Dash +// part of OOAPICheckInInfo. +type OOAPICheckInInfoDash struct { + // ReportID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + +// OOAPICheckInInfoHHFM contains the HHFM +// part of OOAPICheckInInfo. +type OOAPICheckInInfoHHFM struct { + // Report ID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + +// OOAPICheckInInfoHIRL contains the HIRL +// part of OOAPICheckInInfo. +type OOAPICheckInInfoHIRL struct { + // Report ID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + +// OOAPICheckInInfoFacebookMessenger contains the FBMessenger +// part of OOAPICheckInInfo. +type OOAPICheckInInfoFacebookMessenger struct { + // ReportID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + +// OOAPICheckInInfoSignal contains the Signal +// part of OOAPICheckInInfo. +type OOAPICheckInInfoSignal struct { + // ReportID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + +// OOAPICheckInInfoTelegram contains the Telegram +// part of OOAPICheckInInfo. +type OOAPICheckInInfoTelegram struct { + // ReportID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + +// OOAPICheckInInfoWhatsapp contains the Whatsapp +// part of OOAPICheckInInfo. +type OOAPICheckInInfoWhatsapp struct { + // ReportID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + +// OOAPICheckInInfoPsiphon contains the Psiphon +// part of OOAPICheckInInfo. +type OOAPICheckInInfoPsiphon struct { + // ReportID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + +// OOAPICheckInInfoTor contains the Tor +// part of OOAPICheckInInfo. +type OOAPICheckInInfoTor struct { + // ReportID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + +// OOAPICheckInInfoDNSCheck contains the DNSCheck +// part of OOAPICheckInInfo. +type OOAPICheckInInfoDNSCheck struct { + // ReportID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + +// OOAPICheckInInfoStunReachability contains the StunReachability +// part of OOAPICheckInInfo. +type OOAPICheckInInfoStunReachability struct { + // ReportID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + +// OOAPICheckInInfoTorsf contains the Torsf +// part of OOAPICheckInInfo. +type OOAPICheckInInfoTorsf struct { + // ReportID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + +// OOAPICheckInInfoVanillaTor contains the VanillaTor +// part of OOAPICheckInInfo. +type OOAPICheckInInfoVanillaTor struct { + // ReportID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + +// OOAPICheckInInfoRiseupVPN contains the RiseupVPN +// part of OOAPICheckInInfo. +type OOAPICheckInInfoRiseupVPN struct { + // ReportID is the report ID the probe should use. + ReportID string `json:"report_id"` +} + // OOAPICheckInResultNettests contains nettests information // returned by the checkin API call. type OOAPICheckInResultNettests struct { // WebConnectivity contains WebConnectivity related information. WebConnectivity *OOAPICheckInInfoWebConnectivity `json:"web_connectivity"` + + // Dash contains Dash related information. + Dash *OOAPICheckInInfoDash `json:"dash"` + + // NDT contains NDT related information. + NDT *OOAPICheckInInfoNDT `json:"ndt"` + + // HHFM contains the HHFM related information. + HHFM *OOAPICheckInInfoHHFM `json:"http_header_field_manipulation"` + + // HIRL contains the HIRL related information. + HIRL *OOAPICheckInInfoHIRL `json:"http_invalid_request_line"` + + // FacebookMessenger contaings Facebook Messenger related information. + FacebookMessenger *OOAPICheckInInfoFacebookMessenger `json:"facebook_messenger"` + + // Signal contains Signal related information. + // TODO: Add Signal to the check-in API response + Signal *OOAPICheckInInfoSignal `json:"signal"` + + // Telegram contains Telegram related information. + Telegram *OOAPICheckInInfoTelegram `json:"telegram"` + + // Whatsapp contains Whatsapp related information. + Whatsapp *OOAPICheckInInfoWhatsapp `json:"whatsapp"` + + // Psiphon contains Psiphon related information. + Psiphon *OOAPICheckInInfoPsiphon `json:"psiphon"` + + // Tor contains Tor related information. + Tor *OOAPICheckInInfoTor `json:"tor"` + + // DNSCheck contains DNSCheck related information. + DNSChck *OOAPICheckInInfoDNSCheck `json:"dnscheck"` + + // StunReachability contains StunReachability related information. + StunReachability *OOAPICheckInInfoStunReachability `json:"stun_reachability"` + + // Torsf contains Torsf related information. + Torsf *OOAPICheckInInfoTorsf `json:"torsf"` + + // VanillaTor contains VanillaTor related information. + VanillaTor *OOAPICheckInInfoVanillaTor `json:"vanilla_tor"` + + // RiseupVPN contains RiseupVPN related information. + RiseupVPN *OOAPICheckInInfoRiseupVPN `json:"riseupvpn"` } // OOAPICheckInResult is the result returned by the checkin API. diff --git a/internal/ooapi/checkin.go b/internal/ooapi/checkin.go index 89753280fe..0f9ac9316a 100644 --- a/internal/ooapi/checkin.go +++ b/internal/ooapi/checkin.go @@ -25,7 +25,7 @@ func NewDescriptorCheckIn( AcceptEncodingGzip: true, // we want a small response Authorization: "", ContentType: httpapi.ApplicationJSON, - LogBody: false, // we don't want to log psiphon config + LogBody: true, // we don't want to log psiphon config MaxBodySize: 0, Method: http.MethodPost, Request: &httpapi.RequestDescriptor[*model.OOAPICheckInConfig]{ diff --git a/internal/oonirunx/link.go b/internal/oonirunx/link.go new file mode 100644 index 0000000000..989f47f2f0 --- /dev/null +++ b/internal/oonirunx/link.go @@ -0,0 +1,85 @@ +package oonirunx + +// +// OONI Run v1 and v2 links +// + +import ( + "context" + "strings" + + "github.com/ooni/probe-cli/v3/internal/engine" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// LinkConfig contains config for an OONI Run link. You MUST fill all the fields that +// are marked as MANDATORY, or the LinkConfig would cause crashes. +type LinkConfig struct { + // AcceptChanges is OPTIONAL and tells this library that the user is + // okay with running a new or modified OONI Run link without previously + // reviewing what it contains or what has changed. + AcceptChanges bool + + // KVStore is the MANDATORY key-value store to use to keep track of + // OONI Run links and know when they are new or modified. + KVStore model.KeyValueStore + + // NoCollector OPTIONALLY indicates we should not be using any collector. + NoCollector bool + + // NoJSON OPTIONALLY indicates we don't want to save measurements to a JSON file. + NoJSON bool + + // ReportFile is the MANDATORY file in which to save reports, which is only + // used when noJSON is set to false. + ReportFile string + + // Session is the MANDATORY Session to use. + Session *engine.Session + + // DatabaseProps is the MANDATORY database properties to use + DatabaseProps *model.DatabaseProps +} + +// LinkRunner knows how to run an OONI Run v1 or v2 link. +type LinkRunner interface { + Run(ctx context.Context) error +} + +// linkRunner implements LinkRunner. +type linkRunner struct { + config *LinkConfig + f func(ctx context.Context, config *LinkConfig, URL string) error + url string +} + +// Run implements LinkRunner.Run. +func (lr *linkRunner) Run(ctx context.Context) error { + return lr.f(ctx, lr.config, lr.url) +} + +// NewLinkRunner creates a suitable link runner for the current config +// and the given URL, which is one of the following: +// +// 1. OONI Run v1 link with https scheme (e.g., https://run.ooni.io/nettest?...) +// +// 2. OONI Run v1 link with ooni scheme (e.g., ooni://nettest?...) +// +// 3. arbitrary URL of the OONI Run v2 descriptor. +func NewLinkRunner(c *LinkConfig, URL string) LinkRunner { + // TODO(bassosimone): add support for v2 deeplinks. + out := &linkRunner{ + config: c, + f: nil, + url: URL, + } + switch { + case strings.HasPrefix(URL, "https://run.ooni.io/nettest"): + out.f = v1Measure + case strings.HasPrefix(URL, "ooni://nettest"): + out.f = v1Measure + default: + out.f = v2MeasureHTTPS + } + return out +} diff --git a/internal/oonirunx/v1.go b/internal/oonirunx/v1.go new file mode 100644 index 0000000000..dc729b742a --- /dev/null +++ b/internal/oonirunx/v1.go @@ -0,0 +1,98 @@ +package oonirunx + +// +// OONI Run v1 implementation +// + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + + "github.com/ooni/probe-cli/v3/internal/registryx" +) + +var ( + // ErrInvalidV1URLScheme indicates a v1 OONI Run URL has an invalid scheme. + ErrInvalidV1URLScheme = errors.New("oonirun: invalid v1 URL scheme") + + // ErrInvalidV1URLHost indicates a v1 OONI Run URL has an invalid host. + ErrInvalidV1URLHost = errors.New("oonirun: invalid v1 URL host") + + // ErrInvalidV1URLPath indicates a v1 OONI Run URL has an invalid path. + ErrInvalidV1URLPath = errors.New("oonirun: invalid v1 URL path") + + // ErrInvalidV1URLQueryArgument indicates a v1 OONI Run URL query argument is invalid. + ErrInvalidV1URLQueryArgument = errors.New("oonirun: invalid v1 URL query argument") +) + +// v1Arguments contains arguments for a v1 OONI Run URL. These arguments are +// always encoded inside of the "ta" field, which is optional. +type v1Arguments struct { + URLs []string `json:"urls"` +} + +// v1Measure performs a measurement using the given v1 OONI Run URL. +func v1Measure(ctx context.Context, config *LinkConfig, URL string) error { + config.Session.Logger().Infof("oonirun/v1: running %s", URL) + pu, err := url.Parse(URL) + if err != nil { + return err + } + switch pu.Scheme { + case "https": + if pu.Host != "run.ooni.io" { + return ErrInvalidV1URLHost + } + if pu.Path != "/nettest" { + return ErrInvalidV1URLPath + } + case "ooni": + if pu.Host != "nettest" { + return ErrInvalidV1URLHost + } + if pu.Path != "" && pu.Path != "/" { + return ErrInvalidV1URLPath + } + default: + return ErrInvalidV1URLScheme + } + name := pu.Query().Get("tn") + if name == "" { + return fmt.Errorf("%w: empty test name", ErrInvalidV1URLQueryArgument) + } + var inputs []string + if ta := pu.Query().Get("ta"); ta != "" { + inputs, err = v1ParseArguments(ta) + if err != nil { + return err + } + } + if mv := pu.Query().Get("mv"); mv != "1.2.0" { + return fmt.Errorf("%w: unknown minimum version", ErrInvalidV1URLQueryArgument) + } + args := make(map[string]any) + options := registryx.AllExperimentOptions[name] + options.BuildWithOONIRun(inputs, args) + extraOptions := make(map[string]any) // the v1 spec does not allow users to pass experiment options + factory := registryx.AllExperiments[name] + factory.SetArguments(config.Session, config.DatabaseProps, extraOptions) + return factory.Main(ctx) +} + +// v1ParseArguments parses the `ta` field of the query string. +func v1ParseArguments(ta string) ([]string, error) { + var inputs []string + pa, err := url.QueryUnescape(ta) + if err != nil { + return nil, fmt.Errorf("%w: %s", ErrInvalidV1URLQueryArgument, err.Error()) + } + var arguments v1Arguments + if err := json.Unmarshal([]byte(pa), &arguments); err != nil { + return nil, fmt.Errorf("%w: %s", ErrInvalidV1URLQueryArgument, err.Error()) + } + inputs = arguments.URLs + return inputs, nil +} diff --git a/internal/oonirunx/v2.go b/internal/oonirunx/v2.go new file mode 100644 index 0000000000..7a3864b245 --- /dev/null +++ b/internal/oonirunx/v2.go @@ -0,0 +1,250 @@ +package oonirunx + +// +// OONI Run v2 implementation +// + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sync/atomic" + + "github.com/hexops/gotextdiff" + "github.com/hexops/gotextdiff/myers" + "github.com/hexops/gotextdiff/span" + "github.com/ooni/probe-cli/v3/internal/httpx" + "github.com/ooni/probe-cli/v3/internal/kvstore" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/registryx" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +var ( + // v2CountEmptyNettestNames counts the number of cases in which we have been + // given an empty nettest name, which is useful for testing. + v2CountEmptyNettestNames = &atomic.Int64{} + + // v2CountFailedExperiments countes the number of failed experiments + // and is useful when testing this package + v2CountFailedExperiments = &atomic.Int64{} +) + +// V2Descriptor describes a list of nettests to run together. +type V2Descriptor struct { + // Name is the name of this descriptor. + Name string `json:"name"` + + // Description contains a long description. + Description string `json:"description"` + + // Author contains the author's name. + Author string `json:"author"` + + // Nettests contains the list of nettests to run. + Nettests []V2Nettest `json:"nettests"` +} + +// V2Nettest specifies how a nettest should run. +type V2Nettest struct { + // Inputs contains inputs for the experiment. + Inputs []string `json:"inputs"` + + // + Args map[string]any `json:"args"` + + // Options contains the experiment options. Any option name starting with + // `Safe` will be available for the experiment run, but omitted from + // the serialized Measurement that the experiment builder will submit + // to the OONI backend. + Options map[string]any `json:"options"` + + // TestName contains the nettest name. + TestName string `json:"test_name"` +} + +// ErrHTTPRequestFailed indicates that an HTTP request failed. +var ErrHTTPRequestFailed = errors.New("oonirun: HTTP request failed") + +// getV2DescriptorFromHTTPSURL GETs a v2Descriptor instance from +// a static URL (e.g., from a GitHub repo or from a Gist). +func getV2DescriptorFromHTTPSURL(ctx context.Context, client model.HTTPClient, + logger model.Logger, URL string) (*V2Descriptor, error) { + template := httpx.APIClientTemplate{ + Accept: "", + Authorization: "", + BaseURL: URL, + HTTPClient: client, + Host: "", + LogBody: true, + Logger: logger, + UserAgent: model.HTTPHeaderUserAgent, + } + var desc V2Descriptor + if err := template.Build().GetJSON(ctx, "", &desc); err != nil { + return nil, err + } + return &desc, nil +} + +// v2DescriptorCache contains all the known v2Descriptor entries. +type v2DescriptorCache struct { + // Entries contains all the cached descriptors. + Entries map[string]*V2Descriptor +} + +// v2DescriptorCacheKey is the name of the kvstore2 entry keeping +// information about already known v2Descriptor instances. +const v2DescriptorCacheKey = "oonirun-v2.state" + +// v2DescriptorCacheLoad loads the v2DescriptorCache. +func v2DescriptorCacheLoad(fsstore model.KeyValueStore) (*v2DescriptorCache, error) { + data, err := fsstore.Get(v2DescriptorCacheKey) + if err != nil { + if errors.Is(err, kvstore.ErrNoSuchKey) { + cache := &v2DescriptorCache{ + Entries: make(map[string]*V2Descriptor), + } + return cache, nil + } + return nil, err + } + var cache v2DescriptorCache + if err := json.Unmarshal(data, &cache); err != nil { + return nil, err + } + if cache.Entries == nil { + cache.Entries = make(map[string]*V2Descriptor) + } + return &cache, nil +} + +// PullChangesWithoutSideEffects fetches v2Descriptor changes. +// +// This function DOES NOT change the state of the cache. It just returns to +// the caller what changed for a given entry. It is up-to-the-caller to choose +// what to do in case there are changes depending on the CLI flags. +// +// Arguments: +// +// - ctx is the context for deadline/cancellation; +// +// - client is the HTTPClient to use; +// +// - URL is the URL from which to download/update the OONIRun v2Descriptor. +// +// Return values: +// +// - oldValue is the old v2Descriptor, which may be nil; +// +// - newValue is the new v2Descriptor, which may be nil; +// +// - err is the error that occurred, or nil in case of success. +func (cache *v2DescriptorCache) PullChangesWithoutSideEffects( + ctx context.Context, client model.HTTPClient, logger model.Logger, + URL string) (oldValue, newValue *V2Descriptor, err error) { + oldValue = cache.Entries[URL] + newValue, err = getV2DescriptorFromHTTPSURL(ctx, client, logger, URL) + return +} + +// Update updates the given cache entry and writes back onto the disk. +// +// Note: this method modifies cache and is not safe for concurrent usage. +func (cache *v2DescriptorCache) Update( + fsstore model.KeyValueStore, URL string, entry *V2Descriptor) error { + cache.Entries[URL] = entry + data, err := json.Marshal(cache) + runtimex.PanicOnError(err, "json.Marshal failed") + return fsstore.Set(v2DescriptorCacheKey, data) +} + +// ErrNilDescriptor indicates that we have been passed a descriptor that is nil. +var ErrNilDescriptor = errors.New("oonirun: descriptor is nil") + +// V2MeasureDescriptor performs the measurement or measurements +// described by the given list of v2Descriptor. +func V2MeasureDescriptor(ctx context.Context, config *LinkConfig, desc *V2Descriptor) error { + if desc == nil { + // Note: we have a test checking that we can handle a nil + // descriptor, yet adding also this extra safety net feels + // more robust in terms of the implementation. + return ErrNilDescriptor + } + logger := config.Session.Logger() + for _, nettest := range desc.Nettests { + if nettest.TestName == "" { + logger.Warn("oonirun: nettest name cannot be empty") + v2CountEmptyNettestNames.Add(1) + continue + } + // populate options using the V2Nettest struct + options := registryx.AllExperimentOptions[nettest.TestName] + options.BuildWithOONIRun(nettest.Inputs, nettest.Args) + // pass the set options to the experiment factory + factory := registryx.AllExperiments[nettest.TestName] + factory.SetArguments(config.Session, config.DatabaseProps, nettest.Options) + err := factory.Main(ctx) + if err != nil { + logger.Warnf("cannot run experiment: %s", err.Error()) + v2CountFailedExperiments.Add(1) + continue + } + } + return nil +} + +// ErrNeedToAcceptChanges indicates that the user needs to accept +// changes (i.e., a new or modified set of descriptors) before +// we can actually run this set of descriptors. +var ErrNeedToAcceptChanges = errors.New("oonirun: need to accept changes") + +// v2DescriptorDiff shows what changed between the old and the new descriptors. +func v2DescriptorDiff(oldValue, newValue *V2Descriptor, URL string) string { + oldData, err := json.MarshalIndent(oldValue, "", " ") + runtimex.PanicOnError(err, "json.MarshalIndent failed unexpectedly") + newData, err := json.MarshalIndent(newValue, "", " ") + runtimex.PanicOnError(err, "json.MarshalIndent failed unexpectedly") + oldString, newString := string(oldData)+"\n", string(newData)+"\n" + oldFile := "OLD " + URL + newFile := "NEW " + URL + edits := myers.ComputeEdits(span.URIFromPath(oldFile), oldString, newString) + return fmt.Sprint(gotextdiff.ToUnified(oldFile, newFile, oldString, edits)) +} + +// v2MeasureHTTPS performs a measurement using an HTTPS v2 OONI Run URL +// and returns whether performing this measurement failed. +// +// This function maintains an on-disk cache that tracks the status of +// OONI Run v2 links. If there are any changes and the user has not +// provided config.AcceptChanges, this function will log what has changed +// and will return with an ErrNeedToAcceptChanges error. +// +// In such a case, the caller SHOULD print additional information +// explaining how to accept changes and then SHOULD exit 1 or similar. +func v2MeasureHTTPS(ctx context.Context, config *LinkConfig, URL string) error { + logger := config.Session.Logger() + logger.Infof("oonirun/v2: running %s", URL) + cache, err := v2DescriptorCacheLoad(config.KVStore) + if err != nil { + return err + } + clnt := config.Session.DefaultHTTPClient() + oldValue, newValue, err := cache.PullChangesWithoutSideEffects(ctx, clnt, logger, URL) + if err != nil { + return err + } + diff := v2DescriptorDiff(oldValue, newValue, URL) + if !config.AcceptChanges && diff != "" { + logger.Warnf("oonirun: %s changed as follows:\n\n%s", URL, diff) + logger.Warnf("oonirun: we are not going to run this link until you accept changes") + return ErrNeedToAcceptChanges + } + if diff != "" { + if err := cache.Update(config.KVStore, URL, newValue); err != nil { + return err + } + } + return V2MeasureDescriptor(ctx, config, newValue) // handles nil newValue gracefully +} diff --git a/internal/probeservices/collector.go b/internal/probeservices/collector.go index 54818f2749..8779859d33 100644 --- a/internal/probeservices/collector.go +++ b/internal/probeservices/collector.go @@ -97,6 +97,21 @@ func (r reportChan) SubmitMeasurement(ctx context.Context, m *model.Measurement) return nil } +var ErrMissingReportID = errors.New("probeservices: missing report ID") + +func (c Client) SubmitMeasurementV2(ctx context.Context, m *model.Measurement) error { + if m.ReportID == "" { + return ErrMissingReportID + } + var updateResponse model.OOAPICollectorUpdateResponse + return c.APIClientTemplate.WithBodyLogging().Build().PostJSON( + ctx, fmt.Sprintf("/report/%s", m.ReportID), model.OOAPICollectorUpdateRequest{ + Format: "json", + Content: m, + }, &updateResponse, + ) +} + // ReportID returns the report ID. func (r reportChan) ReportID() string { return r.ID diff --git a/internal/registryx/allexperiments.go b/internal/registryx/allexperiments.go new file mode 100644 index 0000000000..718e4f5b3d --- /dev/null +++ b/internal/registryx/allexperiments.go @@ -0,0 +1,16 @@ +package registryx + +import "github.com/ooni/probe-cli/v3/internal/model" + +// Where we register all the available experiments. +var AllExperiments = map[string]Experiment{} + +// ExperimentNames returns the name of all experiments +func ExperimentNames() (names []string) { + for key := range AllExperiments { + names = append(names, key) + } + return +} + +var AllExperimentOptions = map[string]model.ExperimentOptions{} diff --git a/internal/registryx/dash.go b/internal/registryx/dash.go new file mode 100644 index 0000000000..c0b6ba147e --- /dev/null +++ b/internal/registryx/dash.go @@ -0,0 +1,80 @@ +package registryx + +import ( + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/experiment/dash" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" + "github.com/spf13/cobra" +) + +type dashOptions struct { + Annotations []string + ConfigOptions []string +} + +var _ model.ExperimentOptions = &dashOptions{} + +func init() { + options := &dashOptions{} + AllExperimentOptions["dash"] = options + AllExperiments["dash"] = &dash.ExperimentMain{} +} + +// SetArguments +func (do *dashOptions) SetArguments(sess model.ExperimentSession, + db *model.DatabaseProps) *model.ExperimentMainArgs { + annotations := mustMakeMapStringString(do.Annotations) + return &model.ExperimentMainArgs{ + Annotations: annotations, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category + Charging: true, + Callbacks: model.NewPrinterCallbacks(log.Log), + Database: db.Database, + Inputs: nil, + MaxRuntime: 0, + MeasurementDir: db.DatabaseResult.MeasurementDir, + NoCollector: false, + OnWiFi: true, + ResultID: db.DatabaseResult.ID, + RunType: model.RunTypeManual, + Session: sess, + } +} + +// ExtraOptions +func (do *dashOptions) ExtraOptions() map[string]any { + return mustMakeMapStringAny(do.ConfigOptions) +} + +// BuildWithOONIRun +func (do *dashOptions) BuildWithOONIRun(inputs []string, args map[string]any) error { + if err := setter.SetOptionsAny(do, args); err != nil { + return err + } + return nil +} + +// BuildFlags +func (do *dashOptions) BuildFlags(experimentName string, rootCmd *cobra.Command) { + flags := rootCmd.Flags() + config := dash.Config{} + + flags.StringSliceVarP( + &do.Annotations, + "annotation", + "A", + []string{}, + "add KEY=VALUE annotation to the report (can be repeated multiple times)", + ) + + if doc := setter.DocumentationForOptions(experimentName, config); doc != "" { + flags.StringSliceVarP( + &do.ConfigOptions, + "options", + "O", + []string{}, + doc, + ) + } +} diff --git a/internal/registryx/factory.go b/internal/registryx/factory.go new file mode 100644 index 0000000000..d9f307679d --- /dev/null +++ b/internal/registryx/factory.go @@ -0,0 +1,26 @@ +package registryx + +import ( + "context" + + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/spf13/cobra" +) + +// Experiment +type Experiment interface { + // + Main(ctx context.Context) error + + // + SetOptions(options model.ExperimentOptions) + + // + SetArguments(sess model.ExperimentSession, db *model.DatabaseProps, extraOptions map[string]any) error +} + +// Factory is a forwarder for the respective experiment's main +type Factory struct { + // BuildFlags initializes the experiment specific flags + BuildFlags func(experimentName string, rootCmd *cobra.Command) model.ExperimentOptions +} diff --git a/internal/registryx/fbmessenger.go b/internal/registryx/fbmessenger.go new file mode 100644 index 0000000000..129b61229c --- /dev/null +++ b/internal/registryx/fbmessenger.go @@ -0,0 +1,80 @@ +package registryx + +import ( + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/experiment/fbmessenger" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" + "github.com/spf13/cobra" +) + +type fbmessengerOptions struct { + Annotations []string + ConfigOptions []string +} + +var _ model.ExperimentOptions = &fbmessengerOptions{} + +func init() { + options := &fbmessengerOptions{} + AllExperimentOptions["facebook_messenger"] = options + AllExperiments["facebook_messenger"] = &fbmessenger.ExperimentMain{} +} + +// SetAruments +func (fbo *fbmessengerOptions) SetArguments(sess model.ExperimentSession, + db *model.DatabaseProps) *model.ExperimentMainArgs { + annotations := mustMakeMapStringString(fbo.Annotations) + return &model.ExperimentMainArgs{ + Annotations: annotations, + CategoryCodes: nil, // accept any category + Charging: true, + Callbacks: model.NewPrinterCallbacks(log.Log), + Database: db.Database, + Inputs: nil, + MaxRuntime: 0, + MeasurementDir: db.DatabaseResult.MeasurementDir, + NoCollector: false, + OnWiFi: true, + ResultID: db.DatabaseResult.ID, + RunType: model.RunTypeManual, + Session: sess, + } +} + +// ExtraOptions +func (fbo *fbmessengerOptions) ExtraOptions() map[string]any { + return mustMakeMapStringAny(fbo.ConfigOptions) +} + +// BuildWithOONIRun +func (fbo *fbmessengerOptions) BuildWithOONIRun(inputs []string, args map[string]any) error { + if err := setter.SetOptionsAny(fbo, args); err != nil { + return err + } + return nil +} + +// BuildFlags +func (fbo *fbmessengerOptions) BuildFlags(experimentName string, rootCmd *cobra.Command) { + flags := rootCmd.Flags() + config := fbmessenger.Config{} + + flags.StringSliceVarP( + &fbo.Annotations, + "annotation", + "A", + []string{}, + "add KEY=VALUE annotation to the report (can be repeated multiple times)", + ) + + if doc := setter.DocumentationForOptions(experimentName, config); doc != "" { + flags.StringSliceVarP( + &fbo.ConfigOptions, + "options", + "O", + []string{}, + doc, + ) + } +} diff --git a/internal/registryx/hhfm.go b/internal/registryx/hhfm.go new file mode 100644 index 0000000000..a7f91f78ea --- /dev/null +++ b/internal/registryx/hhfm.go @@ -0,0 +1,113 @@ +package registryx + +import ( + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/experiment/hhfm" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" + "github.com/spf13/cobra" +) + +// hhfmOptions contains options for hhfm. +type hhfmOptions struct { + Annotations []string + InputFilePaths []string + Inputs []string + MaxRuntime int64 + Random bool + ConfigOptions []string +} + +var _ model.ExperimentOptions = &hhfmOptions{} + +func init() { + options := &hhfmOptions{} + AllExperimentOptions["hhfm"] = options + AllExperiments["hhfm"] = &hhfm.ExperimentMain{} +} + +func (hhfmo *hhfmOptions) SetArguments(sess model.ExperimentSession, + db *model.DatabaseProps) *model.ExperimentMainArgs { + annotations := mustMakeMapStringString(hhfmo.Annotations) + return &model.ExperimentMainArgs{ + Annotations: annotations, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category + Charging: true, + Callbacks: model.NewPrinterCallbacks(log.Log), + Database: db.Database, + Inputs: hhfmo.Inputs, + MaxRuntime: hhfmo.MaxRuntime, + MeasurementDir: db.DatabaseResult.MeasurementDir, + NoCollector: false, + OnWiFi: true, + ResultID: db.DatabaseResult.ID, + RunType: model.RunTypeManual, + Session: sess, + } +} + +// ExtraOptions +func (hhfmo *hhfmOptions) ExtraOptions() map[string]any { + return mustMakeMapStringAny(hhfmo.ConfigOptions) +} + +// BuildWithOONIRun +func (hhfmo *hhfmOptions) BuildWithOONIRun(inputs []string, args map[string]any) error { + if err := setter.SetOptionsAny(hhfmo, args); err != nil { + return err + } + return nil +} + +func (hhfmo *hhfmOptions) BuildFlags(experimentName string, rootCmd *cobra.Command) { + flags := rootCmd.Flags() + config := hhfm.Config{} + + flags.StringSliceVarP( + &hhfmo.Annotations, + "annotation", + "A", + []string{}, + "add KEY=VALUE annotation to the report (can be repeated multiple times)", + ) + + flags.StringSliceVarP( + &hhfmo.InputFilePaths, + "input-file", + "f", + []string{}, + "path to file to supply test dependent input (may be specified multiple times)", + ) + + flags.StringSliceVarP( + &hhfmo.Inputs, + "input", + "i", + []string{}, + "add test-dependent input (may be specified multiple times)", + ) + + flags.Int64Var( + &hhfmo.MaxRuntime, + "max-runtime", + 0, + "maximum runtime in seconds for the experiment (zero means infinite)", + ) + + flags.BoolVar( + &hhfmo.Random, + "random", + false, + "randomize the inputs list", + ) + + if doc := setter.DocumentationForOptions(experimentName, config); doc != "" { + flags.StringSliceVarP( + &hhfmo.ConfigOptions, + "options", + "O", + []string{}, + doc, + ) + } +} diff --git a/internal/registryx/hirl.go b/internal/registryx/hirl.go new file mode 100644 index 0000000000..7ece146574 --- /dev/null +++ b/internal/registryx/hirl.go @@ -0,0 +1,80 @@ +package registryx + +import ( + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/experiment/hirl" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" + "github.com/spf13/cobra" +) + +type hirlOptions struct { + Annotations []string + ConfigOptions []string +} + +var _ model.ExperimentOptions = &hirlOptions{} + +func init() { + options := &hirlOptions{} + AllExperimentOptions["hirl"] = options + AllExperiments["hirl"] = &hirl.ExperimentMain{} +} + +// SetArguments +func (hirlo *hirlOptions) SetArguments(sess model.ExperimentSession, + db *model.DatabaseProps) *model.ExperimentMainArgs { + annotations := mustMakeMapStringString(hirlo.Annotations) + return &model.ExperimentMainArgs{ + Annotations: annotations, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category + Charging: true, + Callbacks: model.NewPrinterCallbacks(log.Log), + Database: db.Database, + Inputs: nil, + MaxRuntime: 0, + MeasurementDir: db.DatabaseResult.MeasurementDir, + NoCollector: false, + OnWiFi: true, + ResultID: db.DatabaseResult.ID, + RunType: model.RunTypeManual, + Session: sess, + } +} + +// ExtraOptions +func (hirlo *hirlOptions) ExtraOptions() map[string]any { + return mustMakeMapStringAny(hirlo.ConfigOptions) +} + +// BuildWithOONIRun +func (hirlo *hirlOptions) BuildWithOONIRun(inputs []string, args map[string]any) error { + if err := setter.SetOptionsAny(hirlo, args); err != nil { + return err + } + return nil +} + +// BuildFlags +func (hirlo *hirlOptions) BuildFlags(experimentName string, rootCmd *cobra.Command) { + flags := rootCmd.Flags() + config := hirl.Config{} + + flags.StringSliceVarP( + &hirlo.Annotations, + "annotation", + "A", + []string{}, + "add KEY=VALUE annotation to the report (can be repeated multiple times)", + ) + + if doc := setter.DocumentationForOptions(experimentName, config); doc != "" { + flags.StringSliceVarP( + &hirlo.ConfigOptions, + "options", + "O", + []string{}, + doc, + ) + } +} diff --git a/internal/registryx/ndt.go b/internal/registryx/ndt.go new file mode 100644 index 0000000000..91277c016d --- /dev/null +++ b/internal/registryx/ndt.go @@ -0,0 +1,80 @@ +package registryx + +import ( + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/experiment/ndt7" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" + "github.com/spf13/cobra" +) + +type ndtOptions struct { + Annotations []string + ConfigOptions []string +} + +var _ model.ExperimentOptions = &ndtOptions{} + +func init() { + options := &ndtOptions{} + AllExperimentOptions["ndt"] = options + AllExperiments["ndt"] = &ndt7.ExperimentMain{} +} + +// SetArguments +func (ndto *ndtOptions) SetArguments(sess model.ExperimentSession, + db *model.DatabaseProps) *model.ExperimentMainArgs { + annotations := mustMakeMapStringString(ndto.Annotations) + return &model.ExperimentMainArgs{ + Annotations: annotations, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category + Charging: true, + Callbacks: model.NewPrinterCallbacks(log.Log), + Database: db.Database, + Inputs: nil, + MaxRuntime: 0, + MeasurementDir: db.DatabaseResult.MeasurementDir, + NoCollector: false, + OnWiFi: true, + ResultID: db.DatabaseResult.ID, + RunType: model.RunTypeManual, + Session: sess, + } +} + +// ExtraOptions +func (ndto *ndtOptions) ExtraOptions() map[string]any { + return mustMakeMapStringAny(ndto.ConfigOptions) +} + +// BuildWithOONIRun +func (ndto *ndtOptions) BuildWithOONIRun(inputs []string, args map[string]any) error { + if err := setter.SetOptionsAny(ndto, args); err != nil { + return err + } + return nil +} + +// BuildFlags +func (ndto *ndtOptions) BuildFlags(experimentName string, rootCmd *cobra.Command) { + flags := rootCmd.Flags() + config := ndt7.Config{} + + flags.StringSliceVarP( + &ndto.Annotations, + "annotation", + "A", + []string{}, + "add KEY=VALUE annotation to the report (can be repeated multiple times)", + ) + + if doc := setter.DocumentationForOptions(experimentName, config); doc != "" { + flags.StringSliceVarP( + &ndto.ConfigOptions, + "options", + "O", + []string{}, + doc, + ) + } +} diff --git a/internal/registryx/psiphon.go b/internal/registryx/psiphon.go new file mode 100644 index 0000000000..79d52781ce --- /dev/null +++ b/internal/registryx/psiphon.go @@ -0,0 +1,80 @@ +package registryx + +import ( + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/experiment/psiphon" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" + "github.com/spf13/cobra" +) + +type psiphonOptions struct { + Annotations []string + ConfigOptions []string +} + +var _ model.ExperimentOptions = &psiphonOptions{} + +func init() { + options := &psiphonOptions{} + AllExperimentOptions["psiphon"] = options + AllExperiments["psiphon"] = &psiphon.ExperimentMain{} +} + +// SetArguments +func (po *psiphonOptions) SetArguments(sess model.ExperimentSession, + db *model.DatabaseProps) *model.ExperimentMainArgs { + annotations := mustMakeMapStringString(po.Annotations) + return &model.ExperimentMainArgs{ + Annotations: annotations, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category + Charging: true, + Callbacks: model.NewPrinterCallbacks(log.Log), + Database: db.Database, + Inputs: nil, + MaxRuntime: 0, + MeasurementDir: db.DatabaseResult.MeasurementDir, + NoCollector: false, + OnWiFi: true, + ResultID: db.DatabaseResult.ID, + RunType: model.RunTypeManual, + Session: sess, + } +} + +// ExtraOptions +func (po *psiphonOptions) ExtraOptions() map[string]any { + return mustMakeMapStringAny(po.ConfigOptions) +} + +// BuildWithOONIRun +func (po *psiphonOptions) BuildWithOONIRun(inputs []string, args map[string]any) error { + if err := setter.SetOptionsAny(po, args); err != nil { + return err + } + return nil +} + +// BuildFlags +func (po *psiphonOptions) BuildFlags(experimentName string, rootCmd *cobra.Command) { + flags := rootCmd.Flags() + config := psiphon.Config{} + + flags.StringSliceVarP( + &po.Annotations, + "annotation", + "A", + []string{}, + "add KEY=VALUE annotation to the report (can be repeated multiple times)", + ) + + if doc := setter.DocumentationForOptions(experimentName, config); doc != "" { + flags.StringSliceVarP( + &po.ConfigOptions, + "options", + "O", + []string{}, + doc, + ) + } +} diff --git a/internal/registryx/riseupvpn.go b/internal/registryx/riseupvpn.go new file mode 100644 index 0000000000..73ba7451b3 --- /dev/null +++ b/internal/registryx/riseupvpn.go @@ -0,0 +1,80 @@ +package registryx + +import ( + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/experiment/riseupvpn" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" + "github.com/spf13/cobra" +) + +type riseupvpnOptions struct { + Annotations []string + ConfigOptions []string +} + +var _ model.ExperimentOptions = &riseupvpnOptions{} + +func init() { + options := &riseupvpnOptions{} + AllExperimentOptions["riseupvpn"] = options + AllExperiments["riseupvpn"] = &riseupvpn.ExperimentMain{} +} + +// SetArguments +func (ro *riseupvpnOptions) SetArguments(sess model.ExperimentSession, + db *model.DatabaseProps) *model.ExperimentMainArgs { + annotations := mustMakeMapStringString(ro.Annotations) + return &model.ExperimentMainArgs{ + Annotations: annotations, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category + Charging: true, + Callbacks: model.NewPrinterCallbacks(log.Log), + Database: db.Database, + Inputs: nil, + MaxRuntime: 0, + MeasurementDir: db.DatabaseResult.MeasurementDir, + NoCollector: false, + OnWiFi: true, + ResultID: db.DatabaseResult.ID, + RunType: model.RunTypeManual, + Session: sess, + } +} + +// ExtraOptions +func (ro *riseupvpnOptions) ExtraOptions() map[string]any { + return mustMakeMapStringAny(ro.ConfigOptions) +} + +// BuildWithOONIRun +func (ro *riseupvpnOptions) BuildWithOONIRun(inputs []string, args map[string]any) error { + if err := setter.SetOptionsAny(ro, args); err != nil { + return err + } + return nil +} + +// BuildFlags +func (ro *riseupvpnOptions) BuildFlags(experimentName string, rootCmd *cobra.Command) { + flags := rootCmd.Flags() + config := &riseupvpn.Config{} + + flags.StringSliceVarP( + &ro.Annotations, + "annotation", + "A", + []string{}, + "add KEY=VALUE annotation to the report (can be repeated multiple times)", + ) + + if doc := setter.DocumentationForOptions(experimentName, config); doc != "" { + flags.StringSliceVarP( + &ro.ConfigOptions, + "options", + "O", + []string{}, + doc, + ) + } +} diff --git a/internal/registryx/signal.go b/internal/registryx/signal.go new file mode 100644 index 0000000000..ef92dd19fa --- /dev/null +++ b/internal/registryx/signal.go @@ -0,0 +1,80 @@ +package registryx + +import ( + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/experiment/signal" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" + "github.com/spf13/cobra" +) + +type signalOptions struct { + Annotations []string + ConfigOptions []string +} + +var _ model.ExperimentOptions = &signalOptions{} + +func init() { + options := &signalOptions{} + AllExperimentOptions["signal"] = options + AllExperiments["signal"] = &signal.ExperimentMain{} +} + +// SetArguments +func (so *signalOptions) SetArguments(sess model.ExperimentSession, + db *model.DatabaseProps) *model.ExperimentMainArgs { + annotations := mustMakeMapStringString(so.Annotations) + return &model.ExperimentMainArgs{ + Annotations: annotations, + CategoryCodes: nil, // accept any category + Charging: true, + Callbacks: model.NewPrinterCallbacks(log.Log), + Database: db.Database, + Inputs: nil, + MaxRuntime: 0, + MeasurementDir: db.DatabaseResult.MeasurementDir, + NoCollector: false, + OnWiFi: true, + ResultID: db.DatabaseResult.ID, + RunType: model.RunTypeManual, + Session: sess, + } +} + +// ExtraOptions +func (so *signalOptions) ExtraOptions() map[string]any { + return mustMakeMapStringAny(so.ConfigOptions) +} + +// BuildWithOONIRun +func (so *signalOptions) BuildWithOONIRun(inputs []string, args map[string]any) error { + if err := setter.SetOptionsAny(so, args); err != nil { + return err + } + return nil +} + +// BuildFlags +func (so *signalOptions) BuildFlags(experimentName string, rootCmd *cobra.Command) { + flags := rootCmd.Flags() + config := &signal.Config{} + + flags.StringSliceVarP( + &so.Annotations, + "annotation", + "A", + []string{}, + "add KEY=VALUE annotation to the report (can be repeated multiple times)", + ) + + if doc := setter.DocumentationForOptions(experimentName, config); doc != "" { + flags.StringSliceVarP( + &so.ConfigOptions, + "options", + "O", + []string{}, + doc, + ) + } +} diff --git a/internal/registryx/telegram.go b/internal/registryx/telegram.go new file mode 100644 index 0000000000..ee4af7b550 --- /dev/null +++ b/internal/registryx/telegram.go @@ -0,0 +1,80 @@ +package registryx + +import ( + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/experiment/telegram" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" + "github.com/spf13/cobra" +) + +type telegramOptions struct { + Annotations []string + ConfigOptions []string +} + +var _ model.ExperimentOptions = &telegramOptions{} + +func init() { + options := &telegramOptions{} + AllExperimentOptions["telegram"] = options + AllExperiments["telegram"] = &telegram.ExperimentMain{} +} + +// SetArguments +func (to *telegramOptions) SetArguments(sess model.ExperimentSession, + db *model.DatabaseProps) *model.ExperimentMainArgs { + annotations := mustMakeMapStringString(to.Annotations) + return &model.ExperimentMainArgs{ + Annotations: annotations, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category + Charging: true, + Callbacks: model.NewPrinterCallbacks(log.Log), + Database: db.Database, + Inputs: nil, + MaxRuntime: 0, + MeasurementDir: db.DatabaseResult.MeasurementDir, + NoCollector: false, + OnWiFi: true, + ResultID: db.DatabaseResult.ID, + RunType: model.RunTypeManual, + Session: sess, + } +} + +// ExtraOptions +func (to *telegramOptions) ExtraOptions() map[string]any { + return mustMakeMapStringAny(to.ConfigOptions) +} + +// BuildWithOONIRun +func (to *telegramOptions) BuildWithOONIRun(inputs []string, args map[string]any) error { + if err := setter.SetOptionsAny(to, args); err != nil { + return err + } + return nil +} + +// BuildFlags +func (to *telegramOptions) BuildFlags(experimentName string, rootCmd *cobra.Command) { + flags := rootCmd.Flags() + config := &telegram.Config{} + + flags.StringSliceVarP( + &to.Annotations, + "annotation", + "A", + []string{}, + "add KEY=VALUE annotation to the report (can be repeated multiple times)", + ) + + if doc := setter.DocumentationForOptions(experimentName, config); doc != "" { + flags.StringSliceVarP( + &to.ConfigOptions, + "options", + "O", + []string{}, + doc, + ) + } +} diff --git a/internal/registryx/tor.go b/internal/registryx/tor.go new file mode 100644 index 0000000000..13a4d583ec --- /dev/null +++ b/internal/registryx/tor.go @@ -0,0 +1,76 @@ +package registryx + +import ( + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/experiment/tor" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" + "github.com/spf13/cobra" +) + +type torOptions struct { + Annotations []string + ConfigOptions []string +} + +var _ model.ExperimentOptions = &torOptions{} + +func init() { + options := &webConnectivityOptions{} + AllExperimentOptions["tor"] = options + AllExperiments["tor"] = &tor.ExperimentMain{} +} + +func (toro *torOptions) SetArguments(sess model.ExperimentSession, + db *model.DatabaseProps) *model.ExperimentMainArgs { + annotations := mustMakeMapStringString(toro.Annotations) + return &model.ExperimentMainArgs{ + Annotations: annotations, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category + Charging: true, + Callbacks: model.NewPrinterCallbacks(log.Log), + Database: db.Database, + Inputs: nil, + MaxRuntime: 0, + MeasurementDir: db.DatabaseResult.MeasurementDir, + NoCollector: false, + OnWiFi: true, + ResultID: db.DatabaseResult.ID, + RunType: model.RunTypeManual, + Session: sess, + } +} + +func (toro *torOptions) ExtraOptions() map[string]any { + return mustMakeMapStringAny(toro.ConfigOptions) +} + +func (toro *torOptions) BuildWithOONIRun(inputs []string, args map[string]any) error { + if err := setter.SetOptionsAny(toro, args); err != nil { + return err + } + return nil +} + +func (toro *torOptions) BuildFlags(experimentName string, rootCmd *cobra.Command) { + flags := rootCmd.Flags() + config := &tor.Config{} + + flags.StringSliceVarP( + &toro.Annotations, + "annotation", + "A", + []string{}, + "add KEY=VALUE annotation to the report (can be repeated multiple times)", + ) + + if doc := setter.DocumentationForOptions(experimentName, config); doc != "" { + flags.StringSliceVarP( + &toro.ConfigOptions, + "options", + "O", + []string{}, + doc, + ) + } +} diff --git a/internal/registryx/utils.go b/internal/registryx/utils.go new file mode 100644 index 0000000000..3e7703f2dd --- /dev/null +++ b/internal/registryx/utils.go @@ -0,0 +1,42 @@ +package registryx + +import ( + "errors" + "strings" + + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// splitPair takes in input a string in the form KEY=VALUE and splits it. This +// function returns an error if it cannot find the = character to split the string. +func splitPair(s string) (string, string, error) { + v := strings.SplitN(s, "=", 2) + if len(v) != 2 { + return "", "", errors.New("invalid key-value pair") + } + return v[0], v[1], nil +} + +// mustMakeMapStringAny makes a map from string to any using as input a list +// of key-value pairs used to initialize the map, or panics on error +func mustMakeMapStringAny(input []string) (output map[string]any) { + output = make(map[string]any) + for _, opt := range input { + key, value, err := splitPair(opt) + runtimex.PanicOnError(err, "cannot split key-value pair") + output[key] = value + } + return +} + +// mustMakeMapStringString makes a map from string to string using as input a list +// of key-value pairs used to initialize the map, or panics on error +func mustMakeMapStringString(input []string) (output map[string]string) { + output = make(map[string]string) + for _, opt := range input { + key, value, err := splitPair(opt) + runtimex.PanicOnError(err, "cannot split key-value pair") + output[key] = value + } + return +} diff --git a/internal/registryx/webconnectivity.go b/internal/registryx/webconnectivity.go new file mode 100644 index 0000000000..5d460638d8 --- /dev/null +++ b/internal/registryx/webconnectivity.go @@ -0,0 +1,111 @@ +package registryx + +import ( + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/experiment/webconnectivity" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" + "github.com/spf13/cobra" +) + +// webConnectivityOptions contains options for web connectivity. +type webConnectivityOptions struct { + Annotations []string + InputFilePaths []string + Inputs []string + MaxRuntime int64 + Random bool + ConfigOptions []string +} + +var _ model.ExperimentOptions = &webConnectivityOptions{} + +func init() { + options := &webConnectivityOptions{} + AllExperimentOptions["web_connectivity"] = options + AllExperiments["web_connectivity"] = &webconnectivity.ExperimentMain{} +} + +func (wco *webConnectivityOptions) SetArguments(sess model.ExperimentSession, + db *model.DatabaseProps) *model.ExperimentMainArgs { + return &model.ExperimentMainArgs{ + Annotations: map[string]string{}, // TODO(bassosimone): fill + CategoryCodes: nil, // accept any category + Charging: true, + Callbacks: model.NewPrinterCallbacks(log.Log), + Database: db.Database, + Inputs: wco.Inputs, + MaxRuntime: wco.MaxRuntime, + MeasurementDir: db.DatabaseResult.MeasurementDir, + NoCollector: false, + OnWiFi: true, + ResultID: db.DatabaseResult.ID, + RunType: model.RunTypeManual, + Session: sess, + } +} + +func (wco *webConnectivityOptions) ExtraOptions() map[string]any { + return mustMakeMapStringAny(wco.ConfigOptions) +} + +func (wco *webConnectivityOptions) BuildWithOONIRun(inputs []string, args map[string]any) error { + wco.Inputs = inputs + if err := setter.SetOptionsAny(wco, args); err != nil { + return err + } + return nil +} + +func (wco *webConnectivityOptions) BuildFlags(experimentName string, rootCmd *cobra.Command) { + flags := rootCmd.Flags() + config := &webconnectivity.Config{} + + flags.StringSliceVarP( + &wco.Annotations, + "annotation", + "A", + []string{}, + "add KEY=VALUE annotation to the report (can be repeated multiple times)", + ) + + flags.StringSliceVarP( + &wco.InputFilePaths, + "input-file", + "f", + []string{}, + "path to file to supply test dependent input (may be specified multiple times)", + ) + + flags.StringSliceVarP( + &wco.Inputs, + "input", + "i", + []string{}, + "add test-dependent input (may be specified multiple times)", + ) + + flags.Int64Var( + &wco.MaxRuntime, + "max-runtime", + 0, + "maximum runtime in seconds for the experiment (zero means infinite)", + ) + + flags.BoolVar( + &wco.Random, + "random", + false, + "randomize the inputs list", + ) + + if doc := setter.DocumentationForOptions(experimentName, config); doc != "" { + flags.StringSliceVarP( + &wco.ConfigOptions, + "options", + "O", + []string{}, + doc, + ) + } +} diff --git a/internal/registryx/whatsapp.go b/internal/registryx/whatsapp.go new file mode 100644 index 0000000000..6ed34a7d06 --- /dev/null +++ b/internal/registryx/whatsapp.go @@ -0,0 +1,80 @@ +package registryx + +import ( + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/experiment/whatsapp" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/setter" + "github.com/spf13/cobra" +) + +type whatsappOptions struct { + Annotations []string + ConfigOptions []string +} + +var _ model.ExperimentOptions = &whatsappOptions{} + +func init() { + options := &whatsappOptions{} + AllExperimentOptions["whatsapp"] = options + AllExperiments["whatsapp"] = &whatsapp.ExperimentMain{} +} + +// SetArguments +func (wo *whatsappOptions) SetArguments(sess model.ExperimentSession, + db *model.DatabaseProps) *model.ExperimentMainArgs { + annotations := mustMakeMapStringString(wo.Annotations) + return &model.ExperimentMainArgs{ + Annotations: annotations, + CategoryCodes: nil, // accept any category + Charging: true, + Callbacks: model.NewPrinterCallbacks(log.Log), + Database: db.Database, + Inputs: nil, + MaxRuntime: 0, + MeasurementDir: db.DatabaseResult.MeasurementDir, + NoCollector: false, + OnWiFi: true, + ResultID: db.DatabaseResult.ID, + RunType: model.RunTypeManual, + Session: sess, + } +} + +// ExtraOptions +func (wo *whatsappOptions) ExtraOptions() map[string]any { + return mustMakeMapStringAny(wo.ConfigOptions) +} + +// BuildWithOONIRun +func (wo *whatsappOptions) BuildWithOONIRun(inputs []string, args map[string]any) error { + if err := setter.SetOptionsAny(wo, args); err != nil { + return err + } + return nil +} + +// BuildFlags +func (wo *whatsappOptions) BuildFlags(experimentName string, rootCmd *cobra.Command) { + flags := rootCmd.Flags() + config := &whatsapp.Config{} + + flags.StringSliceVarP( + &wo.Annotations, + "annotation", + "A", + []string{}, + "add KEY=VALUE annotation to the report (can be repeated multiple times)", + ) + + if doc := setter.DocumentationForOptions(experimentName, config); doc != "" { + flags.StringSliceVarP( + &wo.ConfigOptions, + "options", + "O", + []string{}, + doc, + ) + } +} diff --git a/internal/setter/setter.go b/internal/setter/setter.go new file mode 100644 index 0000000000..fc5c662002 --- /dev/null +++ b/internal/setter/setter.go @@ -0,0 +1,180 @@ +package setter + +import ( + "errors" + "fmt" + "reflect" + "strconv" + "strings" + + "github.com/ooni/probe-cli/v3/internal/model" +) + +var ( + // ErrConfigIsNotAStructPointer indicates we expected a pointer to struct. + ErrConfigIsNotAStructPointer = errors.New("config is not a struct pointer") + + // ErrNoSuchField indicates there's no field with the given name. + ErrNoSuchField = errors.New("no such field") + + // ErrCannotSetIntegerOption means SetOptionAny couldn't set an integer option. + ErrCannotSetIntegerOption = errors.New("cannot set integer option") + + // ErrInvalidStringRepresentationOfBool indicates the string you passed + // to SetOptionaAny is not a valid string representation of a bool. + ErrInvalidStringRepresentationOfBool = errors.New("invalid string representation of bool") + + // ErrCannotSetBoolOption means SetOptionAny couldn't set a bool option. + ErrCannotSetBoolOption = errors.New("cannot set bool option") + + // ErrCannotSetStringOption means SetOptionAny couldn't set a string option. + ErrCannotSetStringOption = errors.New("cannot set string option") + + // ErrUnsupportedOptionType means we don't support the type passed to + // the SetOptionAny method as an opaque any type. + ErrUnsupportedOptionType = errors.New("unsupported option type") +) + +// DocumentationForOptions +func DocumentationForOptions(name string, config any) string { + var sb strings.Builder + options, err := options(config) + if err != nil || len(options) < 1 { + return "" + } + fmt.Fprint(&sb, "Pass KEY=VALUE options to the experiment. Available options:\n") + for name, info := range options { + if info.Doc == "" { + continue + } + fmt.Fprintf(&sb, "\n") + fmt.Fprintf(&sb, " -O, --option %s=<%s>\n", name, info.Type) + fmt.Fprintf(&sb, " %s\n", info.Doc) + } + return sb.String() +} + +// SetOptionsAny calls SetOptionAny for each entry inside [options]. +func SetOptionsAny(config any, options map[string]any) error { + for key, value := range options { + if err := setOptionAny(config, key, value); err != nil { + return err + } + } + return nil +} + +// options returns the options exposed by this experiment. +func options(config any) (map[string]model.ExperimentOptionInfo, error) { + result := make(map[string]model.ExperimentOptionInfo) + ptrinfo := reflect.ValueOf(config) + if ptrinfo.Kind() != reflect.Ptr { + return nil, ErrConfigIsNotAStructPointer + } + structinfo := ptrinfo.Elem().Type() + if structinfo.Kind() != reflect.Struct { + return nil, ErrConfigIsNotAStructPointer + } + for i := 0; i < structinfo.NumField(); i++ { + field := structinfo.Field(i) + result[field.Name] = model.ExperimentOptionInfo{ + Doc: field.Tag.Get("ooni"), + Type: field.Type.String(), + } + } + return result, nil +} + +// setOptionBool sets a bool option. +func setOptionBool(field reflect.Value, value any) error { + switch v := value.(type) { + case bool: + field.SetBool(v) + return nil + case string: + if v != "true" && v != "false" { + return fmt.Errorf("%w: %s", ErrInvalidStringRepresentationOfBool, v) + } + field.SetBool(v == "true") + return nil + default: + return fmt.Errorf("%w from a value of type %T", ErrCannotSetBoolOption, value) + } +} + +// setOptionInt sets an int option +func setOptionInt(field reflect.Value, value any) error { + switch v := value.(type) { + case int64: + field.SetInt(v) + return nil + case int32: + field.SetInt(int64(v)) + return nil + case int16: + field.SetInt(int64(v)) + return nil + case int8: + field.SetInt(int64(v)) + return nil + case int: + field.SetInt(int64(v)) + return nil + case string: + number, err := strconv.ParseInt(v, 10, 64) + if err != nil { + return fmt.Errorf("%w: %s", ErrCannotSetIntegerOption, err.Error()) + } + field.SetInt(number) + return nil + default: + return fmt.Errorf("%w from a value of type %T", ErrCannotSetIntegerOption, value) + } +} + +// setOptionString sets a string option +func setOptionString(field reflect.Value, value any) error { + switch v := value.(type) { + case string: + field.SetString(v) + return nil + default: + return fmt.Errorf("%w from a value of type %T", ErrCannotSetStringOption, value) + } +} + +// setOptionAny sets an option given any value. +func setOptionAny(config any, key string, value any) error { + field, err := fieldbyname(config, key) + if err != nil { + return err + } + switch field.Kind() { + case reflect.Int64: + return setOptionInt(field, value) + case reflect.Bool: + return setOptionBool(field, value) + case reflect.String: + return setOptionString(field, value) + default: + return fmt.Errorf("%w: %T", ErrUnsupportedOptionType, value) + } +} + +// fieldbyname return v's field whose name is equal to the given key. +func fieldbyname(v interface{}, key string) (reflect.Value, error) { + // See https://stackoverflow.com/a/6396678/4354461 + ptrinfo := reflect.ValueOf(v) + if ptrinfo.Kind() != reflect.Ptr { + return reflect.Value{}, fmt.Errorf("%w but a %T", ErrConfigIsNotAStructPointer, v) + } + structinfo := ptrinfo.Elem() + if structinfo.Kind() != reflect.Struct { + return reflect.Value{}, fmt.Errorf("%w but a %T", ErrConfigIsNotAStructPointer, v) + } + field := structinfo.FieldByName(key) + if !field.IsValid() || !field.CanSet() { + return reflect.Value{}, fmt.Errorf("%w: %s", ErrNoSuchField, key) + } + return field, nil +} diff --git a/internal/setter/setter_test.go b/internal/setter/setter_test.go new file mode 100644 index 0000000000..29e1481cbf --- /dev/null +++ b/internal/setter/setter_test.go @@ -0,0 +1,7 @@ +package setter + +import "testing" + +func TestSetter(t *testing.T) { + // TODO(DecFox): Add tests +}