Declare measurement extensions and version

See #423

experiment.go

@@ -357,6 +357,7 @@ func (e *Experiment) newMeasurement(input string) *model.Measurement {
 	m.AddAnnotation("engine_name", "miniooni")
 	m.AddAnnotation("engine_version", version.Version)
 	m.AddAnnotation("platform", platform.Name())
+	m.Extensions = make(map[string]int64)
 	return &m
 }
 

experiment/psiphon/psiphon.go

@@ -51,6 +51,12 @@ type TestKeys struct {
 	TLSHandshakes oonidatamodel.TLSHandshakesList `json:"tls_handshakes"`
 }
 
+func registerExtensions(m *model.Measurement) {
+	oonidatamodel.ExtHTTP.AddTo(m)
+	oonidatamodel.ExtDNS.AddTo(m)
+	oonidatamodel.ExtTLSHandshake.AddTo(m)
+}
+
 type runner struct {
 	beginning      time.Time
 	callbacks      model.ExperimentCallbacks
@@ -215,6 +221,7 @@ func (m *measurer) Run(
 	wg.Add(1)
 	go m.printprogress(ctx, &wg, maxruntime, callbacks)
 	r := newRunner(m.config, callbacks, measurement.MeasurementStartTimeSaved)
+	registerExtensions(measurement)
 	measurement.TestKeys = r.testkeys
 	r.testkeys.MaxRuntime = maxruntime
 	err = r.run(ctx, sess.Logger(), clnt.FetchPsiphonConfig)

experiment/sniblocking/sniblocking.go

@@ -37,6 +37,7 @@ type Config struct {
 // Subresult contains the keys of a single measurement
 // that targets either the target or the control.
 type Subresult struct {
+	Agent         string                          `json:"agent"`
 	Cached        bool                            `json:"-"`
 	Failure       *string                         `json:"failure"`
 	NetworkEvents oonidatamodel.NetworkEventsList `json:"network_events"`
@@ -48,6 +49,14 @@ type Subresult struct {
 	TLSHandshakes oonidatamodel.TLSHandshakesList `json:"tls_handshakes"`
 }
 
+func registerExtensions(m *model.Measurement) {
+	oonidatamodel.ExtHTTP.AddTo(m)
+	oonidatamodel.ExtNetevents.AddTo(m)
+	oonidatamodel.ExtDNS.AddTo(m)
+	oonidatamodel.ExtTCPConnect.AddTo(m)
+	oonidatamodel.ExtTLSHandshake.AddTo(m)
+}
+
 // TestKeys contains sniblocking test keys.
 type TestKeys struct {
 	Control Subresult `json:"control"`
@@ -136,6 +145,7 @@ func (m *measurer) measureone(
 	})
 	// assemble and publish the results
 	smk := Subresult{
+		Agent:         "redirect",
 		NetworkEvents: oonidatamodel.NewNetworkEventsList(result.TestKeys),
 		Queries:       oonidatamodel.NewDNSQueriesList(result.TestKeys),
 		Requests:      oonidatamodel.NewRequestList(result.TestKeys),
@@ -258,6 +268,7 @@ func (m *measurer) Run(
 			m.config.ControlSNI, "443",
 		)
 	}
+	registerExtensions(measurement)
 	// TODO(bassosimone): if the user has configured DoT or DoH, here we
 	// probably want to perform the name resolution before the measurements
 	// or to make sure that the classify logic is robust to that.

experiment/telegram/telegram.go

@@ -42,6 +42,13 @@ type TestKeys struct {
 	TLSHandshakes        oonidatamodel.TLSHandshakesList `json:"tls_handshakes"`
 }
 
+func registerExtensions(m *model.Measurement) {
+	oonidatamodel.ExtHTTP.AddTo(m)
+	oonidatamodel.ExtDNS.AddTo(m)
+	oonidatamodel.ExtTCPConnect.AddTo(m)
+	oonidatamodel.ExtTLSHandshake.AddTo(m)
+}
+
 type urlMeasurements struct {
 	method  string
 	results *oonitemplates.HTTPDoResults
@@ -150,6 +157,7 @@ func (m *measurer) Run(
 ) error {
 	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
 	defer cancel()
+	registerExtensions(measurement)
 	// setup data container
 	var urlmeasurements = map[string]*urlMeasurements{
 		"http://149.154.175.50/":  &urlMeasurements{method: "POST"},

experiment/tor/tor.go

@@ -52,6 +52,14 @@ type TargetResults struct {
 	TLSHandshakes  oonidatamodel.TLSHandshakesList `json:"tls_handshakes"`
 }
 
+func registerExtensions(m *model.Measurement) {
+	oonidatamodel.ExtHTTP.AddTo(m)
+	oonidatamodel.ExtNetevents.AddTo(m)
+	oonidatamodel.ExtDNS.AddTo(m)
+	oonidatamodel.ExtTCPConnect.AddTo(m)
+	oonidatamodel.ExtTLSHandshake.AddTo(m)
+}
+
 // fillSummary fills the Summary field used by the UI.
 func (tr *TargetResults) fillSummary() {
 	tr.Summary = make(map[string]Summary)
@@ -161,6 +169,7 @@ func (m *measurer) Run(
 		ctx, 15*time.Second*time.Duration(len(targets)),
 	)
 	defer cancel()
+	registerExtensions(measurement)
 	m.measureTargets(ctx, sess, measurement, callbacks, targets)
 	return nil
 }

internal/oonidatamodel/oonidatamodel.go

@@ -16,9 +16,38 @@ import (
 
 	"github.com/ooni/probe-engine/internal/oonitemplates"
 	"github.com/ooni/probe-engine/internal/tlsx"
+	"github.com/ooni/probe-engine/model"
 	"github.com/ooni/probe-engine/netx/modelx"
 )
 
+// ExtSpec describes a data format extension
+type ExtSpec struct {
+	Name string // extension name
+	V    int64  // extension version
+}
+
+// AddTo adds the current ExtSpec to the specified measurement
+func (spec ExtSpec) AddTo(m *model.Measurement) {
+	m.Extensions[spec.Name] = spec.V
+}
+
+var (
+	// ExtDNS is the version of df-002-dnst.md
+	ExtDNS = ExtSpec{Name: "dnst", V: 0}
+
+	// ExtNetevents is the version of df-008-netevents.md
+	ExtNetevents = ExtSpec{Name: "netevents", V: 0}
+
+	// ExtHTTP is the version of df-001-httpt.md
+	ExtHTTP = ExtSpec{Name: "httpt", V: 0}
+
+	// ExtTCPConnect is the version of df-005-tcpconnect.md
+	ExtTCPConnect = ExtSpec{Name: "tcpconnect", V: 0}
+
+	// ExtTLSHandshake is the version of df-006-tlshandshake.md
+	ExtTLSHandshake = ExtSpec{Name: "tlshandshake", V: 0}
+)
+
 // TCPConnectStatus contains the TCP connect status.
 type TCPConnectStatus struct {
 	Failure *string `json:"failure"`

model/model.go

@@ -34,6 +34,10 @@ type Measurement struct {
 	// DataFormatVersion is the version of the data format
 	DataFormatVersion string `json:"data_format_version"`
 
+	// Extensions contains information about the extensions included
+	// into the test_keys of this measurement.
+	Extensions map[string]int64 `json:"extensions,omitempty"`
+
 	// ID is the locally generated measurement ID
 	ID string `json:"id,omitempty"`