Merge pull request ooni#9 from OpenObservatory/feature/web_connectivity

Feature/web connectivity

Makefile

@@ -5,6 +5,11 @@ build:
 	@$(GO) build -i -o dist/ooni cmd/ooni/main.go
 .PHONY: build
 
+update-mk:
+	@echo "updating mk"
+	@dep ensure -update github.com/measurement-kit/go-measurement-kit
+.PHONY: update-mk
+
 bindata:
 	@$(GO) run vendor/github.com/shuLhan/go-bindata/go-bindata/*.go \
 		-nometadata	\

internal/cli/geoip/geoip.go

@@ -1,8 +1,6 @@
 package geoip
 
 import (
-	"path/filepath"
-
 	"github.com/alecthomas/kingpin"
 	"github.com/apex/log"
 	"github.com/openobservatory/gooni/internal/cli/root"
@@ -21,10 +19,10 @@ func init() {
 			return err
 		}
 
-		geoipPath := filepath.Join(ctx.Home, "geoip")
-
+		geoipPath := utils.GeoIPDir(ctx.Home)
 		if *shouldUpdate {
 			utils.DownloadGeoIPDatabaseFiles(geoipPath)
+			utils.DownloadLegacyGeoIPDatabaseFiles(geoipPath)
 		}
 
 		loc, err := utils.GeoIPLookup(geoipPath)

internal/cli/run/run.go

@@ -33,6 +33,12 @@ func init() {
 		}
 		log.Debugf("Running test group %s", group.Label)
 
+		err = ctx.MaybeLocationLookup()
+		if err != nil {
+			log.WithError(err).Error("Failed to lookup the location of the probe")
+			return err
+		}
+
 		result, err := database.CreateResult(ctx.DB, ctx.Home, database.Result{
 			Name:      *nettestGroup,
 			StartTime: time.Now().UTC(),

internal/database/models.go

@@ -15,7 +15,7 @@ import (
 type ResultSummaryFunc func(SummaryMap) (string, error)
 
 // SummaryMap contains a mapping from test name to serialized summary for it
-type SummaryMap map[string]string
+type SummaryMap map[string][]string
 
 // UpdateOne will run the specified update query and check that it only affected one row
 func UpdateOne(db *sqlx.DB, query string, arg interface{}) error {
@@ -129,13 +129,19 @@ func (m *Measurement) WriteSummary(db *sqlx.DB, summary string) error {
 
 // AddToResult adds a measurement to a result
 func (m *Measurement) AddToResult(db *sqlx.DB, result *Result) error {
+	var err error
+
 	m.ResultID = result.ID
 	finalPath := filepath.Join(result.MeasurementDir,
 		filepath.Base(m.ReportFilePath))
 
-	err := os.Rename(m.ReportFilePath, finalPath)
-	if err != nil {
-		return errors.Wrap(err, "moving report file")
+	// If the finalPath already exists, it means it has already been moved there.
+	// This happens in multi input reports
+	if _, err = os.Stat(finalPath); os.IsNotExist(err) {
+		err = os.Rename(m.ReportFilePath, finalPath)
+		if err != nil {
+			return errors.Wrap(err, "moving report file")
+		}
 	}
 	m.ReportFilePath = finalPath
 
@@ -204,7 +210,12 @@ func MakeSummaryMap(db *sqlx.DB, r *Result) (SummaryMap, error) {
 		return nil, errors.Wrap(err, "failed to get measurements")
 	}
 	for _, msmt := range msmts {
-		summaryMap[msmt.Name] = msmt.Summary
+		val, ok := summaryMap[msmt.Name]
+		if ok {
+			summaryMap[msmt.Name] = append(val, msmt.Summary)
+		} else {
+			summaryMap[msmt.Name] = []string{msmt.Summary}
+		}
 	}
 	return summaryMap, nil
 }

nettests/groups/groups.go

@@ -34,7 +34,14 @@ type MiddleboxSummary struct {
 
 // IMSummary is the summary for the im tests
 type IMSummary struct {
-	Detected bool
+	Tested  uint
+	Blocked uint
+}
+
+// WebsitesSummary is the summary for the websites test
+type WebsitesSummary struct {
+	Tested  uint
+	Blocked uint
 }
 
 // NettestGroups that can be run by the user
@@ -45,7 +52,28 @@ var NettestGroups = map[string]NettestGroup{
 			websites.WebConnectivity{},
 		},
 		Summary: func(m database.SummaryMap) (string, error) {
-			return "{}", nil
+			// XXX to generate this I need to create the summary map as a list
+			var summary WebsitesSummary
+			summary.Tested = 0
+			summary.Blocked = 0
+			for _, msmtSummaryStr := range m["WebConnectivity"] {
+				var wcSummary websites.WebConnectivitySummary
+
+				err := json.Unmarshal([]byte(msmtSummaryStr), &wcSummary)
+				if err != nil {
+					log.WithError(err).Error("failed to unmarshal WebConnectivity summary")
+					return "", err
+				}
+				if wcSummary.Blocked {
+					summary.Blocked++
+				}
+				summary.Tested++
+			}
+			summaryBytes, err := json.Marshal(summary)
+			if err != nil {
+				return "", err
+			}
+			return string(summaryBytes), nil
 		},
 	},
 	"performance": NettestGroup{
@@ -61,12 +89,12 @@ var NettestGroups = map[string]NettestGroup{
 				dashSummary performance.DashSummary
 				summary     PerformanceSummary
 			)
-			err = json.Unmarshal([]byte(m["Dash"]), &dashSummary)
+			err = json.Unmarshal([]byte(m["Dash"][0]), &dashSummary)
 			if err != nil {
 				log.WithError(err).Error("failed to unmarshal Dash summary")
 				return "", err
 			}
-			err = json.Unmarshal([]byte(m["Ndt"]), &ndtSummary)
+			err = json.Unmarshal([]byte(m["Ndt"][0]), &ndtSummary)
 			if err != nil {
 				log.WithError(err).Error("failed to unmarshal NDT summary")
 				return "", err
@@ -95,12 +123,12 @@ var NettestGroups = map[string]NettestGroup{
 				hirlSummary middlebox.HTTPInvalidRequestLineSummary
 				summary     MiddleboxSummary
 			)
-			err = json.Unmarshal([]byte(m["HttpHeaderFieldManipulation"]), &hhfmSummary)
+			err = json.Unmarshal([]byte(m["HttpHeaderFieldManipulation"][0]), &hhfmSummary)
 			if err != nil {
 				log.WithError(err).Error("failed to unmarshal hhfm summary")
 				return "", err
 			}
-			err = json.Unmarshal([]byte(m["HttpInvalidRequestLine"]), &hirlSummary)
+			err = json.Unmarshal([]byte(m["HttpInvalidRequestLine"][0]), &hirlSummary)
 			if err != nil {
 				log.WithError(err).Error("failed to unmarshal hirl summary")
 				return "", err
@@ -121,7 +149,47 @@ var NettestGroups = map[string]NettestGroup{
 			im.WhatsApp{},
 		},
 		Summary: func(m database.SummaryMap) (string, error) {
-			return "{}", nil
+			var (
+				err       error
+				waSummary im.WhatsAppSummary
+				tgSummary im.TelegramSummary
+				fbSummary im.FacebookMessengerSummary
+				summary   IMSummary
+			)
+			err = json.Unmarshal([]byte(m["Whatsapp"][0]), &waSummary)
+			if err != nil {
+				log.WithError(err).Error("failed to unmarshal whatsapp summary")
+				return "", err
+			}
+			err = json.Unmarshal([]byte(m["Telegram"][0]), &tgSummary)
+			if err != nil {
+				log.WithError(err).Error("failed to unmarshal telegram summary")
+				return "", err
+			}
+			err = json.Unmarshal([]byte(m["FacebookMessenger"][0]), &fbSummary)
+			if err != nil {
+				log.WithError(err).Error("failed to unmarshal facebook summary")
+				return "", err
+			}
+			// XXX it could actually be that some are not tested when the
+			// configuration is changed.
+			summary.Tested = 3
+			summary.Blocked = 0
+			if fbSummary.Blocked == true {
+				summary.Blocked++
+			}
+			if tgSummary.Blocked == true {
+				summary.Blocked++
+			}
+			if waSummary.Blocked == true {
+				summary.Blocked++
+			}
+
+			summaryBytes, err := json.Marshal(summary)
+			if err != nil {
+				return "", err
+			}
+			return string(summaryBytes), nil
 		},
 	},
 }

nettests/im/facebook_messenger.go

@@ -20,13 +20,31 @@ func (h FacebookMessenger) Run(ctl *nettests.Controller) error {
 type FacebookMessengerSummary struct {
 	DNSBlocking bool
 	TCPBlocking bool
+	Blocked     bool
 }
 
 // Summary generates a summary for a test run
 func (h FacebookMessenger) Summary(tk map[string]interface{}) interface{} {
+	var (
+		dnsBlocking bool
+		tcpBlocking bool
+	)
+	if tk["facebook_dns_blocking"] == nil {
+		dnsBlocking = false
+	} else {
+		dnsBlocking = tk["facebook_dns_blocking"].(bool)
+	}
+
+	if tk["facebook_tcp_blocking"] == nil {
+		tcpBlocking = false
+	} else {
+		tcpBlocking = tk["facebook_tcp_blocking"].(bool)
+	}
+
 	return FacebookMessengerSummary{
-		DNSBlocking: tk["facebook_dns_blocking"].(bool),
-		TCPBlocking: tk["facebook_tcp_blocking"].(bool),
+		DNSBlocking: dnsBlocking,
+		TCPBlocking: tcpBlocking,
+		Blocked:     dnsBlocking || tcpBlocking,
 	}
 }
 

nettests/im/telegram.go

@@ -21,14 +21,38 @@ type TelegramSummary struct {
 	HTTPBlocking bool
 	TCPBlocking  bool
 	WebBlocking  bool
+	Blocked      bool
 }
 
 // Summary generates a summary for a test run
 func (h Telegram) Summary(tk map[string]interface{}) interface{} {
+	var (
+		tcpBlocking  bool
+		httpBlocking bool
+		webBlocking  bool
+	)
+
+	if tk["telegram_tcp_blocking"] == nil {
+		tcpBlocking = false
+	} else {
+		tcpBlocking = tk["telegram_tcp_blocking"].(bool)
+	}
+	if tk["telegram_http_blocking"] == nil {
+		httpBlocking = false
+	} else {
+		httpBlocking = tk["telegram_http_blocking"].(bool)
+	}
+	if tk["telegram_web_status"] == nil {
+		webBlocking = false
+	} else {
+		webBlocking = tk["telegram_web_status"].(string) == "blocked"
+	}
+
 	return TelegramSummary{
-		TCPBlocking:  tk["telegram_tcp_blocking"].(bool) == true,
-		HTTPBlocking: tk["telegram_http_blocking"].(bool) == true,
-		WebBlocking:  tk["telegram_web_status"].(string) == "blocked",
+		TCPBlocking:  tcpBlocking,
+		HTTPBlocking: httpBlocking,
+		WebBlocking:  webBlocking,
+		Blocked:      webBlocking || httpBlocking || tcpBlocking,
 	}
 }
 

nettests/im/whatsapp.go

@@ -21,16 +21,36 @@ type WhatsAppSummary struct {
 	RegistrationServerBlocking bool
 	WebBlocking                bool
 	EndpointsBlocking          bool
+	Blocked                    bool
 }
 
 // Summary generates a summary for a test run
 func (h WhatsApp) Summary(tk map[string]interface{}) interface{} {
-	const blk = "blocked"
+	var (
+		webBlocking          bool
+		registrationBlocking bool
+		endpointsBlocking    bool
+	)
+
+	var computeBlocking = func(key string) bool {
+		const blk = "blocked"
+		if tk[key] == nil {
+			return false
+		}
+		if tk[key].(string) == blk {
+			return true
+		}
+		return false
+	}
+	registrationBlocking = computeBlocking("registration_server_status")
+	webBlocking = computeBlocking("whatsapp_web_status")
+	endpointsBlocking = computeBlocking("whatsapp_endpoints_status")
 
 	return WhatsAppSummary{
-		RegistrationServerBlocking: tk["registration_server_status"].(string) == blk,
-		WebBlocking:                tk["whatsapp_web_status"].(string) == blk,
-		EndpointsBlocking:          tk["whatsapp_endpoints_status"].(string) == blk,
+		RegistrationServerBlocking: registrationBlocking,
+		WebBlocking:                webBlocking,
+		EndpointsBlocking:          endpointsBlocking,
+		Blocked:                    registrationBlocking || webBlocking || endpointsBlocking,
 	}
 }
 

nettests/nettests.go

@@ -3,6 +3,7 @@ package nettests
 import (
 	"encoding/json"
 	"fmt"
+	"path/filepath"
 
 	"github.com/apex/log"
 	"github.com/measurement-kit/go-measurement-kit"
@@ -11,6 +12,7 @@ import (
 	"github.com/openobservatory/gooni/internal/colors"
 	"github.com/openobservatory/gooni/internal/database"
 	"github.com/openobservatory/gooni/internal/output"
+	"github.com/openobservatory/gooni/utils"
 )
 
 // Nettest interface. Every Nettest should implement this.
@@ -43,6 +45,8 @@ type Controller struct {
 // Init should be called once to initialise the nettest
 func (c *Controller) Init(nt *mk.Nettest) error {
 	log.Debugf("Init: %v", nt)
+	c.Ctx.LocationLookup()
+
 	c.msmts = make(map[int64]*database.Measurement)
 
 	msmtTemplate := database.Measurement{
@@ -57,16 +61,21 @@ func (c *Controller) Init(nt *mk.Nettest) error {
 
 	log.Debugf("OutputPath: %s", c.msmtPath)
 	nt.Options = mk.NettestOptions{
-		IncludeIP:        c.Ctx.Config.Sharing.IncludeIP,
-		IncludeASN:       c.Ctx.Config.Sharing.IncludeASN,
-		IncludeCountry:   c.Ctx.Config.Advanced.IncludeCountry,
+		IncludeIP:      c.Ctx.Config.Sharing.IncludeIP,
+		IncludeASN:     c.Ctx.Config.Sharing.IncludeASN,
+		IncludeCountry: c.Ctx.Config.Advanced.IncludeCountry,
+
+		ProbeCC:  c.Ctx.Location.CountryCode,
+		ProbeASN: fmt.Sprintf("AS%d", c.Ctx.Location.ASN),
+		ProbeIP:  c.Ctx.Location.IP,
+
 		DisableCollector: false,
 		SoftwareName:     "ooniprobe",
 		SoftwareVersion:  version.Version,
 
 		// XXX
-		GeoIPCountryPath: "",
-		GeoIPASNPath:     "",
+		GeoIPCountryPath: filepath.Join(utils.GeoIPDir(c.Ctx.Home), "GeoIP.dat"),
+		GeoIPASNPath:     filepath.Join(utils.GeoIPDir(c.Ctx.Home), "GeoIPASNum.dat"),
 		OutputPath:       c.msmtPath,
 		CaBundlePath:     "/etc/ssl/cert.pem",
 	}