From 00859e87a6c106e92a317bb6c2f54a481d222812 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 23 Mar 2018 12:41:06 +0100 Subject: [PATCH 1/6] Get web_connectivity to work with dynamic URL fetching --- Gopkg.lock | 2 +- Makefile | 5 +++ internal/cli/run/run.go | 6 +++ nettests/nettests.go | 13 +++++-- nettests/websites/web_connectivity.go | 53 +++++++++++++++++++++++++++ ooni.go | 33 +++++++++++++++-- 6 files changed, 104 insertions(+), 8 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 719134e79b..56b86e14f7 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -89,7 +89,7 @@ branch = "master" name = "github.com/measurement-kit/go-measurement-kit" packages = ["."] - revision = "6ae2401a8e498a90ccdd3edbda1841add079b70e" + revision = "cbf1c976aeaa2906f4fda278fc3068f7bde63c47" [[projects]] branch = "master" diff --git a/Makefile b/Makefile index 56f819cd07..77309fe6d5 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,11 @@ build: @$(GO) build -i -o dist/ooni cmd/ooni/main.go .PHONY: build +update-mk: + @echo "updating mk" + @dep ensure -update github.com/measurement-kit/go-measurement-kit +.PHONY: update-mk + bindata: @$(GO) run vendor/github.com/shuLhan/go-bindata/go-bindata/*.go \ -nometadata \ diff --git a/internal/cli/run/run.go b/internal/cli/run/run.go index b959ee22b6..d427db1d79 100644 --- a/internal/cli/run/run.go +++ b/internal/cli/run/run.go @@ -33,6 +33,12 @@ func init() { } log.Debugf("Running test group %s", group.Label) + err = ctx.MaybeLocationLookup() + if err != nil { + log.WithError(err).Error("Failed to lookup the location of the probe") + return err + } + result, err := database.CreateResult(ctx.DB, ctx.Home, database.Result{ Name: *nettestGroup, StartTime: time.Now().UTC(), diff --git a/nettests/nettests.go b/nettests/nettests.go index bc1cae1805..ac7528d95b 100644 --- a/nettests/nettests.go +++ b/nettests/nettests.go @@ -43,6 +43,8 @@ type Controller struct { // Init should be called once to initialise the nettest func (c *Controller) Init(nt *mk.Nettest) error { log.Debugf("Init: %v", nt) + c.Ctx.LocationLookup() + c.msmts = make(map[int64]*database.Measurement) msmtTemplate := database.Measurement{ @@ -57,9 +59,14 @@ func (c *Controller) Init(nt *mk.Nettest) error { log.Debugf("OutputPath: %s", c.msmtPath) nt.Options = mk.NettestOptions{ - IncludeIP: c.Ctx.Config.Sharing.IncludeIP, - IncludeASN: c.Ctx.Config.Sharing.IncludeASN, - IncludeCountry: c.Ctx.Config.Advanced.IncludeCountry, + IncludeIP: c.Ctx.Config.Sharing.IncludeIP, + IncludeASN: c.Ctx.Config.Sharing.IncludeASN, + IncludeCountry: c.Ctx.Config.Advanced.IncludeCountry, + + ProbeCC: c.Ctx.Location.CountryCode, + ProbeASN: fmt.Sprintf("AS%d", c.Ctx.Location.ASN), + ProbeIP: c.Ctx.Location.IP, + DisableCollector: false, SoftwareName: "ooniprobe", SoftwareVersion: version.Version, diff --git a/nettests/websites/web_connectivity.go b/nettests/websites/web_connectivity.go index fa0552753e..7b82fb164b 100644 --- a/nettests/websites/web_connectivity.go +++ b/nettests/websites/web_connectivity.go @@ -1,10 +1,56 @@ package websites import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "github.com/measurement-kit/go-measurement-kit" "github.com/openobservatory/gooni/nettests" + "github.com/pkg/errors" ) +type URLInfo struct { + URL string `json:"url"` + CategoryCode string `json:"category_code"` +} + +type URLResponse struct { + Results []URLInfo `json:"results"` +} + +const orchestrateBaseURL = "https://events.proteus.test.ooni.io" + +func lookupURLs(ctl *nettests.Controller) ([]string, error) { + var ( + parsed = new(URLResponse) + urls []string + ) + reqURL := fmt.Sprintf("%s/api/v1/urls?probe_cc=%s", + orchestrateBaseURL, + ctl.Ctx.Location.CountryCode) + + resp, err := http.Get(reqURL) + if err != nil { + return urls, errors.Wrap(err, "failed to perform request") + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return urls, errors.Wrap(err, "failed to read response body") + } + err = json.Unmarshal([]byte(body), &parsed) + if err != nil { + return urls, errors.Wrap(err, "failed to parse json") + } + + for _, url := range parsed.Results { + urls = append(urls, url.URL) + } + return urls, nil +} + // WebConnectivity test implementation type WebConnectivity struct { } @@ -13,6 +59,13 @@ type WebConnectivity struct { func (n WebConnectivity) Run(ctl *nettests.Controller) error { nt := mk.NewNettest("WebConnectivity") ctl.Init(nt) + + urls, err := lookupURLs(ctl) + if err != nil { + return err + } + nt.Options.Inputs = urls + return nt.Run() } diff --git a/ooni.go b/ooni.go index 56db62d431..903ae99d36 100644 --- a/ooni.go +++ b/ooni.go @@ -13,6 +13,7 @@ import ( "github.com/openobservatory/gooni/config" "github.com/openobservatory/gooni/internal/database" "github.com/openobservatory/gooni/internal/legacy" + "github.com/openobservatory/gooni/utils" "github.com/pkg/errors" ) @@ -35,15 +36,39 @@ func Onboarding(c *Config) error { // Context for OONI Probe type Context struct { - Config *Config - DB *sqlx.DB + Config *Config + DB *sqlx.DB + Location *utils.LocationInfo + + Home string + TempDir string - Home string - TempDir string dbPath string configPath string } +// MaybeLocationLookup will lookup the location of the user unless it's already cached +func (c *Context) MaybeLocationLookup() error { + if c.Location == nil { + return c.LocationLookup() + } + return nil +} + +// LocationLookup lookup the location of the user via geoip +func (c *Context) LocationLookup() error { + var err error + + dbPath := filepath.Join(c.Home, "geoip") + + c.Location, err = utils.GeoIPLookup(dbPath) + if err != nil { + return err + } + + return nil +} + // Init the OONI manager func (c *Context) Init() error { var err error From f5ff3ac87a46a28a6db78cac2d3bc262d9517f48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 23 Mar 2018 13:17:39 +0100 Subject: [PATCH 2/6] Add support for generating the Measurement summary To generate the result summary we need to refactor how the SummaryMap works --- internal/database/models.go | 12 ++++++-- nettests/groups/groups.go | 7 +++++ nettests/websites/web_connectivity.go | 41 ++++++++++++++++++++++++++- 3 files changed, 56 insertions(+), 4 deletions(-) diff --git a/internal/database/models.go b/internal/database/models.go index 89d007b33a..bd704a4964 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -129,13 +129,19 @@ func (m *Measurement) WriteSummary(db *sqlx.DB, summary string) error { // AddToResult adds a measurement to a result func (m *Measurement) AddToResult(db *sqlx.DB, result *Result) error { + var err error + m.ResultID = result.ID finalPath := filepath.Join(result.MeasurementDir, filepath.Base(m.ReportFilePath)) - err := os.Rename(m.ReportFilePath, finalPath) - if err != nil { - return errors.Wrap(err, "moving report file") + // If the finalPath already exists, it means it has already been moved there. + // This happens in multi input reports + if _, err = os.Stat(finalPath); os.IsNotExist(err) { + err = os.Rename(m.ReportFilePath, finalPath) + if err != nil { + return errors.Wrap(err, "moving report file") + } } m.ReportFilePath = finalPath diff --git a/nettests/groups/groups.go b/nettests/groups/groups.go index 2eb0440285..6a82f74af4 100644 --- a/nettests/groups/groups.go +++ b/nettests/groups/groups.go @@ -37,6 +37,12 @@ type IMSummary struct { Detected bool } +// WebsitesSummary is the summary for the websites test +type WebsitesSummary struct { + Tested uint + Blocked uint +} + // NettestGroups that can be run by the user var NettestGroups = map[string]NettestGroup{ "websites": NettestGroup{ @@ -45,6 +51,7 @@ var NettestGroups = map[string]NettestGroup{ websites.WebConnectivity{}, }, Summary: func(m database.SummaryMap) (string, error) { + // XXX to generate this I need to create the summary map as a list return "{}", nil }, }, diff --git a/nettests/websites/web_connectivity.go b/nettests/websites/web_connectivity.go index 7b82fb164b..dd3394f488 100644 --- a/nettests/websites/web_connectivity.go +++ b/nettests/websites/web_connectivity.go @@ -11,11 +11,13 @@ import ( "github.com/pkg/errors" ) +// URLInfo contains the URL and the citizenlab category code for that URL type URLInfo struct { URL string `json:"url"` CategoryCode string `json:"category_code"` } +// URLResponse is the orchestrate url response containing a list of URLs type URLResponse struct { Results []URLInfo `json:"results"` } @@ -69,9 +71,46 @@ func (n WebConnectivity) Run(ctl *nettests.Controller) error { return nt.Run() } +// WebConnectivitySummary for the test +type WebConnectivitySummary struct { + Accessible bool + Blocking string + Blocked bool +} + // Summary generates a summary for a test run func (n WebConnectivity) Summary(tk map[string]interface{}) interface{} { - return nil + var ( + blocked bool + blocking string + accessible bool + ) + + // We need to do these complicated type assertions, because some of the fields + // are "nullable" and/or can be of different types + switch v := tk["blocking"].(type) { + case bool: + blocked = false + blocking = "none" + case string: + blocked = true + blocking = v + default: + blocked = false + blocking = "none" + } + + if tk["accessible"] == nil { + accessible = false + } else { + accessible = tk["accessible"].(bool) + } + + return WebConnectivitySummary{ + Accessible: accessible, + Blocking: blocking, + Blocked: blocked, + } } // LogSummary writes the summary to the standard output From 98e95b8b7e51630e62587d4210b425aa1de97bc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 23 Mar 2018 14:58:33 +0100 Subject: [PATCH 3/6] Add support for generating the web connectivity summary --- internal/database/models.go | 9 +++++++-- nettests/groups/groups.go | 30 +++++++++++++++++++++++++----- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/internal/database/models.go b/internal/database/models.go index bd704a4964..be101c3cd1 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -15,7 +15,7 @@ import ( type ResultSummaryFunc func(SummaryMap) (string, error) // SummaryMap contains a mapping from test name to serialized summary for it -type SummaryMap map[string]string +type SummaryMap map[string][]string // UpdateOne will run the specified update query and check that it only affected one row func UpdateOne(db *sqlx.DB, query string, arg interface{}) error { @@ -210,7 +210,12 @@ func MakeSummaryMap(db *sqlx.DB, r *Result) (SummaryMap, error) { return nil, errors.Wrap(err, "failed to get measurements") } for _, msmt := range msmts { - summaryMap[msmt.Name] = msmt.Summary + val, ok := summaryMap[msmt.Name] + if ok { + summaryMap[msmt.Name] = append(val, msmt.Summary) + } else { + summaryMap[msmt.Name] = []string{msmt.Summary} + } } return summaryMap, nil } diff --git a/nettests/groups/groups.go b/nettests/groups/groups.go index 6a82f74af4..5ab291ab0a 100644 --- a/nettests/groups/groups.go +++ b/nettests/groups/groups.go @@ -52,7 +52,27 @@ var NettestGroups = map[string]NettestGroup{ }, Summary: func(m database.SummaryMap) (string, error) { // XXX to generate this I need to create the summary map as a list - return "{}", nil + var summary WebsitesSummary + summary.Tested = 0 + summary.Blocked = 0 + for _, msmtSummaryStr := range m["WebConnectivity"] { + var wcSummary websites.WebConnectivitySummary + + err := json.Unmarshal([]byte(msmtSummaryStr), &wcSummary) + if err != nil { + log.WithError(err).Error("failed to unmarshal WebConnectivity summary") + return "", err + } + if wcSummary.Blocked { + summary.Blocked++ + } + summary.Tested++ + } + summaryBytes, err := json.Marshal(summary) + if err != nil { + return "", err + } + return string(summaryBytes), nil }, }, "performance": NettestGroup{ @@ -68,12 +88,12 @@ var NettestGroups = map[string]NettestGroup{ dashSummary performance.DashSummary summary PerformanceSummary ) - err = json.Unmarshal([]byte(m["Dash"]), &dashSummary) + err = json.Unmarshal([]byte(m["Dash"][0]), &dashSummary) if err != nil { log.WithError(err).Error("failed to unmarshal Dash summary") return "", err } - err = json.Unmarshal([]byte(m["Ndt"]), &ndtSummary) + err = json.Unmarshal([]byte(m["Ndt"][0]), &ndtSummary) if err != nil { log.WithError(err).Error("failed to unmarshal NDT summary") return "", err @@ -102,12 +122,12 @@ var NettestGroups = map[string]NettestGroup{ hirlSummary middlebox.HTTPInvalidRequestLineSummary summary MiddleboxSummary ) - err = json.Unmarshal([]byte(m["HttpHeaderFieldManipulation"]), &hhfmSummary) + err = json.Unmarshal([]byte(m["HttpHeaderFieldManipulation"][0]), &hhfmSummary) if err != nil { log.WithError(err).Error("failed to unmarshal hhfm summary") return "", err } - err = json.Unmarshal([]byte(m["HttpInvalidRequestLine"]), &hirlSummary) + err = json.Unmarshal([]byte(m["HttpInvalidRequestLine"][0]), &hirlSummary) if err != nil { log.WithError(err).Error("failed to unmarshal hirl summary") return "", err From e56ce730064c01bbbfacf092a6c54a00fc0e3bea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 23 Mar 2018 15:22:58 +0100 Subject: [PATCH 4/6] Add result summary support for IM tests --- nettests/groups/groups.go | 45 +++++++++++++++++++++++++++++-- nettests/im/facebook_messenger.go | 22 +++++++++++++-- nettests/im/telegram.go | 30 ++++++++++++++++++--- nettests/im/whatsapp.go | 28 ++++++++++++++++--- 4 files changed, 114 insertions(+), 11 deletions(-) diff --git a/nettests/groups/groups.go b/nettests/groups/groups.go index 5ab291ab0a..c3f9f1a2d1 100644 --- a/nettests/groups/groups.go +++ b/nettests/groups/groups.go @@ -34,7 +34,8 @@ type MiddleboxSummary struct { // IMSummary is the summary for the im tests type IMSummary struct { - Detected bool + Tested uint + Blocked uint } // WebsitesSummary is the summary for the websites test @@ -148,7 +149,47 @@ var NettestGroups = map[string]NettestGroup{ im.WhatsApp{}, }, Summary: func(m database.SummaryMap) (string, error) { - return "{}", nil + var ( + err error + waSummary im.WhatsAppSummary + tgSummary im.TelegramSummary + fbSummary im.FacebookMessengerSummary + summary IMSummary + ) + err = json.Unmarshal([]byte(m["Whatsapp"][0]), &waSummary) + if err != nil { + log.WithError(err).Error("failed to unmarshal whatsapp summary") + return "", err + } + err = json.Unmarshal([]byte(m["Telegram"][0]), &tgSummary) + if err != nil { + log.WithError(err).Error("failed to unmarshal telegram summary") + return "", err + } + err = json.Unmarshal([]byte(m["FacebookMessenger"][0]), &fbSummary) + if err != nil { + log.WithError(err).Error("failed to unmarshal facebook summary") + return "", err + } + // XXX it could actually be that some are not tested when the + // configuration is changed. + summary.Tested = 3 + summary.Blocked = 0 + if fbSummary.Blocked == true { + summary.Blocked++ + } + if tgSummary.Blocked == true { + summary.Blocked++ + } + if waSummary.Blocked == true { + summary.Blocked++ + } + + summaryBytes, err := json.Marshal(summary) + if err != nil { + return "", err + } + return string(summaryBytes), nil }, }, } diff --git a/nettests/im/facebook_messenger.go b/nettests/im/facebook_messenger.go index bb7fcacbbc..0108824031 100644 --- a/nettests/im/facebook_messenger.go +++ b/nettests/im/facebook_messenger.go @@ -20,13 +20,31 @@ func (h FacebookMessenger) Run(ctl *nettests.Controller) error { type FacebookMessengerSummary struct { DNSBlocking bool TCPBlocking bool + Blocked bool } // Summary generates a summary for a test run func (h FacebookMessenger) Summary(tk map[string]interface{}) interface{} { + var ( + dnsBlocking bool + tcpBlocking bool + ) + if tk["facebook_dns_blocking"] == nil { + dnsBlocking = false + } else { + dnsBlocking = tk["facebook_dns_blocking"].(bool) + } + + if tk["facebook_tcp_blocking"] == nil { + tcpBlocking = false + } else { + tcpBlocking = tk["facebook_tcp_blocking"].(bool) + } + return FacebookMessengerSummary{ - DNSBlocking: tk["facebook_dns_blocking"].(bool), - TCPBlocking: tk["facebook_tcp_blocking"].(bool), + DNSBlocking: dnsBlocking, + TCPBlocking: tcpBlocking, + Blocked: dnsBlocking || tcpBlocking, } } diff --git a/nettests/im/telegram.go b/nettests/im/telegram.go index 503d55c4b7..8d3f45053a 100644 --- a/nettests/im/telegram.go +++ b/nettests/im/telegram.go @@ -21,14 +21,38 @@ type TelegramSummary struct { HTTPBlocking bool TCPBlocking bool WebBlocking bool + Blocked bool } // Summary generates a summary for a test run func (h Telegram) Summary(tk map[string]interface{}) interface{} { + var ( + tcpBlocking bool + httpBlocking bool + webBlocking bool + ) + + if tk["telegram_tcp_blocking"] == nil { + tcpBlocking = false + } else { + tcpBlocking = tk["telegram_tcp_blocking"].(bool) + } + if tk["telegram_http_blocking"] == nil { + httpBlocking = false + } else { + httpBlocking = tk["telegram_http_blocking"].(bool) + } + if tk["telegram_web_status"] == nil { + webBlocking = false + } else { + webBlocking = tk["telegram_web_status"].(string) == "blocked" + } + return TelegramSummary{ - TCPBlocking: tk["telegram_tcp_blocking"].(bool) == true, - HTTPBlocking: tk["telegram_http_blocking"].(bool) == true, - WebBlocking: tk["telegram_web_status"].(string) == "blocked", + TCPBlocking: tcpBlocking, + HTTPBlocking: httpBlocking, + WebBlocking: webBlocking, + Blocked: webBlocking || httpBlocking || tcpBlocking, } } diff --git a/nettests/im/whatsapp.go b/nettests/im/whatsapp.go index 5dc26ba90a..24c8b598a7 100644 --- a/nettests/im/whatsapp.go +++ b/nettests/im/whatsapp.go @@ -21,16 +21,36 @@ type WhatsAppSummary struct { RegistrationServerBlocking bool WebBlocking bool EndpointsBlocking bool + Blocked bool } // Summary generates a summary for a test run func (h WhatsApp) Summary(tk map[string]interface{}) interface{} { - const blk = "blocked" + var ( + webBlocking bool + registrationBlocking bool + endpointsBlocking bool + ) + + var computeBlocking = func(key string) bool { + const blk = "blocked" + if tk[key] == nil { + return false + } + if tk[key].(string) == blk { + return true + } + return false + } + registrationBlocking = computeBlocking("registration_server_status") + webBlocking = computeBlocking("whatsapp_web_status") + endpointsBlocking = computeBlocking("whatsapp_endpoints_status") return WhatsAppSummary{ - RegistrationServerBlocking: tk["registration_server_status"].(string) == blk, - WebBlocking: tk["whatsapp_web_status"].(string) == blk, - EndpointsBlocking: tk["whatsapp_endpoints_status"].(string) == blk, + RegistrationServerBlocking: registrationBlocking, + WebBlocking: webBlocking, + EndpointsBlocking: endpointsBlocking, + Blocked: registrationBlocking || webBlocking || endpointsBlocking, } } From 749cc665affe35493319665ccd41cad9b0245dc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 23 Mar 2018 15:26:25 +0100 Subject: [PATCH 5/6] Add support for downloading legacy geoip databases Pass the geoip databases to measurement-kit --- internal/cli/geoip/geoip.go | 6 +-- nettests/nettests.go | 6 ++- utils/geoip.go | 79 ++++++++++++++++++++++++++++++------- 3 files changed, 71 insertions(+), 20 deletions(-) diff --git a/internal/cli/geoip/geoip.go b/internal/cli/geoip/geoip.go index 803d117fee..acdb458dc9 100644 --- a/internal/cli/geoip/geoip.go +++ b/internal/cli/geoip/geoip.go @@ -1,8 +1,6 @@ package geoip import ( - "path/filepath" - "github.com/alecthomas/kingpin" "github.com/apex/log" "github.com/openobservatory/gooni/internal/cli/root" @@ -21,10 +19,10 @@ func init() { return err } - geoipPath := filepath.Join(ctx.Home, "geoip") - + geoipPath := utils.GeoIPDir(ctx.Home) if *shouldUpdate { utils.DownloadGeoIPDatabaseFiles(geoipPath) + utils.DownloadLegacyGeoIPDatabaseFiles(geoipPath) } loc, err := utils.GeoIPLookup(geoipPath) diff --git a/nettests/nettests.go b/nettests/nettests.go index ac7528d95b..b93f3575a9 100644 --- a/nettests/nettests.go +++ b/nettests/nettests.go @@ -3,6 +3,7 @@ package nettests import ( "encoding/json" "fmt" + "path/filepath" "github.com/apex/log" "github.com/measurement-kit/go-measurement-kit" @@ -11,6 +12,7 @@ import ( "github.com/openobservatory/gooni/internal/colors" "github.com/openobservatory/gooni/internal/database" "github.com/openobservatory/gooni/internal/output" + "github.com/openobservatory/gooni/utils" ) // Nettest interface. Every Nettest should implement this. @@ -72,8 +74,8 @@ func (c *Controller) Init(nt *mk.Nettest) error { SoftwareVersion: version.Version, // XXX - GeoIPCountryPath: "", - GeoIPASNPath: "", + GeoIPCountryPath: filepath.Join(utils.GeoIPDir(c.Ctx.Home), "GeoIP.dat"), + GeoIPASNPath: filepath.Join(utils.GeoIPDir(c.Ctx.Home), "GeoIPASNum.dat"), OutputPath: c.msmtPath, CaBundlePath: "/etc/ssl/cert.pem", } diff --git a/utils/geoip.go b/utils/geoip.go index 1bb77301f7..9294dc87a0 100644 --- a/utils/geoip.go +++ b/utils/geoip.go @@ -31,31 +31,82 @@ var geoipFiles = map[string]string{ "GeoLite2-Country.mmdb": "http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz", } -// DownloadGeoIPDatabaseFiles into the target directory -func DownloadGeoIPDatabaseFiles(dir string) error { - for filename, url := range geoipFiles { +var legacyGeoipFiles = map[string]string{ + "GeoIPASNum.dat": "http://download.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz", + "GeoIP.dat": "http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz", +} + +// GeoIPDir returns the geoip data dir for the given OONI Home +func GeoIPDir(home string) string { + return filepath.Join(home, "geoip") +} + +// Download the file to a temporary location +func downloadToTemp(url string) (string, error) { + out, err := ioutil.TempFile(os.TempDir(), "maxmind") + if err != nil { + return "", errors.Wrap(err, "failed to create temporary directory") + } + resp, err := http.Get(url) + if err != nil { + return "", errors.Wrap(err, "failed to fetch URL") + } + + _, err = io.Copy(out, resp.Body) + if err != nil { + return "", errors.Wrap(err, "failed to copy response body") + } + out.Close() + resp.Body.Close() + return out.Name(), nil +} + +// DownloadLegacyGeoIPDatabaseFiles into the target directory +func DownloadLegacyGeoIPDatabaseFiles(dir string) error { + for filename, url := range legacyGeoipFiles { dstPath := filepath.Join(dir, filename) - // Download the file to a temporary location - out, err := ioutil.TempFile(os.TempDir(), "maxmind") + tmpPath, err := downloadToTemp(url) if err != nil { - return errors.Wrap(err, "failed to create temporary directory") + return err } - resp, err := http.Get(url) + + // Extract the tar.gz file + f, err := os.Open(tmpPath) + defer f.Close() if err != nil { - return errors.Wrap(err, "failed to fetch URL") + return errors.Wrap(err, "failed to read file") } - _, err = io.Copy(out, resp.Body) + gzf, err := gzip.NewReader(f) if err != nil { - return errors.Wrap(err, "failed to copy response body") + return errors.Wrap(err, "failed to create gzip reader") } - out.Close() - resp.Body.Close() - // Extract the tar.gz file + outFile, err := os.Create(dstPath) + if err != nil { + return errors.Wrap(err, "error creating file") + } + if _, err := io.Copy(outFile, gzf); err != nil { + return errors.Wrap(err, "error reading file from gzip") + } + outFile.Close() + } + return nil +} - f, err := os.Open(out.Name()) +// DownloadGeoIPDatabaseFiles into the target directory +func DownloadGeoIPDatabaseFiles(dir string) error { + for filename, url := range geoipFiles { + dstPath := filepath.Join(dir, filename) + + tmpPath, err := downloadToTemp(url) + if err != nil { + return err + } + + // Extract the tar.gz file + f, err := os.Open(tmpPath) if err != nil { return errors.Wrap(err, "failed to read file") } From 9fe917f294d5fa19c8660c04a58f7de033ed8a09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 23 Mar 2018 15:38:25 +0100 Subject: [PATCH 6/6] On first run also download the geoip data files --- ooni.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/ooni.go b/ooni.go index 903ae99d36..8000c0b1a1 100644 --- a/ooni.go +++ b/ooni.go @@ -77,7 +77,7 @@ func (c *Context) Init() error { return errors.Wrap(err, "migrating home") } - if err = CreateHomeDirs(c.Home); err != nil { + if err = MaybeInitializeHome(c.Home); err != nil { return err } @@ -212,16 +212,28 @@ func ParseConfig(b []byte) (*Config, error) { return c, nil } -// CreateHomeDirs creates the OONI home subdirectories -func CreateHomeDirs(home string) error { +// MaybeInitializeHome does the setup for a new OONI Home +func MaybeInitializeHome(home string) error { + firstRun := false requiredDirs := []string{"db", "msmts", "geoip"} for _, d := range requiredDirs { if _, e := os.Stat(filepath.Join(home, d)); e != nil { + firstRun = true if err := os.MkdirAll(filepath.Join(home, d), 0700); err != nil { return err } } } + if firstRun == true { + log.Info("This is the first time you are running OONI Probe. Downloading some files.") + geoipDir := utils.GeoIPDir(home) + if err := utils.DownloadGeoIPDatabaseFiles(geoipDir); err != nil { + return err + } + if err := utils.DownloadLegacyGeoIPDatabaseFiles(geoipDir); err != nil { + return err + } + } return nil }