Merge pull request #8 from OpenObservatory/feature/geoiplookup

Feature/geoiplookup

.gitignore

@@ -1,3 +1,3 @@
-vendor/
-/ooni
+/vendor
+/dist
 *.njson

Gopkg.toml

@@ -61,3 +61,7 @@ required = ["github.com/shuLhan/go-bindata/go-bindata"]
 [[constraint]]
   name = "github.com/shuLhan/go-bindata"
   version = "3.3.0"
+
+[[constraint]]
+  name = "github.com/oschwald/geoip2-golang"
+  version = "1.2.1"

Makefile

@@ -2,7 +2,7 @@ GO ?= go
 
 build:
 	@echo "Building ./ooni"
-	@$(GO) build -i -o ooni cmd/ooni/main.go
+	@$(GO) build -i -o dist/ooni cmd/ooni/main.go
 .PHONY: build
 
 bindata:

cmd/ooni/main.go

@@ -4,6 +4,7 @@ import (
 	// commands
 	"github.com/apex/log"
 
+	_ "github.com/openobservatory/gooni/internal/cli/geoip"
 	_ "github.com/openobservatory/gooni/internal/cli/info"
 	_ "github.com/openobservatory/gooni/internal/cli/list"
 	_ "github.com/openobservatory/gooni/internal/cli/nettest"

internal/cli/geoip/geoip.go

@@ -0,0 +1,44 @@
+package geoip
+
+import (
+	"path/filepath"
+
+	"github.com/alecthomas/kingpin"
+	"github.com/apex/log"
+	"github.com/openobservatory/gooni/internal/cli/root"
+	"github.com/openobservatory/gooni/utils"
+)
+
+func init() {
+	cmd := root.Command("geoip", "Perform a geoip lookup")
+
+	shouldUpdate := cmd.Flag("update", "Update the geoip database").Bool()
+
+	cmd.Action(func(_ *kingpin.ParseContext) error {
+		log.Info("geoip")
+		ctx, err := root.Init()
+		if err != nil {
+			return err
+		}
+
+		geoipPath := filepath.Join(ctx.Home, "geoip")
+
+		if *shouldUpdate {
+			utils.DownloadGeoIPDatabaseFiles(geoipPath)
+		}
+
+		loc, err := utils.GeoIPLookup(geoipPath)
+		if err != nil {
+			return err
+		}
+
+		log.WithFields(log.Fields{
+			"asn":          loc.ASN,
+			"network_name": loc.NetworkName,
+			"country_code": loc.CountryCode,
+			"ip":           loc.IP,
+		}).Info("Looked up your location")
+
+		return nil
+	})
+}

internal/cli/root/root.go

@@ -4,7 +4,6 @@ import (
 	"github.com/alecthomas/kingpin"
 	"github.com/apex/log"
 	ooni "github.com/openobservatory/gooni"
-	"github.com/openobservatory/gooni/internal/database"
 	"github.com/openobservatory/gooni/internal/log/handlers/batch"
 	"github.com/openobservatory/gooni/internal/log/handlers/cli"
 	"github.com/prometheus/common/version"
@@ -17,7 +16,7 @@ var Cmd = kingpin.New("ooni", "")
 var Command = Cmd.Command
 
 // Init should be called by all subcommand that care to have a ooni.OONI instance
-var Init func() (*ooni.Config, *ooni.Context, error)
+var Init func() (*ooni.Context, error)
 
 func init() {
 	configPath := Cmd.Flag("config", "Set a custom config file path").Short('c').String()
@@ -36,39 +35,21 @@ func init() {
 			log.Debugf("ooni version %s", version.Version)
 		}
 
-		Init = func() (*ooni.Config, *ooni.Context, error) {
-			var config *ooni.Config
+		Init = func() (*ooni.Context, error) {
 			var err error
 
-			if *configPath != "" {
-				log.Debugf("Reading config file from %s", *configPath)
-				config, err = ooni.ReadConfig(*configPath)
-			} else {
-				log.Debug("Reading default config file")
-				config, err = ooni.ReadDefaultConfigPaths()
-			}
-			if err != nil {
-				return nil, nil, err
-			}
-
-			dbPath, err := database.DefaultDatabasePath()
-			if err != nil {
-				return nil, nil, err
-			}
-
-			log.Debugf("Connecting to database sqlite3://%s", dbPath)
-			db, err := database.Connect(dbPath)
+			homePath, err := ooni.GetOONIHome()
 			if err != nil {
-				return nil, nil, err
+				return nil, err
 			}
 
-			ctx := ooni.New(config, db)
+			ctx := ooni.NewContext(*configPath, homePath)
 			err = ctx.Init()
 			if err != nil {
-				return nil, nil, err
+				return nil, err
 			}
 
-			return config, ctx, nil
+			return ctx, nil
 		}
 
 		return nil

internal/cli/run/run.go

@@ -21,7 +21,7 @@ func init() {
 
 	cmd.Action(func(_ *kingpin.ParseContext) error {
 		log.Infof("Starting %s", *nettestGroup)
-		_, ctx, err := root.Init()
+		ctx, err := root.Init()
 		if err != nil {
 			log.Errorf("%s", err)
 			return err
@@ -33,7 +33,7 @@ func init() {
 		}
 		log.Debugf("Running test group %s", group.Label)
 
-		result, err := database.CreateResult(ctx.DB, database.Result{
+		result, err := database.CreateResult(ctx.DB, ctx.Home, database.Result{
 			Name:      *nettestGroup,
 			StartTime: time.Now().UTC(),
 		})

internal/database/database.go

@@ -1,14 +1,10 @@
 package database
 
 import (
-	"path/filepath"
-
 	"github.com/apex/log"
 	"github.com/jmoiron/sqlx"
 	_ "github.com/mattn/go-sqlite3" // this is needed to load the sqlite3 driver
-	ooni "github.com/openobservatory/gooni"
 	"github.com/openobservatory/gooni/internal/bindata"
-	"github.com/pkg/errors"
 	migrate "github.com/rubenv/sql-migrate"
 )
 
@@ -41,12 +37,3 @@ func Connect(path string) (db *sqlx.DB, err error) {
 	}
 	return
 }
-
-// DefaultDatabasePath for the main database
-func DefaultDatabasePath() (string, error) {
-	home, err := ooni.GetOONIHome()
-	if err != nil {
-		return "", errors.Wrap(err, "default database path")
-	}
-	return filepath.Join(home, "db", "main.sqlite3"), nil
-}

internal/database/models.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/apex/log"
 	"github.com/jmoiron/sqlx"
-	ooni "github.com/openobservatory/gooni"
 	"github.com/pkg/errors"
 )
 
@@ -238,11 +237,7 @@ func (r *Result) Finished(db *sqlx.DB, makeSummary ResultSummaryFunc) error {
 }
 
 // MakeResultsPath creates and returns a directory for the result
-func MakeResultsPath(r *Result) (string, error) {
-	home, err := ooni.GetOONIHome()
-	if err != nil {
-		return "", errors.Wrap(err, "default measurements path")
-	}
+func MakeResultsPath(home string, r *Result) (string, error) {
 	p := filepath.Join(home, "msmts",
 		fmt.Sprintf("%s-%s", r.Name, r.StartTime.Format(time.RFC3339Nano)))
 
@@ -251,7 +246,7 @@ func MakeResultsPath(r *Result) (string, error) {
 	if _, e := os.Stat(p); e == nil {
 		return "", errors.New("results path already exists")
 	}
-	err = os.MkdirAll(p, 0700)
+	err := os.MkdirAll(p, 0700)
 	if err != nil {
 		return "", err
 	}
@@ -260,10 +255,10 @@ func MakeResultsPath(r *Result) (string, error) {
 
 // CreateResult writes the Result to the database a returns a pointer
 // to the Result
-func CreateResult(db *sqlx.DB, r Result) (*Result, error) {
+func CreateResult(db *sqlx.DB, homePath string, r Result) (*Result, error) {
 	log.Debugf("Creating result %v", r)
 
-	p, err := MakeResultsPath(&r)
+	p, err := MakeResultsPath(homePath, &r)
 	if err != nil {
 		return nil, err
 	}

ooni.go

@@ -11,6 +11,7 @@ import (
 	"github.com/jmoiron/sqlx"
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/openobservatory/gooni/config"
+	"github.com/openobservatory/gooni/internal/database"
 	"github.com/openobservatory/gooni/internal/legacy"
 	"github.com/pkg/errors"
 )
@@ -34,22 +35,52 @@ func Onboarding(c *Config) error {
 
 // Context for OONI Probe
 type Context struct {
-	Config  *Config
-	DB      *sqlx.DB
-	TempDir string
+	Config *Config
+	DB     *sqlx.DB
+
+	Home       string
+	TempDir    string
+	dbPath     string
+	configPath string
 }
 
 // Init the OONI manager
 func (c *Context) Init() error {
-	if err := legacy.MaybeMigrateHome(); err != nil {
+	var err error
+
+	if err = legacy.MaybeMigrateHome(); err != nil {
 		return errors.Wrap(err, "migrating home")
 	}
+
+	if err = CreateHomeDirs(c.Home); err != nil {
+		return err
+	}
+
+	if c.configPath != "" {
+		log.Debugf("Reading config file from %s", c.configPath)
+		c.Config, err = ReadConfig(c.configPath)
+	} else {
+		log.Debug("Reading default config file")
+		c.Config, err = ReadDefaultConfigPaths(c.Home)
+	}
+	if err != nil {
+		return err
+	}
+
+	c.dbPath = filepath.Join(c.Home, "db", "main.sqlite3")
 	if c.Config.InformedConsent == false {
-		if err := Onboarding(c.Config); err != nil {
+		if err = Onboarding(c.Config); err != nil {
 			return errors.Wrap(err, "onboarding")
 		}
 	}
 
+	log.Debugf("Connecting to database sqlite3://%s", c.dbPath)
+	db, err := database.Connect(c.dbPath)
+	if err != nil {
+		return err
+	}
+	c.DB = db
+
 	tempDir, err := ioutil.TempDir("", "ooni")
 	if err != nil {
 		return errors.Wrap(err, "creating TempDir")
@@ -59,11 +90,12 @@ func (c *Context) Init() error {
 	return nil
 }
 
-// New Context instance.
-func New(c *Config, d *sqlx.DB) *Context {
+// NewContext instance.
+func NewContext(configPath string, homePath string) *Context {
 	return &Context{
-		Config: c,
-		DB:     d,
+		Home:       homePath,
+		Config:     &Config{},
+		configPath: configPath,
 	}
 }
 
@@ -155,24 +187,18 @@ func ParseConfig(b []byte) (*Config, error) {
 	return c, nil
 }
 
-//EnsureDefaultOONIHomeDir makes sure the paths to the OONI Home exist
-func EnsureDefaultOONIHomeDir() (string, error) {
-	home, err := GetOONIHome()
-	if err != nil {
-		return "", err
-	}
-
-	requiredDirs := []string{"db", "msmts"}
+// CreateHomeDirs creates the OONI home subdirectories
+func CreateHomeDirs(home string) error {
+	requiredDirs := []string{"db", "msmts", "geoip"}
 	for _, d := range requiredDirs {
 		if _, e := os.Stat(filepath.Join(home, d)); e != nil {
-			err = os.MkdirAll(filepath.Join(home, d), 0700)
-			if err != nil {
-				return "", err
+			if err := os.MkdirAll(filepath.Join(home, d), 0700); err != nil {
+				return err
 			}
 		}
 	}
 
-	return home, nil
+	return nil
 }
 
 // ReadConfig reads the configuration from the path
@@ -206,11 +232,7 @@ func ReadConfig(path string) (*Config, error) {
 }
 
 // ReadDefaultConfigPaths from common locations.
-func ReadDefaultConfigPaths() (*Config, error) {
-	home, err := EnsureDefaultOONIHomeDir()
-	if err != nil {
-		return nil, errors.Wrap(err, "reading default config paths")
-	}
+func ReadDefaultConfigPaths(home string) (*Config, error) {
 	var paths = []string{
 		filepath.Join(home, "config.json"),
 	}