Add google_webmaster collector.

cmd/scollector/collectors/google_analytics.go

@@ -50,7 +50,7 @@ func c_google_analytics(clientid string, secret string, tokenstr string, sites [
 	var md opentsdb.MultiDataPoint
 	var mErr multiError
 
-	c, err := analyticsClient(clientid, secret, tokenstr)
+	c, err := googleAPIClient(clientid, secret, tokenstr, []string{analytics.AnalyticsScope})
 	if err != nil {
 		return nil, err
 	}
@@ -183,16 +183,17 @@ func getPageviews(md *opentsdb.MultiDataPoint, svc *analytics.Service, site conf
 	return nil
 }
 
-// analyticsClient() takes in a clientid, secret, and a base64'd gob representing the cached oauth token.
-// Generating the token is left as an exercise to the reader. (TODO)
-func analyticsClient(clientid string, secret string, tokenstr string) (*http.Client, error) {
+// googleAPIClient() takes in a clientid, secret, a base64'd gob representing
+// the cached oauth token, and a list of oauth scopes.  Generating the token is
+// left as an exercise to the reader.
+func googleAPIClient(clientid string, secret string, tokenstr string, scopes []string) (*http.Client, error) {
 	ctx := context.Background()
 
 	config := &oauth2.Config{
 		ClientID:     clientid,
 		ClientSecret: secret,
 		Endpoint:     google.Endpoint,
-		Scopes:       []string{analytics.AnalyticsScope},
+		Scopes:       scopes,
 	}
 
 	token := new(oauth2.Token)

cmd/scollector/collectors/google_webmaster.go

@@ -0,0 +1,91 @@
+package collectors
+
+import (
+	"net/url"
+	"time"
+
+	"bosun.org/cmd/scollector/conf"
+	"bosun.org/metadata"
+	"bosun.org/opentsdb"
+	"bosun.org/slog"
+
+	"google.golang.org/api/webmasters/v3"
+)
+
+func init() {
+	registerInit(func(c *conf.Conf) {
+		for _, g := range c.GoogleWebmaster {
+			collectors = append(collectors, &IntervalCollector{
+				F: func() (opentsdb.MultiDataPoint, error) {
+					return c_google_webmaster(g.ClientID, g.Secret, g.Token)
+				},
+				name:     "c_google_webmaster",
+				Interval: time.Hour * 1,
+			})
+		}
+	})
+}
+
+func c_google_webmaster(clientID, secret, tokenStr string) (opentsdb.MultiDataPoint, error) {
+	c, err := googleAPIClient(clientID, secret, tokenStr, []string{webmasters.WebmastersReadonlyScope})
+	if err != nil {
+		return nil, err
+	}
+
+	svc, err := webmasters.New(c)
+	if err != nil {
+		return nil, err
+	}
+
+	md, err := getWebmasterErrorsMetrics(svc)
+	if err != nil {
+		return nil, err
+	}
+
+	return *md, nil
+}
+
+// getWebmasterErrorsMetric utilizes the webmasters API to list all sites
+// associated with an authenticated account, fetch the time-series data error
+// metrics for each of those sites, and builds opentsdb datapoints from that
+// data.
+func getWebmasterErrorsMetrics(svc *webmasters.Service) (*opentsdb.MultiDataPoint, error) {
+	md := &opentsdb.MultiDataPoint{}
+
+	sites, err := svc.Sites.List().Do()
+	if err != nil {
+		return nil, err
+	}
+
+	metricName := "google.webmaster.errors"
+	for _, site := range sites.SiteEntry {
+		u, err := url.Parse(site.SiteUrl)
+
+		if err != nil {
+			return nil, err
+		}
+		if site.PermissionLevel == "siteUnverifiedUser" {
+			slog.Errorf("Lack permission to fetch error metrics for site %s. Skipping.\n", u.Host)
+			continue
+		}
+		tags := opentsdb.TagSet{}
+		tags["site"] = u.Host
+		crawlErrors, err := svc.Urlcrawlerrorscounts.Query(site.SiteUrl).LatestCountsOnly(true).Do()
+		if err != nil {
+			return nil, err
+		}
+		for _, e := range crawlErrors.CountPerTypes {
+			tags["platform"] = e.Platform
+			tags["category"] = e.Category
+			for _, entry := range e.Entries {
+				t, err := time.Parse(time.RFC3339, entry.Timestamp)
+				if err != nil {
+					return md, err
+				}
+				AddTS(md, metricName, t.Unix(), entry.Count, tags, metadata.Gauge, metadata.Error, "")
+			}
+		}
+	}
+
+	return md, nil
+}

cmd/scollector/conf/conf.go

@@ -74,6 +74,7 @@ type Conf struct {
 	RabbitMQ            []RabbitMQ
 	Nexpose             []Nexpose
 	GoogleAnalytics     []GoogleAnalytics
+	GoogleWebmaster     []GoogleWebmaster
 	Cadvisor            []Cadvisor
 	RedisCounters       []RedisCounters
 	ExtraHop            []ExtraHop
@@ -109,6 +110,12 @@ type GoogleAnalytics struct {
 	Sites    []GoogleAnalyticsSite
 }
 
+type GoogleWebmaster struct {
+	ClientID string
+	Secret   string
+	Token    string
+}
+
 type Fastly struct {
 	Key            string
 	StatusBaseAddr string