adding script to find popular, but unused projects

Issue: #14

Author: adamdecaf

README.md

@@ -15,3 +15,6 @@ If you feel there's a project that would benefit from our help, please open an
 issue above. The issue itself will have some questions that you can answer to
 help us discover more about the project. From there we can begin to evaluate the
 project, and our ability to properly support it.
+
+We've also started writing a script that tries to find projects which are highly
+used, but also lacking recent updates.

go.mod

@@ -0,0 +1,11 @@
+module github.com/gofrs/help-requests
+
+require (
+	github.com/golang/protobuf v1.2.0 // indirect
+	github.com/google/go-github v17.0.0+incompatible
+	github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 // indirect
+	golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3
+	golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
+	golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f // indirect
+	google.golang.org/appengine v1.1.0 // indirect
+)

go.sum

@@ -0,0 +1,14 @@
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
+github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
+github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135 h1:zLTLjkaOFEFIOxY5BWLFLwh+cL8vOBW4XJ2aqLE/Tf0=
+github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 h1:czFLhve3vsQetD6JOJ8NZZvGQIXlnN3/yXxbT6/awxI=
+golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=

main.go

@@ -0,0 +1,115 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math"
+	"net/http"
+	"os"
+	"strconv"
+	"strings"
+	"text/tabwriter"
+	"time"
+
+	"github.com/google/go-github/github"
+	"golang.org/x/net/html"
+	"golang.org/x/oauth2"
+)
+
+func main() {
+	ghClient, err := createGithubClient(context.Background())
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "ERROR: problem creating github client: %v", err)
+	}
+
+	query := "stars:>100 pushed:<2018-01-01 language:Go"
+	repoRes, res, err := ghClient.Search.Repositories(context.Background(), query, &github.SearchOptions{
+		Sort:  "stars",
+		Order: "desc",
+		ListOptions: github.ListOptions{
+			PerPage: 25,
+			Page:    1,
+		},
+	})
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "ERROR: problem reading github repositories: %v", err)
+	}
+	res.Close = true
+
+	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
+	fmt.Fprintf(w, "name\tstars\tlast commit (days)\timporters\n")
+	defer w.Flush()
+	for i := range repoRes.Repositories {
+		repo := repoRes.Repositories[i]
+
+		cleanName := strings.Replace(*repo.HTMLURL, `https://`, "", 1)
+
+		// TODO(adam): goroutines + sync.WaitGroup
+		importers, err := scrapeGodocImports(cleanName)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "ERROR: problem grabbing %s godoc importers: %v\n", cleanName, err)
+		}
+
+		days := int(math.Abs(float64(repo.PushedAt.Sub(time.Now()).Hours()) / 24.0))
+		fmt.Fprintf(w, "%s\t%d\t%d\t%d\n", cleanName, *repo.StargazersCount, days, importers)
+	}
+}
+
+func createGithubClient(ctx context.Context) (*github.Client, error) {
+	v := os.Getenv("GITHUB_TOKEN")
+	if v == "" {
+		return nil, errors.New("environment variable GITHUB_TOKEN is required")
+	}
+	ts := oauth2.StaticTokenSource(&oauth2.Token{
+		AccessToken: v,
+	})
+	tc := oauth2.NewClient(ctx, ts)
+	return github.NewClient(tc), nil
+}
+
+func scrapeGodocImports(importPath string) (int, error) {
+	req, err := http.NewRequest("GET", "https://godoc.org/"+importPath, nil)
+	if err != nil {
+		return -1, fmt.Errorf("problem loading godoc.org: %v", err)
+	}
+	req.Header.Set("User-Agent", "Gofrs popstalerepo bot")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return -1, fmt.Errorf("problem loading %s: %v", req.URL, err)
+	}
+	defer resp.Body.Close()
+
+	// recursive search, from /x/net/html docs
+	var f func(n *html.Node) (int, error)
+	f = func(n *html.Node) (int, error) {
+		if n.Type == html.ElementNode && n.Data == "a" {
+			for _, a := range n.Attr {
+				// TODO(adam): we should try and refresh importers
+				// when running into errors.
+				if a.Key == "href" && strings.Contains(a.Val, "?importers") {
+					parts := strings.Fields(n.FirstChild.Data)
+					n, err := strconv.Atoi(parts[0])
+					if err != nil {
+						return -1, fmt.Errorf("couldn't parse %q: %v", parts[0], err)
+					}
+					return n, nil
+				}
+			}
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			n, err := f(c)
+			if err == nil && n > 0 {
+				return n, err
+			}
+		}
+		return -1, errors.New(`didn't find <a href="?importers">`)
+	}
+
+	doc, err := html.Parse(resp.Body)
+	if err != nil {
+		return -1, fmt.Errorf("couldn't parse html: %v", err)
+	}
+	return f(doc)
+}