|
| 1 | +package main |
| 2 | + |
| 3 | +import ( |
| 4 | + "context" |
| 5 | + "errors" |
| 6 | + "fmt" |
| 7 | + "math" |
| 8 | + "net/http" |
| 9 | + "os" |
| 10 | + "strconv" |
| 11 | + "strings" |
| 12 | + "text/tabwriter" |
| 13 | + "time" |
| 14 | + |
| 15 | + "github.com/google/go-github/github" |
| 16 | + "golang.org/x/net/html" |
| 17 | + "golang.org/x/oauth2" |
| 18 | +) |
| 19 | + |
| 20 | +func main() { |
| 21 | + ghClient, err := createGithubClient(context.Background()) |
| 22 | + if err != nil { |
| 23 | + fmt.Fprintf(os.Stderr, "ERROR: problem creating github client: %v", err) |
| 24 | + } |
| 25 | + |
| 26 | + query := "stars:>100 pushed:<2018-01-01 language:Go" |
| 27 | + repoRes, res, err := ghClient.Search.Repositories(context.Background(), query, &github.SearchOptions{ |
| 28 | + Sort: "stars", |
| 29 | + Order: "desc", |
| 30 | + ListOptions: github.ListOptions{ |
| 31 | + PerPage: 25, |
| 32 | + Page: 1, |
| 33 | + }, |
| 34 | + }) |
| 35 | + if err != nil { |
| 36 | + fmt.Fprintf(os.Stderr, "ERROR: problem reading github repositories: %v", err) |
| 37 | + } |
| 38 | + res.Close = true |
| 39 | + |
| 40 | + w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0) |
| 41 | + fmt.Fprintf(w, "name\tstars\tlast commit (days)\timporters\n") |
| 42 | + defer w.Flush() |
| 43 | + for i := range repoRes.Repositories { |
| 44 | + repo := repoRes.Repositories[i] |
| 45 | + |
| 46 | + cleanName := strings.Replace(*repo.HTMLURL, `https://`, "", 1) |
| 47 | + |
| 48 | + // TODO(adam): goroutines + sync.WaitGroup |
| 49 | + importers, err := scrapeGodocImports(cleanName) |
| 50 | + if err != nil { |
| 51 | + fmt.Fprintf(os.Stderr, "ERROR: problem grabbing %s godoc importers: %v\n", cleanName, err) |
| 52 | + } |
| 53 | + |
| 54 | + days := int(math.Abs(float64(repo.PushedAt.Sub(time.Now()).Hours()) / 24.0)) |
| 55 | + fmt.Fprintf(w, "%s\t%d\t%d\t%d\n", cleanName, *repo.StargazersCount, days, importers) |
| 56 | + } |
| 57 | +} |
| 58 | + |
| 59 | +func createGithubClient(ctx context.Context) (*github.Client, error) { |
| 60 | + v := os.Getenv("GITHUB_TOKEN") |
| 61 | + if v == "" { |
| 62 | + return nil, errors.New("environment variable GITHUB_TOKEN is required") |
| 63 | + } |
| 64 | + ts := oauth2.StaticTokenSource(&oauth2.Token{ |
| 65 | + AccessToken: v, |
| 66 | + }) |
| 67 | + tc := oauth2.NewClient(ctx, ts) |
| 68 | + return github.NewClient(tc), nil |
| 69 | +} |
| 70 | + |
| 71 | +func scrapeGodocImports(importPath string) (int, error) { |
| 72 | + req, err := http.NewRequest("GET", "https://godoc.org/"+importPath, nil) |
| 73 | + if err != nil { |
| 74 | + return -1, fmt.Errorf("problem loading godoc.org: %v", err) |
| 75 | + } |
| 76 | + req.Header.Set("User-Agent", "Gofrs popstalerepo bot") |
| 77 | + |
| 78 | + resp, err := http.DefaultClient.Do(req) |
| 79 | + if err != nil { |
| 80 | + return -1, fmt.Errorf("problem loading %s: %v", req.URL, err) |
| 81 | + } |
| 82 | + defer resp.Body.Close() |
| 83 | + |
| 84 | + // recursive search, from /x/net/html docs |
| 85 | + var f func(n *html.Node) (int, error) |
| 86 | + f = func(n *html.Node) (int, error) { |
| 87 | + if n.Type == html.ElementNode && n.Data == "a" { |
| 88 | + for _, a := range n.Attr { |
| 89 | + // TODO(adam): we should try and refresh importers |
| 90 | + // when running into errors. |
| 91 | + if a.Key == "href" && strings.Contains(a.Val, "?importers") { |
| 92 | + parts := strings.Fields(n.FirstChild.Data) |
| 93 | + n, err := strconv.Atoi(parts[0]) |
| 94 | + if err != nil { |
| 95 | + return -1, fmt.Errorf("couldn't parse %q: %v", parts[0], err) |
| 96 | + } |
| 97 | + return n, nil |
| 98 | + } |
| 99 | + } |
| 100 | + } |
| 101 | + for c := n.FirstChild; c != nil; c = c.NextSibling { |
| 102 | + n, err := f(c) |
| 103 | + if err == nil && n > 0 { |
| 104 | + return n, err |
| 105 | + } |
| 106 | + } |
| 107 | + return -1, errors.New(`didn't find <a href="?importers">`) |
| 108 | + } |
| 109 | + |
| 110 | + doc, err := html.Parse(resp.Body) |
| 111 | + if err != nil { |
| 112 | + return -1, fmt.Errorf("couldn't parse html: %v", err) |
| 113 | + } |
| 114 | + return f(doc) |
| 115 | +} |
0 commit comments