cmd/modinfo: a tool for getting module info

Give this command-line tool a module path and it will print the number
of importers for each package in the module.

Change-Id: Ib93eb6bec1b7f049bda43c5f7f1b05a1281fb8e0
Reviewed-on: https://go-review.googlesource.com/c/vulndb/+/513499
Run-TryBot: Jonathan Amsterdam <jba@google.com>
Reviewed-by: Tatiana Bradley <tatianabradley@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>

cmd/modinfo/internal/pkgsitedb/db.go

@@ -0,0 +1,112 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !plan9
+
+// Package pkgsitedb provides functionality for connecting to the pkgsite
+// database.
+package pkgsitedb
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "regexp"
+
+ secretmanager "cloud.google.com/go/secretmanager/apiv1"
+ smpb "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb"
+ _ "github.com/lib/pq"
+)
+
+type Config struct {
+ User string
+ PasswordSecret string
+ Password string
+ Host string
+ Port string
+ DBName string
+}
+
+// Open creates a connection to the pkgsite database.
+func Open(ctx context.Context, cfg Config) (_ *sql.DB, err error) {
+ if cfg.Password == "" {
+ var err error
+ cfg.Password, err = getSecret(ctx, cfg.PasswordSecret)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ connString := fmt.Sprintf(
+ "user='%s' password='%s' host='%s' port=%s dbname='%s' sslmode='disable'",
+ cfg.User, cfg.Password, cfg.Host, cfg.Port, cfg.DBName)
+ db, err := sql.Open("postgres", connString)
+ if err != nil {
+ return nil, err
+ }
+ if err := db.PingContext(ctx); err != nil {
+ return nil, err
+ }
+ return db, nil
+}
+
+type Module struct {
+ Path string
+ Packages []*Package
+}
+
+type Package struct {
+ Path string
+ Version string
+ NumImporters int
+}
+
+func QueryModule(ctx context.Context, db *sql.DB, modulePath string) (*Module, error) {
+ query := `
+ SELECT package_path, version, imported_by_count
+ FROM search_documents
+ WHERE module_path = $1
+ ORDER BY 3 DESC
+ `
+ rows, err := db.QueryContext(ctx, query, modulePath)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+ m := &Module{Path: modulePath}
+ for rows.Next() {
+ var p Package
+ if err := rows.Scan(&p.Path, &p.Version, &p.NumImporters); err != nil {
+ return nil, err
+ }
+ m.Packages = append(m.Packages, &p)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+ return m, nil
+}
+
+var passwordRegexp = regexp.MustCompile(`password=\S+`)
+
+func redactPassword(dbinfo string) string {
+ return passwordRegexp.ReplaceAllLiteralString(dbinfo, "password=REDACTED")
+}
+
+// getSecret retrieves a secret from the GCP Secret Manager.
+// secretFullName should be of the form "projects/PROJECT/secrets/NAME".
+func getSecret(ctx context.Context, secretFullName string) (_ string, err error) {
+ client, err := secretmanager.NewClient(ctx)
+ if err != nil {
+ return "", err
+ }
+ defer client.Close()
+ result, err := client.AccessSecretVersion(ctx, &smpb.AccessSecretVersionRequest{
+ Name: secretFullName + "/versions/latest",
+ })
+ if err != nil {
+ return "", err
+ }
+ return string(result.Payload.Data), nil
+}

cmd/modinfo/internal/pkgsitedb/db_plan9.go

@@ -0,0 +1,26 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build plan9
+
+package pkgsitedb
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+
+ "golang.org/x/pkgsite-metrics/internal/config"
+ "golang.org/x/pkgsite-metrics/internal/scan"
+)
+
+var errDoesNotCompile = errors.New("github.com/lib/pq does not compile on plan9")
+
+func Open(ctx context.Context, cfg *config.Config) (_ *sql.DB, err error) {
+ return nil, errDoesNotCompile
+}
+
+func ModuleSpecs(ctx context.Context, db *sql.DB, minImportedByCount int) (specs []scan.ModuleSpec, err error) {
+ return nil, errDoesNotCompile
+}

cmd/modinfo/internal/pkgsitedb/db_test.go

@@ -0,0 +1,54 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !plan9
+
+package pkgsitedb
+
+import (
+ // imported to register the postgres database driver
+
+ "context"
+ "encoding/json"
+ "flag"
+ "fmt"
+ "os"
+ "testing"
+
+ _ "github.com/lib/pq"
+)
+
+var dbConfigFile = flag.String("dbconf", "", "filename with db config as JSON")
+
+func TestQueryModule(t *testing.T) {
+ if *dbConfigFile == "" {
+ t.Skip("missing -dbconf")
+ }
+ f, err := os.Open(*dbConfigFile)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer f.Close()
+ var cfg Config
+ if err := json.NewDecoder(f).Decode(&cfg); err != nil {
+ t.Fatal(err)
+ }
+ ctx := context.Background()
+
+ db, err := Open(ctx, cfg)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ if err := db.PingContext(ctx); err != nil {
+ t.Fatal(err)
+ }
+ m, err := QueryModule(ctx, db, "golang.org/x/tools")
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, p := range m.Packages {
+ fmt.Printf("%+v\n", p)
+ }
+}

cmd/modinfo/internal/pkgsitedb/secrets.go

@@ -0,0 +1,5 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package pkgsitedb

cmd/modinfo/main.go

@@ -0,0 +1,95 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Command modinfo displays module info from the pkgsite database.
+//
+// One-time setup: install the cloud sql proxy:
+//
+// > curl -o cloud-sql-proxy https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v2.6.0/cloud-sql-proxy.linux.amd64
+// > chmod +x cloud-sql-proxy
+// > ./cloud-sql-proxy $MODINFO_DB?port=5429 &
+//
+// cloud-sql-proxy must be running in the background to run this command.
+// If it stops, restart it by running the last command above.
+//
+// You will also need to do
+//
+// > gcloud auth application-default login
+//
+// if you haven't already.
+package main
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "os"
+ "sort"
+ "strings"
+ "text/tabwriter"
+
+ "golang.org/x/exp/maps"
+ "golang.org/x/vulndb/cmd/modinfo/internal/pkgsitedb"
+)
+
+func main() {
+ args := os.Args[1:]
+ if len(args) == 0 {
+ log.Fatal("missing module paths")
+ }
+ var cfg pkgsitedb.Config
+ cfg.User = mustEnv("MODINFO_USER")
+ cfg.Password = mustEnv("MODINFO_PASSWORD")
+ cfg.Host = "127.0.0.1"
+ cfg.Port = "5429"
+ cfg.DBName = mustEnv("MODINFO_DBNAME")
+ ctx := context.Background()
+ db, err := pkgsitedb.Open(ctx, cfg)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, `Could not open DB. You may need to run
+
+ cloud-sql-proxy $MODINFO_DB?port=5429 &
+
+Details: %v`, err)
+ fmt.Fprintln(os.Stderr)
+ os.Exit(1)
+ }
+ defer db.Close()
+ for _, modpath := range args {
+ mod, err := pkgsitedb.QueryModule(ctx, db, modpath)
+ if err != nil {
+ log.Fatal(err)
+ }
+ display(mod)
+ }
+}
+
+func mustEnv(ev string) string {
+ if r := os.Getenv(ev); r != "" {
+ return r
+ }
+ fmt.Fprintf(os.Stderr, "need value for environment variable %s\n", ev)
+ os.Exit(1)
+ return ""
+}
+
+func display(m *pkgsitedb.Module) {
+ if len(m.Packages) == 0 {
+ fmt.Printf("No packages for module %s; maybe it doesn't exist?\n", m.Path)
+ return
+ }
+ versionMap := map[string]bool{}
+ for _, p := range m.Packages {
+ versionMap[p.Version] = true
+ }
+ versions := maps.Keys(versionMap)
+ sort.Strings(versions)
+ fmt.Printf("==== %s @ %s ====\n", m.Path, strings.Join(versions, ", "))
+ tw := tabwriter.NewWriter(os.Stdout, 2, 4, 1, ' ', 0)
+ fmt.Fprintf(tw, "%s\t%s\n", "Import Path", "Importers")
+ for _, p := range m.Packages {
+ fmt.Fprintf(tw, "%s\t%5d\n", p.Path, p.NumImporters)
+ }
+ tw.Flush()
+}