Allow ignoring versions in fetcher

The grpc/java plugin had release v1.68.0 go out by accident. Update the
fetcher to allow specifying versions to ignore when detecting new
plugins.

This is partially implemented across all source types - it still needs
to be implemented for Go/NPM but those should be easy follow ups when we
need them.

cmd/download-plugins/main.go

@@ -17,7 +17,7 @@ import (
  "time"
 
  "aead.dev/minisign"
- "github.com/google/go-github/v63/github"
+ "github.com/google/go-github/v66/github"
  "golang.org/x/mod/semver"
 
  "github.com/bufbuild/plugins/internal/plugin"

go.mod

@@ -6,12 +6,11 @@ require (
  aead.dev/minisign v0.3.0
  github.com/bufbuild/buf v1.44.0
  github.com/google/go-containerregistry v0.20.2
- github.com/google/go-github/v63 v63.0.0
+ github.com/google/go-github/v66 v66.0.0
  github.com/hashicorp/go-retryablehttp v0.7.7
  github.com/sethvargo/go-envconfig v1.1.0
  github.com/spf13/pflag v1.0.5
  github.com/stretchr/testify v1.9.0
- go.uber.org/multierr v1.11.0
  golang.org/x/mod v0.21.0
  golang.org/x/oauth2 v0.23.0
  golang.org/x/sync v0.8.0
@@ -51,6 +50,7 @@ require (
  go.opentelemetry.io/otel/sdk v1.30.0 // indirect
  go.opentelemetry.io/otel/trace v1.30.0 // indirect
  go.uber.org/atomic v1.11.0 // indirect
+ go.uber.org/multierr v1.11.0 // indirect
  go.uber.org/zap v1.27.0 // indirect
  golang.org/x/crypto v0.27.0 // indirect
  golang.org/x/sys v0.25.0 // indirect

go.sum

@@ -46,8 +46,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-containerregistry v0.20.2 h1:B1wPJ1SN/S7pB+ZAimcciVD+r+yV/l/DSArMxlbwseo=
 github.com/google/go-containerregistry v0.20.2/go.mod h1:z38EKdKh4h7IP2gSfUUqEvalZBqs6AoLeWfUy34nQC8=
-github.com/google/go-github/v63 v63.0.0 h1:13xwK/wk9alSokujB9lJkuzdmQuVn2QCPeck76wR3nE=
-github.com/google/go-github/v63 v63.0.0/go.mod h1:IqbcrgUmIcEaioWrGYei/09o+ge5vhffGOcxrO0AfmA=
+github.com/google/go-github/v66 v66.0.0 h1:ADJsaXj9UotwdgK8/iFZtv7MLc8E8WBl62WLd/D/9+M=
+github.com/google/go-github/v66 v66.0.0/go.mod h1:+4SO9Zkuyf8ytMj0csN1NR/5OTR+MfqPp8P8dVlcvY4=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=

internal/cmd/fetcher/main.go

@@ -16,7 +16,6 @@ import (
  "time"
 
  "github.com/bufbuild/buf/private/pkg/interrupt"
- "go.uber.org/multierr"
  "golang.org/x/mod/semver"
 
  "github.com/bufbuild/plugins/internal/docker"
@@ -226,7 +225,7 @@ func copyDirectory(
  }
  defer func() {
  if retErr != nil {
- retErr = multierr.Append(retErr, os.RemoveAll(target))
+ retErr = errors.Join(retErr, os.RemoveAll(target))
  }
  }()
  for _, file := range entries {
@@ -257,7 +256,7 @@ func createPluginDir(
  }
  defer func() {
  if retErr != nil {
- retErr = multierr.Append(retErr, os.RemoveAll(filepath.Join(dir, newVersion)))
+ retErr = errors.Join(retErr, os.RemoveAll(filepath.Join(dir, newVersion)))
  }
  }()
  return copyDirectory(

internal/cmd/release/main.go

@@ -25,7 +25,7 @@ import (
  githubkeychain "github.com/google/go-containerregistry/pkg/authn/github"
  "github.com/google/go-containerregistry/pkg/name"
  "github.com/google/go-containerregistry/pkg/v1/remote"
- "github.com/google/go-github/v63/github"
+ "github.com/google/go-github/v66/github"
  "golang.org/x/mod/semver"
 
  "github.com/bufbuild/plugins/internal/plugin"

internal/fetchclient/fetchclient.go

@@ -11,11 +11,12 @@ import (
  "os"
  "strings"
 
- "github.com/google/go-github/v63/github"
+ "github.com/google/go-github/v66/github"
  "github.com/hashicorp/go-retryablehttp"
  "golang.org/x/mod/semver"
  "golang.org/x/oauth2"
 
+ "github.com/bufbuild/buf/private/pkg/slicesext"
  "github.com/bufbuild/plugins/internal/source"
 )
 
@@ -79,24 +80,29 @@ func (c *Client) Fetch(ctx context.Context, config *source.Config) (string, erro
 }
 
 func (c *Client) fetch(ctx context.Context, config *source.Config) (string, error) {
+ ignoreVersions := slicesext.ToStructMap(config.Source.IgnoreVersions)
  switch {
  case config.Source.GitHub != nil:
- return c.fetchGithub(ctx, config.Source.GitHub.Owner, config.Source.GitHub.Repository)
+ return c.fetchGithub(ctx, config.Source.GitHub.Owner, config.Source.GitHub.Repository, ignoreVersions)
  case config.Source.DartFlutter != nil:
- return c.fetchDartFlutter(ctx, config.Source.DartFlutter.Name)
+ return c.fetchDartFlutter(ctx, config.Source.DartFlutter.Name, ignoreVersions)
  case config.Source.GoProxy != nil:
- return c.fetchGoProxy(ctx, config.Source.GoProxy.Name)
+ return c.fetchGoProxy(ctx, config.Source.GoProxy.Name, ignoreVersions)
  case config.Source.NPMRegistry != nil:
- return c.fetchNPMRegistry(ctx, config.Source.NPMRegistry.Name)
+ return c.fetchNPMRegistry(ctx, config.Source.NPMRegistry.Name, ignoreVersions)
  case config.Source.Maven != nil:
- return c.fetchMaven(ctx, config.Source.Maven.Group, config.Source.Maven.Name)
+ return c.fetchMaven(ctx, config.Source.Maven.Group, config.Source.Maven.Name, ignoreVersions)
  case config.Source.Crates != nil:
- return c.fetchCrate(ctx, config.Source.Crates.CrateName)
+ return c.fetchCrate(ctx, config.Source.Crates.CrateName, ignoreVersions)
  }
  return "", errors.New("failed to match a source")
 }
 
-func (c *Client) fetchDartFlutter(ctx context.Context, name string) (string, error) {
+func (c *Client) fetchDartFlutter(
+ ctx context.Context,
+ name string,
+ ignoreVersions map[string]struct{},
+) (string, error) {
  request, err := http.NewRequestWithContext(
  ctx,
  http.MethodGet,
@@ -119,14 +125,37 @@ func (c *Client) fetchDartFlutter(ctx context.Context, name string) (string, err
  Latest struct {
  Version string `json:"version"`
  } `json:"latest"`
+ Versions []struct {
+ Version string `json:"version"`
+ } `json:"versions"`
  }
  if err := json.NewDecoder(response.Body).Decode(&data); err != nil {
  return "", err
  }
- return data.Latest.Version, nil
+ if len(ignoreVersions) == 0 {
+ return data.Latest.Version, nil
+ }
+ var latestVersion string
+ for _, version := range data.Versions {
+ version, ok := ensureSemverPrefix(version.Version)
+ if !ok {
+ continue
+ }
+ if _, ok := ignoreVersions[version]; ok {
+ continue
+ }
+ if latestVersion == "" || semver.Compare(latestVersion, version) < 0 {
+ latestVersion = version
+ }
+ }
+ // Shouldn't be possible unless we've ignored all versions
+ if latestVersion == "" {
+ return "", fmt.Errorf("failed to calculate latest version for dart source %s", name)
+ }
+ return latestVersion, nil
 }
 
-func (c *Client) fetchCrate(ctx context.Context, name string) (string, error) {
+func (c *Client) fetchCrate(ctx context.Context, name string, ignoreVersions map[string]struct{}) (string, error) {
  request, err := http.NewRequestWithContext(
  ctx,
  http.MethodGet,
@@ -164,7 +193,11 @@ func (c *Client) fetchCrate(ctx context.Context, name string) (string, error) {
  // from the server's index.
  continue
  }
- if v, ok := ensureSemverPrefix(version.Num); ok {
+ v, ok := ensureSemverPrefix(version.Num)
+ if !ok {
+ continue
+ }
+ if _, ok := ignoreVersions[v]; !ok {
  versions = append(versions, v)
  }
  }
@@ -175,7 +208,10 @@ func (c *Client) fetchCrate(ctx context.Context, name string) (string, error) {
  return versions[len(versions)-1], nil
 }
 
-func (c *Client) fetchGoProxy(ctx context.Context, name string) (string, error) {
+func (c *Client) fetchGoProxy(ctx context.Context, name string, ignoreVersions map[string]struct{}) (string, error) {
+ if len(ignoreVersions) > 0 {
+ return "", errors.New("ignore_versions not supported yet for go sources")
+ }
  request, err := http.NewRequestWithContext(
  ctx,
  http.MethodGet,
@@ -203,7 +239,10 @@ func (c *Client) fetchGoProxy(ctx context.Context, name string) (string, error)
  return data.Version, nil
 }
 
-func (c *Client) fetchNPMRegistry(ctx context.Context, name string) (string, error) {
+func (c *Client) fetchNPMRegistry(ctx context.Context, name string, ignoreVersions map[string]struct{}) (string, error) {
+ if len(ignoreVersions) > 0 {
+ return "", errors.New("ignore_versions not supported yet for NPM sources")
+ }
  request, err := http.NewRequestWithContext(
  ctx,
  http.MethodGet,
@@ -233,7 +272,12 @@ func (c *Client) fetchNPMRegistry(ctx context.Context, name string) (string, err
  return data.DistTags.Latest, nil
 }
 
-func (c *Client) fetchMaven(ctx context.Context, group string, name string) (string, error) {
+func (c *Client) fetchMaven(
+ ctx context.Context,
+ group string,
+ name string,
+ ignoreVersions map[string]struct{},
+) (string, error) {
  groupComponents := strings.Split(group, ".")
  targetURL, err := url.JoinPath(mavenURL, append(groupComponents, name, "maven-metadata.xml")...)
  if err != nil {
@@ -271,6 +315,9 @@ func (c *Client) fetchMaven(ctx context.Context, group string, name string) (str
  continue
  }
  v = semver.Canonical(v)
+ if _, ok := ignoreVersions[v]; ok {
+ continue
+ }
  if latestVersion == "" || semver.Compare(latestVersion, v) < 0 {
  latestVersion = v
  }
@@ -281,7 +328,12 @@ func (c *Client) fetchMaven(ctx context.Context, group string, name string) (str
  return latestVersion, nil
 }
 
-func (c *Client) fetchGithub(ctx context.Context, owner string, repository string) (string, error) {
+func (c *Client) fetchGithub(
+ ctx context.Context,
+ owner string,
+ repository string,
+ ignoreVersions map[string]struct{},
+) (string, error) {
  // With the GitHub API we have a few options:
  //
  // ✅ 1. list all git tags
@@ -305,7 +357,9 @@ func (c *Client) fetchGithub(ctx context.Context, owner string, repository strin
  continue
  }
  if v, ok := ensureSemverPrefix(*tag.Name); ok {
- versions = append(versions, v)
+ if _, ok := ignoreVersions[v]; !ok {
+ versions = append(versions, v)
+ }
  }
  }
  page = response.NextPage

internal/release/github.go

@@ -14,7 +14,7 @@ import (
  "time"
 
  "aead.dev/minisign"
- "github.com/google/go-github/v63/github"
+ "github.com/google/go-github/v66/github"
  "github.com/hashicorp/go-retryablehttp"
  "golang.org/x/oauth2"
 )

internal/source/config.go

@@ -45,6 +45,8 @@ type Source struct {
  NPMRegistry *NPMRegistryConfig `yaml:"npm_registry"`
  Maven *MavenConfig `yaml:"maven"`
  Crates *CratesConfig `yaml:"crates"`
+ // IgnoreVersions is a list of versions to ignore when fetching.
+ IgnoreVersions []string `yaml:"ignore_versions"`
 }
 
 var _ Cacheable = (*Source)(nil)

internal/source/source.go

@@ -6,8 +6,6 @@ import (
  "os"
  "path/filepath"
  "strings"
-
- "go.uber.org/multierr"
 )
 
 var (
@@ -36,7 +34,7 @@ func loadConfigFile(filename string) (_ *Config, retErr error) {
  return nil, err
  }
  defer func() {
- retErr = multierr.Append(retErr, file.Close())
+ retErr = errors.Join(retErr, file.Close())
  }()
  config, err := NewConfig(file)
  if err != nil {

plugins/grpc/java/source.yaml

@@ -2,3 +2,5 @@ source:
  maven:
  group: io.grpc
  name: protoc-gen-grpc-java
+ ignore_versions:
+ - v1.68.0