Add buftarget package

TODO.txt

@@ -12,15 +12,3 @@ NOTE: We are not allowing cross-workspace finding for include_package_files=true
 
 - Changed protoc-gen-buf-lint and protoc-gen-buf-breaking configuration, deprecated old configurations
 - Need to manually test protoc-gen-buf-breaking
-
-$ bothbuf build google/ads/admob/v1/admob_api.proto
-OLD
-OLD STDOUT:
-OLD EXIT CODE: 100
-OLD STDERR:
-google/ads/admob/v1/admob_api.proto:19:8:stat google/ads/admob/v1/admob_resources.proto: file does not exist
-NEW
-NEW STDOUT:
-NEW EXIT CODE: 100
-NEW STDERR:
-/Users/pedge/git/googleapis/google/ads/admob/v1/admob_api.proto:19:8:stat google/ads/admob/v1/admob_resources.proto: file does not exist

private/buf/bufctl/controller.go

@@ -792,7 +792,7 @@ func (c *controller) getWorkspaceForProtoFileRef(
  // TODO FUTURE: Feed flag names through to here.
  return nil, fmt.Errorf("--exclude-path is not valid for use with .proto file references")
  }
- readBucketCloser, err := c.buffetchReader.GetSourceReadBucketCloser(
+ readBucketCloser, bucketTargeting, err := c.buffetchReader.GetSourceReadBucketCloser(
  ctx,
  c.container,
  protoFileRef,
@@ -804,19 +804,9 @@ func (c *controller) getWorkspaceForProtoFileRef(
  defer func() {
  retErr = multierr.Append(retErr, readBucketCloser.Close())
  }()
- // The ProtoFilePath is still relative to the input bucket, not the bucket
- // retrieved from buffetch. Treat the path just as we do with targetPaths
- // and externalPaths in withPathsForBucketExtender.
- protoFilePath, err := readBucketCloser.PathForExternalPath(protoFileRef.ProtoFilePath())
- if err != nil {
- return nil, err
- }
  options := []bufworkspace.WorkspaceBucketOption{
- bufworkspace.WithTargetSubDirPath(
- readBucketCloser.SubDirPath(),
- ),
  bufworkspace.WithProtoFileTargetPath(
- protoFilePath,
+ protoFileRef.ProtoFilePath(),
  protoFileRef.IncludePackageFiles(),
  ),
  bufworkspace.WithConfigOverride(
@@ -832,6 +822,7 @@ func (c *controller) getWorkspaceForProtoFileRef(
  return c.workspaceProvider.GetWorkspaceForBucket(
  ctx,
  readBucketCloser,
+ bucketTargeting,
  options...,
  )
 }
@@ -841,7 +832,7 @@ func (c *controller) getWorkspaceForSourceRef(
  sourceRef buffetch.SourceRef,
  functionOptions *functionOptions,
 ) (_ bufworkspace.Workspace, retErr error) {
- readBucketCloser, err := c.buffetchReader.GetSourceReadBucketCloser(
+ readBucketCloser, bucketTargeting, err := c.buffetchReader.GetSourceReadBucketCloser(
  ctx,
  c.container,
  sourceRef,
@@ -853,18 +844,7 @@ func (c *controller) getWorkspaceForSourceRef(
  defer func() {
  retErr = multierr.Append(retErr, readBucketCloser.Close())
  }()
- functionOptions, err = functionOptions.withPathsForBucketExtender(readBucketCloser)
- if err != nil {
- return nil, err
- }
  options := []bufworkspace.WorkspaceBucketOption{
- bufworkspace.WithTargetSubDirPath(
- readBucketCloser.SubDirPath(),
- ),
- bufworkspace.WithTargetPaths(
- functionOptions.targetPaths,
- functionOptions.targetExcludePaths,
- ),
  bufworkspace.WithConfigOverride(
  functionOptions.configOverride,
  ),
@@ -878,6 +858,7 @@ func (c *controller) getWorkspaceForSourceRef(
  return c.workspaceProvider.GetWorkspaceForBucket(
  ctx,
  readBucketCloser,
+ bucketTargeting,
  options...,
  )
 }
@@ -887,7 +868,7 @@ func (c *controller) getWorkspaceDepManagerForDirRef(
  dirRef buffetch.DirRef,
  functionOptions *functionOptions,
 ) (_ bufworkspace.WorkspaceDepManager, retErr error) {
- readWriteBucket, err := c.buffetchReader.GetDirReadWriteBucket(
+ readWriteBucket, bucketTargeting, err := c.buffetchReader.GetDirReadWriteBucket(
  ctx,
  c.container,
  dirRef,
@@ -901,9 +882,7 @@ func (c *controller) getWorkspaceDepManagerForDirRef(
  return c.workspaceDepManagerProvider.GetWorkspaceDepManager(
  ctx,
  readWriteBucket,
- bufworkspace.WithTargetSubDirPath(
- readWriteBucket.SubDirPath(),
- ),
+ bucketTargeting,
  )
 }
 

private/buf/bufctl/option.go

@@ -14,7 +14,9 @@
 
 package bufctl
 
-import "github.com/bufbuild/buf/private/buf/buffetch"
+import (
+ "github.com/bufbuild/buf/private/buf/buffetch"
+)
 
 type ControllerOption func(*controller)
 
@@ -144,28 +146,6 @@ func newFunctionOptions(controller *controller) *functionOptions {
  }
 }
 
-func (f *functionOptions) withPathsForBucketExtender(
- bucketExtender buffetch.BucketExtender,
-) (*functionOptions, error) {
- deref := *f
- c := &deref
- for i, inputTargetPath := range c.targetPaths {
- targetPath, err := bucketExtender.PathForExternalPath(inputTargetPath)
- if err != nil {
- return nil, err
- }
- c.targetPaths[i] = targetPath
- }
- for i, inputTargetExcludePath := range c.targetExcludePaths {
- targetExcludePath, err := bucketExtender.PathForExternalPath(inputTargetExcludePath)
- if err != nil {
- return nil, err
- }
- c.targetExcludePaths[i] = targetExcludePath
- }
- return c, nil
-}
-
 func (f *functionOptions) getGetReadBucketCloserOptions() []buffetch.GetReadBucketCloserOption {
  var getReadBucketCloserOptions []buffetch.GetReadBucketCloserOption
  if f.copyToInMemory {
@@ -174,6 +154,18 @@ func (f *functionOptions) getGetReadBucketCloserOptions() []buffetch.GetReadBuck
  buffetch.GetReadBucketCloserWithCopyToInMemory(),
  )
  }
+ if len(f.targetPaths) > 0 {
+ getReadBucketCloserOptions = append(
+ getReadBucketCloserOptions,
+ buffetch.GetReadBucketCloserWithTargetPaths(f.targetPaths),
+ )
+ }
+ if len(f.targetExcludePaths) > 0 {
+ getReadBucketCloserOptions = append(
+ getReadBucketCloserOptions,
+ buffetch.GetReadBucketCloserWithTargetExcludePaths(f.targetExcludePaths),
+ )
+ }
  if f.configOverride != "" {
  // If we have a config override, we do not search for buf.yamls or buf.work.yamls,
  // instead acting as if the config override was the only configuration file available.

private/buf/buffetch/buffetch.go

@@ -22,6 +22,7 @@ import (
  "net/http"
 
  "github.com/bufbuild/buf/private/buf/buffetch/internal"
+ "github.com/bufbuild/buf/private/buf/buftarget"
  "github.com/bufbuild/buf/private/bufpkg/bufconfig"
  "github.com/bufbuild/buf/private/bufpkg/bufmodule"
  "github.com/bufbuild/buf/private/pkg/app"
@@ -321,7 +322,7 @@ type SourceReader interface {
  container app.EnvStdinContainer,
  sourceRef SourceRef,
  options ...GetReadBucketCloserOption,
- ) (ReadBucketCloser, error)
+ ) (ReadBucketCloser, buftarget.BucketTargeting, error)
 }
 
 // GetReadBucketCloserOption is an option for a GetSourceReadBucketCloser call.
@@ -345,6 +346,22 @@ func GetReadBucketCloserWithNoSearch() GetReadBucketCloserOption {
  }
 }
 
+// GetReadBucketCloserWithTargetPaths sets the targets paths for bucket targeting information
+// returned with the bucket.
+func GetReadBucketCloserWithTargetPaths(targetPaths []string) GetReadBucketCloserOption {
+ return func(getReadBucketCloserOptions *getReadBucketCloserOptions) {
+ getReadBucketCloserOptions.targetPaths = targetPaths
+ }
+}
+
+// GetReadBucketCloserWithTargetExcludePaths sets the target exclude paths for bucket targeting
+// information returned with the bucket.
+func GetReadBucketCloserWithTargetExcludePaths(targetExcludePaths []string) GetReadBucketCloserOption {
+ return func(getReadBucketCloserOptions *getReadBucketCloserOptions) {
+ getReadBucketCloserOptions.targetExcludePaths = targetExcludePaths
+ }
+}
+
 // DirReader is a dir reader.
 type DirReader interface {
  // GetDirReadWriteBucket gets the dir bucket.
@@ -353,7 +370,7 @@ type DirReader interface {
  container app.EnvStdinContainer,
  dirRef DirRef,
  options ...GetReadWriteBucketOption,
- ) (ReadWriteBucket, error)
+ ) (ReadWriteBucket, buftarget.BucketTargeting, error)
 }
 
 // GetReadWriteBucketOption is an option for a GetDirReadWriteBucket call.
@@ -369,6 +386,22 @@ func GetReadWriteBucketWithNoSearch() GetReadWriteBucketOption {
  }
 }
 
+// GetReadWriteBucketWithTargetPaths sets the target paths for the bucket targeting information
+// returned with the bucket.
+func GetReadWriteBucketWithTargetPaths(targetPaths []string) GetReadWriteBucketOption {
+ return func(getReadWriteBucketOptions *getReadWriteBucketOptions) {
+ getReadWriteBucketOptions.targetPaths = targetPaths
+ }
+}
+
+// GetReadWriteBucketWithTargetExcludePaths sets the target exclude paths for the bucket
+// targeting information returned with the bucket.
+func GetReadWriteBucketWithTargetExcludePaths(targetExcludePaths []string) GetReadWriteBucketOption {
+ return func(getReadWriteBucketOptions *getReadWriteBucketOptions) {
+ getReadWriteBucketOptions.targetExcludePaths = targetExcludePaths
+ }
+}
+
 // ModuleFetcher is a module fetcher.
 type ModuleFetcher interface {
  // GetModuleKey gets the ModuleKey.
@@ -544,16 +577,20 @@ func GetInputConfigForString(
 }
 
 type getReadBucketCloserOptions struct {
- noSearch bool
- copyToInMemory bool
+ noSearch bool
+ copyToInMemory bool
+ targetPaths []string
+ targetExcludePaths []string
 }
 
 func newGetReadBucketCloserOptions() *getReadBucketCloserOptions {
  return &getReadBucketCloserOptions{}
 }
 
 type getReadWriteBucketOptions struct {
- noSearch bool
+ noSearch bool
+ targetPaths []string
+ targetExcludePaths []string
 }
 
 func newGetReadWriteBucketOptions() *getReadWriteBucketOptions {

private/buf/buffetch/internal/internal.go

@@ -21,6 +21,7 @@ import (
  "net/http"
  "strconv"
 
+ "github.com/bufbuild/buf/private/buf/buftarget"
  "github.com/bufbuild/buf/private/bufpkg/bufconfig"
  "github.com/bufbuild/buf/private/bufpkg/bufmodule"
  "github.com/bufbuild/buf/private/pkg/app"
@@ -183,7 +184,7 @@ type ProtoFileRef interface {
  BucketRef
  // Path is the normalized path to the file reference.
  Path() string
- // IncludePackageFiles says to include the same package files TODO update comment
+ // IncludePackageFiles says to include the files from the same package files
  IncludePackageFiles() bool
  FileScheme() FileScheme
  protoFileRef()
@@ -411,12 +412,6 @@ type BucketExtender interface {
  // the directory that contained this terminate file, and the subDirPath will be the sub-direftory of
  // the actual asset relative to the terminate file.
  SubDirPath() string
-
- // PathForExternalPath takes a path external to the asset and converts it to
- // a path that is relative to the asset.
- //
- // The returned path will be normalized and validated.
- PathForExternalPath(externalPath string) (string, error)
 }
 
 // ReadBucketCloser is a bucket returned from GetReadBucketCloser.
@@ -443,23 +438,19 @@ type Reader interface {
  options ...GetFileOption,
  ) (io.ReadCloser, error)
  // GetReadBucketCloser gets the bucket.
- //
- // If done for a ProtoFileRef, the Bucket will be for the enclosing module or workspace via
- // terminateFileNames or protoFileTerminateFileNames. No filtering of the Bucket is performed, this is
- // the responsibility of the caller.
  GetReadBucketCloser(
  ctx context.Context,
  container app.EnvStdinContainer,
  bucketRef BucketRef,
  options ...GetReadBucketCloserOption,
- ) (ReadBucketCloser, error)
+ ) (ReadBucketCloser, buftarget.BucketTargeting, error)
  // GetReadWriteBucket gets the bucket.
  GetReadWriteBucket(
  ctx context.Context,
  container app.EnvStdinContainer,
  dirRef DirRef,
  options ...GetReadWriteBucketOption,
- ) (ReadWriteBucket, error)
+ ) (ReadWriteBucket, buftarget.BucketTargeting, error)
  // GetModuleKey gets the ModuleKey.
  GetModuleKey(
  ctx context.Context,
@@ -829,21 +820,23 @@ func WithGetReadBucketCloserCopyToInMemory() GetReadBucketCloserOption {
 // See bufconfig.TerminateAtControllingWorkspace, which is the only thing that uses this.
 // This is used by both non-ProtoFileRefs to find the controlling workspace, AND ProtoFileRefs
 // to find the controlling workspace of an enclosing module or workspace.
-func WithGetReadBucketCloserTerminateFunc(terminateFunc TerminateFunc) GetReadBucketCloserOption {
+func WithGetReadBucketCloserTerminateFunc(terminateFunc buftarget.TerminateFunc) GetReadBucketCloserOption {
  return func(getReadBucketCloserOptions *getReadBucketCloserOptions) {
  getReadBucketCloserOptions.terminateFunc = terminateFunc
  }
 }
 
-// WithGetReadBucketCloserProtoFileTerminateFunc is like WithGetReadBucketCloserTerminateFunc, but determines
-// where to stop searching for the enclosing module or workspace when given a ProtoFileRef.
-//
-// See bufconfig.TerminateAtEnclosingModuleOrWorkspaceForProtoFileRef, which is the only thing that uses this.
-// This finds the enclosing module or workspace.
-// This is only used for ProtoFileRefs.
-func WithGetReadBucketCloserProtoFileTerminateFunc(protoFileTerminateFunc TerminateFunc) GetReadBucketCloserOption {
+// WithGetReadBucketCloserTargetPaths sets the target paths for the bucket targeting information.
+func WithGetReadBucketCloserTargetPaths(targetPaths []string) GetReadBucketCloserOption {
  return func(getReadBucketCloserOptions *getReadBucketCloserOptions) {
- getReadBucketCloserOptions.protoFileTerminateFunc = protoFileTerminateFunc
+ getReadBucketCloserOptions.targetPaths = targetPaths
+ }
+}
+
+// WithGetReadBucketCloserTargetExcludePaths sets the target exclude paths for the bucket targeting information.
+func WithGetReadBucketCloserTargetExcludePaths(targetExcludePaths []string) GetReadBucketCloserOption {
+ return func(getReadBucketCloserOptions *getReadBucketCloserOptions) {
+ getReadBucketCloserOptions.targetExcludePaths = targetExcludePaths
  }
 }
 
@@ -857,24 +850,24 @@ type GetReadWriteBucketOption func(*getReadWriteBucketOptions)
 // See bufconfig.TerminateAtControllingWorkspace, which is the only thing that uses this.
 // This is used by both non-ProtoFileRefs to find the controlling workspace, AND ProtoFileRefs
 // to find the controlling workspace of an enclosing module or workspace.
-func WithGetReadWriteBucketTerminateFunc(terminateFunc TerminateFunc) GetReadWriteBucketOption {
+func WithGetReadWriteBucketTerminateFunc(terminateFunc buftarget.TerminateFunc) GetReadWriteBucketOption {
  return func(getReadWriteBucketOptions *getReadWriteBucketOptions) {
  getReadWriteBucketOptions.terminateFunc = terminateFunc
  }
 }
 
-// TerminateFunc is a termination function.
-//
-// This function should return true if the search should terminate at the prefix, that is
-// the prefix should be where the bucket is mapped onto.
-//
-// The original subDirPath is also given to the TerminateFunc.
-type TerminateFunc func(
- ctx context.Context,
- bucket storage.ReadBucket,
- prefix string,
- originalSubDirPath string,
-) (terminate bool, err error)
+// WithGetReadWriteBucketTargetPaths sets the target paths for the bucket targeting information.
+func WithGetReadWriteBucketTargetPaths(targetPaths []string) GetReadWriteBucketOption {
+ return func(getReadWriteBucketOptions *getReadWriteBucketOptions) {
+ getReadWriteBucketOptions.targetPaths = targetPaths
+ }
+}
+
+func WithGetReadWriteBucketTargetExcludePaths(targetExcludePaths []string) GetReadWriteBucketOption {
+ return func(getReadWriteBucketOptions *getReadWriteBucketOptions) {
+  getReadWriteBucketOptions.targetExcludePaths = targetExcludePaths
+ }
+}
 
 // PutFileOption is a PutFile option.
 type PutFileOption func(*putFileOptions)