diff --git a/cmd/root_test.go b/cmd/root_test.go index c658fca8..0bcee35b 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -2484,6 +2484,35 @@ func TestConcurrentInvocation(t *testing.T) { as.NoError(eg.Wait()) } +func TestMaxBatchSize(t *testing.T) { + tempDir := test.TempExamples(t) + configPath := filepath.Join(tempDir, "/treefmt.toml") + + test.ChangeWorkDir(t, tempDir) + + maxBatchSize := 1 + cfg := &config.Config{ + FormatterConfigs: map[string]*config.Formatter{ + "echo": { + Command: "test-fmt-only-one-file-at-a-time", + Includes: []string{"*"}, + MaxBatchSize: &maxBatchSize, + }, + }, + } + + treefmt(t, + withConfig(configPath, cfg), + withNoError(t), + withStats(t, map[stats.Type]int{ + stats.Traversed: 33, + stats.Matched: 33, + stats.Formatted: 33, + stats.Changed: 33, + }), + ) +} + type options struct { args []string env map[string]string diff --git a/config/config.go b/config/config.go index b3ce635c..e87dc5a3 100644 --- a/config/config.go +++ b/config/config.go @@ -62,6 +62,8 @@ type Formatter struct { Excludes []string `mapstructure:"excludes,omitempty" toml:"excludes,omitempty"` // Indicates the order of precedence when executing this Formatter in a sequence of Formatters. Priority int `mapstructure:"priority,omitempty" toml:"priority,omitempty"` + // The maximum number of files we should pass to this Formatter at once. + MaxBatchSize *int `mapstructure:"max-batch-size" toml:"max-batch-size"` } // SetFlags appends our flags to the provided flag set. diff --git a/format/formatter.go b/format/formatter.go index 0885f7da..8d620dad 100644 --- a/format/formatter.go +++ b/format/formatter.go @@ -46,6 +46,14 @@ func (f *Formatter) Name() string { return f.name } +func (f *Formatter) MaxBatchSize() int { + if f.config.MaxBatchSize == nil { + return 1024 //<<< + } else { + return *f.config.MaxBatchSize + } +} + func (f *Formatter) Priority() int { return f.config.Priority } @@ -78,6 +86,24 @@ func (f *Formatter) Hash(h hash.Hash) error { } func (f *Formatter) Apply(ctx context.Context, files []*walk.File) error { + for i := 0; i < len(files); i += f.MaxBatchSize() { + end := min(i+f.MaxBatchSize(), len(files)) + if err := f.apply(ctx, files[i:end]); err != nil { + return err + } + } + + return nil +} + +func (f *Formatter) apply(ctx context.Context, files []*walk.File) error { + if len(files) > f.MaxBatchSize() { + //<<< learn me some go >>> + //<<< this error gets swallowed by scheduler.schedule and turned into a generic "Error: failed to finalise formatting: formatting failures detected" >>> + //<<< should we update that code to print more information? or should this be a panic instead? >>> + return fmt.Errorf("formatter cannot format %d files at once (max batch size: %d)", len(files), f.MaxBatchSize()) + } + start := time.Now() // construct args, starting with config diff --git a/format/scheduler.go b/format/scheduler.go index beac3292..fbb23b6c 100644 --- a/format/scheduler.go +++ b/format/scheduler.go @@ -6,6 +6,7 @@ import ( "context" "crypto/md5" //nolint:gosec "fmt" + "os" "runtime" "slices" "strings" @@ -127,6 +128,7 @@ func (s *scheduler) submit( s.batches[key] = append(s.batches[key], file) // schedule the batch for processing if it's full + // <<< TODO: impedance mismatch? instead compute LCM (up to some max) of max-batch-size of all matched formatters? >>> if len(s.batches[key]) == s.batchSize { s.schedule(ctx, key, s.batches[key]) // reset the batch @@ -146,6 +148,7 @@ func (s *scheduler) schedule(ctx context.Context, key batchKey, batch []*walk.Fi formatter := s.formatters[name] if err := formatter.Apply(ctx, batch); err != nil { + fmt.Fprintf(os.Stderr, "formatter failed with error: %s\n", err) //<<< formatErrors = append(formatErrors, err) } } diff --git a/nix/packages/treefmt/formatters.nix b/nix/packages/treefmt/formatters.nix index dc155347..d9b68262 100644 --- a/nix/packages/treefmt/formatters.nix +++ b/nix/packages/treefmt/formatters.nix @@ -58,4 +58,15 @@ with pkgs; [ test-fmt-append "$@" ''; }) + (pkgs.writeShellApplication { + name = "test-fmt-only-one-file-at-a-time"; + text = '' + if [ $# -ne 1 ]; then + echo "I only support formatting exactly 1 file at a time" + exit 1 + fi + + test-fmt-append "suffix" "$1" + ''; + }) ]