Add --workers flag

.goreleaser.yml

@@ -58,6 +58,7 @@ dockers:
       - internal
       - go.mod
       - go.sum
+      - Makefile
   - image_templates:
       ["ghcr.io/cloudflare/{{ .ProjectName }}:{{ .Version }}-arm64"]
     goarch: arm64
@@ -78,6 +79,7 @@ dockers:
       - internal
       - go.mod
       - go.sum
+      - Makefile
 docker_manifests:
   - name_template: ghcr.io/cloudflare/{{ .ProjectName }}:{{ .Version }}
     image_templates:

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## v0.7.3
+
+### Added
+
+- `--workers` flag to control the number of worker threads for running checks.
+
 ## v0.7.2
 
 ### Changed

Makefile

@@ -47,6 +47,10 @@ test:
 		-timeout=5m \
 		./...
 
+.PHONY: debug-testscript
+debug-testscript:
+	for I in ./cmd/pint/tests/*.txt ; do T=`basename "$${I}" | cut -d. -f1`; echo ">>> $${T}" ; go test -count=1 -timeout=30s -v -run=TestScript/$${T} ./cmd/pint ; done
+
 .PHONY: cover
 cover: test
 	go tool cover -func=$(COVER_PROFILE)

cmd/pint/ci.go

@@ -31,6 +31,11 @@ func actionCI(c *cli.Context) (err error) {
 		return fmt.Errorf("failed to set log level: %w", err)
 	}
 
+	workers := c.Int(workersFlag)
+	if workers < 1 {
+		return fmt.Errorf("--%s flag must be > 0", workersFlag)
+	}
+
 	cfg, err := config.Load(c.Path(configFlag), c.IsSet(configFlag))
 	if err != nil {
 		return fmt.Errorf("failed to load config file %q: %w", c.Path(configFlag), err)
@@ -71,7 +76,7 @@ func actionCI(c *cli.Context) (err error) {
 
 	gitBlame := discovery.NewGitBlameLineFinder(git.RunGit, toScan.Commits())
 	ctx := context.WithValue(context.Background(), config.CommandKey, config.CICommand)
-	summary := scanFiles(ctx, cfg, toScan, gitBlame)
+	summary := scanFiles(ctx, workers, cfg, toScan, gitBlame)
 
 	reps := []reporter.Reporter{
 		reporter.NewConsoleReporter(os.Stderr),

cmd/pint/lint.go

@@ -26,6 +26,11 @@ func actionLint(c *cli.Context) (err error) {
 		return fmt.Errorf("failed to set log level: %w", err)
 	}
 
+	workers := c.Int(workersFlag)
+	if workers < 1 {
+		return fmt.Errorf("--%s flag must be > 0", workersFlag)
+	}
+
 	paths := c.Args().Slice()
 	if len(paths) == 0 {
 		return fmt.Errorf("at least one file or directory required")
@@ -51,7 +56,7 @@ func actionLint(c *cli.Context) (err error) {
 	}
 
 	ctx := context.WithValue(context.Background(), config.CommandKey, config.LintCommand)
-	summary := scanFiles(ctx, cfg, toScan, &discovery.NoopLineFinder{})
+	summary := scanFiles(ctx, workers, cfg, toScan, &discovery.NoopLineFinder{})
 
 	r := reporter.NewConsoleReporter(os.Stderr)
 	err = r.Submit(summary)

cmd/pint/main.go

@@ -16,6 +16,7 @@ const (
 	disabledFlag = "disabled"
 	offlineFlag  = "offline"
 	noColorFlag  = "no-color"
+	workersFlag  = "workers"
 )
 
 var (
@@ -33,6 +34,12 @@ func newApp() *cli.App {
 				Value:   ".pint.hcl",
 				Usage:   "Configuration file to use",
 			},
+			&cli.IntFlag{
+				Name:    workersFlag,
+				Aliases: []string{"w"},
+				Value:   10,
+				Usage:   "Number of worker threads for running checks",
+			},
 			&cli.StringFlag{
 				Name:    logLevelFlag,
 				Aliases: []string{"l"},

cmd/pint/scan.go

@@ -45,7 +45,7 @@ func scanProblem(path string, err error) reporter.Report {
 	}
 }
 
-func scanFiles(ctx context.Context, cfg config.Config, fcs discovery.FileFindResults, ld discovery.LineFinder) (summary reporter.Summary) {
+func scanFiles(ctx context.Context, workers int, cfg config.Config, fcs discovery.FileFindResults, ld discovery.LineFinder) (summary reporter.Summary) {
 	summary.FileChanges = fcs
 
 	scanJobs := []scanJob{}
@@ -119,12 +119,11 @@ func scanFiles(ctx context.Context, cfg config.Config, fcs discovery.FileFindRes
 		}
 	}
 
-	jobs := make(chan scanJob, 100)
-	results := make(chan reporter.Report, 100)
+	jobs := make(chan scanJob, workers*5)
+	results := make(chan reporter.Report, workers*5)
 	wg := sync.WaitGroup{}
 
-	// run 10 workers
-	for w := 1; w <= 10; w++ {
+	for w := 1; w <= workers; w++ {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()

cmd/pint/watch.go

@@ -63,6 +63,11 @@ func actionWatch(c *cli.Context) (err error) {
 		return fmt.Errorf("failed to set log level: %w", err)
 	}
 
+	workers := c.Int(workersFlag)
+	if workers < 1 {
+		return fmt.Errorf("--%s flag must be > 0", workersFlag)
+	}
+
 	paths := c.Args().Slice()
 	if len(paths) == 0 {
 		return fmt.Errorf("at least one file or directory required")
@@ -118,7 +123,7 @@ func actionWatch(c *cli.Context) (err error) {
 	// start timer to run every $interval
 	interval := c.Duration(intervalFlag)
 	ack := make(chan bool, 1)
-	stop := startTimer(mainCtx, cfg, interval, ack, collector)
+	stop := startTimer(mainCtx, cfg, workers, interval, ack, collector)
 
 	quit := make(chan os.Signal, 1)
 	signal.Notify(quit, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
@@ -140,7 +145,7 @@ func actionWatch(c *cli.Context) (err error) {
 	return nil
 }
 
-func startTimer(ctx context.Context, cfg config.Config, interval time.Duration, ack chan bool, collector *problemCollector) chan bool {
+func startTimer(ctx context.Context, cfg config.Config, workers int, interval time.Duration, ack chan bool, collector *problemCollector) chan bool {
 	ticker := time.NewTicker(time.Second)
 	stop := make(chan bool, 1)
 	wasBootstrapped := false
@@ -156,7 +161,7 @@ func startTimer(ctx context.Context, cfg config.Config, interval time.Duration,
 					ticker.Reset(interval)
 					wasBootstrapped = true
 				}
-				if err := collector.scan(ctx); err != nil {
+				if err := collector.scan(ctx, workers); err != nil {
 					log.Error().Err(err).Msg("Got an error when running checks")
 				}
 				checkIterationsTotal.Inc()
@@ -194,7 +199,7 @@ func newProblemCollector(cfg config.Config, paths []string) *problemCollector {
 	}
 }
 
-func (c *problemCollector) scan(ctx context.Context) error {
+func (c *problemCollector) scan(ctx context.Context, workers int) error {
 	d := discovery.NewGlobFileFinder()
 	toScan, err := d.Find(c.paths...)
 	if err != nil {
@@ -205,7 +210,7 @@ func (c *problemCollector) scan(ctx context.Context) error {
 		return fmt.Errorf("no matching files")
 	}
 
-	s := scanFiles(ctx, c.cfg, toScan, &discovery.NoopLineFinder{})
+	s := scanFiles(ctx, workers, c.cfg, toScan, &discovery.NoopLineFinder{})
 
 	c.lock.Lock()
 	c.summary = &s