Add --log-order option

cli/internal/cmdutil/cmdutil.go

@@ -66,14 +66,22 @@ func (h *Helper) Cleanup(cliConfig *turbostate.ParsedArgsFromRust) {
 }
 
 func (h *Helper) getUI(cliArgs *turbostate.ParsedArgsFromRust) cli.Ui {
+ factory := h.getUIFactory(cliArgs)
+ return factory.Build(os.Stdout, os.Stdin, os.Stderr)
+}
+
+func (h *Helper) getUIFactory(cliArgs *turbostate.ParsedArgsFromRust) ui.Factory {
  colorMode := ui.GetColorModeFromEnv()
  if cliArgs.NoColor {
  colorMode = ui.ColorModeSuppressed
  }
  if cliArgs.Color {
  colorMode = ui.ColorModeForced
  }
- return ui.BuildColoredUi(colorMode)
+ return &ui.ColoredUIFactory{
+ ColorMode: colorMode,
+ Base: &ui.BasicUIFactory{},
+ }
 }
 
 func (h *Helper) getLogger() (hclog.Logger, error) {
@@ -126,7 +134,8 @@ func NewHelper(turboVersion string, args *turbostate.ParsedArgsFromRust) *Helper
 // It additionally returns a mechanism to set an error, so
 func (h *Helper) GetCmdBase(executionState *turbostate.ExecutionState) (*CmdBase, error) {
  // terminal is for color/no-color output
- terminal := h.getUI(&executionState.CLIArgs)
+ uiFactory := h.getUIFactory(&executionState.CLIArgs)
+ terminal := uiFactory.Build(os.Stdin, os.Stdout, os.Stderr)
  // logger is configured with verbosity level using --verbosity flag from end users
  logger, err := h.getLogger()
  if err != nil {
@@ -155,6 +164,7 @@ func (h *Helper) GetCmdBase(executionState *turbostate.ExecutionState) (*CmdBase
 
  return &CmdBase{
  UI: terminal,
+ UIFactory: uiFactory,
  Logger: logger,
  RepoRoot: repoRoot,
  APIClient: apiClient,
@@ -165,6 +175,7 @@ func (h *Helper) GetCmdBase(executionState *turbostate.ExecutionState) (*CmdBase
 // CmdBase encompasses configured components common to all turbo commands.
 type CmdBase struct {
  UI cli.Ui
+ UIFactory ui.Factory
  Logger hclog.Logger
  RepoRoot turbopath.AbsoluteSystemPath
  APIClient *client.APIClient

cli/internal/logstreamer/logstreamer.go

@@ -140,9 +140,9 @@ type PrettyStdoutWriter struct {
 var _ io.Writer = (*PrettyStdoutWriter)(nil)
 
 // NewPrettyStdoutWriter returns an instance of PrettyStdoutWriter
-func NewPrettyStdoutWriter(prefix string) *PrettyStdoutWriter {
+func NewPrettyIoWriter(prefix string, ioWriter io.Writer) *PrettyStdoutWriter {
  return &PrettyStdoutWriter{
- w: os.Stdout,
+ w: ioWriter,
  Prefix: prefix,
  }
 }

cli/internal/run/real_run.go

@@ -1,9 +1,12 @@
 package run
 
 import (
+ "bytes"
  gocontext "context"
  "fmt"
+ "io"
  "log"
+ "os"
  "os/exec"
  "strings"
  "sync"
@@ -75,11 +78,15 @@ func RealRun(
 
  runCache := runcache.New(turboCache, base.RepoRoot, rs.Opts.runcacheOpts, colorCache)
 
+ concurrentUIFactory := ui.ConcurrentUIFactory{
+ Base: base.UIFactory,
+ }
+
  ec := &execContext{
  colorCache: colorCache,
  runSummary: runSummary,
  rs: rs,
- ui: &cli.ConcurrentUi{Ui: base.UI},
+ ui: concurrentUIFactory.Build(os.Stdin, os.Stdout, os.Stderr),
  runCache: runCache,
  env: globalEnv,
  passThroughEnv: globalPassThroughEnv,
@@ -97,10 +104,54 @@ func RealRun(
  Concurrency: rs.Opts.runOpts.Concurrency,
  }
 
- mu := sync.Mutex{}
+ taskCount := len(engine.TaskGraph.Vertices())
+ logChan := make(chan taskLogContext, taskCount)
+ logWaitGroup := sync.WaitGroup{}
+ isGrouped := rs.Opts.runOpts.LogOrder == "grouped"
+
+ var outputLogs func()
+
+ if isGrouped {
+ outputLogs = func() {
+ for i := 1; i < taskCount; i++ {
+ logContext := <-logChan
+
+ outBytes := logContext.outBuf.Bytes()
+ errBytes := logContext.errBuf.Bytes()
+
+ _, errOut := os.Stdout.Write(outBytes)
+ _, errErr := os.Stderr.Write(errBytes)
+
+ if errOut != nil || errErr != nil {
+ ec.ui.Error("Failed to output some of the logs.")
+ }
+
+ logWaitGroup.Done()
+ }
+ }
+ }
+ if isGrouped {
+ go outputLogs()
+ }
+
+ taskSummaryMutex := sync.Mutex{}
  taskSummaries := []*runsummary.TaskSummary{}
  execFunc := func(ctx gocontext.Context, packageTask *nodes.PackageTask, taskSummary *runsummary.TaskSummary) error {
- taskExecutionSummary, err := ec.exec(ctx, packageTask)
+ logWaitGroup.Add(1)
+ outBuf := &bytes.Buffer{}
+ errBuf := &bytes.Buffer{}
+
+ var outWriter io.Writer = os.Stdout
+ var errWriter io.Writer = os.Stderr
+
+ if isGrouped {
+ outWriter = outBuf
+ errWriter = errBuf
+ }
+
+ ui := concurrentUIFactory.Build(os.Stdin, outWriter, errWriter)
+
+ taskExecutionSummary, err := ec.exec(ctx, packageTask, ui, outWriter)
 
  // taskExecutionSummary will be nil if the task never executed
  // (i.e. if the workspace didn't implement the script corresponding to the task)
@@ -111,13 +162,19 @@ func RealRun(
  taskSummary.CacheSummary = taskHashTracker.GetCacheStatus(taskSummary.TaskID)
 
  // lock since multiple things to be appending to this array at the same time
- mu.Lock()
+ taskSummaryMutex.Lock()
  taskSummaries = append(taskSummaries, taskSummary)
  // not using defer, just release the lock
- mu.Unlock()
+ taskSummaryMutex.Unlock()
 
  runSummary.CloseTask(taskSummary)
  }
+ if isGrouped {
+ logChan <- taskLogContext{
+ outBuf: outBuf,
+ errBuf: errBuf,
+ }
+ }
 
  // Return the error when there is one
  if err != nil {
@@ -176,6 +233,10 @@ func RealRun(
  }
  }
 
+ if isGrouped {
+ logWaitGroup.Wait()
+ }
+
  if err := runSummary.Close(ctx, exitCode, g.WorkspaceInfos); err != nil {
  // We don't need to throw an error, but we can warn on this.
  // Note: this method doesn't actually return an error for Real Runs at the time of writing.
@@ -190,6 +251,11 @@ func RealRun(
  return nil
 }
 
+type taskLogContext struct {
+ outBuf *bytes.Buffer
+ errBuf *bytes.Buffer
+}
+
 type execContext struct {
  colorCache *colorcache.ColorCache
  runSummary runsummary.Meta
@@ -216,7 +282,7 @@ func (ec *execContext) logError(prefix string, err error) {
  ec.ui.Error(fmt.Sprintf("%s%s%s", ui.ERROR_PREFIX, prefix, color.RedString(" %v", err)))
 }
 
-func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTask) (*runsummary.TaskExecutionSummary, error) {
+func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTask, ui cli.Ui, outWriter io.Writer) (*runsummary.TaskExecutionSummary, error) {
  // Setup tracer. Every time tracer() is called the taskExecutionSummary's duration is updated
  // So make sure to call it before returning.
  successExitCode := 0 // We won't use this till later
@@ -257,7 +323,7 @@ func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTas
  taskCache := ec.runCache.TaskCache(packageTask, hash)
  // Create a logger for replaying
  prefixedUI := &cli.PrefixedUi{
- Ui: ec.ui,
+ Ui: ui,
  OutputPrefix: prettyPrefix,
  InfoPrefix: prettyPrefix,
  ErrorPrefix: prettyPrefix,
@@ -328,7 +394,7 @@ func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTas
  // Setup stdout/stderr
  // If we are not caching anything, then we don't need to write logs to disk
  // be careful about this conditional given the default of cache = true
- writer, err := taskCache.OutputWriter(prettyPrefix)
+ writer, err := taskCache.OutputWriter(prettyPrefix, outWriter)
  if err != nil {
  tracer(runsummary.TargetBuildFailed, err, nil)
 

cli/internal/run/run.go

@@ -93,12 +93,14 @@ func optsFromArgs(args *turbostate.ParsedArgsFromRust) (*Opts, error) {
  }
  opts.runOpts.Concurrency = concurrency
  }
+
  opts.runOpts.Parallel = runPayload.Parallel
  opts.runOpts.Profile = runPayload.Profile
  opts.runOpts.ContinueOnError = runPayload.ContinueExecution
  opts.runOpts.Only = runPayload.Only
  opts.runOpts.NoDaemon = runPayload.NoDaemon
  opts.runOpts.SinglePackage = args.Command.Run.SinglePackage
+ opts.runOpts.LogOrder = args.Command.Run.LogOrder
 
  // See comment on Graph in turbostate.go for an explanation on Graph's representation.
  // If flag is passed...

cli/internal/runcache/runcache.go

@@ -224,12 +224,12 @@ func (fwc *fileWriterCloser) Close() error {
 
 // OutputWriter creates a sink suitable for handling the output of the command associated
 // with this task.
-func (tc TaskCache) OutputWriter(prefix string) (io.WriteCloser, error) {
+func (tc TaskCache) OutputWriter(prefix string, ioWriter io.Writer) (io.WriteCloser, error) {
  // an os.Stdout wrapper that will add prefixes before printing to stdout
- stdoutWriter := logstreamer.NewPrettyStdoutWriter(prefix)
+ prettyIoWriter := logstreamer.NewPrettyIoWriter(prefix, ioWriter)
 
  if tc.cachingDisabled || tc.rc.writesDisabled {
- return nopWriteCloser{stdoutWriter}, nil
+ return nopWriteCloser{prettyIoWriter}, nil
  }
  // Setup log file
  if err := tc.LogFileName.EnsureDir(); err != nil {
@@ -250,7 +250,7 @@ func (tc TaskCache) OutputWriter(prefix string) (io.WriteCloser, error) {
  // only write to log file, not to stdout
  fwc.Writer = bufWriter
  } else {
- fwc.Writer = io.MultiWriter(stdoutWriter, bufWriter)
+ fwc.Writer = io.MultiWriter(prettyIoWriter, bufWriter)
  }
 
  return fwc, nil

cli/internal/scope/scope_test.go

@@ -84,7 +84,10 @@ func TestResolvePackages(t *testing.T) {
  if err != nil {
  t.Fatalf("cwd: %v", err)
  }
- tui := ui.Default()
+ defaultUIFactory := ui.ColoredUIFactory{
+ Base: &ui.BasicUIFactory{},
+ }
+ tui := defaultUIFactory.Build(os.Stdin, os.Stdout, os.Stderr)
  logger := hclog.Default()
  // Dependency graph:
  //

cli/internal/turbostate/turbostate.go

@@ -47,6 +47,7 @@ type RunPayload struct {
  NoDeps bool `json:"no_deps"`
  Only bool `json:"only"`
  OutputLogs string `json:"output_logs"`
+ LogOrder string `json:"log_order"`
  PassThroughArgs []string `json:"pass_through_args"`
  Parallel bool `json:"parallel"`
  Profile string `json:"profile"`

cli/internal/ui/ui.go

@@ -10,7 +10,6 @@ import (
 
  "github.com/fatih/color"
  "github.com/mattn/go-isatty"
- "github.com/mitchellh/cli"
  "github.com/vercel/turbo/cli/internal/ci"
 )
 
@@ -88,34 +87,3 @@ func (into *stripAnsiWriter) Write(p []byte) (int, error) {
  // written.
  return len(p), nil
 }
-
-// Default returns the default colored ui
-func Default() *cli.ColoredUi {
- return BuildColoredUi(ColorModeUndefined)
-}
-
-func BuildColoredUi(colorMode ColorMode) *cli.ColoredUi {
- colorMode = applyColorMode(colorMode)
-
- var outWriter, errWriter io.Writer
-
- if colorMode == ColorModeSuppressed {
- outWriter = &stripAnsiWriter{wrappedWriter: os.Stdout}
- errWriter = &stripAnsiWriter{wrappedWriter: os.Stderr}
- } else {
- outWriter = os.Stdout
- errWriter = os.Stderr
- }
-
- return &cli.ColoredUi{
- Ui: &cli.BasicUi{
- Reader: os.Stdin,
- Writer: outWriter,
- ErrorWriter: errWriter,
- },
- OutputColor: cli.UiColorNone,
- InfoColor: cli.UiColorNone,
- WarnColor: cli.UiColor{Code: int(color.FgYellow), Bold: false},
- ErrorColor: cli.UiColorRed,
- }
-}