From 7ef10c4735e6a7d10c0497fec245b60e188016c7 Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Tue, 21 Feb 2023 20:51:14 +0000 Subject: [PATCH 01/36] Add log order flag to rust --- crates/turborepo-lib/src/cli.rs | 45 ++++++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/crates/turborepo-lib/src/cli.rs b/crates/turborepo-lib/src/cli.rs index 0473e2c3ebc25..8d5e309fd9041 100644 --- a/crates/turborepo-lib/src/cli.rs +++ b/crates/turborepo-lib/src/cli.rs @@ -44,6 +44,20 @@ impl Default for OutputLogsMode { } } +#[derive(Copy, Clone, Debug, PartialEq, Serialize, ValueEnum)] +pub enum LogOrder { + #[serde(rename = "stream")] + Stream, + #[serde(rename = "grouped")] + Grouped, +} + +impl Default for LogOrder { + fn default() -> Self { + Self::Stream + } +} + // NOTE: These *must* be kept in sync with the `_dryRunJSONValue` // and `_dryRunTextValue` constants in run.go. #[derive(Copy, Clone, Debug, PartialEq, Serialize, ValueEnum)] @@ -327,6 +341,11 @@ pub struct RunArgs { /// output. (default full) #[clap(long, value_enum)] pub output_logs: Option, + /// Set type of process output order. Use "stream" to show + /// output as soon as it is available. Use "grouped" to + /// show output when a command has finished execution. (default stream) + #[clap(long, value_enum)] + pub log_order: Option, #[clap(long, hide = true)] pub only: bool, /// Execute all tasks in parallel. @@ -569,7 +588,7 @@ mod test { use anyhow::Result; - use crate::cli::{Args, Command, DryRunMode, OutputLogsMode, RunArgs, Verbosity}; + use crate::cli::{Args, Command, DryRunMode, LogOrder, OutputLogsMode, RunArgs, Verbosity}; #[test] fn test_parse_run() -> Result<()> { @@ -875,6 +894,30 @@ mod test { } ); + assert_eq!( + Args::try_parse_from(["turbo", "run", "build", "--log-order", "stream"]).unwrap(), + Args { + command: Some(Command::Run(Box::new(RunArgs { + tasks: vec!["build".to_string()], + log_order: Some(LogOrder::Stream), + ..get_default_run_args() + }))), + ..Args::default() + } + ); + + assert_eq!( + Args::try_parse_from(["turbo", "run", "build", "--log-order", "grouped"]).unwrap(), + Args { + command: Some(Command::Run(Box::new(RunArgs { + tasks: vec!["build".to_string()], + log_order: Some(LogOrder::Grouped), + ..get_default_run_args() + }))), + ..Args::default() + } + ); + assert_eq!( Args::try_parse_from(["turbo", "run", "build", "--parallel"]).unwrap(), Args { From 55a49bb54de8f6300322e4a5afaa70902896d227 Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Tue, 21 Feb 2023 21:26:38 +0000 Subject: [PATCH 02/36] Add basic functionality for grouping outputs --- cli/internal/logstreamer/logstreamer.go | 15 +++++++++++---- cli/internal/run/real_run.go | 4 ++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/cli/internal/logstreamer/logstreamer.go b/cli/internal/logstreamer/logstreamer.go index 4379c2597d6ea..cb43d4bd09f98 100644 --- a/cli/internal/logstreamer/logstreamer.go +++ b/cli/internal/logstreamer/logstreamer.go @@ -27,9 +27,11 @@ type Logstreamer struct { colorOkay string colorFail string colorReset string + + grouped bool } -func NewLogstreamer(logger *log.Logger, prefix string, record bool) *Logstreamer { +func NewLogstreamer(logger *log.Logger, prefix string, record bool, grouped bool) *Logstreamer { streamer := &Logstreamer{ Logger: logger, buf: bytes.NewBuffer([]byte("")), @@ -39,6 +41,7 @@ func NewLogstreamer(logger *log.Logger, prefix string, record bool) *Logstreamer colorOkay: "", colorFail: "", colorReset: "", + grouped: grouped, } if strings.HasPrefix(os.Getenv("TERM"), "xterm") { @@ -55,6 +58,11 @@ func (l *Logstreamer) Write(p []byte) (n int, err error) { return } + // If we want logs to be grouped, do not output them when we receive them, only when they are flushed + if l.grouped { + return + } + err = l.OutputLines() return } @@ -68,12 +76,11 @@ func (l *Logstreamer) Close() error { } func (l *Logstreamer) Flush() error { - p := make([]byte, l.buf.Len()) - if _, err := l.buf.Read(p); err != nil { + err := l.OutputLines() + if err != nil { return err } - l.out(string(p)) return nil } diff --git a/cli/internal/run/real_run.go b/cli/internal/run/real_run.go index b8088971de606..b19353ee5f7fc 100644 --- a/cli/internal/run/real_run.go +++ b/cli/internal/run/real_run.go @@ -223,9 +223,9 @@ func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTas // Create a logger logger := log.New(writer, "", 0) // Setup a streamer that we'll pipe cmd.Stdout to - logStreamerOut := logstreamer.NewLogstreamer(logger, prettyPrefix, false) + logStreamerOut := logstreamer.NewLogstreamer(logger, prettyPrefix, false, false) // Setup a streamer that we'll pipe cmd.Stderr to. - logStreamerErr := logstreamer.NewLogstreamer(logger, prettyPrefix, false) + logStreamerErr := logstreamer.NewLogstreamer(logger, prettyPrefix, false, false) cmd.Stderr = logStreamerErr cmd.Stdout = logStreamerOut // Flush/Reset any error we recorded From 69206015adc94ea3582be19a9ae68f093a52f55b Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Wed, 22 Feb 2023 14:19:36 +0000 Subject: [PATCH 03/36] Implement usage of the flag --- cli/internal/run/real_run.go | 6 ++++-- cli/internal/run/run.go | 1 + cli/internal/run/run_spec.go | 2 ++ cli/internal/turbostate/turbostate.go | 1 + 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/cli/internal/run/real_run.go b/cli/internal/run/real_run.go index b19353ee5f7fc..82dd84c841b1a 100644 --- a/cli/internal/run/real_run.go +++ b/cli/internal/run/real_run.go @@ -220,12 +220,14 @@ func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTas } } + grouped := ec.rs.Opts.runOpts.logOrder == "grouped" + // Create a logger logger := log.New(writer, "", 0) // Setup a streamer that we'll pipe cmd.Stdout to - logStreamerOut := logstreamer.NewLogstreamer(logger, prettyPrefix, false, false) + logStreamerOut := logstreamer.NewLogstreamer(logger, prettyPrefix, false, grouped) // Setup a streamer that we'll pipe cmd.Stderr to. - logStreamerErr := logstreamer.NewLogstreamer(logger, prettyPrefix, false, false) + logStreamerErr := logstreamer.NewLogstreamer(logger, prettyPrefix, false, grouped) cmd.Stderr = logStreamerErr cmd.Stdout = logStreamerOut // Flush/Reset any error we recorded diff --git a/cli/internal/run/run.go b/cli/internal/run/run.go index 7f275b7c6c185..41148079f31e5 100644 --- a/cli/internal/run/run.go +++ b/cli/internal/run/run.go @@ -102,6 +102,7 @@ func optsFromArgs(args *turbostate.ParsedArgsFromRust) (*Opts, error) { opts.runOpts.only = runPayload.Only opts.runOpts.noDaemon = runPayload.NoDaemon opts.runOpts.singlePackage = args.Command.Run.SinglePackage + opts.runOpts.logOrder = args.Command.Run.LogOrder // See comment on Graph in turbostate.go for an explanation on Graph's representation. // If flag is passed... diff --git a/cli/internal/run/run_spec.go b/cli/internal/run/run_spec.go index b837f9af8258e..cea950383bb2a 100644 --- a/cli/internal/run/run_spec.go +++ b/cli/internal/run/run_spec.go @@ -74,4 +74,6 @@ type runOpts struct { graphFile string noDaemon bool singlePackage bool + // The order of the logs, either 'grouped' or 'stream' + logOrder string } diff --git a/cli/internal/turbostate/turbostate.go b/cli/internal/turbostate/turbostate.go index 0fb86d15c1c31..a615d2c65a715 100644 --- a/cli/internal/turbostate/turbostate.go +++ b/cli/internal/turbostate/turbostate.go @@ -51,6 +51,7 @@ type RunPayload struct { NoDeps bool `json:"no_deps"` Only bool `json:"only"` OutputLogs string `json:"output_logs"` + LogOrder string `json:"log_order"` PassThroughArgs []string `json:"pass_through_args"` Parallel bool `json:"parallel"` Profile string `json:"profile"` From 5ca0d14508d88b433e33dd1b485416405d368a02 Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Wed, 22 Feb 2023 14:51:32 +0000 Subject: [PATCH 04/36] Fix tests --- cli/internal/logstreamer/logstreamer_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cli/internal/logstreamer/logstreamer_test.go b/cli/internal/logstreamer/logstreamer_test.go index 94d8a8283716c..62d2003560fc1 100644 --- a/cli/internal/logstreamer/logstreamer_test.go +++ b/cli/internal/logstreamer/logstreamer_test.go @@ -19,11 +19,11 @@ func TestLogstreamerOk(t *testing.T) { logger := log.New(os.Stdout, "--> ", log.Ldate|log.Ltime) // Setup a streamer that we'll pipe cmd.Stdout to - logStreamerOut := NewLogstreamer(logger, "stdout", false) + logStreamerOut := NewLogstreamer(logger, "stdout", false, false) defer logStreamerOut.Close() // Setup a streamer that we'll pipe cmd.Stderr to. // We want to record/buffer anything that's written to this (3rd argument true) - logStreamerErr := NewLogstreamer(logger, "stderr", true) + logStreamerErr := NewLogstreamer(logger, "stderr", true, false) defer logStreamerErr.Close() // Execute something that succeeds @@ -57,11 +57,11 @@ func TestLogstreamerErr(t *testing.T) { logger := log.New(os.Stdout, "--> ", log.Ldate|log.Ltime) // Setup a streamer that we'll pipe cmd.Stdout to - logStreamerOut := NewLogstreamer(logger, "stdout", false) + logStreamerOut := NewLogstreamer(logger, "stdout", false, false) defer logStreamerOut.Close() // Setup a streamer that we'll pipe cmd.Stderr to. // We want to record/buffer anything that's written to this (3rd argument true) - logStreamerErr := NewLogstreamer(logger, "stderr", true) + logStreamerErr := NewLogstreamer(logger, "stderr", true, false) defer logStreamerErr.Close() // Execute something that succeeds @@ -99,7 +99,7 @@ func TestLogstreamerFlush(t *testing.T) { byteWriter := bufio.NewWriter(&buffer) logger := log.New(byteWriter, "", 0) - logStreamerOut := NewLogstreamer(logger, "", false) + logStreamerOut := NewLogstreamer(logger, "", false, false) defer logStreamerOut.Close() logStreamerOut.Write([]byte(text)) From a04aac4afa36e98a992b091b4278c51aa61f3ab8 Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Wed, 22 Feb 2023 14:51:41 +0000 Subject: [PATCH 05/36] Fix logger --- cli/internal/logstreamer/logstreamer.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cli/internal/logstreamer/logstreamer.go b/cli/internal/logstreamer/logstreamer.go index cb43d4bd09f98..3403324bfe8dd 100644 --- a/cli/internal/logstreamer/logstreamer.go +++ b/cli/internal/logstreamer/logstreamer.go @@ -81,6 +81,12 @@ func (l *Logstreamer) Flush() error { return err } + p := make([]byte, l.buf.Len()) + if _, err := l.buf.Read(p); err != nil { + return err + } + + l.out(string(p)) return nil } From 5ddbd7b17282f4bbac705d998a21cc9defa1044a Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Wed, 22 Feb 2023 17:54:58 +0000 Subject: [PATCH 06/36] Add new flag to existing tests --- cli/integration_tests/bad_flag.t | 2 +- cli/integration_tests/no_args.t | 1 + cli/integration_tests/turbo_help.t | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cli/integration_tests/bad_flag.t b/cli/integration_tests/bad_flag.t index 827ca26859e19..a886c308b02b0 100644 --- a/cli/integration_tests/bad_flag.t +++ b/cli/integration_tests/bad_flag.t @@ -19,7 +19,7 @@ Bad flag with an implied run command should display run flags note: to pass '--bad-flag' as a value, use '-- --bad-flag' - Usage: turbo <--cache-dir |--cache-workers |--concurrency |--continue|--dry-run []|--single-package|--filter |--force|--global-deps |--graph []|--ignore |--include-dependencies|--no-cache|--no-daemon|--no-deps|--output-logs |--only|--parallel|--pkg-inference-root |--profile |--remote-only|--scope |--since |TASKS|PASS_THROUGH_ARGS> + Usage: turbo <--cache-dir |--cache-workers |--concurrency |--continue|--dry-run []|--single-package|--filter |--force|--global-deps |--graph []|--ignore |--include-dependencies|--no-cache|--no-daemon|--no-deps|--output-logs |--log-order |--only|--parallel|--pkg-inference-root |--profile |--remote-only|--scope |--since |--log-prefix |TASKS|PASS_THROUGH_ARGS> For more information, try '--help'. diff --git a/cli/integration_tests/no_args.t b/cli/integration_tests/no_args.t index 354eb328f12e6..5ed1bdc138d5a 100644 --- a/cli/integration_tests/no_args.t +++ b/cli/integration_tests/no_args.t @@ -52,6 +52,7 @@ Make sure exit code is 2 when no args are passed --no-daemon Run without using turbo's daemon process --no-deps Exclude dependent task consumers from execution --output-logs Set type of process output logging. Use "full" to show all output. Use "hash-only" to show only turbo-computed task hashes. Use "new-only" to show only new output with only hashes for cached tasks. Use "none" to hide process output. (default full) [possible values: full, none, hash-only, new-only, errors-only] + --log-order Set type of process output order. Use "stream" to show output as soon as it is available. Use "grouped" to show output when a command has finished execution. (default stream) [possible values: stream, grouped] --parallel Execute all tasks in parallel --profile File to write turbo's performance profile output into. You can load the file up in chrome://tracing to see which parts of your build were slow --remote-only Ignore the local filesystem cache for all tasks. Only allow reading and caching artifacts using the remote cache diff --git a/cli/integration_tests/turbo_help.t b/cli/integration_tests/turbo_help.t index e639c7dce2472..7d9b5f30b939f 100644 --- a/cli/integration_tests/turbo_help.t +++ b/cli/integration_tests/turbo_help.t @@ -52,6 +52,7 @@ Test help flag --no-daemon Run without using turbo's daemon process --no-deps Exclude dependent task consumers from execution --output-logs Set type of process output logging. Use "full" to show all output. Use "hash-only" to show only turbo-computed task hashes. Use "new-only" to show only new output with only hashes for cached tasks. Use "none" to hide process output. (default full) [possible values: full, none, hash-only, new-only, errors-only] + --log-order Set type of process output order. Use "stream" to show output as soon as it is available. Use "grouped" to show output when a command has finished execution. (default stream) [possible values: stream, grouped] --parallel Execute all tasks in parallel --profile File to write turbo's performance profile output into. You can load the file up in chrome://tracing to see which parts of your build were slow --remote-only Ignore the local filesystem cache for all tasks. Only allow reading and caching artifacts using the remote cache @@ -113,6 +114,7 @@ Test help flag --no-daemon Run without using turbo's daemon process --no-deps Exclude dependent task consumers from execution --output-logs Set type of process output logging. Use "full" to show all output. Use "hash-only" to show only turbo-computed task hashes. Use "new-only" to show only new output with only hashes for cached tasks. Use "none" to hide process output. (default full) [possible values: full, none, hash-only, new-only, errors-only] + --log-order Set type of process output order. Use "stream" to show output as soon as it is available. Use "grouped" to show output when a command has finished execution. (default stream) [possible values: stream, grouped] --parallel Execute all tasks in parallel --profile File to write turbo's performance profile output into. You can load the file up in chrome://tracing to see which parts of your build were slow --remote-only Ignore the local filesystem cache for all tasks. Only allow reading and caching artifacts using the remote cache From 3246e1b18703cae2d0604a434437f571286350fd Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Wed, 22 Feb 2023 19:25:59 +0000 Subject: [PATCH 07/36] Add integration tests --- cli/integration_tests/ordered/grouped.t | 28 +++++++++++++++++++ .../ordered/monorepo/.gitignore | 3 ++ .../ordered/monorepo/apps/my-app/package.json | 9 ++++++ .../ordered/monorepo/package.json | 7 +++++ .../monorepo/packages/util/package.json | 6 ++++ .../ordered/monorepo/turbo.json | 8 ++++++ cli/integration_tests/ordered/setup.sh | 6 ++++ cli/integration_tests/ordered/stream.t | 28 +++++++++++++++++++ 8 files changed, 95 insertions(+) create mode 100644 cli/integration_tests/ordered/grouped.t create mode 100644 cli/integration_tests/ordered/monorepo/.gitignore create mode 100644 cli/integration_tests/ordered/monorepo/apps/my-app/package.json create mode 100644 cli/integration_tests/ordered/monorepo/package.json create mode 100644 cli/integration_tests/ordered/monorepo/packages/util/package.json create mode 100644 cli/integration_tests/ordered/monorepo/turbo.json create mode 100755 cli/integration_tests/ordered/setup.sh create mode 100644 cli/integration_tests/ordered/stream.t diff --git a/cli/integration_tests/ordered/grouped.t b/cli/integration_tests/ordered/grouped.t new file mode 100644 index 0000000000000..4f19decd98211 --- /dev/null +++ b/cli/integration_tests/ordered/grouped.t @@ -0,0 +1,28 @@ +Setup + $ . ${TESTDIR}/../setup.sh + $ . ${TESTDIR}/setup.sh $(pwd) + +# Build in grouped order. + $ ${TURBO} run build --log-order grouped --force + \xe2\x80\xa2 Packages in scope: my-app, util (esc) + \xe2\x80\xa2 Running build in 2 packages (esc) + \xe2\x80\xa2 Remote caching disabled (esc) + (my-app|util):build: cache bypass, force executing (f1ea8c68bf163f6b|8107080a88b155ef) (re) + (my-app|util):build: cache bypass, force executing (f1ea8c68bf163f6b|8107080a88b155ef) (re) + my-app:build: + my-app:build: > build + my-app:build: > echo 'building' && sleep 1 && echo 'done' + my-app:build: + my-app:build: building + my-app:build: done + util:build: + util:build: > build + util:build: > sleep 0.5 && echo 'building' && sleep 1 && echo 'completed' + util:build: + util:build: building + util:build: completed + + Tasks: 2 successful, 2 total + Cached: 0 cached, 2 total + Time:\s*[\.0-9]+m?s (re) + diff --git a/cli/integration_tests/ordered/monorepo/.gitignore b/cli/integration_tests/ordered/monorepo/.gitignore new file mode 100644 index 0000000000000..77af9fc60321d --- /dev/null +++ b/cli/integration_tests/ordered/monorepo/.gitignore @@ -0,0 +1,3 @@ +node_modules/ +.turbo +.npmrc diff --git a/cli/integration_tests/ordered/monorepo/apps/my-app/package.json b/cli/integration_tests/ordered/monorepo/apps/my-app/package.json new file mode 100644 index 0000000000000..8134a20f0f1b2 --- /dev/null +++ b/cli/integration_tests/ordered/monorepo/apps/my-app/package.json @@ -0,0 +1,9 @@ +{ + "name": "my-app", + "scripts": { + "build": "echo 'building' && sleep 1 && echo 'done'" + }, + "dependencies": { + "util": "*" + } +} diff --git a/cli/integration_tests/ordered/monorepo/package.json b/cli/integration_tests/ordered/monorepo/package.json new file mode 100644 index 0000000000000..85175c18a49b0 --- /dev/null +++ b/cli/integration_tests/ordered/monorepo/package.json @@ -0,0 +1,7 @@ +{ + "name": "monorepo", + "workspaces": [ + "apps/**", + "packages/**" + ] +} diff --git a/cli/integration_tests/ordered/monorepo/packages/util/package.json b/cli/integration_tests/ordered/monorepo/packages/util/package.json new file mode 100644 index 0000000000000..c512fc681619a --- /dev/null +++ b/cli/integration_tests/ordered/monorepo/packages/util/package.json @@ -0,0 +1,6 @@ +{ + "name": "util", + "scripts": { + "build": "sleep 0.5 && echo 'building' && sleep 1 && echo 'completed'" + } +} diff --git a/cli/integration_tests/ordered/monorepo/turbo.json b/cli/integration_tests/ordered/monorepo/turbo.json new file mode 100644 index 0000000000000..0d547fb58901a --- /dev/null +++ b/cli/integration_tests/ordered/monorepo/turbo.json @@ -0,0 +1,8 @@ +{ + "$schema": "https://turbo.build/schema.json", + "pipeline": { + "build": { + "outputs": [] + } + } +} diff --git a/cli/integration_tests/ordered/setup.sh b/cli/integration_tests/ordered/setup.sh new file mode 100755 index 0000000000000..864b7a64a37a6 --- /dev/null +++ b/cli/integration_tests/ordered/setup.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) +TARGET_DIR=$1 +cp -a ${SCRIPT_DIR}/monorepo/. ${TARGET_DIR}/ +${SCRIPT_DIR}/../setup_git.sh ${TARGET_DIR} diff --git a/cli/integration_tests/ordered/stream.t b/cli/integration_tests/ordered/stream.t new file mode 100644 index 0000000000000..16934e60efcc7 --- /dev/null +++ b/cli/integration_tests/ordered/stream.t @@ -0,0 +1,28 @@ +Setup + $ . ${TESTDIR}/../setup.sh + $ . ${TESTDIR}/setup.sh $(pwd) + +# Build in stream order. All the .*'s are unpredictable lines, however the amount of lines is predictable. + $ ${TURBO} run build --log-order stream --force + \xe2\x80\xa2 Packages in scope: my-app, util (esc) + \xe2\x80\xa2 Running build in 2 packages (esc) + \xe2\x80\xa2 Remote caching disabled (esc) + (my-app|util):build: cache bypass, force executing (f1ea8c68bf163f6b|8107080a88b155ef) (re) + (my-app|util):build: cache bypass, force executing (f1ea8c68bf163f6b|8107080a88b155ef) (re) + .* (re) + .* (re) + .* (re) + .* (re) + .* (re) + .* (re) + .* (re) + .* (re) + .* (re) + util:build: building + my-app:build: done + util:build: completed + + Tasks: 2 successful, 2 total + Cached: 0 cached, 2 total + Time:\s*[\.0-9]+m?s (re) + \ No newline at end of file From 7cac2fd06ef0633bd7346dd6b97702eb709a3357 Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Sat, 25 Feb 2023 23:06:44 +0000 Subject: [PATCH 08/36] Use a mutex to guarantee logorder inside a OutputLines call --- cli/internal/logstreamer/logstreamer.go | 10 +++++++++- cli/internal/run/real_run.go | 12 ++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/cli/internal/logstreamer/logstreamer.go b/cli/internal/logstreamer/logstreamer.go index 3403324bfe8dd..3e262fb095f34 100644 --- a/cli/internal/logstreamer/logstreamer.go +++ b/cli/internal/logstreamer/logstreamer.go @@ -9,6 +9,7 @@ import ( "log" "os" "strings" + "sync" ) type Logstreamer struct { @@ -29,9 +30,11 @@ type Logstreamer struct { colorReset string grouped bool + + logMutex *sync.Mutex } -func NewLogstreamer(logger *log.Logger, prefix string, record bool, grouped bool) *Logstreamer { +func NewLogstreamer(logger *log.Logger, prefix string, record bool, grouped bool, mutex *sync.Mutex) *Logstreamer { streamer := &Logstreamer{ Logger: logger, buf: bytes.NewBuffer([]byte("")), @@ -42,6 +45,7 @@ func NewLogstreamer(logger *log.Logger, prefix string, record bool, grouped bool colorFail: "", colorReset: "", grouped: grouped, + logMutex: mutex, } if strings.HasPrefix(os.Getenv("TERM"), "xterm") { @@ -91,6 +95,7 @@ func (l *Logstreamer) Flush() error { } func (l *Logstreamer) OutputLines() error { + l.logMutex.Lock() for { line, err := l.buf.ReadString('\n') @@ -102,6 +107,7 @@ func (l *Logstreamer) OutputLines() error { // Close() or Flush() have to be used to flush out // the last remaining line if it does not end with a newline if _, err := l.buf.WriteString(line); err != nil { + l.logMutex.Unlock() return err } } @@ -112,9 +118,11 @@ func (l *Logstreamer) OutputLines() error { } if err != nil { + l.logMutex.Unlock() return err } } + l.logMutex.Unlock() return nil } diff --git a/cli/internal/run/real_run.go b/cli/internal/run/real_run.go index 99501ec67d803..33c5d9c97bdd9 100644 --- a/cli/internal/run/real_run.go +++ b/cli/internal/run/real_run.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "strings" + "sync" "time" "github.com/fatih/color" @@ -88,10 +89,13 @@ func RealRun( Concurrency: rs.Opts.runOpts.concurrency, } + var logMutex sync.Mutex + fmt.Println("new mutex") + execFunc := func(ctx gocontext.Context, packageTask *nodes.PackageTask) error { deps := engine.TaskGraph.DownEdges(packageTask.TaskID) // deps here are passed in to calculate the task hash - return ec.exec(ctx, packageTask, deps) + return ec.exec(ctx, packageTask, deps, &logMutex) } visitorFn := g.GetPackageTaskVisitor(ctx, execFunc) @@ -148,7 +152,7 @@ func (ec *execContext) logError(log hclog.Logger, prefix string, err error) { ec.ui.Error(fmt.Sprintf("%s%s%s", ui.ERROR_PREFIX, prefix, color.RedString(" %v", err))) } -func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTask, deps dag.Set) error { +func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTask, deps dag.Set, logMutex *sync.Mutex) error { cmdTime := time.Now() progressLogger := ec.logger.Named("") @@ -233,9 +237,9 @@ func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTas // Create a logger logger := log.New(writer, "", 0) // Setup a streamer that we'll pipe cmd.Stdout to - logStreamerOut := logstreamer.NewLogstreamer(logger, prettyPrefix, false, grouped) + logStreamerOut := logstreamer.NewLogstreamer(logger, prettyPrefix, false, grouped, logMutex) // Setup a streamer that we'll pipe cmd.Stderr to. - logStreamerErr := logstreamer.NewLogstreamer(logger, prettyPrefix, false, grouped) + logStreamerErr := logstreamer.NewLogstreamer(logger, prettyPrefix, false, grouped, logMutex) cmd.Stderr = logStreamerErr cmd.Stdout = logStreamerOut // Flush/Reset any error we recorded From e668dc5abc1f907110c7e62b0b9a57cef1d25a69 Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Tue, 7 Mar 2023 23:59:34 +0000 Subject: [PATCH 09/36] Group turbo messages with the other output --- cli/integration_tests/ordered/grouped.t | 4 +-- cli/internal/logstreamer/logstreamer.go | 10 +----- cli/internal/run/real_run.go | 43 +++++++++++++++++++------ cli/internal/runcache/runcache.go | 43 +++++++++++++++++++++---- 4 files changed, 72 insertions(+), 28 deletions(-) diff --git a/cli/integration_tests/ordered/grouped.t b/cli/integration_tests/ordered/grouped.t index 4f19decd98211..1c7377c639393 100644 --- a/cli/integration_tests/ordered/grouped.t +++ b/cli/integration_tests/ordered/grouped.t @@ -7,14 +7,14 @@ Setup \xe2\x80\xa2 Packages in scope: my-app, util (esc) \xe2\x80\xa2 Running build in 2 packages (esc) \xe2\x80\xa2 Remote caching disabled (esc) - (my-app|util):build: cache bypass, force executing (f1ea8c68bf163f6b|8107080a88b155ef) (re) - (my-app|util):build: cache bypass, force executing (f1ea8c68bf163f6b|8107080a88b155ef) (re) + my-app:build: cache bypass, force executing 8107080a88b155ef my-app:build: my-app:build: > build my-app:build: > echo 'building' && sleep 1 && echo 'done' my-app:build: my-app:build: building my-app:build: done + util:build: cache bypass, force executing f1ea8c68bf163f6b util:build: util:build: > build util:build: > sleep 0.5 && echo 'building' && sleep 1 && echo 'completed' diff --git a/cli/internal/logstreamer/logstreamer.go b/cli/internal/logstreamer/logstreamer.go index 3e262fb095f34..3403324bfe8dd 100644 --- a/cli/internal/logstreamer/logstreamer.go +++ b/cli/internal/logstreamer/logstreamer.go @@ -9,7 +9,6 @@ import ( "log" "os" "strings" - "sync" ) type Logstreamer struct { @@ -30,11 +29,9 @@ type Logstreamer struct { colorReset string grouped bool - - logMutex *sync.Mutex } -func NewLogstreamer(logger *log.Logger, prefix string, record bool, grouped bool, mutex *sync.Mutex) *Logstreamer { +func NewLogstreamer(logger *log.Logger, prefix string, record bool, grouped bool) *Logstreamer { streamer := &Logstreamer{ Logger: logger, buf: bytes.NewBuffer([]byte("")), @@ -45,7 +42,6 @@ func NewLogstreamer(logger *log.Logger, prefix string, record bool, grouped bool colorFail: "", colorReset: "", grouped: grouped, - logMutex: mutex, } if strings.HasPrefix(os.Getenv("TERM"), "xterm") { @@ -95,7 +91,6 @@ func (l *Logstreamer) Flush() error { } func (l *Logstreamer) OutputLines() error { - l.logMutex.Lock() for { line, err := l.buf.ReadString('\n') @@ -107,7 +102,6 @@ func (l *Logstreamer) OutputLines() error { // Close() or Flush() have to be used to flush out // the last remaining line if it does not end with a newline if _, err := l.buf.WriteString(line); err != nil { - l.logMutex.Unlock() return err } } @@ -118,11 +112,9 @@ func (l *Logstreamer) OutputLines() error { } if err != nil { - l.logMutex.Unlock() return err } } - l.logMutex.Unlock() return nil } diff --git a/cli/internal/run/real_run.go b/cli/internal/run/real_run.go index 1388eb21a465f..776540263ac51 100644 --- a/cli/internal/run/real_run.go +++ b/cli/internal/run/real_run.go @@ -179,6 +179,8 @@ func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTas // Setup tracer tracer := ec.runState.Run(packageTask.TaskID) + grouped := ec.rs.Opts.runOpts.logOrder == "grouped" + passThroughArgs := ec.rs.ArgsForTask(packageTask.Task) hash := packageTask.Hash ec.logger.Debug("task hash", "value", hash) @@ -213,12 +215,23 @@ func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTas ErrorPrefix: prettyPrefix, WarnPrefix: prettyPrefix, } - hit, err := taskCache.RestoreOutputs(ctx, prefixedUI, progressLogger) - if err != nil { - prefixedUI.Error(fmt.Sprintf("error fetching from cache: %s", err)) - } else if hit { - tracer(TargetCached, nil) - return nil + + cacheHit := taskCache.IsCacheHit(ctx) + outputtedTurboLogs := false + + if cacheHit || !grouped { + outputtedTurboLogs = true + logMutex.Lock() + hit, err := taskCache.RestoreOutputs(ctx, prefixedUI, progressLogger, false) + if err != nil { + prefixedUI.Error(fmt.Sprintf("error fetching from cache: %s", err)) + } else if hit { + tracer(TargetCached, nil) + logMutex.Unlock() + return nil + } + + logMutex.Unlock() } // Setup command execution @@ -246,14 +259,12 @@ func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTas } } - grouped := ec.rs.Opts.runOpts.logOrder == "grouped" - // Create a logger logger := log.New(writer, "", 0) // Setup a streamer that we'll pipe cmd.Stdout to - logStreamerOut := logstreamer.NewLogstreamer(logger, prettyPrefix, false, grouped, logMutex) + logStreamerOut := logstreamer.NewLogstreamer(logger, prettyPrefix, false, grouped) // Setup a streamer that we'll pipe cmd.Stderr to. - logStreamerErr := logstreamer.NewLogstreamer(logger, prettyPrefix, false, grouped, logMutex) + logStreamerErr := logstreamer.NewLogstreamer(logger, prettyPrefix, false, grouped) cmd.Stderr = logStreamerErr cmd.Stdout = logStreamerOut // Flush/Reset any error we recorded @@ -261,6 +272,16 @@ func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTas logStreamerOut.FlushRecord() closeOutputs := func() error { + logMutex.Lock() + if !outputtedTurboLogs { + hit, err := taskCache.RestoreOutputs(ctx, prefixedUI, progressLogger, true) + if err != nil { + prefixedUI.Error(fmt.Sprintf("error fetching from cache: %s", err)) + } else if hit { + return fmt.Errorf("Hit cache while expecting cache miss") + } + } + var closeErrors []error if err := logStreamerOut.Close(); err != nil { @@ -278,8 +299,10 @@ func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTas for i, err := range closeErrors { msgs[i] = err.Error() } + logMutex.Unlock() return fmt.Errorf("could not flush log output: %v", strings.Join(msgs, ", ")) } + logMutex.Unlock() return nil } diff --git a/cli/internal/runcache/runcache.go b/cli/internal/runcache/runcache.go index aa60b18aeb7b6..4049f8854faa8 100644 --- a/cli/internal/runcache/runcache.go +++ b/cli/internal/runcache/runcache.go @@ -106,9 +106,27 @@ type TaskCache struct { LogFileName turbopath.AbsoluteSystemPath } +// IsCacheHit returns true if we have a cache hit, false if we have a cache miss +func (tc TaskCache) IsCacheHit(ctx context.Context) bool { + if tc.cachingDisabled || tc.rc.readsDisabled { + return false + } + changedOutputGlobs, err := tc.rc.outputWatcher.GetChangedOutputs(ctx, tc.hash, tc.repoRelativeGlobs.Inclusions) + if err != nil { + changedOutputGlobs = tc.repoRelativeGlobs.Inclusions + } + + hasChangedOutputs := len(changedOutputGlobs) > 0 + if hasChangedOutputs { + hitStatus := tc.rc.cache.Exists(tc.hash) + return hitStatus.Local || hitStatus.Remote + } + + return true +} + // RestoreOutputs attempts to restore output for the corresponding task from the cache. -// Returns true if successful. -func (tc TaskCache) RestoreOutputs(ctx context.Context, prefixedUI *cli.PrefixedUi, progressLogger hclog.Logger) (bool, error) { +func (tc TaskCache) RestoreOutputs(ctx context.Context, prefixedUI *cli.PrefixedUi, progressLogger hclog.Logger, logStatusOnly bool) (bool, error) { if tc.cachingDisabled || tc.rc.readsDisabled { if tc.taskOutputMode != util.NoTaskOutput && tc.taskOutputMode != util.ErrorTaskOutput { prefixedUI.Output(fmt.Sprintf("cache bypass, force executing %s", ui.Dim(tc.hash))) @@ -127,10 +145,19 @@ func (tc TaskCache) RestoreOutputs(ctx context.Context, prefixedUI *cli.Prefixed // Note that we currently don't use the output globs when restoring, but we could in the // future to avoid doing unnecessary file I/O. We also need to pass along the exclusion // globs as well. - hit, _, _, err := tc.rc.cache.Fetch(tc.rc.repoRoot, tc.hash, nil) - if err != nil { - return false, err - } else if !hit { + hit := false + if logStatusOnly { + cacheStatus := tc.rc.cache.Exists(tc.hash) + hit = cacheStatus.Local || cacheStatus.Remote + } else { + fetchHit, _, _, err := tc.rc.cache.Fetch(tc.rc.repoRoot, tc.hash, nil) + if err != nil { + return false, err + } + hit = fetchHit + } + + if !hit { if tc.taskOutputMode != util.NoTaskOutput && tc.taskOutputMode != util.ErrorTaskOutput { prefixedUI.Output(fmt.Sprintf("cache miss, executing %s", ui.Dim(tc.hash))) } @@ -154,7 +181,9 @@ func (tc TaskCache) RestoreOutputs(ctx context.Context, prefixedUI *cli.Prefixed case util.FullTaskOutput: progressLogger.Debug("log file", "path", tc.LogFileName) prefixedUI.Info(fmt.Sprintf("cache hit, replaying output %s", ui.Dim(tc.hash))) - tc.ReplayLogFile(prefixedUI, progressLogger) + if !logStatusOnly { + tc.ReplayLogFile(prefixedUI, progressLogger) + } case util.ErrorTaskOutput: // The task succeeded, so we don't output anything in this case default: From b847e682385f2d09dc59d53b6c75d5ad2c70aa52 Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Wed, 8 Mar 2023 00:03:53 +0000 Subject: [PATCH 10/36] Replaced hashes in integration tests to make them pass --- cli/integration_tests/inference/no-workspaces.t | 6 +++--- cli/integration_tests/single_package/dry-run.t | 6 +++--- cli/integration_tests/single_package/run.t | 4 ++-- cli/integration_tests/single_package_deps/dry-run.t | 12 ++++++------ cli/integration_tests/single_package_deps/run.t | 12 ++++++------ .../single_package_no_config/dry-run.t | 6 +++--- cli/integration_tests/single_package_no_config/run.t | 4 ++-- 7 files changed, 25 insertions(+), 25 deletions(-) diff --git a/cli/integration_tests/inference/no-workspaces.t b/cli/integration_tests/inference/no-workspaces.t index b57cc7df2f97e..a595dca53221d 100644 --- a/cli/integration_tests/inference/no-workspaces.t +++ b/cli/integration_tests/inference/no-workspaces.t @@ -11,7 +11,7 @@ Setup [-0-9:.TWZ+]+ \[DEBUG] turbo: Found go binary at "[\-\w\/]+" (re) [-0-9:.TWZ+]+ \[INFO] turbo: skipping turbod since we appear to be in a non-interactive context (re) [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash env vars: vars=\["VERCEL_ANALYTICS_ID"] (re) - [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash: value=4c73ae6c71119fad (re) + [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash: value=476a4927e552ab88 (re) [-0-9:.TWZ+]+ \[DEBUG] turbo: local cache folder: path="" (re) \xe2\x80\xa2 Running build (esc) \xe2\x80\xa2 Remote caching disabled (esc) @@ -31,7 +31,7 @@ Setup [-0-9:.TWZ+]+ \[DEBUG] turbo: Found go binary at "[\-\w\/]+" (re) [-0-9:.TWZ+]+ \[INFO] turbo: skipping turbod since we appear to be in a non-interactive context (re) [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash env vars: vars=\["VERCEL_ANALYTICS_ID"] (re) - [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash: value=96793eaf8b145bde (re) + [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash: value=d82bc5a0d9ca1a6b (re) [-0-9:.TWZ+]+ \[DEBUG] turbo: local cache folder: path="" (re) \xe2\x80\xa2 Running build (esc) \xe2\x80\xa2 Remote caching disabled (esc) @@ -51,7 +51,7 @@ Setup [-0-9:.TWZ+]+ \[DEBUG] turbo: Found go binary at "[\-\w\/]+" (re) [-0-9:.TWZ+]+ \[INFO] turbo: skipping turbod since we appear to be in a non-interactive context (re) [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash env vars: vars=\["VERCEL_ANALYTICS_ID"] (re) - [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash: value=67527326c7931f8b (re) + [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash: value=8169caaf504488a2 (re) [-0-9:.TWZ+]+ \[DEBUG] turbo: local cache folder: path="" (re) \xe2\x80\xa2 Running build (esc) \xe2\x80\xa2 Remote caching disabled (esc) diff --git a/cli/integration_tests/single_package/dry-run.t b/cli/integration_tests/single_package/dry-run.t index ac7829471f58a..8a104f5f4fe61 100644 --- a/cli/integration_tests/single_package/dry-run.t +++ b/cli/integration_tests/single_package/dry-run.t @@ -14,7 +14,7 @@ Check Tasks to Run build Task = build - Hash = 7bf32e1dedb04a5d + Hash = fef49cd3cf47af84 Cached (Local) = false Cached (Remote) = false Command = echo 'building' > foo @@ -34,7 +34,7 @@ Check "tasks": [ { "task": "build", - "hash": "7bf32e1dedb04a5d", + "hash": "fef49cd3cf47af84", "cacheState": { "local": false, "remote": false @@ -60,7 +60,7 @@ Check }, "expandedInputs": { ".gitignore": "6f23ff6842b5526da43ab38f4a5bf3b0158eeb42", - "package-lock.json": "8db0df575e6509336a6719094b63eb23d2c649c1", + "package-lock.json": "1c117cce37347befafe3a9cba1b8a609b3600021", "package.json": "185771929d92c3865ce06c863c07d357500d3364", "turbo.json": "2b9b71e8eca61cda6f4c14e07067feac9c1f9862" }, diff --git a/cli/integration_tests/single_package/run.t b/cli/integration_tests/single_package/run.t index 582ffacbb1acc..3032a6e48a90c 100644 --- a/cli/integration_tests/single_package/run.t +++ b/cli/integration_tests/single_package/run.t @@ -6,7 +6,7 @@ Check $ ${TURBO} run build --single-package \xe2\x80\xa2 Running build (esc) \xe2\x80\xa2 Remote caching disabled (esc) - build: cache miss, executing 7bf32e1dedb04a5d + build: cache miss, executing fef49cd3cf47af84 build: build: > build build: > echo 'building' > foo @@ -20,7 +20,7 @@ Run a second time, verify caching works because there is a config $ ${TURBO} run build --single-package \xe2\x80\xa2 Running build (esc) \xe2\x80\xa2 Remote caching disabled (esc) - build: cache hit, replaying output 7bf32e1dedb04a5d + build: cache hit, replaying output fef49cd3cf47af84 build: build: > build build: > echo 'building' > foo diff --git a/cli/integration_tests/single_package_deps/dry-run.t b/cli/integration_tests/single_package_deps/dry-run.t index ad0571e5a4445..35972bd1a0992 100644 --- a/cli/integration_tests/single_package_deps/dry-run.t +++ b/cli/integration_tests/single_package_deps/dry-run.t @@ -14,7 +14,7 @@ Check Tasks to Run build Task = build - Hash = 8fc80cfff3b64237 + Hash = 02cd45da7a08ba05 Cached (Local) = false Cached (Remote) = false Command = echo 'building' > foo @@ -30,7 +30,7 @@ Check Framework = test Task = test - Hash = c71366ccd6a86465 + Hash = 6aadbf9bca4ae1e0 Cached (Local) = false Cached (Remote) = false Command = [[ ( -f foo ) && $(cat foo) == 'building' ]] @@ -50,7 +50,7 @@ Check "tasks": [ { "task": "build", - "hash": "8fc80cfff3b64237", + "hash": "02cd45da7a08ba05", "cacheState": { "local": false, "remote": false @@ -78,7 +78,7 @@ Check }, "expandedInputs": { ".gitignore": "6f23ff6842b5526da43ab38f4a5bf3b0158eeb42", - "package-lock.json": "8db0df575e6509336a6719094b63eb23d2c649c1", + "package-lock.json": "1c117cce37347befafe3a9cba1b8a609b3600021", "package.json": "bc24e5c5b8bd13d419e0742ae3e92a2bf61c53d0", "turbo.json": "e1fe3e5402fe019ef3845cc63a736878a68934c7" }, @@ -93,7 +93,7 @@ Check }, { "task": "test", - "hash": "c71366ccd6a86465", + "hash": "6aadbf9bca4ae1e0", "cacheState": { "local": false, "remote": false @@ -119,7 +119,7 @@ Check }, "expandedInputs": { ".gitignore": "6f23ff6842b5526da43ab38f4a5bf3b0158eeb42", - "package-lock.json": "8db0df575e6509336a6719094b63eb23d2c649c1", + "package-lock.json": "1c117cce37347befafe3a9cba1b8a609b3600021", "package.json": "bc24e5c5b8bd13d419e0742ae3e92a2bf61c53d0", "turbo.json": "e1fe3e5402fe019ef3845cc63a736878a68934c7" }, diff --git a/cli/integration_tests/single_package_deps/run.t b/cli/integration_tests/single_package_deps/run.t index e004bbfd89524..0a86d6690c0d0 100644 --- a/cli/integration_tests/single_package_deps/run.t +++ b/cli/integration_tests/single_package_deps/run.t @@ -6,12 +6,12 @@ Check $ ${TURBO} run test --single-package \xe2\x80\xa2 Running test (esc) \xe2\x80\xa2 Remote caching disabled (esc) - build: cache miss, executing 8fc80cfff3b64237 + build: cache miss, executing 02cd45da7a08ba05 build: build: > build build: > echo 'building' > foo build: - test: cache miss, executing c71366ccd6a86465 + test: cache miss, executing 6aadbf9bca4ae1e0 test: test: > test test: > [[ ( -f foo ) && $(cat foo) == 'building' ]] @@ -25,12 +25,12 @@ Run a second time, verify caching works because there is a config $ ${TURBO} run test --single-package \xe2\x80\xa2 Running test (esc) \xe2\x80\xa2 Remote caching disabled (esc) - build: cache hit, replaying output 8fc80cfff3b64237 + build: cache hit, replaying output 02cd45da7a08ba05 build: build: > build build: > echo 'building' > foo build: - test: cache hit, replaying output c71366ccd6a86465 + test: cache hit, replaying output 6aadbf9bca4ae1e0 test: test: > test test: > [[ ( -f foo ) && $(cat foo) == 'building' ]] @@ -44,8 +44,8 @@ Run with --output-logs=hash-only $ ${TURBO} run test --single-package --output-logs=hash-only \xe2\x80\xa2 Running test (esc) \xe2\x80\xa2 Remote caching disabled (esc) - build: cache hit, suppressing output 8fc80cfff3b64237 - test: cache hit, suppressing output c71366ccd6a86465 + build: cache hit, suppressing output 02cd45da7a08ba05 + test: cache hit, suppressing output 6aadbf9bca4ae1e0 Tasks: 2 successful, 2 total Cached: 2 cached, 2 total diff --git a/cli/integration_tests/single_package_no_config/dry-run.t b/cli/integration_tests/single_package_no_config/dry-run.t index ddc35b5950c4c..077ef3afd82d7 100644 --- a/cli/integration_tests/single_package_no_config/dry-run.t +++ b/cli/integration_tests/single_package_no_config/dry-run.t @@ -14,7 +14,7 @@ Check Tasks to Run build Task = build - Hash = c7223f212c321d3b + Hash = 569c206a6fa456c6 Cached (Local) = false Cached (Remote) = false Command = echo 'building' @@ -34,7 +34,7 @@ Check "tasks": [ { "task": "build", - "hash": "c7223f212c321d3b", + "hash": "569c206a6fa456c6", "cacheState": { "local": false, "remote": false @@ -56,7 +56,7 @@ Check }, "expandedInputs": { ".gitignore": "38548b0538f2fc563d6bacf70dd42798c6fd9a35", - "package-lock.json": "8db0df575e6509336a6719094b63eb23d2c649c1", + "package-lock.json": "1c117cce37347befafe3a9cba1b8a609b3600021", "package.json": "581fe2b8dcba5b03cbe51d78a973143eb6d33e3a" }, "framework": "\u003cNO FRAMEWORK DETECTED\u003e", diff --git a/cli/integration_tests/single_package_no_config/run.t b/cli/integration_tests/single_package_no_config/run.t index 1633b7c973162..8f2507b7c25e6 100644 --- a/cli/integration_tests/single_package_no_config/run.t +++ b/cli/integration_tests/single_package_no_config/run.t @@ -6,7 +6,7 @@ Check $ ${TURBO} run build --single-package \xe2\x80\xa2 Running build (esc) \xe2\x80\xa2 Remote caching disabled (esc) - build: cache bypass, force executing c7223f212c321d3b + build: cache bypass, force executing 569c206a6fa456c6 build: build: > build build: > echo 'building' @@ -21,7 +21,7 @@ Run a second time, verify no caching because there is no config $ ${TURBO} run build --single-package \xe2\x80\xa2 Running build (esc) \xe2\x80\xa2 Remote caching disabled (esc) - build: cache bypass, force executing c7223f212c321d3b + build: cache bypass, force executing 569c206a6fa456c6 build: build: > build build: > echo 'building' From bcf332f58dbe5ecf6dfe017fa568f3dd99312786 Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Wed, 8 Mar 2023 00:14:04 +0000 Subject: [PATCH 11/36] Add a comment --- cli/internal/run/real_run.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cli/internal/run/real_run.go b/cli/internal/run/real_run.go index 776540263ac51..87ec9cb3e1834 100644 --- a/cli/internal/run/real_run.go +++ b/cli/internal/run/real_run.go @@ -219,6 +219,9 @@ func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTas cacheHit := taskCache.IsCacheHit(ctx) outputtedTurboLogs := false + // If we want grouped log ordering we want to either: + // - restore + // - not display any logs and do this when we have finished execution if we have a cache mis all output and display logs if we have a cache hit.s. if cacheHit || !grouped { outputtedTurboLogs = true logMutex.Lock() From 092aa3536917262602bfd72b25ce38f889fa2772 Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Wed, 8 Mar 2023 00:15:00 +0000 Subject: [PATCH 12/36] Clarify name of variable --- cli/internal/run/real_run.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/internal/run/real_run.go b/cli/internal/run/real_run.go index 87ec9cb3e1834..e46e720ab5ad3 100644 --- a/cli/internal/run/real_run.go +++ b/cli/internal/run/real_run.go @@ -217,13 +217,13 @@ func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTas } cacheHit := taskCache.IsCacheHit(ctx) - outputtedTurboLogs := false + outputtedCacheStatusLogs := false // If we want grouped log ordering we want to either: // - restore // - not display any logs and do this when we have finished execution if we have a cache mis all output and display logs if we have a cache hit.s. if cacheHit || !grouped { - outputtedTurboLogs = true + outputtedCacheStatusLogs = true logMutex.Lock() hit, err := taskCache.RestoreOutputs(ctx, prefixedUI, progressLogger, false) if err != nil { @@ -276,7 +276,7 @@ func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTas closeOutputs := func() error { logMutex.Lock() - if !outputtedTurboLogs { + if !outputtedCacheStatusLogs { hit, err := taskCache.RestoreOutputs(ctx, prefixedUI, progressLogger, true) if err != nil { prefixedUI.Error(fmt.Sprintf("error fetching from cache: %s", err)) From a444856df5269cafa592d254f81862260916f147 Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Fri, 10 Mar 2023 00:04:06 +0000 Subject: [PATCH 13/36] Implement a factory for cli.Uis --- cli/internal/ui/ui_factory.go | 91 +++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 cli/internal/ui/ui_factory.go diff --git a/cli/internal/ui/ui_factory.go b/cli/internal/ui/ui_factory.go new file mode 100644 index 0000000000000..80f69f6cb12b5 --- /dev/null +++ b/cli/internal/ui/ui_factory.go @@ -0,0 +1,91 @@ +package ui + +import ( + "io" + + "github.com/fatih/color" + "github.com/mitchellh/cli" +) + +// UiFactory provides an interface for creating cli.Ui instances from input, output and error IOs +type UiFactory interface { + Build(in io.Reader, out io.Writer, err io.Writer) cli.Ui +} + +// BasicUiFactory provides a method for creating a cli.BasicUi from input, output and error IOs +type BasicUiFactory struct { +} + +// Build builds a cli.BasicUi from input, output and error IOs +func (factory *BasicUiFactory) Build(in io.Reader, out io.Writer, err io.Writer) cli.Ui { + return &cli.BasicUi{ + Reader: in, + Writer: out, + ErrorWriter: err, + } +} + +// ColoredUiFactory provides a method for creating a cli.ColoredUi from input, output and error IOs +type ColoredUiFactory struct { + ColorMode ColorMode + Base UiFactory +} + +// Build builds a cli.ColoredUi from input, output and error IOs +func (factory *ColoredUiFactory) Build(in io.Reader, out io.Writer, err io.Writer) cli.Ui { + factory.ColorMode = applyColorMode(factory.ColorMode) + + var outWriter, errWriter io.Writer + + if factory.ColorMode == ColorModeSuppressed { + outWriter = &stripAnsiWriter{wrappedWriter: out} + errWriter = &stripAnsiWriter{wrappedWriter: err} + } else { + outWriter = out + errWriter = err + } + + return &cli.ColoredUi{ + Ui: factory.Base.Build(in, outWriter, errWriter), + OutputColor: cli.UiColorNone, + InfoColor: cli.UiColorNone, + WarnColor: cli.UiColor{Code: int(color.FgYellow), Bold: false}, + ErrorColor: cli.UiColorRed, + } +} + +// ConcurrentUiFactory provides a method for creating a cli.ConcurrentUi from input, output and error IOs +type ConcurrentUiFactory struct { + Base UiFactory +} + +// Build builds a cli.ConcurrentUi from input, output and error IOs +func (factory *ConcurrentUiFactory) Build(in io.Reader, out io.Writer, err io.Writer) cli.Ui { + return &cli.ConcurrentUi{ + Ui: factory.Base.Build(in, out, err), + } +} + +// PrefixedUiFactory provides a method for creating a cli.PrefixedUi from input, output and error IOs +type PrefixedUiFactory struct { + Base UiFactory + + AskPrefix string + AskSecretPrefix string + OutputPrefix string + InfoPrefix string + ErrorPrefix string + WarnPrefix string +} + +// Build builds a cli.PrefixedUi from input, output and error IOs +func (factory *PrefixedUiFactory) Build(in io.Reader, out io.Writer, err io.Writer) cli.Ui { + return &cli.PrefixedUi{ + AskPrefix: factory.AskPrefix, + AskSecretPrefix: factory.AskSecretPrefix, + OutputPrefix: factory.OutputPrefix, + InfoPrefix: factory.InfoPrefix, + ErrorPrefix: factory.ErrorPrefix, + WarnPrefix: factory.WarnPrefix, + } +} From f761e723f834ad6fe098c9bef7aef283eed12be7 Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Fri, 10 Mar 2023 00:15:01 +0000 Subject: [PATCH 14/36] WIP --- cli/internal/cmdutil/cmdutil.go | 15 +++++++++++++-- cli/internal/run/real_run.go | 8 +++++++- cli/internal/ui/ui.go | 32 -------------------------------- 3 files changed, 20 insertions(+), 35 deletions(-) diff --git a/cli/internal/cmdutil/cmdutil.go b/cli/internal/cmdutil/cmdutil.go index 9eff34be81906..92fd5749eb695 100644 --- a/cli/internal/cmdutil/cmdutil.go +++ b/cli/internal/cmdutil/cmdutil.go @@ -75,6 +75,11 @@ func (h *Helper) Cleanup(cliConfig *turbostate.ParsedArgsFromRust) { } func (h *Helper) getUI(cliConfig *turbostate.ParsedArgsFromRust) cli.Ui { + factory := h.getUIFactory(cliConfig) + return factory.Build(os.Stdout, os.Stdin, os.Stderr) +} + +func (h *Helper) getUIFactory(cliConfig *turbostate.ParsedArgsFromRust) ui.UiFactory { colorMode := ui.GetColorModeFromEnv() if cliConfig.GetNoColor() { colorMode = ui.ColorModeSuppressed @@ -82,7 +87,10 @@ func (h *Helper) getUI(cliConfig *turbostate.ParsedArgsFromRust) cli.Ui { if cliConfig.GetColor() { colorMode = ui.ColorModeForced } - return ui.BuildColoredUi(colorMode) + return &ui.ColoredUiFactory{ + ColorMode: colorMode, + Base: &ui.BasicUiFactory{}, + } } func (h *Helper) getLogger() (hclog.Logger, error) { @@ -136,7 +144,8 @@ func NewHelper(turboVersion string, args *turbostate.ParsedArgsFromRust) *Helper // It additionally returns a mechanism to set an error, so func (h *Helper) GetCmdBase(cliConfig *turbostate.ParsedArgsFromRust) (*CmdBase, error) { // terminal is for color/no-color output - terminal := h.getUI(cliConfig) + uiFactory := h.getUIFactory(cliConfig) + terminal := uiFactory.Build(os.Stdin, os.Stdout, os.Stderr) // logger is configured with verbosity level using --verbosity flag from end users logger, err := h.getLogger() if err != nil { @@ -198,6 +207,7 @@ func (h *Helper) GetCmdBase(cliConfig *turbostate.ParsedArgsFromRust) (*CmdBase, return &CmdBase{ UI: terminal, + UIFactory: uiFactory, Logger: logger, RepoRoot: repoRoot, APIClient: apiClient, @@ -211,6 +221,7 @@ func (h *Helper) GetCmdBase(cliConfig *turbostate.ParsedArgsFromRust) (*CmdBase, // CmdBase encompasses configured components common to all turbo commands. type CmdBase struct { UI cli.Ui + UIFactory ui.UiFactory Logger hclog.Logger RepoRoot turbopath.AbsoluteSystemPath APIClient *client.ApiClient diff --git a/cli/internal/run/real_run.go b/cli/internal/run/real_run.go index e46e720ab5ad3..60523d5f0e62c 100644 --- a/cli/internal/run/real_run.go +++ b/cli/internal/run/real_run.go @@ -71,11 +71,16 @@ func RealRun( runCache := runcache.New(turboCache, base.RepoRoot, rs.Opts.runcacheOpts, colorCache) + concurrentUIFactory := ui.ConcurrentUiFactory{ + Base: base.UIFactory, + } + ec := &execContext{ colorCache: colorCache, runState: runState, rs: rs, - ui: &cli.ConcurrentUi{Ui: base.UI}, + ui: concurrentUIFactory.Build(os.Stdin, os.Stdin, os.Stderr), + uiFactory: &concurrentUIFactory, runCache: runCache, logger: base.Logger, packageManager: packageManager, @@ -151,6 +156,7 @@ type execContext struct { runState *RunState rs *runSpec ui cli.Ui + uiFactory ui.UiFactory runCache *runcache.RunCache logger hclog.Logger packageManager *packagemanager.PackageManager diff --git a/cli/internal/ui/ui.go b/cli/internal/ui/ui.go index 9084c76c5a1c9..f4deaf7e80ad1 100644 --- a/cli/internal/ui/ui.go +++ b/cli/internal/ui/ui.go @@ -10,7 +10,6 @@ import ( "github.com/fatih/color" "github.com/mattn/go-isatty" - "github.com/mitchellh/cli" "github.com/vercel/turbo/cli/internal/ci" ) @@ -88,34 +87,3 @@ func (into *stripAnsiWriter) Write(p []byte) (int, error) { // written. return len(p), nil } - -// Default returns the default colored ui -func Default() *cli.ColoredUi { - return BuildColoredUi(ColorModeUndefined) -} - -func BuildColoredUi(colorMode ColorMode) *cli.ColoredUi { - colorMode = applyColorMode(colorMode) - - var outWriter, errWriter io.Writer - - if colorMode == ColorModeSuppressed { - outWriter = &stripAnsiWriter{wrappedWriter: os.Stdout} - errWriter = &stripAnsiWriter{wrappedWriter: os.Stderr} - } else { - outWriter = os.Stdout - errWriter = os.Stderr - } - - return &cli.ColoredUi{ - Ui: &cli.BasicUi{ - Reader: os.Stdin, - Writer: outWriter, - ErrorWriter: errWriter, - }, - OutputColor: cli.UiColorNone, - InfoColor: cli.UiColorNone, - WarnColor: cli.UiColor{Code: int(color.FgYellow), Bold: false}, - ErrorColor: cli.UiColorRed, - } -} From 90aca5f1e267caca5e4b4a0a8cfb80c64bd5aa6f Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Thu, 23 Mar 2023 20:50:04 +0000 Subject: [PATCH 15/36] Add queued ui --- cli/internal/ui/queued_ui.go | 48 +++++++++++++++++++++++++++++++++++ cli/internal/ui/ui_factory.go | 21 +++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 cli/internal/ui/queued_ui.go diff --git a/cli/internal/ui/queued_ui.go b/cli/internal/ui/queued_ui.go new file mode 100644 index 0000000000000..8aa35bd1289f1 --- /dev/null +++ b/cli/internal/ui/queued_ui.go @@ -0,0 +1,48 @@ +package ui + +import ( + "bytes" + "io" + + "github.com/mitchellh/cli" +) + +type QueuedUi struct { + out io.Writer + err io.Writer + in io.Reader + + OutBuffer *bytes.Buffer + ErrBuffer *bytes.Buffer + + ui cli.Ui +} + +func (u *QueuedUi) Ask(query string) (string, error) { + return u.ui.Ask(query) +} + +func (u *QueuedUi) AskSecret(query string) (string, error) { + return u.ui.AskSecret(query) +} + +func (u *QueuedUi) Error(message string) { + u.ui.Error(message) +} + +func (u *QueuedUi) Info(message string) { + u.ui.Info(message) +} + +func (u *QueuedUi) Output(message string) { + u.ui.Output(message) +} + +func (u *QueuedUi) Warn(message string) { + u.ui.Warn(message) +} + +func (u *QueuedUi) WriteOutput() { + u.out.Write(u.OutBuffer.Bytes()) + u.err.Write(u.ErrBuffer.Bytes()) +} diff --git a/cli/internal/ui/ui_factory.go b/cli/internal/ui/ui_factory.go index 80f69f6cb12b5..6eb45307d583c 100644 --- a/cli/internal/ui/ui_factory.go +++ b/cli/internal/ui/ui_factory.go @@ -1,6 +1,7 @@ package ui import ( + "bytes" "io" "github.com/fatih/color" @@ -87,5 +88,25 @@ func (factory *PrefixedUiFactory) Build(in io.Reader, out io.Writer, err io.Writ InfoPrefix: factory.InfoPrefix, ErrorPrefix: factory.ErrorPrefix, WarnPrefix: factory.WarnPrefix, + Ui: factory.Base.Build(in, out, err), + } +} + +type QueuedUiFactory struct { + Base UiFactory +} + +func (factory *QueuedUiFactory) Build(in io.Reader, out io.Writer, err io.Writer) *QueuedUi { + outBuf := &bytes.Buffer{} + errBuf := &bytes.Buffer{} + + return &QueuedUi{ + out: out, + err: err, + in: in, + + OutBuffer: outBuf, + ErrBuffer: errBuf, + ui: factory.Base.Build(in, io.Writer(outBuf), io.Writer(errBuf)), } } From 00ca62e527d938a8a2048048c1e134485f2fad71 Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Thu, 23 Mar 2023 21:21:21 +0000 Subject: [PATCH 16/36] WIP --- cli/internal/logstreamer/logstreamer.go | 16 ++++---- cli/internal/run/real_run.go | 52 +++++++++++++++++++++---- cli/internal/runcache/runcache.go | 20 +++++----- cli/internal/ui/queued_ui.go | 5 --- cli/internal/ui/ui_factory.go | 13 +++---- package.json | 2 +- 6 files changed, 68 insertions(+), 40 deletions(-) diff --git a/cli/internal/logstreamer/logstreamer.go b/cli/internal/logstreamer/logstreamer.go index 3403324bfe8dd..797c71288a448 100644 --- a/cli/internal/logstreamer/logstreamer.go +++ b/cli/internal/logstreamer/logstreamer.go @@ -143,24 +143,24 @@ func (l *Logstreamer) out(str string) { l.Logger.Print(str) } -// PrettyStdoutWriter wraps an ioWriter so it can add string +// PrettyIoWriter wraps an ioWriter so it can add string // prefixes to every message it writes to stdout. -type PrettyStdoutWriter struct { +type PrettyIoWriter struct { w io.Writer Prefix string } -var _ io.Writer = (*PrettyStdoutWriter)(nil) +var _ io.Writer = (*PrettyIoWriter)(nil) -// NewPrettyStdoutWriter returns an instance of PrettyStdoutWriter -func NewPrettyStdoutWriter(prefix string) *PrettyStdoutWriter { - return &PrettyStdoutWriter{ - w: os.Stdout, +// NewPrettyIoWriter returns an instance of PrettyStdoutWriter +func NewPrettyIoWriter(prefix string, ioWriter io.Writer) *PrettyIoWriter { + return &PrettyIoWriter{ + w: ioWriter, Prefix: prefix, } } -func (psw *PrettyStdoutWriter) Write(p []byte) (int, error) { +func (psw *PrettyIoWriter) Write(p []byte) (int, error) { str := psw.Prefix + string(p) n, err := psw.w.Write([]byte(str)) diff --git a/cli/internal/run/real_run.go b/cli/internal/run/real_run.go index 60523d5f0e62c..dda77079004a1 100644 --- a/cli/internal/run/real_run.go +++ b/cli/internal/run/real_run.go @@ -1,8 +1,10 @@ package run import ( + "bytes" gocontext "context" "fmt" + "io" "log" "os" "os/exec" @@ -75,12 +77,39 @@ func RealRun( Base: base.UIFactory, } + grouped := rs.Opts.runOpts.logOrder == "grouped" + + var uiFactory ui.UiFactory + outBuf := &bytes.Buffer{} + errBuf := &bytes.Buffer{} + + var outWriter io.Writer + var errWriter io.Writer + + if grouped { + queuedUiFactory := ui.QueuedUiFactory{ + OutBuf: outBuf, + ErrBuf: errBuf, + Base: &concurrentUIFactory, + } + + uiFactory = &queuedUiFactory + + outWriter = outBuf + errWriter = errBuf + } else { + uiFactory = &concurrentUIFactory + + outWriter = os.Stdout + errWriter = os.Stderr + } + ec := &execContext{ colorCache: colorCache, runState: runState, rs: rs, - ui: concurrentUIFactory.Build(os.Stdin, os.Stdin, os.Stderr), - uiFactory: &concurrentUIFactory, + ui: uiFactory.Build(os.Stdin, os.Stdout, os.Stderr), + uiFactory: uiFactory, runCache: runCache, logger: base.Logger, packageManager: packageManager, @@ -88,6 +117,8 @@ func RealRun( taskHashTracker: taskHashTracker, repoRoot: base.RepoRoot, isSinglePackage: singlePackage, + outWriter: outWriter, + errWriter: errWriter, } // run the thing @@ -103,7 +134,12 @@ func RealRun( deps := engine.TaskGraph.DownEdges(packageTask.TaskID) taskSummaries = append(taskSummaries, taskSummary) // deps here are passed in to calculate the task hash - return ec.exec(ctx, packageTask, deps, &logMutex) + err := ec.exec(ctx, packageTask, deps, &logMutex) + + os.Stdout.Write(outBuf.Bytes()) + os.Stdout.Write(errBuf.Bytes()) + + return err } getArgs := func(taskID string) []string { @@ -164,6 +200,8 @@ type execContext struct { taskHashTracker *taskhash.Tracker repoRoot turbopath.AbsoluteSystemPath isSinglePackage bool + outWriter io.Writer + errWriter io.Writer } func (ec *execContext) logError(log hclog.Logger, prefix string, err error) { @@ -185,8 +223,6 @@ func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTas // Setup tracer tracer := ec.runState.Run(packageTask.TaskID) - grouped := ec.rs.Opts.runOpts.logOrder == "grouped" - passThroughArgs := ec.rs.ArgsForTask(packageTask.Task) hash := packageTask.Hash ec.logger.Debug("task hash", "value", hash) @@ -259,7 +295,7 @@ func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTas // Setup stdout/stderr // If we are not caching anything, then we don't need to write logs to disk // be careful about this conditional given the default of cache = true - writer, err := taskCache.OutputWriter(prettyPrefix) + writer, err := taskCache.OutputWriter(prettyPrefix, ec.outWriter) if err != nil { tracer(TargetBuildFailed, err) ec.logError(progressLogger, prettyPrefix, err) @@ -271,9 +307,9 @@ func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTas // Create a logger logger := log.New(writer, "", 0) // Setup a streamer that we'll pipe cmd.Stdout to - logStreamerOut := logstreamer.NewLogstreamer(logger, prettyPrefix, false, grouped) + logStreamerOut := logstreamer.NewLogstreamer(logger, prettyPrefix, false) // Setup a streamer that we'll pipe cmd.Stderr to. - logStreamerErr := logstreamer.NewLogstreamer(logger, prettyPrefix, false, grouped) + logStreamerErr := logstreamer.NewLogstreamer(logger, prettyPrefix, false) cmd.Stderr = logStreamerErr cmd.Stdout = logStreamerOut // Flush/Reset any error we recorded diff --git a/cli/internal/runcache/runcache.go b/cli/internal/runcache/runcache.go index 4049f8854faa8..1185a87628986 100644 --- a/cli/internal/runcache/runcache.go +++ b/cli/internal/runcache/runcache.go @@ -24,7 +24,7 @@ import ( ) // LogReplayer is a function that is responsible for replaying the contents of a given log file -type LogReplayer = func(logger hclog.Logger, output *cli.PrefixedUi, logFile turbopath.AbsoluteSystemPath) +type LogReplayer = func(logger hclog.Logger, output cli.Ui, logFile turbopath.AbsoluteSystemPath) // Opts holds the configurable options for a RunCache instance type Opts struct { @@ -126,7 +126,7 @@ func (tc TaskCache) IsCacheHit(ctx context.Context) bool { } // RestoreOutputs attempts to restore output for the corresponding task from the cache. -func (tc TaskCache) RestoreOutputs(ctx context.Context, prefixedUI *cli.PrefixedUi, progressLogger hclog.Logger, logStatusOnly bool) (bool, error) { +func (tc TaskCache) RestoreOutputs(ctx context.Context, prefixedUI cli.Ui, progressLogger hclog.Logger, logStatusOnly bool) (bool, error) { if tc.cachingDisabled || tc.rc.readsDisabled { if tc.taskOutputMode != util.NoTaskOutput && tc.taskOutputMode != util.ErrorTaskOutput { prefixedUI.Output(fmt.Sprintf("cache bypass, force executing %s", ui.Dim(tc.hash))) @@ -194,7 +194,7 @@ func (tc TaskCache) RestoreOutputs(ctx context.Context, prefixedUI *cli.Prefixed } // ReplayLogFile writes out the stored logfile to the terminal -func (tc TaskCache) ReplayLogFile(prefixedUI *cli.PrefixedUi, progressLogger hclog.Logger) { +func (tc TaskCache) ReplayLogFile(prefixedUI cli.Ui, progressLogger hclog.Logger) { if tc.LogFileName.FileExists() { tc.rc.logReplayer(progressLogger, prefixedUI, tc.LogFileName) } @@ -202,7 +202,7 @@ func (tc TaskCache) ReplayLogFile(prefixedUI *cli.PrefixedUi, progressLogger hcl // OnError replays the logfile if --output-mode=errors-only. // This is called if the task exited with an non-zero error code. -func (tc TaskCache) OnError(terminal *cli.PrefixedUi, logger hclog.Logger) { +func (tc TaskCache) OnError(terminal cli.Ui, logger hclog.Logger) { if tc.taskOutputMode == util.ErrorTaskOutput { tc.ReplayLogFile(terminal, logger) } @@ -230,12 +230,12 @@ func (fwc *fileWriterCloser) Close() error { // OutputWriter creates a sink suitable for handling the output of the command associated // with this task. -func (tc TaskCache) OutputWriter(prefix string) (io.WriteCloser, error) { +func (tc TaskCache) OutputWriter(prefix string, ioWriter io.Writer) (io.WriteCloser, error) { // an os.Stdout wrapper that will add prefixes before printing to stdout - stdoutWriter := logstreamer.NewPrettyStdoutWriter(prefix) + prettyIoWriter := logstreamer.NewPrettyIoWriter(prefix, ioWriter) if tc.cachingDisabled || tc.rc.writesDisabled { - return nopWriteCloser{stdoutWriter}, nil + return nopWriteCloser{prettyIoWriter}, nil } // Setup log file if err := tc.LogFileName.EnsureDir(); err != nil { @@ -256,7 +256,7 @@ func (tc TaskCache) OutputWriter(prefix string) (io.WriteCloser, error) { // only write to log file, not to stdout fwc.Writer = bufWriter } else { - fwc.Writer = io.MultiWriter(stdoutWriter, bufWriter) + fwc.Writer = io.MultiWriter(prettyIoWriter, bufWriter) } return fwc, nil @@ -336,7 +336,7 @@ func (rc *RunCache) TaskCache(pt *nodes.PackageTask, hash string) TaskCache { } // defaultLogReplayer will try to replay logs back to the given Ui instance -func defaultLogReplayer(logger hclog.Logger, output *cli.PrefixedUi, logFileName turbopath.AbsoluteSystemPath) { +func defaultLogReplayer(logger hclog.Logger, output cli.Ui, logFileName turbopath.AbsoluteSystemPath) { logger.Debug("start replaying logs") f, err := logFileName.Open() if err != nil { @@ -353,7 +353,7 @@ func defaultLogReplayer(logger hclog.Logger, output *cli.PrefixedUi, logFileName if str == "" { // Just output the prefix if the current line is a blank string // Note: output.OutputPrefix is also a colored prefix already - output.Ui.Output(output.OutputPrefix) + output.Output("") } else { // Writing to Stdout output.Output(str) diff --git a/cli/internal/ui/queued_ui.go b/cli/internal/ui/queued_ui.go index 8aa35bd1289f1..23236a2c53bda 100644 --- a/cli/internal/ui/queued_ui.go +++ b/cli/internal/ui/queued_ui.go @@ -41,8 +41,3 @@ func (u *QueuedUi) Output(message string) { func (u *QueuedUi) Warn(message string) { u.ui.Warn(message) } - -func (u *QueuedUi) WriteOutput() { - u.out.Write(u.OutBuffer.Bytes()) - u.err.Write(u.ErrBuffer.Bytes()) -} diff --git a/cli/internal/ui/ui_factory.go b/cli/internal/ui/ui_factory.go index 6eb45307d583c..333cc93ab36f4 100644 --- a/cli/internal/ui/ui_factory.go +++ b/cli/internal/ui/ui_factory.go @@ -93,20 +93,17 @@ func (factory *PrefixedUiFactory) Build(in io.Reader, out io.Writer, err io.Writ } type QueuedUiFactory struct { - Base UiFactory + Base UiFactory + OutBuf *bytes.Buffer + ErrBuf *bytes.Buffer } -func (factory *QueuedUiFactory) Build(in io.Reader, out io.Writer, err io.Writer) *QueuedUi { - outBuf := &bytes.Buffer{} - errBuf := &bytes.Buffer{} - +func (factory *QueuedUiFactory) Build(in io.Reader, out io.Writer, err io.Writer) cli.Ui { return &QueuedUi{ out: out, err: err, in: in, - OutBuffer: outBuf, - ErrBuffer: errBuf, - ui: factory.Base.Build(in, io.Writer(outBuf), io.Writer(errBuf)), + ui: factory.Base.Build(in, io.Writer(factory.OutBuf), io.Writer(factory.ErrBuf)), } } diff --git a/package.json b/package.json index 22c09016b8d58..c59e76b4bfc27 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "0.0.0", "private": true, "scripts": { - "build": "pnpm -- turbo run build --filter=docs", + "build": "pnpm -- turbo run build --filter=docs --force", "build:turbo": "pnpm run --filter=cli build", "build:ts": "tsc -b tsconfig.project.json", "check:prettier": "prettier -c .", From d4252b4e0656522931c237c43b71c175fbbfc6cb Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Thu, 23 Mar 2023 23:25:44 +0000 Subject: [PATCH 17/36] Functional implementation --- cli/internal/logstreamer/logstreamer.go | 24 ++--- cli/internal/run/real_run.go | 116 +++++++++--------------- cli/internal/runcache/runcache.go | 53 +++-------- cli/internal/ui/queued_ui.go | 4 - package.json | 2 +- 5 files changed, 63 insertions(+), 136 deletions(-) diff --git a/cli/internal/logstreamer/logstreamer.go b/cli/internal/logstreamer/logstreamer.go index 797c71288a448..e832f728f5544 100644 --- a/cli/internal/logstreamer/logstreamer.go +++ b/cli/internal/logstreamer/logstreamer.go @@ -27,11 +27,9 @@ type Logstreamer struct { colorOkay string colorFail string colorReset string - - grouped bool } -func NewLogstreamer(logger *log.Logger, prefix string, record bool, grouped bool) *Logstreamer { +func NewLogstreamer(logger *log.Logger, prefix string, record bool) *Logstreamer { streamer := &Logstreamer{ Logger: logger, buf: bytes.NewBuffer([]byte("")), @@ -41,7 +39,6 @@ func NewLogstreamer(logger *log.Logger, prefix string, record bool, grouped bool colorOkay: "", colorFail: "", colorReset: "", - grouped: grouped, } if strings.HasPrefix(os.Getenv("TERM"), "xterm") { @@ -58,11 +55,6 @@ func (l *Logstreamer) Write(p []byte) (n int, err error) { return } - // If we want logs to be grouped, do not output them when we receive them, only when they are flushed - if l.grouped { - return - } - err = l.OutputLines() return } @@ -143,24 +135,24 @@ func (l *Logstreamer) out(str string) { l.Logger.Print(str) } -// PrettyIoWriter wraps an ioWriter so it can add string +// PrettyStdoutWriter wraps an ioWriter so it can add string // prefixes to every message it writes to stdout. -type PrettyIoWriter struct { +type PrettyStdoutWriter struct { w io.Writer Prefix string } -var _ io.Writer = (*PrettyIoWriter)(nil) +var _ io.Writer = (*PrettyStdoutWriter)(nil) -// NewPrettyIoWriter returns an instance of PrettyStdoutWriter -func NewPrettyIoWriter(prefix string, ioWriter io.Writer) *PrettyIoWriter { - return &PrettyIoWriter{ +// NewPrettyStdoutWriter returns an instance of PrettyStdoutWriter +func NewPrettyIoWriter(prefix string, ioWriter io.Writer) *PrettyStdoutWriter { + return &PrettyStdoutWriter{ w: ioWriter, Prefix: prefix, } } -func (psw *PrettyIoWriter) Write(p []byte) (int, error) { +func (psw *PrettyStdoutWriter) Write(p []byte) (int, error) { str := psw.Prefix + string(p) n, err := psw.w.Write([]byte(str)) diff --git a/cli/internal/run/real_run.go b/cli/internal/run/real_run.go index dda77079004a1..78cf40173a5bc 100644 --- a/cli/internal/run/real_run.go +++ b/cli/internal/run/real_run.go @@ -9,7 +9,6 @@ import ( "os" "os/exec" "strings" - "sync" "time" "github.com/fatih/color" @@ -77,39 +76,11 @@ func RealRun( Base: base.UIFactory, } - grouped := rs.Opts.runOpts.logOrder == "grouped" - - var uiFactory ui.UiFactory - outBuf := &bytes.Buffer{} - errBuf := &bytes.Buffer{} - - var outWriter io.Writer - var errWriter io.Writer - - if grouped { - queuedUiFactory := ui.QueuedUiFactory{ - OutBuf: outBuf, - ErrBuf: errBuf, - Base: &concurrentUIFactory, - } - - uiFactory = &queuedUiFactory - - outWriter = outBuf - errWriter = errBuf - } else { - uiFactory = &concurrentUIFactory - - outWriter = os.Stdout - errWriter = os.Stderr - } - ec := &execContext{ colorCache: colorCache, runState: runState, rs: rs, - ui: uiFactory.Build(os.Stdin, os.Stdout, os.Stderr), - uiFactory: uiFactory, + ui: base.UI, runCache: runCache, logger: base.Logger, packageManager: packageManager, @@ -117,8 +88,6 @@ func RealRun( taskHashTracker: taskHashTracker, repoRoot: base.RepoRoot, isSinglePackage: singlePackage, - outWriter: outWriter, - errWriter: errWriter, } // run the thing @@ -127,17 +96,45 @@ func RealRun( Concurrency: rs.Opts.runOpts.concurrency, } - var logMutex sync.Mutex - taskSummaries := []*runsummary.TaskSummary{} execFunc := func(ctx gocontext.Context, packageTask *nodes.PackageTask, taskSummary *runsummary.TaskSummary) error { + + grouped := rs.Opts.runOpts.logOrder == "grouped" + + var uiFactory ui.UiFactory + outBuf := &bytes.Buffer{} + errBuf := &bytes.Buffer{} + + var outWriter io.Writer + var errWriter io.Writer + + if grouped { + queuedUiFactory := ui.QueuedUiFactory{ + OutBuf: outBuf, + ErrBuf: errBuf, + Base: &concurrentUIFactory, + } + + uiFactory = &queuedUiFactory + + outWriter = outBuf + errWriter = errBuf + } else { + uiFactory = &concurrentUIFactory + + outWriter = os.Stdout + errWriter = os.Stderr + } + + ui := uiFactory.Build(os.Stdin, os.Stdout, os.Stderr) + deps := engine.TaskGraph.DownEdges(packageTask.TaskID) taskSummaries = append(taskSummaries, taskSummary) // deps here are passed in to calculate the task hash - err := ec.exec(ctx, packageTask, deps, &logMutex) + err := ec.exec(ctx, packageTask, deps, ui, outWriter, errWriter) os.Stdout.Write(outBuf.Bytes()) - os.Stdout.Write(errBuf.Bytes()) + os.Stderr.Write(errBuf.Bytes()) return err } @@ -192,7 +189,6 @@ type execContext struct { runState *RunState rs *runSpec ui cli.Ui - uiFactory ui.UiFactory runCache *runcache.RunCache logger hclog.Logger packageManager *packagemanager.PackageManager @@ -200,8 +196,6 @@ type execContext struct { taskHashTracker *taskhash.Tracker repoRoot turbopath.AbsoluteSystemPath isSinglePackage bool - outWriter io.Writer - errWriter io.Writer } func (ec *execContext) logError(log hclog.Logger, prefix string, err error) { @@ -214,7 +208,7 @@ func (ec *execContext) logError(log hclog.Logger, prefix string, err error) { ec.ui.Error(fmt.Sprintf("%s%s%s", ui.ERROR_PREFIX, prefix, color.RedString(" %v", err))) } -func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTask, deps dag.Set, logMutex *sync.Mutex) error { +func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTask, deps dag.Set, ui cli.Ui, outWriter io.Writer, errWriter io.Writer) error { cmdTime := time.Now() progressLogger := ec.logger.Named("") @@ -251,32 +245,18 @@ func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTas taskCache := ec.runCache.TaskCache(packageTask, hash) // Create a logger for replaying prefixedUI := &cli.PrefixedUi{ - Ui: ec.ui, + Ui: ui, OutputPrefix: prettyPrefix, InfoPrefix: prettyPrefix, ErrorPrefix: prettyPrefix, WarnPrefix: prettyPrefix, } - - cacheHit := taskCache.IsCacheHit(ctx) - outputtedCacheStatusLogs := false - - // If we want grouped log ordering we want to either: - // - restore - // - not display any logs and do this when we have finished execution if we have a cache mis all output and display logs if we have a cache hit.s. - if cacheHit || !grouped { - outputtedCacheStatusLogs = true - logMutex.Lock() - hit, err := taskCache.RestoreOutputs(ctx, prefixedUI, progressLogger, false) - if err != nil { - prefixedUI.Error(fmt.Sprintf("error fetching from cache: %s", err)) - } else if hit { - tracer(TargetCached, nil) - logMutex.Unlock() - return nil - } - - logMutex.Unlock() + hit, err := taskCache.RestoreOutputs(ctx, prefixedUI, progressLogger) + if err != nil { + prefixedUI.Error(fmt.Sprintf("error fetching from cache: %s", err)) + } else if hit { + tracer(TargetCached, nil) + return nil } // Setup command execution @@ -295,7 +275,7 @@ func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTas // Setup stdout/stderr // If we are not caching anything, then we don't need to write logs to disk // be careful about this conditional given the default of cache = true - writer, err := taskCache.OutputWriter(prettyPrefix, ec.outWriter) + writer, err := taskCache.OutputWriter(prettyPrefix, outWriter) if err != nil { tracer(TargetBuildFailed, err) ec.logError(progressLogger, prettyPrefix, err) @@ -317,16 +297,6 @@ func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTas logStreamerOut.FlushRecord() closeOutputs := func() error { - logMutex.Lock() - if !outputtedCacheStatusLogs { - hit, err := taskCache.RestoreOutputs(ctx, prefixedUI, progressLogger, true) - if err != nil { - prefixedUI.Error(fmt.Sprintf("error fetching from cache: %s", err)) - } else if hit { - return fmt.Errorf("Hit cache while expecting cache miss") - } - } - var closeErrors []error if err := logStreamerOut.Close(); err != nil { @@ -344,10 +314,8 @@ func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTas for i, err := range closeErrors { msgs[i] = err.Error() } - logMutex.Unlock() return fmt.Errorf("could not flush log output: %v", strings.Join(msgs, ", ")) } - logMutex.Unlock() return nil } diff --git a/cli/internal/runcache/runcache.go b/cli/internal/runcache/runcache.go index 1185a87628986..8a1916b6dd523 100644 --- a/cli/internal/runcache/runcache.go +++ b/cli/internal/runcache/runcache.go @@ -24,7 +24,7 @@ import ( ) // LogReplayer is a function that is responsible for replaying the contents of a given log file -type LogReplayer = func(logger hclog.Logger, output cli.Ui, logFile turbopath.AbsoluteSystemPath) +type LogReplayer = func(logger hclog.Logger, output *cli.PrefixedUi, logFile turbopath.AbsoluteSystemPath) // Opts holds the configurable options for a RunCache instance type Opts struct { @@ -106,27 +106,9 @@ type TaskCache struct { LogFileName turbopath.AbsoluteSystemPath } -// IsCacheHit returns true if we have a cache hit, false if we have a cache miss -func (tc TaskCache) IsCacheHit(ctx context.Context) bool { - if tc.cachingDisabled || tc.rc.readsDisabled { - return false - } - changedOutputGlobs, err := tc.rc.outputWatcher.GetChangedOutputs(ctx, tc.hash, tc.repoRelativeGlobs.Inclusions) - if err != nil { - changedOutputGlobs = tc.repoRelativeGlobs.Inclusions - } - - hasChangedOutputs := len(changedOutputGlobs) > 0 - if hasChangedOutputs { - hitStatus := tc.rc.cache.Exists(tc.hash) - return hitStatus.Local || hitStatus.Remote - } - - return true -} - // RestoreOutputs attempts to restore output for the corresponding task from the cache. -func (tc TaskCache) RestoreOutputs(ctx context.Context, prefixedUI cli.Ui, progressLogger hclog.Logger, logStatusOnly bool) (bool, error) { +// Returns true if successful. +func (tc TaskCache) RestoreOutputs(ctx context.Context, prefixedUI *cli.PrefixedUi, progressLogger hclog.Logger) (bool, error) { if tc.cachingDisabled || tc.rc.readsDisabled { if tc.taskOutputMode != util.NoTaskOutput && tc.taskOutputMode != util.ErrorTaskOutput { prefixedUI.Output(fmt.Sprintf("cache bypass, force executing %s", ui.Dim(tc.hash))) @@ -145,19 +127,10 @@ func (tc TaskCache) RestoreOutputs(ctx context.Context, prefixedUI cli.Ui, progr // Note that we currently don't use the output globs when restoring, but we could in the // future to avoid doing unnecessary file I/O. We also need to pass along the exclusion // globs as well. - hit := false - if logStatusOnly { - cacheStatus := tc.rc.cache.Exists(tc.hash) - hit = cacheStatus.Local || cacheStatus.Remote - } else { - fetchHit, _, _, err := tc.rc.cache.Fetch(tc.rc.repoRoot, tc.hash, nil) - if err != nil { - return false, err - } - hit = fetchHit - } - - if !hit { + hit, _, _, err := tc.rc.cache.Fetch(tc.rc.repoRoot, tc.hash, nil) + if err != nil { + return false, err + } else if !hit { if tc.taskOutputMode != util.NoTaskOutput && tc.taskOutputMode != util.ErrorTaskOutput { prefixedUI.Output(fmt.Sprintf("cache miss, executing %s", ui.Dim(tc.hash))) } @@ -181,9 +154,7 @@ func (tc TaskCache) RestoreOutputs(ctx context.Context, prefixedUI cli.Ui, progr case util.FullTaskOutput: progressLogger.Debug("log file", "path", tc.LogFileName) prefixedUI.Info(fmt.Sprintf("cache hit, replaying output %s", ui.Dim(tc.hash))) - if !logStatusOnly { - tc.ReplayLogFile(prefixedUI, progressLogger) - } + tc.ReplayLogFile(prefixedUI, progressLogger) case util.ErrorTaskOutput: // The task succeeded, so we don't output anything in this case default: @@ -194,7 +165,7 @@ func (tc TaskCache) RestoreOutputs(ctx context.Context, prefixedUI cli.Ui, progr } // ReplayLogFile writes out the stored logfile to the terminal -func (tc TaskCache) ReplayLogFile(prefixedUI cli.Ui, progressLogger hclog.Logger) { +func (tc TaskCache) ReplayLogFile(prefixedUI *cli.PrefixedUi, progressLogger hclog.Logger) { if tc.LogFileName.FileExists() { tc.rc.logReplayer(progressLogger, prefixedUI, tc.LogFileName) } @@ -202,7 +173,7 @@ func (tc TaskCache) ReplayLogFile(prefixedUI cli.Ui, progressLogger hclog.Logger // OnError replays the logfile if --output-mode=errors-only. // This is called if the task exited with an non-zero error code. -func (tc TaskCache) OnError(terminal cli.Ui, logger hclog.Logger) { +func (tc TaskCache) OnError(terminal *cli.PrefixedUi, logger hclog.Logger) { if tc.taskOutputMode == util.ErrorTaskOutput { tc.ReplayLogFile(terminal, logger) } @@ -336,7 +307,7 @@ func (rc *RunCache) TaskCache(pt *nodes.PackageTask, hash string) TaskCache { } // defaultLogReplayer will try to replay logs back to the given Ui instance -func defaultLogReplayer(logger hclog.Logger, output cli.Ui, logFileName turbopath.AbsoluteSystemPath) { +func defaultLogReplayer(logger hclog.Logger, output *cli.PrefixedUi, logFileName turbopath.AbsoluteSystemPath) { logger.Debug("start replaying logs") f, err := logFileName.Open() if err != nil { @@ -353,7 +324,7 @@ func defaultLogReplayer(logger hclog.Logger, output cli.Ui, logFileName turbopat if str == "" { // Just output the prefix if the current line is a blank string // Note: output.OutputPrefix is also a colored prefix already - output.Output("") + output.Ui.Output(output.OutputPrefix) } else { // Writing to Stdout output.Output(str) diff --git a/cli/internal/ui/queued_ui.go b/cli/internal/ui/queued_ui.go index 23236a2c53bda..59d3aab91f47e 100644 --- a/cli/internal/ui/queued_ui.go +++ b/cli/internal/ui/queued_ui.go @@ -1,7 +1,6 @@ package ui import ( - "bytes" "io" "github.com/mitchellh/cli" @@ -12,9 +11,6 @@ type QueuedUi struct { err io.Writer in io.Reader - OutBuffer *bytes.Buffer - ErrBuffer *bytes.Buffer - ui cli.Ui } diff --git a/package.json b/package.json index c59e76b4bfc27..22c09016b8d58 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "0.0.0", "private": true, "scripts": { - "build": "pnpm -- turbo run build --filter=docs --force", + "build": "pnpm -- turbo run build --filter=docs", "build:turbo": "pnpm run --filter=cli build", "build:ts": "tsc -b tsconfig.project.json", "check:prettier": "prettier -c .", From ba1595a6112110f0a7325995f0fce8c9c4701acc Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Fri, 24 Mar 2023 18:02:17 +0000 Subject: [PATCH 18/36] Some improvements --- cli/internal/logstreamer/logstreamer.go | 5 --- cli/internal/logstreamer/logstreamer_test.go | 10 ++--- cli/internal/run/real_run.go | 32 ++++++++-------- cli/internal/ui/queued_ui.go | 39 -------------------- cli/internal/ui/ui_factory.go | 17 --------- 5 files changed, 20 insertions(+), 83 deletions(-) delete mode 100644 cli/internal/ui/queued_ui.go diff --git a/cli/internal/logstreamer/logstreamer.go b/cli/internal/logstreamer/logstreamer.go index e832f728f5544..4368a8b44fbac 100644 --- a/cli/internal/logstreamer/logstreamer.go +++ b/cli/internal/logstreamer/logstreamer.go @@ -68,11 +68,6 @@ func (l *Logstreamer) Close() error { } func (l *Logstreamer) Flush() error { - err := l.OutputLines() - if err != nil { - return err - } - p := make([]byte, l.buf.Len()) if _, err := l.buf.Read(p); err != nil { return err diff --git a/cli/internal/logstreamer/logstreamer_test.go b/cli/internal/logstreamer/logstreamer_test.go index 62d2003560fc1..94d8a8283716c 100644 --- a/cli/internal/logstreamer/logstreamer_test.go +++ b/cli/internal/logstreamer/logstreamer_test.go @@ -19,11 +19,11 @@ func TestLogstreamerOk(t *testing.T) { logger := log.New(os.Stdout, "--> ", log.Ldate|log.Ltime) // Setup a streamer that we'll pipe cmd.Stdout to - logStreamerOut := NewLogstreamer(logger, "stdout", false, false) + logStreamerOut := NewLogstreamer(logger, "stdout", false) defer logStreamerOut.Close() // Setup a streamer that we'll pipe cmd.Stderr to. // We want to record/buffer anything that's written to this (3rd argument true) - logStreamerErr := NewLogstreamer(logger, "stderr", true, false) + logStreamerErr := NewLogstreamer(logger, "stderr", true) defer logStreamerErr.Close() // Execute something that succeeds @@ -57,11 +57,11 @@ func TestLogstreamerErr(t *testing.T) { logger := log.New(os.Stdout, "--> ", log.Ldate|log.Ltime) // Setup a streamer that we'll pipe cmd.Stdout to - logStreamerOut := NewLogstreamer(logger, "stdout", false, false) + logStreamerOut := NewLogstreamer(logger, "stdout", false) defer logStreamerOut.Close() // Setup a streamer that we'll pipe cmd.Stderr to. // We want to record/buffer anything that's written to this (3rd argument true) - logStreamerErr := NewLogstreamer(logger, "stderr", true, false) + logStreamerErr := NewLogstreamer(logger, "stderr", true) defer logStreamerErr.Close() // Execute something that succeeds @@ -99,7 +99,7 @@ func TestLogstreamerFlush(t *testing.T) { byteWriter := bufio.NewWriter(&buffer) logger := log.New(byteWriter, "", 0) - logStreamerOut := NewLogstreamer(logger, "", false, false) + logStreamerOut := NewLogstreamer(logger, "", false) defer logStreamerOut.Close() logStreamerOut.Write([]byte(text)) diff --git a/cli/internal/run/real_run.go b/cli/internal/run/real_run.go index 78cf40173a5bc..bc17f2b1f801d 100644 --- a/cli/internal/run/real_run.go +++ b/cli/internal/run/real_run.go @@ -80,7 +80,7 @@ func RealRun( colorCache: colorCache, runState: runState, rs: rs, - ui: base.UI, + ui: concurrentUIFactory.Build(os.Stdin, os.Stdout, os.Stderr), runCache: runCache, logger: base.Logger, packageManager: packageManager, @@ -101,40 +101,38 @@ func RealRun( grouped := rs.Opts.runOpts.logOrder == "grouped" - var uiFactory ui.UiFactory outBuf := &bytes.Buffer{} errBuf := &bytes.Buffer{} var outWriter io.Writer var errWriter io.Writer - if grouped { - queuedUiFactory := ui.QueuedUiFactory{ - OutBuf: outBuf, - ErrBuf: errBuf, - Base: &concurrentUIFactory, - } - - uiFactory = &queuedUiFactory + defaultOutWriter := os.Stdout + defaultErrWriter := os.Stderr + if grouped { outWriter = outBuf errWriter = errBuf } else { - uiFactory = &concurrentUIFactory - - outWriter = os.Stdout - errWriter = os.Stderr + outWriter = defaultOutWriter + errWriter = defaultErrWriter } - ui := uiFactory.Build(os.Stdin, os.Stdout, os.Stderr) + ui := concurrentUIFactory.Build(os.Stdin, outWriter, errWriter) deps := engine.TaskGraph.DownEdges(packageTask.TaskID) taskSummaries = append(taskSummaries, taskSummary) // deps here are passed in to calculate the task hash err := ec.exec(ctx, packageTask, deps, ui, outWriter, errWriter) - os.Stdout.Write(outBuf.Bytes()) - os.Stderr.Write(errBuf.Bytes()) + if grouped { + _, outErr := defaultOutWriter.Write(outBuf.Bytes()) + _, errErr := defaultErrWriter.Write(errBuf.Bytes()) + + if outErr != nil || errErr != nil { + base.UI.Error("Failed to write part of the output to terminal") + } + } return err } diff --git a/cli/internal/ui/queued_ui.go b/cli/internal/ui/queued_ui.go deleted file mode 100644 index 59d3aab91f47e..0000000000000 --- a/cli/internal/ui/queued_ui.go +++ /dev/null @@ -1,39 +0,0 @@ -package ui - -import ( - "io" - - "github.com/mitchellh/cli" -) - -type QueuedUi struct { - out io.Writer - err io.Writer - in io.Reader - - ui cli.Ui -} - -func (u *QueuedUi) Ask(query string) (string, error) { - return u.ui.Ask(query) -} - -func (u *QueuedUi) AskSecret(query string) (string, error) { - return u.ui.AskSecret(query) -} - -func (u *QueuedUi) Error(message string) { - u.ui.Error(message) -} - -func (u *QueuedUi) Info(message string) { - u.ui.Info(message) -} - -func (u *QueuedUi) Output(message string) { - u.ui.Output(message) -} - -func (u *QueuedUi) Warn(message string) { - u.ui.Warn(message) -} diff --git a/cli/internal/ui/ui_factory.go b/cli/internal/ui/ui_factory.go index 333cc93ab36f4..b83fff604dfa4 100644 --- a/cli/internal/ui/ui_factory.go +++ b/cli/internal/ui/ui_factory.go @@ -1,7 +1,6 @@ package ui import ( - "bytes" "io" "github.com/fatih/color" @@ -91,19 +90,3 @@ func (factory *PrefixedUiFactory) Build(in io.Reader, out io.Writer, err io.Writ Ui: factory.Base.Build(in, out, err), } } - -type QueuedUiFactory struct { - Base UiFactory - OutBuf *bytes.Buffer - ErrBuf *bytes.Buffer -} - -func (factory *QueuedUiFactory) Build(in io.Reader, out io.Writer, err io.Writer) cli.Ui { - return &QueuedUi{ - out: out, - err: err, - in: in, - - ui: factory.Base.Build(in, io.Writer(factory.OutBuf), io.Writer(factory.ErrBuf)), - } -} From 34976a871fd306eb2a44725781f70315ea117a80 Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Sun, 2 Apr 2023 14:32:45 +0000 Subject: [PATCH 19/36] Use mutex to prevent rare log interleaving When two tasks finish at the same time --- cli/internal/run/real_run.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cli/internal/run/real_run.go b/cli/internal/run/real_run.go index bc17f2b1f801d..868b35c74d83e 100644 --- a/cli/internal/run/real_run.go +++ b/cli/internal/run/real_run.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "strings" + "sync" "time" "github.com/fatih/color" @@ -96,6 +97,8 @@ func RealRun( Concurrency: rs.Opts.runOpts.concurrency, } + var logMutex sync.Mutex + taskSummaries := []*runsummary.TaskSummary{} execFunc := func(ctx gocontext.Context, packageTask *nodes.PackageTask, taskSummary *runsummary.TaskSummary) error { @@ -126,9 +129,10 @@ func RealRun( err := ec.exec(ctx, packageTask, deps, ui, outWriter, errWriter) if grouped { + logMutex.Lock() _, outErr := defaultOutWriter.Write(outBuf.Bytes()) _, errErr := defaultErrWriter.Write(errBuf.Bytes()) - + logMutex.Unlock() if outErr != nil || errErr != nil { base.UI.Error("Failed to write part of the output to terminal") } From a55535934ede97380a2bf9861ce27c1b79ef1937 Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Sun, 2 Apr 2023 15:48:34 +0000 Subject: [PATCH 20/36] Fix test --- cli/internal/scope/scope_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cli/internal/scope/scope_test.go b/cli/internal/scope/scope_test.go index f79f2a3c7e8ef..7651448b7cb08 100644 --- a/cli/internal/scope/scope_test.go +++ b/cli/internal/scope/scope_test.go @@ -84,7 +84,10 @@ func TestResolvePackages(t *testing.T) { if err != nil { t.Fatalf("cwd: %v", err) } - tui := ui.Default() + defaultUIFactory := ui.ColoredUiFactory{ + Base: &ui.BasicUiFactory{}, + } + tui := defaultUIFactory.Build(os.Stdin, os.Stdout, os.Stderr) logger := hclog.Default() // Dependency graph: // From 39c610eb53bfff19f5dbdd4ca60f552da971fd09 Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Sun, 2 Apr 2023 15:50:46 +0000 Subject: [PATCH 21/36] Fix naming issues --- cli/internal/cmdutil/cmdutil.go | 8 ++++---- cli/internal/run/real_run.go | 2 +- cli/internal/scope/scope_test.go | 4 ++-- cli/internal/ui/ui_factory.go | 34 ++++++++++++++++---------------- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/cli/internal/cmdutil/cmdutil.go b/cli/internal/cmdutil/cmdutil.go index 92fd5749eb695..ef506c064c504 100644 --- a/cli/internal/cmdutil/cmdutil.go +++ b/cli/internal/cmdutil/cmdutil.go @@ -79,7 +79,7 @@ func (h *Helper) getUI(cliConfig *turbostate.ParsedArgsFromRust) cli.Ui { return factory.Build(os.Stdout, os.Stdin, os.Stderr) } -func (h *Helper) getUIFactory(cliConfig *turbostate.ParsedArgsFromRust) ui.UiFactory { +func (h *Helper) getUIFactory(cliConfig *turbostate.ParsedArgsFromRust) ui.UIFactory { colorMode := ui.GetColorModeFromEnv() if cliConfig.GetNoColor() { colorMode = ui.ColorModeSuppressed @@ -87,9 +87,9 @@ func (h *Helper) getUIFactory(cliConfig *turbostate.ParsedArgsFromRust) ui.UiFac if cliConfig.GetColor() { colorMode = ui.ColorModeForced } - return &ui.ColoredUiFactory{ + return &ui.ColoredUIFactory{ ColorMode: colorMode, - Base: &ui.BasicUiFactory{}, + Base: &ui.BasicUIFactory{}, } } @@ -221,7 +221,7 @@ func (h *Helper) GetCmdBase(cliConfig *turbostate.ParsedArgsFromRust) (*CmdBase, // CmdBase encompasses configured components common to all turbo commands. type CmdBase struct { UI cli.Ui - UIFactory ui.UiFactory + UIFactory ui.UIFactory Logger hclog.Logger RepoRoot turbopath.AbsoluteSystemPath APIClient *client.ApiClient diff --git a/cli/internal/run/real_run.go b/cli/internal/run/real_run.go index 868b35c74d83e..2d4a9492be0ef 100644 --- a/cli/internal/run/real_run.go +++ b/cli/internal/run/real_run.go @@ -73,7 +73,7 @@ func RealRun( runCache := runcache.New(turboCache, base.RepoRoot, rs.Opts.runcacheOpts, colorCache) - concurrentUIFactory := ui.ConcurrentUiFactory{ + concurrentUIFactory := ui.ConcurrentUIFactory{ Base: base.UIFactory, } diff --git a/cli/internal/scope/scope_test.go b/cli/internal/scope/scope_test.go index 7651448b7cb08..0d103a4e1d940 100644 --- a/cli/internal/scope/scope_test.go +++ b/cli/internal/scope/scope_test.go @@ -84,8 +84,8 @@ func TestResolvePackages(t *testing.T) { if err != nil { t.Fatalf("cwd: %v", err) } - defaultUIFactory := ui.ColoredUiFactory{ - Base: &ui.BasicUiFactory{}, + defaultUIFactory := ui.ColoredUIFactory{ + Base: &ui.BasicUIFactory{}, } tui := defaultUIFactory.Build(os.Stdin, os.Stdout, os.Stderr) logger := hclog.Default() diff --git a/cli/internal/ui/ui_factory.go b/cli/internal/ui/ui_factory.go index b83fff604dfa4..3532936f468e3 100644 --- a/cli/internal/ui/ui_factory.go +++ b/cli/internal/ui/ui_factory.go @@ -7,17 +7,17 @@ import ( "github.com/mitchellh/cli" ) -// UiFactory provides an interface for creating cli.Ui instances from input, output and error IOs -type UiFactory interface { +// UIFactory provides an interface for creating cli.Ui instances from input, output and error IOs +type UIFactory interface { Build(in io.Reader, out io.Writer, err io.Writer) cli.Ui } -// BasicUiFactory provides a method for creating a cli.BasicUi from input, output and error IOs -type BasicUiFactory struct { +// BasicUIFactory provides a method for creating a cli.BasicUi from input, output and error IOs +type BasicUIFactory struct { } // Build builds a cli.BasicUi from input, output and error IOs -func (factory *BasicUiFactory) Build(in io.Reader, out io.Writer, err io.Writer) cli.Ui { +func (factory *BasicUIFactory) Build(in io.Reader, out io.Writer, err io.Writer) cli.Ui { return &cli.BasicUi{ Reader: in, Writer: out, @@ -25,14 +25,14 @@ func (factory *BasicUiFactory) Build(in io.Reader, out io.Writer, err io.Writer) } } -// ColoredUiFactory provides a method for creating a cli.ColoredUi from input, output and error IOs -type ColoredUiFactory struct { +// ColoredUIFactory provides a method for creating a cli.ColoredUi from input, output and error IOs +type ColoredUIFactory struct { ColorMode ColorMode - Base UiFactory + Base UIFactory } // Build builds a cli.ColoredUi from input, output and error IOs -func (factory *ColoredUiFactory) Build(in io.Reader, out io.Writer, err io.Writer) cli.Ui { +func (factory *ColoredUIFactory) Build(in io.Reader, out io.Writer, err io.Writer) cli.Ui { factory.ColorMode = applyColorMode(factory.ColorMode) var outWriter, errWriter io.Writer @@ -54,21 +54,21 @@ func (factory *ColoredUiFactory) Build(in io.Reader, out io.Writer, err io.Write } } -// ConcurrentUiFactory provides a method for creating a cli.ConcurrentUi from input, output and error IOs -type ConcurrentUiFactory struct { - Base UiFactory +// ConcurrentUIFactory provides a method for creating a cli.ConcurrentUi from input, output and error IOs +type ConcurrentUIFactory struct { + Base UIFactory } // Build builds a cli.ConcurrentUi from input, output and error IOs -func (factory *ConcurrentUiFactory) Build(in io.Reader, out io.Writer, err io.Writer) cli.Ui { +func (factory *ConcurrentUIFactory) Build(in io.Reader, out io.Writer, err io.Writer) cli.Ui { return &cli.ConcurrentUi{ Ui: factory.Base.Build(in, out, err), } } -// PrefixedUiFactory provides a method for creating a cli.PrefixedUi from input, output and error IOs -type PrefixedUiFactory struct { - Base UiFactory +// PrefixedUIFactory provides a method for creating a cli.PrefixedUi from input, output and error IOs +type PrefixedUIFactory struct { + Base UIFactory AskPrefix string AskSecretPrefix string @@ -79,7 +79,7 @@ type PrefixedUiFactory struct { } // Build builds a cli.PrefixedUi from input, output and error IOs -func (factory *PrefixedUiFactory) Build(in io.Reader, out io.Writer, err io.Writer) cli.Ui { +func (factory *PrefixedUIFactory) Build(in io.Reader, out io.Writer, err io.Writer) cli.Ui { return &cli.PrefixedUi{ AskPrefix: factory.AskPrefix, AskSecretPrefix: factory.AskSecretPrefix, From 2fa8a6bc2bd1f2b178bd6d16b3cb7f8ff29766e9 Mon Sep 17 00:00:00 2001 From: rafaeltab Date: Sun, 2 Apr 2023 19:07:04 +0000 Subject: [PATCH 22/36] Fix integration tests --- cli/integration_tests/inference/no_workspaces.t | 6 +++--- cli/integration_tests/single_package/dry-run.t | 8 ++++---- cli/integration_tests/single_package/run.t | 4 ++-- .../single_package_deps/dry-run.t | 14 +++++++------- cli/integration_tests/single_package_deps/run.t | 12 ++++++------ .../single_package_no_config/dry-run.t | 8 ++++---- .../single_package_no_config/run.t | 4 ++-- 7 files changed, 28 insertions(+), 28 deletions(-) diff --git a/cli/integration_tests/inference/no_workspaces.t b/cli/integration_tests/inference/no_workspaces.t index 5d575054251c7..5b58c07f2016c 100644 --- a/cli/integration_tests/inference/no_workspaces.t +++ b/cli/integration_tests/inference/no_workspaces.t @@ -13,7 +13,7 @@ Setup [-0-9:.TWZ+]+ \[DEBUG] turbo: build tag: (go|rust) (re) [-0-9:.TWZ+]+ \[INFO] turbo: skipping turbod since we appear to be in a non-interactive context (re) [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash env vars: vars=\["VERCEL_ANALYTICS_ID"] (re) - [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash: value=476a4927e552ab88 (re) + [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash: value=4c73ae6c71119fad (re) [-0-9:.TWZ+]+ \[DEBUG] turbo: local cache folder: path="" (re) \xe2\x80\xa2 Running build (esc) \xe2\x80\xa2 Remote caching disabled (esc) @@ -35,7 +35,7 @@ Setup [-0-9:.TWZ+]+ \[DEBUG] turbo: build tag: (go|rust) (re) [-0-9:.TWZ+]+ \[INFO] turbo: skipping turbod since we appear to be in a non-interactive context (re) [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash env vars: vars=\["VERCEL_ANALYTICS_ID"] (re) - [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash: value=d82bc5a0d9ca1a6b (re) + [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash: value=96793eaf8b145bde (re) [-0-9:.TWZ+]+ \[DEBUG] turbo: local cache folder: path="" (re) \xe2\x80\xa2 Running build (esc) \xe2\x80\xa2 Remote caching disabled (esc) @@ -57,7 +57,7 @@ Setup [-0-9:.TWZ+]+ \[DEBUG] turbo: build tag: (go|rust) (re) [-0-9:.TWZ+]+ \[INFO] turbo: skipping turbod since we appear to be in a non-interactive context (re) [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash env vars: vars=\["VERCEL_ANALYTICS_ID"] (re) - [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash: value=8169caaf504488a2 (re) + [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash: value=67527326c7931f8b (re) [-0-9:.TWZ+]+ \[DEBUG] turbo: local cache folder: path="" (re) \xe2\x80\xa2 Running build (esc) \xe2\x80\xa2 Remote caching disabled (esc) diff --git a/cli/integration_tests/single_package/dry-run.t b/cli/integration_tests/single_package/dry-run.t index 3a2ce9ffc09df..92aaf826734be 100644 --- a/cli/integration_tests/single_package/dry-run.t +++ b/cli/integration_tests/single_package/dry-run.t @@ -14,7 +14,7 @@ Check Tasks to Run build Task = build - Hash = acd1f716a22c635b + Hash = dd4a9a7b508b0e38 Cached (Local) = false Cached (Remote) = false Command = echo 'building' > foo @@ -36,7 +36,7 @@ Check "turboVersion": "[a-z0-9\.-]+", (re) "globalHashSummary": { "globalFileHashMap": { - "package-lock.json": "1c117cce37347befafe3a9cba1b8a609b3600021", + "package-lock.json": "8db0df575e6509336a6719094b63eb23d2c649c1", "package.json": "185771929d92c3865ce06c863c07d357500d3364", "somefile.txt": "45b983be36b73c0788dc9cbcb76cbb80fc7bb057" }, @@ -59,7 +59,7 @@ Check "tasks": [ { "task": "build", - "hash": "acd1f716a22c635b", + "hash": "dd4a9a7b508b0e38", "cacheState": { "local": false, "remote": false @@ -86,7 +86,7 @@ Check }, "expandedInputs": { ".gitignore": "6f23ff6842b5526da43ab38f4a5bf3b0158eeb42", - "package-lock.json": "1c117cce37347befafe3a9cba1b8a609b3600021", + "package-lock.json": "8db0df575e6509336a6719094b63eb23d2c649c1", "package.json": "185771929d92c3865ce06c863c07d357500d3364", "somefile.txt": "45b983be36b73c0788dc9cbcb76cbb80fc7bb057", "turbo.json": "505752e75c10f9e7a0d2538cf8b6f0fcfb8980a0" diff --git a/cli/integration_tests/single_package/run.t b/cli/integration_tests/single_package/run.t index 816e77facacda..0481a939bd55c 100644 --- a/cli/integration_tests/single_package/run.t +++ b/cli/integration_tests/single_package/run.t @@ -6,7 +6,7 @@ Check $ ${TURBO} run build --single-package \xe2\x80\xa2 Running build (esc) \xe2\x80\xa2 Remote caching disabled (esc) - build: cache miss, executing acd1f716a22c635b + build: cache miss, executing dd4a9a7b508b0e38 build: build: > build build: > echo 'building' > foo @@ -23,7 +23,7 @@ Run a second time, verify caching works because there is a config $ ${TURBO} run build --single-package \xe2\x80\xa2 Running build (esc) \xe2\x80\xa2 Remote caching disabled (esc) - build: cache hit, replaying output acd1f716a22c635b + build: cache hit, replaying output dd4a9a7b508b0e38 build: build: > build build: > echo 'building' > foo diff --git a/cli/integration_tests/single_package_deps/dry-run.t b/cli/integration_tests/single_package_deps/dry-run.t index 2fbdcebf604b3..59d6483b5d0f1 100644 --- a/cli/integration_tests/single_package_deps/dry-run.t +++ b/cli/integration_tests/single_package_deps/dry-run.t @@ -14,7 +14,7 @@ Check Tasks to Run build Task = build - Hash = 02cd45da7a08ba05 + Hash = 8fc80cfff3b64237 Cached (Local) = false Cached (Remote) = false Command = echo 'building' > foo @@ -30,7 +30,7 @@ Check Framework = test Task = test - Hash = 6aadbf9bca4ae1e0 + Hash = c71366ccd6a86465 Cached (Local) = false Cached (Remote) = false Command = [[ ( -f foo ) && $(cat foo) == 'building' ]] @@ -52,7 +52,7 @@ Check "turboVersion": "[a-z0-9\.-]+", (re) "globalHashSummary": { "globalFileHashMap": { - "package-lock.json": "1c117cce37347befafe3a9cba1b8a609b3600021", + "package-lock.json": "8db0df575e6509336a6719094b63eb23d2c649c1", "package.json": "bc24e5c5b8bd13d419e0742ae3e92a2bf61c53d0" }, "rootExternalDepsHash": "", @@ -85,7 +85,7 @@ Check "tasks": [ { "task": "build", - "hash": "02cd45da7a08ba05", + "hash": "8fc80cfff3b64237", "cacheState": { "local": false, "remote": false @@ -114,7 +114,7 @@ Check }, "expandedInputs": { ".gitignore": "6f23ff6842b5526da43ab38f4a5bf3b0158eeb42", - "package-lock.json": "1c117cce37347befafe3a9cba1b8a609b3600021", + "package-lock.json": "8db0df575e6509336a6719094b63eb23d2c649c1", "package.json": "bc24e5c5b8bd13d419e0742ae3e92a2bf61c53d0", "turbo.json": "e1fe3e5402fe019ef3845cc63a736878a68934c7" }, @@ -131,7 +131,7 @@ Check }, { "task": "test", - "hash": "6aadbf9bca4ae1e0", + "hash": "c71366ccd6a86465", "cacheState": { "local": false, "remote": false @@ -158,7 +158,7 @@ Check }, "expandedInputs": { ".gitignore": "6f23ff6842b5526da43ab38f4a5bf3b0158eeb42", - "package-lock.json": "1c117cce37347befafe3a9cba1b8a609b3600021", + "package-lock.json": "8db0df575e6509336a6719094b63eb23d2c649c1", "package.json": "bc24e5c5b8bd13d419e0742ae3e92a2bf61c53d0", "turbo.json": "e1fe3e5402fe019ef3845cc63a736878a68934c7" }, diff --git a/cli/integration_tests/single_package_deps/run.t b/cli/integration_tests/single_package_deps/run.t index 0a86d6690c0d0..e004bbfd89524 100644 --- a/cli/integration_tests/single_package_deps/run.t +++ b/cli/integration_tests/single_package_deps/run.t @@ -6,12 +6,12 @@ Check $ ${TURBO} run test --single-package \xe2\x80\xa2 Running test (esc) \xe2\x80\xa2 Remote caching disabled (esc) - build: cache miss, executing 02cd45da7a08ba05 + build: cache miss, executing 8fc80cfff3b64237 build: build: > build build: > echo 'building' > foo build: - test: cache miss, executing 6aadbf9bca4ae1e0 + test: cache miss, executing c71366ccd6a86465 test: test: > test test: > [[ ( -f foo ) && $(cat foo) == 'building' ]] @@ -25,12 +25,12 @@ Run a second time, verify caching works because there is a config $ ${TURBO} run test --single-package \xe2\x80\xa2 Running test (esc) \xe2\x80\xa2 Remote caching disabled (esc) - build: cache hit, replaying output 02cd45da7a08ba05 + build: cache hit, replaying output 8fc80cfff3b64237 build: build: > build build: > echo 'building' > foo build: - test: cache hit, replaying output 6aadbf9bca4ae1e0 + test: cache hit, replaying output c71366ccd6a86465 test: test: > test test: > [[ ( -f foo ) && $(cat foo) == 'building' ]] @@ -44,8 +44,8 @@ Run with --output-logs=hash-only $ ${TURBO} run test --single-package --output-logs=hash-only \xe2\x80\xa2 Running test (esc) \xe2\x80\xa2 Remote caching disabled (esc) - build: cache hit, suppressing output 02cd45da7a08ba05 - test: cache hit, suppressing output 6aadbf9bca4ae1e0 + build: cache hit, suppressing output 8fc80cfff3b64237 + test: cache hit, suppressing output c71366ccd6a86465 Tasks: 2 successful, 2 total Cached: 2 cached, 2 total diff --git a/cli/integration_tests/single_package_no_config/dry-run.t b/cli/integration_tests/single_package_no_config/dry-run.t index 703e031819f7a..3d540766b0b01 100644 --- a/cli/integration_tests/single_package_no_config/dry-run.t +++ b/cli/integration_tests/single_package_no_config/dry-run.t @@ -14,7 +14,7 @@ Check Tasks to Run build Task = build - Hash = 569c206a6fa456c6 + Hash = c7223f212c321d3b Cached (Local) = false Cached (Remote) = false Command = echo 'building' @@ -36,7 +36,7 @@ Check "turboVersion": "[a-z0-9\.-]+", (re) "globalHashSummary": { "globalFileHashMap": { - "package-lock.json": "1c117cce37347befafe3a9cba1b8a609b3600021", + "package-lock.json": "8db0df575e6509336a6719094b63eb23d2c649c1", "package.json": "581fe2b8dcba5b03cbe51d78a973143eb6d33e3a" }, "rootExternalDepsHash": "", @@ -56,7 +56,7 @@ Check "tasks": [ { "task": "build", - "hash": "569c206a6fa456c6", + "hash": "c7223f212c321d3b", "cacheState": { "local": false, "remote": false @@ -79,7 +79,7 @@ Check }, "expandedInputs": { ".gitignore": "38548b0538f2fc563d6bacf70dd42798c6fd9a35", - "package-lock.json": "1c117cce37347befafe3a9cba1b8a609b3600021", + "package-lock.json": "8db0df575e6509336a6719094b63eb23d2c649c1", "package.json": "581fe2b8dcba5b03cbe51d78a973143eb6d33e3a" }, "expandedOutputs": [], diff --git a/cli/integration_tests/single_package_no_config/run.t b/cli/integration_tests/single_package_no_config/run.t index 8f2507b7c25e6..1633b7c973162 100644 --- a/cli/integration_tests/single_package_no_config/run.t +++ b/cli/integration_tests/single_package_no_config/run.t @@ -6,7 +6,7 @@ Check $ ${TURBO} run build --single-package \xe2\x80\xa2 Running build (esc) \xe2\x80\xa2 Remote caching disabled (esc) - build: cache bypass, force executing 569c206a6fa456c6 + build: cache bypass, force executing c7223f212c321d3b build: build: > build build: > echo 'building' @@ -21,7 +21,7 @@ Run a second time, verify no caching because there is no config $ ${TURBO} run build --single-package \xe2\x80\xa2 Running build (esc) \xe2\x80\xa2 Remote caching disabled (esc) - build: cache bypass, force executing 569c206a6fa456c6 + build: cache bypass, force executing c7223f212c321d3b build: build: > build build: > echo 'building' From cd6cad7ca89705cc51b9012d1cf5e36021177afb Mon Sep 17 00:00:00 2001 From: Rafael Date: Wed, 12 Apr 2023 17:23:55 +0000 Subject: [PATCH 23/36] Output logs in goroutine --- cli/internal/run/real_run.go | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/cli/internal/run/real_run.go b/cli/internal/run/real_run.go index f7b51e514dbb7..c7a3a5f1fba93 100644 --- a/cli/internal/run/real_run.go +++ b/cli/internal/run/real_run.go @@ -96,6 +96,21 @@ func RealRun( } logMutex := sync.Mutex{} + logWaitGroup := sync.WaitGroup{} + + outputLogsFunc := func(grouped bool, defaultOutWriter io.Writer, defaultErrWriter io.Writer, outBuf bytes.Buffer, errBuf bytes.Buffer) { + if grouped { + logMutex.Lock() + _, outErr := defaultOutWriter.Write(outBuf.Bytes()) + _, errErr := defaultErrWriter.Write(errBuf.Bytes()) + logMutex.Unlock() + if outErr != nil || errErr != nil { + base.UI.Error("Failed to write part of the output to terminal") + } + } + logWaitGroup.Done() + } + taskSummaryMutex := sync.Mutex{} taskSummaries := []*runsummary.TaskSummary{} execFunc := func(ctx gocontext.Context, packageTask *nodes.PackageTask, taskSummary *runsummary.TaskSummary) error { @@ -138,15 +153,8 @@ func RealRun( taskSummaryMutex.Unlock() } - if grouped { - logMutex.Lock() - _, outErr := defaultOutWriter.Write(outBuf.Bytes()) - _, errErr := defaultErrWriter.Write(errBuf.Bytes()) - logMutex.Unlock() - if outErr != nil || errErr != nil { - base.UI.Error("Failed to write part of the output to terminal") - } - } + logWaitGroup.Add(1) + go outputLogsFunc(grouped, defaultOutWriter, defaultErrWriter, *outBuf, *errBuf) return err } @@ -200,6 +208,8 @@ func RealRun( } } + logWaitGroup.Wait() + if err := runSummary.Close(exitCode, g.WorkspaceInfos); err != nil { // We don't need to throw an error, but we can warn on this. // Note: this method doesn't actually return an error for Real Runs at the time of writing. From 47924d66e00305077671be1f019d960e0909ba7d Mon Sep 17 00:00:00 2001 From: Rafael Date: Wed, 12 Apr 2023 18:09:39 +0000 Subject: [PATCH 24/36] Fix integration tests --- cli/integration_tests/inference/no_workspaces.t | 6 +++--- cli/integration_tests/no_args.t | 2 +- cli/integration_tests/single_package/dry-run.t | 8 ++++---- cli/integration_tests/single_package/run.t | 4 ++-- .../single_package_deps/dry-run.t | 14 +++++++------- cli/integration_tests/single_package_deps/run.t | 12 ++++++------ .../single_package_no_config/dry-run.t | 8 ++++---- .../single_package_no_config/run.t | 4 ++-- cli/integration_tests/turbo_help.t | 4 ++-- 9 files changed, 31 insertions(+), 31 deletions(-) diff --git a/cli/integration_tests/inference/no_workspaces.t b/cli/integration_tests/inference/no_workspaces.t index 5b58c07f2016c..5d575054251c7 100644 --- a/cli/integration_tests/inference/no_workspaces.t +++ b/cli/integration_tests/inference/no_workspaces.t @@ -13,7 +13,7 @@ Setup [-0-9:.TWZ+]+ \[DEBUG] turbo: build tag: (go|rust) (re) [-0-9:.TWZ+]+ \[INFO] turbo: skipping turbod since we appear to be in a non-interactive context (re) [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash env vars: vars=\["VERCEL_ANALYTICS_ID"] (re) - [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash: value=4c73ae6c71119fad (re) + [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash: value=476a4927e552ab88 (re) [-0-9:.TWZ+]+ \[DEBUG] turbo: local cache folder: path="" (re) \xe2\x80\xa2 Running build (esc) \xe2\x80\xa2 Remote caching disabled (esc) @@ -35,7 +35,7 @@ Setup [-0-9:.TWZ+]+ \[DEBUG] turbo: build tag: (go|rust) (re) [-0-9:.TWZ+]+ \[INFO] turbo: skipping turbod since we appear to be in a non-interactive context (re) [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash env vars: vars=\["VERCEL_ANALYTICS_ID"] (re) - [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash: value=96793eaf8b145bde (re) + [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash: value=d82bc5a0d9ca1a6b (re) [-0-9:.TWZ+]+ \[DEBUG] turbo: local cache folder: path="" (re) \xe2\x80\xa2 Running build (esc) \xe2\x80\xa2 Remote caching disabled (esc) @@ -57,7 +57,7 @@ Setup [-0-9:.TWZ+]+ \[DEBUG] turbo: build tag: (go|rust) (re) [-0-9:.TWZ+]+ \[INFO] turbo: skipping turbod since we appear to be in a non-interactive context (re) [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash env vars: vars=\["VERCEL_ANALYTICS_ID"] (re) - [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash: value=67527326c7931f8b (re) + [-0-9:.TWZ+]+ \[DEBUG] turbo: global hash: value=8169caaf504488a2 (re) [-0-9:.TWZ+]+ \[DEBUG] turbo: local cache folder: path="" (re) \xe2\x80\xa2 Running build (esc) \xe2\x80\xa2 Remote caching disabled (esc) diff --git a/cli/integration_tests/no_args.t b/cli/integration_tests/no_args.t index a9af7e7a63c72..427d40057c8e6 100644 --- a/cli/integration_tests/no_args.t +++ b/cli/integration_tests/no_args.t @@ -54,7 +54,7 @@ Make sure exit code is 2 when no args are passed --no-daemon Run without using turbo's daemon process --no-deps Exclude dependent task consumers from execution --output-logs Set type of process output logging. Use "full" to show all output. Use "hash-only" to show only turbo-computed task hashes. Use "new-only" to show only new output with only hashes for cached tasks. Use "none" to hide process output. (default full) [possible values: full, none, hash-only, new-only, errors-only] - --log-order Set type of process output order. Use "stream" to show output as soon as it is available. Use "grouped" to show output when a command has finished execution. (default stream) [possible values: stream, grouped] + --log-order Set type of process output order. Use "stream" to show output as soon as it is available. Use "grouped" to show output when a command has finished execution. (default stream) [default: stream] [possible values: stream, grouped] --parallel Execute all tasks in parallel --profile File to write turbo's performance profile output into. You can load the file up in chrome://tracing to see which parts of your build were slow --remote-only Ignore the local filesystem cache for all tasks. Only allow reading and caching artifacts using the remote cache diff --git a/cli/integration_tests/single_package/dry-run.t b/cli/integration_tests/single_package/dry-run.t index 92aaf826734be..3a2ce9ffc09df 100644 --- a/cli/integration_tests/single_package/dry-run.t +++ b/cli/integration_tests/single_package/dry-run.t @@ -14,7 +14,7 @@ Check Tasks to Run build Task = build - Hash = dd4a9a7b508b0e38 + Hash = acd1f716a22c635b Cached (Local) = false Cached (Remote) = false Command = echo 'building' > foo @@ -36,7 +36,7 @@ Check "turboVersion": "[a-z0-9\.-]+", (re) "globalHashSummary": { "globalFileHashMap": { - "package-lock.json": "8db0df575e6509336a6719094b63eb23d2c649c1", + "package-lock.json": "1c117cce37347befafe3a9cba1b8a609b3600021", "package.json": "185771929d92c3865ce06c863c07d357500d3364", "somefile.txt": "45b983be36b73c0788dc9cbcb76cbb80fc7bb057" }, @@ -59,7 +59,7 @@ Check "tasks": [ { "task": "build", - "hash": "dd4a9a7b508b0e38", + "hash": "acd1f716a22c635b", "cacheState": { "local": false, "remote": false @@ -86,7 +86,7 @@ Check }, "expandedInputs": { ".gitignore": "6f23ff6842b5526da43ab38f4a5bf3b0158eeb42", - "package-lock.json": "8db0df575e6509336a6719094b63eb23d2c649c1", + "package-lock.json": "1c117cce37347befafe3a9cba1b8a609b3600021", "package.json": "185771929d92c3865ce06c863c07d357500d3364", "somefile.txt": "45b983be36b73c0788dc9cbcb76cbb80fc7bb057", "turbo.json": "505752e75c10f9e7a0d2538cf8b6f0fcfb8980a0" diff --git a/cli/integration_tests/single_package/run.t b/cli/integration_tests/single_package/run.t index 0481a939bd55c..816e77facacda 100644 --- a/cli/integration_tests/single_package/run.t +++ b/cli/integration_tests/single_package/run.t @@ -6,7 +6,7 @@ Check $ ${TURBO} run build --single-package \xe2\x80\xa2 Running build (esc) \xe2\x80\xa2 Remote caching disabled (esc) - build: cache miss, executing dd4a9a7b508b0e38 + build: cache miss, executing acd1f716a22c635b build: build: > build build: > echo 'building' > foo @@ -23,7 +23,7 @@ Run a second time, verify caching works because there is a config $ ${TURBO} run build --single-package \xe2\x80\xa2 Running build (esc) \xe2\x80\xa2 Remote caching disabled (esc) - build: cache hit, replaying output dd4a9a7b508b0e38 + build: cache hit, replaying output acd1f716a22c635b build: build: > build build: > echo 'building' > foo diff --git a/cli/integration_tests/single_package_deps/dry-run.t b/cli/integration_tests/single_package_deps/dry-run.t index 59d6483b5d0f1..2fbdcebf604b3 100644 --- a/cli/integration_tests/single_package_deps/dry-run.t +++ b/cli/integration_tests/single_package_deps/dry-run.t @@ -14,7 +14,7 @@ Check Tasks to Run build Task = build - Hash = 8fc80cfff3b64237 + Hash = 02cd45da7a08ba05 Cached (Local) = false Cached (Remote) = false Command = echo 'building' > foo @@ -30,7 +30,7 @@ Check Framework = test Task = test - Hash = c71366ccd6a86465 + Hash = 6aadbf9bca4ae1e0 Cached (Local) = false Cached (Remote) = false Command = [[ ( -f foo ) && $(cat foo) == 'building' ]] @@ -52,7 +52,7 @@ Check "turboVersion": "[a-z0-9\.-]+", (re) "globalHashSummary": { "globalFileHashMap": { - "package-lock.json": "8db0df575e6509336a6719094b63eb23d2c649c1", + "package-lock.json": "1c117cce37347befafe3a9cba1b8a609b3600021", "package.json": "bc24e5c5b8bd13d419e0742ae3e92a2bf61c53d0" }, "rootExternalDepsHash": "", @@ -85,7 +85,7 @@ Check "tasks": [ { "task": "build", - "hash": "8fc80cfff3b64237", + "hash": "02cd45da7a08ba05", "cacheState": { "local": false, "remote": false @@ -114,7 +114,7 @@ Check }, "expandedInputs": { ".gitignore": "6f23ff6842b5526da43ab38f4a5bf3b0158eeb42", - "package-lock.json": "8db0df575e6509336a6719094b63eb23d2c649c1", + "package-lock.json": "1c117cce37347befafe3a9cba1b8a609b3600021", "package.json": "bc24e5c5b8bd13d419e0742ae3e92a2bf61c53d0", "turbo.json": "e1fe3e5402fe019ef3845cc63a736878a68934c7" }, @@ -131,7 +131,7 @@ Check }, { "task": "test", - "hash": "c71366ccd6a86465", + "hash": "6aadbf9bca4ae1e0", "cacheState": { "local": false, "remote": false @@ -158,7 +158,7 @@ Check }, "expandedInputs": { ".gitignore": "6f23ff6842b5526da43ab38f4a5bf3b0158eeb42", - "package-lock.json": "8db0df575e6509336a6719094b63eb23d2c649c1", + "package-lock.json": "1c117cce37347befafe3a9cba1b8a609b3600021", "package.json": "bc24e5c5b8bd13d419e0742ae3e92a2bf61c53d0", "turbo.json": "e1fe3e5402fe019ef3845cc63a736878a68934c7" }, diff --git a/cli/integration_tests/single_package_deps/run.t b/cli/integration_tests/single_package_deps/run.t index e004bbfd89524..0a86d6690c0d0 100644 --- a/cli/integration_tests/single_package_deps/run.t +++ b/cli/integration_tests/single_package_deps/run.t @@ -6,12 +6,12 @@ Check $ ${TURBO} run test --single-package \xe2\x80\xa2 Running test (esc) \xe2\x80\xa2 Remote caching disabled (esc) - build: cache miss, executing 8fc80cfff3b64237 + build: cache miss, executing 02cd45da7a08ba05 build: build: > build build: > echo 'building' > foo build: - test: cache miss, executing c71366ccd6a86465 + test: cache miss, executing 6aadbf9bca4ae1e0 test: test: > test test: > [[ ( -f foo ) && $(cat foo) == 'building' ]] @@ -25,12 +25,12 @@ Run a second time, verify caching works because there is a config $ ${TURBO} run test --single-package \xe2\x80\xa2 Running test (esc) \xe2\x80\xa2 Remote caching disabled (esc) - build: cache hit, replaying output 8fc80cfff3b64237 + build: cache hit, replaying output 02cd45da7a08ba05 build: build: > build build: > echo 'building' > foo build: - test: cache hit, replaying output c71366ccd6a86465 + test: cache hit, replaying output 6aadbf9bca4ae1e0 test: test: > test test: > [[ ( -f foo ) && $(cat foo) == 'building' ]] @@ -44,8 +44,8 @@ Run with --output-logs=hash-only $ ${TURBO} run test --single-package --output-logs=hash-only \xe2\x80\xa2 Running test (esc) \xe2\x80\xa2 Remote caching disabled (esc) - build: cache hit, suppressing output 8fc80cfff3b64237 - test: cache hit, suppressing output c71366ccd6a86465 + build: cache hit, suppressing output 02cd45da7a08ba05 + test: cache hit, suppressing output 6aadbf9bca4ae1e0 Tasks: 2 successful, 2 total Cached: 2 cached, 2 total diff --git a/cli/integration_tests/single_package_no_config/dry-run.t b/cli/integration_tests/single_package_no_config/dry-run.t index 3d540766b0b01..703e031819f7a 100644 --- a/cli/integration_tests/single_package_no_config/dry-run.t +++ b/cli/integration_tests/single_package_no_config/dry-run.t @@ -14,7 +14,7 @@ Check Tasks to Run build Task = build - Hash = c7223f212c321d3b + Hash = 569c206a6fa456c6 Cached (Local) = false Cached (Remote) = false Command = echo 'building' @@ -36,7 +36,7 @@ Check "turboVersion": "[a-z0-9\.-]+", (re) "globalHashSummary": { "globalFileHashMap": { - "package-lock.json": "8db0df575e6509336a6719094b63eb23d2c649c1", + "package-lock.json": "1c117cce37347befafe3a9cba1b8a609b3600021", "package.json": "581fe2b8dcba5b03cbe51d78a973143eb6d33e3a" }, "rootExternalDepsHash": "", @@ -56,7 +56,7 @@ Check "tasks": [ { "task": "build", - "hash": "c7223f212c321d3b", + "hash": "569c206a6fa456c6", "cacheState": { "local": false, "remote": false @@ -79,7 +79,7 @@ Check }, "expandedInputs": { ".gitignore": "38548b0538f2fc563d6bacf70dd42798c6fd9a35", - "package-lock.json": "8db0df575e6509336a6719094b63eb23d2c649c1", + "package-lock.json": "1c117cce37347befafe3a9cba1b8a609b3600021", "package.json": "581fe2b8dcba5b03cbe51d78a973143eb6d33e3a" }, "expandedOutputs": [], diff --git a/cli/integration_tests/single_package_no_config/run.t b/cli/integration_tests/single_package_no_config/run.t index 1633b7c973162..8f2507b7c25e6 100644 --- a/cli/integration_tests/single_package_no_config/run.t +++ b/cli/integration_tests/single_package_no_config/run.t @@ -6,7 +6,7 @@ Check $ ${TURBO} run build --single-package \xe2\x80\xa2 Running build (esc) \xe2\x80\xa2 Remote caching disabled (esc) - build: cache bypass, force executing c7223f212c321d3b + build: cache bypass, force executing 569c206a6fa456c6 build: build: > build build: > echo 'building' @@ -21,7 +21,7 @@ Run a second time, verify no caching because there is no config $ ${TURBO} run build --single-package \xe2\x80\xa2 Running build (esc) \xe2\x80\xa2 Remote caching disabled (esc) - build: cache bypass, force executing c7223f212c321d3b + build: cache bypass, force executing 569c206a6fa456c6 build: build: > build build: > echo 'building' diff --git a/cli/integration_tests/turbo_help.t b/cli/integration_tests/turbo_help.t index 1d828f918904f..9eac1cd5c5191 100644 --- a/cli/integration_tests/turbo_help.t +++ b/cli/integration_tests/turbo_help.t @@ -54,7 +54,7 @@ Test help flag --no-daemon Run without using turbo's daemon process --no-deps Exclude dependent task consumers from execution --output-logs Set type of process output logging. Use "full" to show all output. Use "hash-only" to show only turbo-computed task hashes. Use "new-only" to show only new output with only hashes for cached tasks. Use "none" to hide process output. (default full) [possible values: full, none, hash-only, new-only, errors-only] - --log-order Set type of process output order. Use "stream" to show output as soon as it is available. Use "grouped" to show output when a command has finished execution. (default stream) [possible values: stream, grouped] + --log-order Set type of process output order. Use "stream" to show output as soon as it is available. Use "grouped" to show output when a command has finished execution. (default stream) [default: stream] [possible values: stream, grouped] --parallel Execute all tasks in parallel --profile File to write turbo's performance profile output into. You can load the file up in chrome://tracing to see which parts of your build were slow --remote-only Ignore the local filesystem cache for all tasks. Only allow reading and caching artifacts using the remote cache @@ -120,7 +120,7 @@ Test help flag --no-daemon Run without using turbo's daemon process --no-deps Exclude dependent task consumers from execution --output-logs Set type of process output logging. Use "full" to show all output. Use "hash-only" to show only turbo-computed task hashes. Use "new-only" to show only new output with only hashes for cached tasks. Use "none" to hide process output. (default full) [possible values: full, none, hash-only, new-only, errors-only] - --log-order Set type of process output order. Use "stream" to show output as soon as it is available. Use "grouped" to show output when a command has finished execution. (default stream) [possible values: stream, grouped] + --log-order Set type of process output order. Use "stream" to show output as soon as it is available. Use "grouped" to show output when a command has finished execution. (default stream) [default: stream] [possible values: stream, grouped] --parallel Execute all tasks in parallel --profile File to write turbo's performance profile output into. You can load the file up in chrome://tracing to see which parts of your build were slow --remote-only Ignore the local filesystem cache for all tasks. Only allow reading and caching artifacts using the remote cache From 06eb777c9ca331bebdb398805d5d5a8eae579027 Mon Sep 17 00:00:00 2001 From: Rafael Date: Wed, 12 Apr 2023 18:09:53 +0000 Subject: [PATCH 25/36] Use default_value_t --- crates/turborepo-lib/src/cli.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/turborepo-lib/src/cli.rs b/crates/turborepo-lib/src/cli.rs index 9fdf35f2d73c5..993910eae5f57 100644 --- a/crates/turborepo-lib/src/cli.rs +++ b/crates/turborepo-lib/src/cli.rs @@ -350,8 +350,8 @@ pub struct RunArgs { /// Set type of process output order. Use "stream" to show /// output as soon as it is available. Use "grouped" to /// show output when a command has finished execution. (default stream) - #[clap(long, value_enum)] - pub log_order: Option, + #[clap(long, value_enum, default_value_t = LogOrder::Stream)] + pub log_order: LogOrder, #[clap(long, hide = true)] pub only: bool, /// Execute all tasks in parallel. From 7a024f57527f982424ccf5fee4ccc6efc1190831 Mon Sep 17 00:00:00 2001 From: Rafael Date: Wed, 12 Apr 2023 20:11:51 +0000 Subject: [PATCH 26/36] Use channel WIP --- cli/internal/run/real_run.go | 54 +++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/cli/internal/run/real_run.go b/cli/internal/run/real_run.go index c7a3a5f1fba93..bbfa59c039072 100644 --- a/cli/internal/run/real_run.go +++ b/cli/internal/run/real_run.go @@ -33,6 +33,11 @@ import ( "github.com/vercel/turbo/cli/internal/ui" ) +type taskLogContext struct { + outBuf *bytes.Buffer + errBuf *bytes.Buffer +} + // RealRun executes a set of tasks func RealRun( ctx gocontext.Context, @@ -95,43 +100,41 @@ func RealRun( Concurrency: rs.Opts.runOpts.Concurrency, } - logMutex := sync.Mutex{} + taskCount := len(engine.TaskGraph.Vertices()) + logChan := make(chan taskLogContext, taskCount) logWaitGroup := sync.WaitGroup{} + grouped := rs.Opts.runOpts.LogOrder == "grouped" - outputLogsFunc := func(grouped bool, defaultOutWriter io.Writer, defaultErrWriter io.Writer, outBuf bytes.Buffer, errBuf bytes.Buffer) { - if grouped { - logMutex.Lock() - _, outErr := defaultOutWriter.Write(outBuf.Bytes()) - _, errErr := defaultErrWriter.Write(errBuf.Bytes()) - logMutex.Unlock() - if outErr != nil || errErr != nil { - base.UI.Error("Failed to write part of the output to terminal") - } + outputLogs := func() { + for i := 1; i < taskCount; i++ { + logContext := <-logChan + + os.Stdout.Write(logContext.outBuf.Bytes()) + os.Stderr.Write(logContext.errBuf.Bytes()) + logWaitGroup.Done() } - logWaitGroup.Done() + } + + if grouped { + go outputLogs() } taskSummaryMutex := sync.Mutex{} taskSummaries := []*runsummary.TaskSummary{} execFunc := func(ctx gocontext.Context, packageTask *nodes.PackageTask, taskSummary *runsummary.TaskSummary) error { - - grouped := rs.Opts.runOpts.LogOrder == "grouped" - + logWaitGroup.Add(1) outBuf := &bytes.Buffer{} errBuf := &bytes.Buffer{} var outWriter io.Writer var errWriter io.Writer - defaultOutWriter := os.Stdout - defaultErrWriter := os.Stderr - if grouped { outWriter = outBuf errWriter = errBuf } else { - outWriter = defaultOutWriter - errWriter = defaultErrWriter + outWriter = os.Stdout + errWriter = os.Stderr } ui := concurrentUIFactory.Build(os.Stdin, outWriter, errWriter) @@ -152,9 +155,12 @@ func RealRun( // not using defer, just release the lock taskSummaryMutex.Unlock() } - - logWaitGroup.Add(1) - go outputLogsFunc(grouped, defaultOutWriter, defaultErrWriter, *outBuf, *errBuf) + if grouped { + logChan <- taskLogContext{ + outBuf: outBuf, + errBuf: errBuf, + } + } return err } @@ -208,7 +214,9 @@ func RealRun( } } - logWaitGroup.Wait() + if grouped { + logWaitGroup.Wait() + } if err := runSummary.Close(exitCode, g.WorkspaceInfos); err != nil { // We don't need to throw an error, but we can warn on this. From e9f2b17a8b9557bb73b7294c8ddfe62b3c3debb9 Mon Sep 17 00:00:00 2001 From: Rafael Date: Sat, 15 Apr 2023 15:32:20 +0000 Subject: [PATCH 27/36] Small improvements --- cli/internal/run/real_run.go | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/cli/internal/run/real_run.go b/cli/internal/run/real_run.go index bbfa59c039072..ff2f665203f9c 100644 --- a/cli/internal/run/real_run.go +++ b/cli/internal/run/real_run.go @@ -33,11 +33,6 @@ import ( "github.com/vercel/turbo/cli/internal/ui" ) -type taskLogContext struct { - outBuf *bytes.Buffer - errBuf *bytes.Buffer -} - // RealRun executes a set of tasks func RealRun( ctx gocontext.Context, @@ -109,8 +104,16 @@ func RealRun( for i := 1; i < taskCount; i++ { logContext := <-logChan - os.Stdout.Write(logContext.outBuf.Bytes()) - os.Stderr.Write(logContext.errBuf.Bytes()) + outBytes := logContext.outBuf.Bytes() + errBytes := logContext.errBuf.Bytes() + + _, errOut := os.Stdout.Write(outBytes) + _, errErr := os.Stderr.Write(errBytes) + + if errOut != nil || errErr != nil { + ec.ui.Error("Failed to output some of the logs.") + } + logWaitGroup.Done() } } @@ -232,6 +235,11 @@ func RealRun( return nil } +type taskLogContext struct { + outBuf *bytes.Buffer + errBuf *bytes.Buffer +} + type execContext struct { colorCache *colorcache.ColorCache runSummary runsummary.Meta From dc89f321050ee819151ed51138a892cf8caf9df3 Mon Sep 17 00:00:00 2001 From: Rafael Date: Tue, 18 Apr 2023 18:21:52 +0000 Subject: [PATCH 28/36] Fix merge mistake --- cli/internal/run/real_run.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/cli/internal/run/real_run.go b/cli/internal/run/real_run.go index b99b0ffb9f376..e55e94ee04418 100644 --- a/cli/internal/run/real_run.go +++ b/cli/internal/run/real_run.go @@ -177,12 +177,7 @@ func RealRun( return err } - // Return the error when there is one - if err != nil { - return err - } - - return err + return nil } getArgs := func(taskID string) []string { From 8d2451b44a77f3ee1f1e1675648396095387615d Mon Sep 17 00:00:00 2001 From: Rafael Date: Sun, 28 May 2023 16:43:51 +0000 Subject: [PATCH 29/36] Fix go lint problems --- cli/internal/cmdutil/cmdutil.go | 4 ++-- cli/internal/run/real_run.go | 4 ++-- cli/internal/ui/ui_factory.go | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cli/internal/cmdutil/cmdutil.go b/cli/internal/cmdutil/cmdutil.go index 6421a3c3aead0..bbd0cbcc5a746 100644 --- a/cli/internal/cmdutil/cmdutil.go +++ b/cli/internal/cmdutil/cmdutil.go @@ -76,7 +76,7 @@ func (h *Helper) getUI(cliArgs *turbostate.ParsedArgsFromRust) cli.Ui { return factory.Build(os.Stdout, os.Stdin, os.Stderr) } -func (h *Helper) getUIFactory(cliArgs *turbostate.ParsedArgsFromRust) ui.UIFactory { +func (h *Helper) getUIFactory(cliArgs *turbostate.ParsedArgsFromRust) ui.Factory { colorMode := ui.GetColorModeFromEnv() if cliArgs.NoColor { colorMode = ui.ColorModeSuppressed @@ -182,7 +182,7 @@ func (h *Helper) GetCmdBase(executionState *turbostate.ExecutionState) (*CmdBase // CmdBase encompasses configured components common to all turbo commands. type CmdBase struct { UI cli.Ui - UIFactory ui.UIFactory + UIFactory ui.Factory Logger hclog.Logger RepoRoot turbopath.AbsoluteSystemPath APIClient *client.APIClient diff --git a/cli/internal/run/real_run.go b/cli/internal/run/real_run.go index 6ca6ec11d1399..a7eee02d5ee2d 100644 --- a/cli/internal/run/real_run.go +++ b/cli/internal/run/real_run.go @@ -151,7 +151,7 @@ func RealRun( ui := concurrentUIFactory.Build(os.Stdin, outWriter, errWriter) - taskExecutionSummary, err := ec.exec(ctx, packageTask, ui, outWriter, errWriter) + taskExecutionSummary, err := ec.exec(ctx, packageTask, ui, outWriter) // taskExecutionSummary will be nil if the task never executed // (i.e. if the workspace didn't implement the script corresponding to the task) @@ -282,7 +282,7 @@ func (ec *execContext) logError(prefix string, err error) { ec.ui.Error(fmt.Sprintf("%s%s%s", ui.ERROR_PREFIX, prefix, color.RedString(" %v", err))) } -func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTask, ui cli.Ui, outWriter io.Writer, errWriter io.Writer) (*runsummary.TaskExecutionSummary, error) { +func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTask, ui cli.Ui, outWriter io.Writer) (*runsummary.TaskExecutionSummary, error) { // Setup tracer. Every time tracer() is called the taskExecutionSummary's duration is updated // So make sure to call it before returning. successExitCode := 0 // We won't use this till later diff --git a/cli/internal/ui/ui_factory.go b/cli/internal/ui/ui_factory.go index 3532936f468e3..cc6f7f457a3cf 100644 --- a/cli/internal/ui/ui_factory.go +++ b/cli/internal/ui/ui_factory.go @@ -7,8 +7,8 @@ import ( "github.com/mitchellh/cli" ) -// UIFactory provides an interface for creating cli.Ui instances from input, output and error IOs -type UIFactory interface { +// Factory provides an interface for creating cli.Ui instances from input, output and error IOs +type Factory interface { Build(in io.Reader, out io.Writer, err io.Writer) cli.Ui } @@ -28,7 +28,7 @@ func (factory *BasicUIFactory) Build(in io.Reader, out io.Writer, err io.Writer) // ColoredUIFactory provides a method for creating a cli.ColoredUi from input, output and error IOs type ColoredUIFactory struct { ColorMode ColorMode - Base UIFactory + Base Factory } // Build builds a cli.ColoredUi from input, output and error IOs @@ -56,7 +56,7 @@ func (factory *ColoredUIFactory) Build(in io.Reader, out io.Writer, err io.Write // ConcurrentUIFactory provides a method for creating a cli.ConcurrentUi from input, output and error IOs type ConcurrentUIFactory struct { - Base UIFactory + Base Factory } // Build builds a cli.ConcurrentUi from input, output and error IOs @@ -68,7 +68,7 @@ func (factory *ConcurrentUIFactory) Build(in io.Reader, out io.Writer, err io.Wr // PrefixedUIFactory provides a method for creating a cli.PrefixedUi from input, output and error IOs type PrefixedUIFactory struct { - Base UIFactory + Base Factory AskPrefix string AskSecretPrefix string From 3c40e8f91cd38643669badab8d046f162853592c Mon Sep 17 00:00:00 2001 From: Rafael Date: Sun, 28 May 2023 17:06:22 +0000 Subject: [PATCH 30/36] Fix rust mismatched types --- crates/turborepo-lib/src/cli.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/turborepo-lib/src/cli.rs b/crates/turborepo-lib/src/cli.rs index dd1fd40c66152..530056072b737 100644 --- a/crates/turborepo-lib/src/cli.rs +++ b/crates/turborepo-lib/src/cli.rs @@ -1272,7 +1272,7 @@ mod test { Args { command: Some(Command::Run(Box::new(RunArgs { tasks: vec!["build".to_string()], - log_order: Some(LogOrder::Stream), + log_order: LogOrder::Stream, ..get_default_run_args() }))), ..Args::default() @@ -1284,7 +1284,7 @@ mod test { Args { command: Some(Command::Run(Box::new(RunArgs { tasks: vec!["build".to_string()], - log_order: Some(LogOrder::Grouped), + log_order: LogOrder::Grouped, ..get_default_run_args() }))), ..Args::default() From e0de43c3301d92b888e65f74d426c6f12046caad Mon Sep 17 00:00:00 2001 From: Rafael Date: Sun, 28 May 2023 19:33:20 +0000 Subject: [PATCH 31/36] Fix integration tests --- cli/integration_tests/ordered/setup.sh | 6 ------ .../integration/tests/_fixtures/ordered}/.gitignore | 0 .../_fixtures/ordered}/apps/my-app/package.json | 0 .../tests/_fixtures/ordered}/package.json | 0 .../_fixtures/ordered}/packages/util/package.json | 0 .../integration/tests/_fixtures/ordered}/turbo.json | 0 turborepo-tests/integration/tests/bad_flag.t | 2 +- .../integration/tests}/ordered/grouped.t | 10 +++++----- .../integration/tests}/ordered/stream.t | 12 ++++++------ 9 files changed, 12 insertions(+), 18 deletions(-) delete mode 100755 cli/integration_tests/ordered/setup.sh rename {cli/integration_tests/ordered/monorepo => turborepo-tests/integration/tests/_fixtures/ordered}/.gitignore (100%) rename {cli/integration_tests/ordered/monorepo => turborepo-tests/integration/tests/_fixtures/ordered}/apps/my-app/package.json (100%) rename {cli/integration_tests/ordered/monorepo => turborepo-tests/integration/tests/_fixtures/ordered}/package.json (100%) rename {cli/integration_tests/ordered/monorepo => turborepo-tests/integration/tests/_fixtures/ordered}/packages/util/package.json (100%) rename {cli/integration_tests/ordered/monorepo => turborepo-tests/integration/tests/_fixtures/ordered}/turbo.json (100%) rename {cli/integration_tests => turborepo-tests/integration/tests}/ordered/grouped.t (74%) rename {cli/integration_tests => turborepo-tests/integration/tests}/ordered/stream.t (68%) diff --git a/cli/integration_tests/ordered/setup.sh b/cli/integration_tests/ordered/setup.sh deleted file mode 100755 index 864b7a64a37a6..0000000000000 --- a/cli/integration_tests/ordered/setup.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) -TARGET_DIR=$1 -cp -a ${SCRIPT_DIR}/monorepo/. ${TARGET_DIR}/ -${SCRIPT_DIR}/../setup_git.sh ${TARGET_DIR} diff --git a/cli/integration_tests/ordered/monorepo/.gitignore b/turborepo-tests/integration/tests/_fixtures/ordered/.gitignore similarity index 100% rename from cli/integration_tests/ordered/monorepo/.gitignore rename to turborepo-tests/integration/tests/_fixtures/ordered/.gitignore diff --git a/cli/integration_tests/ordered/monorepo/apps/my-app/package.json b/turborepo-tests/integration/tests/_fixtures/ordered/apps/my-app/package.json similarity index 100% rename from cli/integration_tests/ordered/monorepo/apps/my-app/package.json rename to turborepo-tests/integration/tests/_fixtures/ordered/apps/my-app/package.json diff --git a/cli/integration_tests/ordered/monorepo/package.json b/turborepo-tests/integration/tests/_fixtures/ordered/package.json similarity index 100% rename from cli/integration_tests/ordered/monorepo/package.json rename to turborepo-tests/integration/tests/_fixtures/ordered/package.json diff --git a/cli/integration_tests/ordered/monorepo/packages/util/package.json b/turborepo-tests/integration/tests/_fixtures/ordered/packages/util/package.json similarity index 100% rename from cli/integration_tests/ordered/monorepo/packages/util/package.json rename to turborepo-tests/integration/tests/_fixtures/ordered/packages/util/package.json diff --git a/cli/integration_tests/ordered/monorepo/turbo.json b/turborepo-tests/integration/tests/_fixtures/ordered/turbo.json similarity index 100% rename from cli/integration_tests/ordered/monorepo/turbo.json rename to turborepo-tests/integration/tests/_fixtures/ordered/turbo.json diff --git a/turborepo-tests/integration/tests/bad_flag.t b/turborepo-tests/integration/tests/bad_flag.t index f6b44d0fd38c2..0ddb58d774d37 100644 --- a/turborepo-tests/integration/tests/bad_flag.t +++ b/turborepo-tests/integration/tests/bad_flag.t @@ -19,7 +19,7 @@ Bad flag with an implied run command should display run flags note: to pass '--bad-flag' as a value, use '-- --bad-flag' - Usage: turbo <--cache-dir |--cache-workers |--concurrency |--continue|--dry-run []|--single-package|--filter |--force []|--framework-inference []|--global-deps |--graph []|--env-mode []|--ignore |--include-dependencies|--no-cache|--no-daemon|--no-deps|--output-logs |--only|--parallel|--pkg-inference-root |--profile |--remote-only|--scope |--since |--summarize []|--log-prefix |TASKS|PASS_THROUGH_ARGS|--experimental-space-id > + Usage: turbo <--cache-dir |--cache-workers |--concurrency |--continue|--dry-run []|--single-package|--filter |--force []|--framework-inference []|--global-deps |--graph []|--env-mode []|--ignore |--include-dependencies|--no-cache|--no-daemon|--no-deps|--output-logs |--log-order |--only|--parallel|--pkg-inference-root |--profile |--remote-only|--scope |--since |--summarize []|--log-prefix |TASKS|PASS_THROUGH_ARGS|--experimental-space-id > For more information, try '--help'. diff --git a/cli/integration_tests/ordered/grouped.t b/turborepo-tests/integration/tests/ordered/grouped.t similarity index 74% rename from cli/integration_tests/ordered/grouped.t rename to turborepo-tests/integration/tests/ordered/grouped.t index 1c7377c639393..751317dee2826 100644 --- a/cli/integration_tests/ordered/grouped.t +++ b/turborepo-tests/integration/tests/ordered/grouped.t @@ -1,20 +1,20 @@ -Setup - $ . ${TESTDIR}/../setup.sh - $ . ${TESTDIR}/setup.sh $(pwd) +# Setup + $ . ${TESTDIR}/../../../helpers/setup.sh + $ . ${TESTDIR}/../_helpers/setup_monorepo.sh $(pwd) ordered # Build in grouped order. $ ${TURBO} run build --log-order grouped --force \xe2\x80\xa2 Packages in scope: my-app, util (esc) \xe2\x80\xa2 Running build in 2 packages (esc) \xe2\x80\xa2 Remote caching disabled (esc) - my-app:build: cache bypass, force executing 8107080a88b155ef + my-app:build: cache bypass, force executing [0-9a-f]+ (re) my-app:build: my-app:build: > build my-app:build: > echo 'building' && sleep 1 && echo 'done' my-app:build: my-app:build: building my-app:build: done - util:build: cache bypass, force executing f1ea8c68bf163f6b + util:build: cache bypass, force executing [0-9a-f]+ (re) util:build: util:build: > build util:build: > sleep 0.5 && echo 'building' && sleep 1 && echo 'completed' diff --git a/cli/integration_tests/ordered/stream.t b/turborepo-tests/integration/tests/ordered/stream.t similarity index 68% rename from cli/integration_tests/ordered/stream.t rename to turborepo-tests/integration/tests/ordered/stream.t index 16934e60efcc7..13fa16d6d2a1f 100644 --- a/cli/integration_tests/ordered/stream.t +++ b/turborepo-tests/integration/tests/ordered/stream.t @@ -1,14 +1,14 @@ -Setup - $ . ${TESTDIR}/../setup.sh - $ . ${TESTDIR}/setup.sh $(pwd) +# Setup + $ . ${TESTDIR}/../../../helpers/setup.sh + $ . ${TESTDIR}/../_helpers/setup_monorepo.sh $(pwd) ordered # Build in stream order. All the .*'s are unpredictable lines, however the amount of lines is predictable. $ ${TURBO} run build --log-order stream --force \xe2\x80\xa2 Packages in scope: my-app, util (esc) \xe2\x80\xa2 Running build in 2 packages (esc) \xe2\x80\xa2 Remote caching disabled (esc) - (my-app|util):build: cache bypass, force executing (f1ea8c68bf163f6b|8107080a88b155ef) (re) - (my-app|util):build: cache bypass, force executing (f1ea8c68bf163f6b|8107080a88b155ef) (re) + (my-app|util):build: cache bypass, force executing [0-9a-f]+ (re) + (my-app|util):build: cache bypass, force executing [0-9a-f]+ (re) .* (re) .* (re) .* (re) @@ -25,4 +25,4 @@ Setup Tasks: 2 successful, 2 total Cached: 0 cached, 2 total Time:\s*[\.0-9]+m?s (re) - \ No newline at end of file + From 61535e9cf94474f84a4ceb1efc3a0d7d4272c08a Mon Sep 17 00:00:00 2001 From: Rafael Date: Thu, 8 Jun 2023 19:15:46 +0000 Subject: [PATCH 32/36] Unnecessary null check --- cli/internal/run/real_run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/internal/run/real_run.go b/cli/internal/run/real_run.go index a7eee02d5ee2d..07c54ef8cfa0e 100644 --- a/cli/internal/run/real_run.go +++ b/cli/internal/run/real_run.go @@ -156,7 +156,7 @@ func RealRun( // taskExecutionSummary will be nil if the task never executed // (i.e. if the workspace didn't implement the script corresponding to the task) // We don't need to collect any of the outputs or execution if the task didn't execute. - if err == nil && taskExecutionSummary != nil { + if taskExecutionSummary != nil { taskSummary.ExpandedOutputs = taskHashTracker.GetExpandedOutputs(taskSummary.TaskID) taskSummary.Execution = taskExecutionSummary taskSummary.CacheSummary = taskHashTracker.GetCacheStatus(taskSummary.TaskID) From 1f73aeed054344ae16e3388d62b9435543b6c987 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 9 Jun 2023 17:10:43 +0000 Subject: [PATCH 33/36] Apply comments --- cli/internal/run/real_run.go | 44 ++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/cli/internal/run/real_run.go b/cli/internal/run/real_run.go index 71ade69719697..fce188d58378c 100644 --- a/cli/internal/run/real_run.go +++ b/cli/internal/run/real_run.go @@ -107,27 +107,30 @@ func RealRun( taskCount := len(engine.TaskGraph.Vertices()) logChan := make(chan taskLogContext, taskCount) logWaitGroup := sync.WaitGroup{} - grouped := rs.Opts.runOpts.LogOrder == "grouped" + isGrouped := rs.Opts.runOpts.LogOrder == "grouped" - outputLogs := func() { - for i := 1; i < taskCount; i++ { - logContext := <-logChan + var outputLogs func() - outBytes := logContext.outBuf.Bytes() - errBytes := logContext.errBuf.Bytes() + if isGrouped { + outputLogs = func() { + for i := 1; i < taskCount; i++ { + logContext := <-logChan - _, errOut := os.Stdout.Write(outBytes) - _, errErr := os.Stderr.Write(errBytes) + outBytes := logContext.outBuf.Bytes() + errBytes := logContext.errBuf.Bytes() - if errOut != nil || errErr != nil { - ec.ui.Error("Failed to output some of the logs.") - } + _, errOut := os.Stdout.Write(outBytes) + _, errErr := os.Stderr.Write(errBytes) + + if errOut != nil || errErr != nil { + ec.ui.Error("Failed to output some of the logs.") + } - logWaitGroup.Done() + logWaitGroup.Done() + } } } - - if grouped { + if isGrouped { go outputLogs() } @@ -138,15 +141,12 @@ func RealRun( outBuf := &bytes.Buffer{} errBuf := &bytes.Buffer{} - var outWriter io.Writer - var errWriter io.Writer + var outWriter io.Writer = os.Stdout + var errWriter io.Writer = os.Stderr - if grouped { + if isGrouped { outWriter = outBuf errWriter = errBuf - } else { - outWriter = os.Stdout - errWriter = os.Stderr } ui := concurrentUIFactory.Build(os.Stdin, outWriter, errWriter) @@ -169,7 +169,7 @@ func RealRun( runSummary.CloseTask(taskSummary) } - if grouped { + if isGrouped { logChan <- taskLogContext{ outBuf: outBuf, errBuf: errBuf, @@ -233,7 +233,7 @@ func RealRun( } } - if grouped { + if isGrouped { logWaitGroup.Wait() } From 21a7922248e82dac416eefde8834fddd718cd5aa Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 9 Jun 2023 17:45:16 +0000 Subject: [PATCH 34/36] Add test for default --- crates/turborepo-lib/src/cli.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/crates/turborepo-lib/src/cli.rs b/crates/turborepo-lib/src/cli.rs index 0a0309a71c386..1cfc482732d4e 100644 --- a/crates/turborepo-lib/src/cli.rs +++ b/crates/turborepo-lib/src/cli.rs @@ -1286,6 +1286,18 @@ mod test { } ); + assert_eq!( + Args::try_parse_from(["turbo", "run", "build"]).unwrap(), + Args { + command: Some(Command::Run(Box::new(RunArgs { + tasks: vec!["build".to_string()], + log_order: LogOrder::Stream, + ..get_default_run_args() + }))), + ..Args::default() + } + ); + assert_eq!( Args::try_parse_from(["turbo", "run", "build", "--parallel"]).unwrap(), Args { From 9af7cae631e2fc3d6e7999bbd8f5d4fbcc43a5fa Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 9 Jun 2023 17:46:52 +0000 Subject: [PATCH 35/36] Remove unnecessary empty output list --- turborepo-tests/integration/tests/_fixtures/ordered/turbo.json | 1 - 1 file changed, 1 deletion(-) diff --git a/turborepo-tests/integration/tests/_fixtures/ordered/turbo.json b/turborepo-tests/integration/tests/_fixtures/ordered/turbo.json index 0d547fb58901a..1047f99e18906 100644 --- a/turborepo-tests/integration/tests/_fixtures/ordered/turbo.json +++ b/turborepo-tests/integration/tests/_fixtures/ordered/turbo.json @@ -2,7 +2,6 @@ "$schema": "https://turbo.build/schema.json", "pipeline": { "build": { - "outputs": [] } } } From 46cb415d13ebd2e247d4c5212a7fed1110baf984 Mon Sep 17 00:00:00 2001 From: Rafael Date: Fri, 9 Jun 2023 17:47:45 +0000 Subject: [PATCH 36/36] whitespace --- cli/internal/ui/ui_factory.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cli/internal/ui/ui_factory.go b/cli/internal/ui/ui_factory.go index cc6f7f457a3cf..e136fbca915bf 100644 --- a/cli/internal/ui/ui_factory.go +++ b/cli/internal/ui/ui_factory.go @@ -68,8 +68,7 @@ func (factory *ConcurrentUIFactory) Build(in io.Reader, out io.Writer, err io.Wr // PrefixedUIFactory provides a method for creating a cli.PrefixedUi from input, output and error IOs type PrefixedUIFactory struct { - Base Factory - + Base Factory AskPrefix string AskSecretPrefix string OutputPrefix string