From a44672dfefe835bc65f7e8d051df37deba2fa55a Mon Sep 17 00:00:00 2001 From: Ce Gao Date: Tue, 10 May 2022 16:31:49 +0800 Subject: [PATCH] chore: show the compile stage progress similar to build stage (#79) * feat: Print progress when compile Signed-off-by: Ce Gao * chore: Update license Signed-off-by: Ce Gao * fix: Update license Signed-off-by: Ce Gao --- .gitignore | 1 + Makefile | 3 +- build.MIDI | 1 + go.mod | 21 +- go.sum | 5 - pkg/builder/builder.go | 52 +- pkg/builder/builder_test.go | 32 +- pkg/lang/ir/consts.go | 14 + pkg/lang/ir/graph.go | 8 + pkg/lang/ir/interface.go | 14 + pkg/lang/ir/types.go | 3 + pkg/progress/color.go | 48 - pkg/progress/compileui/display.go | 120 +++ pkg/progress/compileui/mock/mock.go | 58 ++ pkg/progress/{util.go => compileui/trace.go} | 30 +- pkg/progress/logging.go | 148 --- pkg/progress/monitor.go | 130 --- pkg/progress/progressui/display.go | 932 +++++++++++++++++++ pkg/progress/progressui/display_test.go | 183 ++++ pkg/progress/progressui/printer.go | 348 +++++++ pkg/progress/progressui/term.go | 27 + pkg/progress/progressui/term_windows.go | 27 + pkg/progress/progresswriter/printer.go | 110 +++ pkg/progress/progresswriter/writer.go | 61 ++ 24 files changed, 1990 insertions(+), 386 deletions(-) delete mode 100644 pkg/progress/color.go create mode 100644 pkg/progress/compileui/display.go create mode 100644 pkg/progress/compileui/mock/mock.go rename pkg/progress/{util.go => compileui/trace.go} (61%) delete mode 100644 pkg/progress/logging.go delete mode 100644 pkg/progress/monitor.go create mode 100644 pkg/progress/progressui/display.go create mode 100644 pkg/progress/progressui/display_test.go create mode 100644 pkg/progress/progressui/printer.go create mode 100644 pkg/progress/progressui/term.go create mode 100644 pkg/progress/progressui/term_windows.go create mode 100644 pkg/progress/progresswriter/printer.go create mode 100644 pkg/progress/progresswriter/writer.go diff --git a/.gitignore b/.gitignore index 1176de809..b7e9faa36 100644 --- a/.gitignore +++ b/.gitignore @@ -33,5 +33,6 @@ bin/ debug-bin/ /build.MIDI .ipynb_checkpoints/ +cover.html dist/ diff --git a/Makefile b/Makefile index 1d0b21490..a2d1c38c6 100644 --- a/Makefile +++ b/Makefile @@ -117,6 +117,7 @@ build-local: generate: mockgen-install @mockgen -source pkg/buildkitd/buildkitd.go -destination pkg/buildkitd/mock/mock.go -package mock @mockgen -source pkg/lang/frontend/starlark/interpreter.go -destination pkg/lang/frontend/starlark/mock/mock.go -package mock + @mockgen -source pkg/progress/compileui/display.go -destination pkg/progress/compileui/mock/mock.go -package mock # It is used by vscode to attach into the process. debug-local: @@ -128,7 +129,7 @@ debug-local: done addlicense: - addlicense -c "The MIDI Authors" **/*.go **/**/*.go + addlicense -c "The MIDI Authors" **/*.go **/**/*.go **/**/**/*.go test: generate @go test -race -coverprofile=coverage.out ./... diff --git a/build.MIDI b/build.MIDI index 2658634ce..6884875e5 100644 --- a/build.MIDI +++ b/build.MIDI @@ -5,3 +5,4 @@ install_package(name = [ cuda(version="11.6", cudnn="8") shell("zsh") jupyter(password="", port=8888) +vscode(plugins = ["dbaeumer.vscode-eslint-2.2.3"]) diff --git a/go.mod b/go.mod index 11b231a1a..58d9d0899 100644 --- a/go.mod +++ b/go.mod @@ -5,26 +5,33 @@ go 1.17 require ( github.com/alessio/shellescape v1.4.1 github.com/cockroachdb/errors v1.9.0 + github.com/containerd/console v1.0.3 github.com/containerd/containerd v1.6.3-0.20220401172941-5ff8fce1fcc6 github.com/creack/pty v1.1.18 github.com/docker/docker v20.10.7+incompatible - github.com/fatih/color v1.13.0 + github.com/docker/go-connections v0.4.0 github.com/gliderlabs/ssh v0.3.3 github.com/go-git/go-git/v5 v5.4.2 github.com/golang/mock v1.6.0 github.com/google/uuid v1.3.0 github.com/moby/buildkit v0.10.1 + github.com/morikuni/aec v1.0.0 github.com/onsi/ginkgo/v2 v2.1.4 github.com/onsi/gomega v1.19.0 github.com/opencontainers/go-digest v1.0.0 + github.com/pkg/errors v0.9.1 github.com/pkg/sftp v1.13.4 github.com/sirupsen/logrus v1.8.1 github.com/spf13/viper v1.4.0 + github.com/stretchr/testify v1.7.0 + github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea + github.com/tonistiigi/vt100 v0.0.0-20210615222946-8066bb97264f github.com/urfave/cli/v2 v2.4.0 go.starlark.net v0.0.0-20220328144851-d1966c6b9fcd golang.org/x/crypto v0.0.0-20211202192323-5770296d904e golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 + golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac ) require ( @@ -38,13 +45,12 @@ require ( github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect github.com/cockroachdb/redact v1.1.3 // indirect - github.com/containerd/console v1.0.3 // indirect github.com/containerd/continuity v0.2.3-0.20220330195504-d132b287edc8 // indirect github.com/containerd/typeurl v1.0.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/cli v20.10.13+incompatible // indirect github.com/docker/distribution v2.8.0+incompatible // indirect - github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/emirpasic/gods v1.12.0 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect @@ -69,15 +75,12 @@ require ( github.com/kr/pretty v0.3.0 // indirect github.com/kr/text v0.2.0 // indirect github.com/magiconair/properties v1.8.1 // indirect - github.com/mattn/go-colorable v0.1.11 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.1.2 // indirect github.com/moby/sys/signal v0.6.0 // indirect - github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect github.com/pelletier/go-toml v1.9.4 // indirect - github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.8.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sergi/go-diff v1.1.0 // indirect @@ -86,8 +89,6 @@ require ( github.com/spf13/jwalterweatherman v1.0.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tonistiigi/fsutil v0.0.0-20220115021204-b19f7f9cb274 // indirect - github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea // indirect - github.com/tonistiigi/vt100 v0.0.0-20210615222946-8066bb97264f // indirect github.com/xanzy/ssh-agent v0.3.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.29.0 // indirect go.opentelemetry.io/otel v1.4.1 // indirect @@ -98,13 +99,13 @@ require ( golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 // indirect golang.org/x/text v0.3.7 // indirect - golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect google.golang.org/grpc v1.45.0 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) // Copied from buildkit to make github.com/tonistiigi/fsutil happy. diff --git a/go.sum b/go.sum index 56b4ed3a2..9a72f9c5b 100644 --- a/go.sum +++ b/go.sum @@ -168,8 +168,6 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go. github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= @@ -389,14 +387,11 @@ github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A= github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.11 h1:nQ+aFkoE2TMGc0b68U2OKSexC+eq46+XwZzWXHRmPYs= github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= diff --git a/pkg/builder/builder.go b/pkg/builder/builder.go index 5f1f4903d..213e78ec0 100644 --- a/pkg/builder/builder.go +++ b/pkg/builder/builder.go @@ -21,7 +21,7 @@ import ( "github.com/cockroachdb/errors" "github.com/moby/buildkit/client" - "github.com/moby/buildkit/util/progress/progresswriter" + "github.com/moby/buildkit/client/llb" "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" @@ -31,6 +31,7 @@ import ( "github.com/tensorchord/MIDI/pkg/home" "github.com/tensorchord/MIDI/pkg/lang/frontend/starlark" "github.com/tensorchord/MIDI/pkg/lang/ir" + "github.com/tensorchord/MIDI/pkg/progress/progresswriter" ) type Builder interface { @@ -47,7 +48,6 @@ type generalBuilder struct { logger *logrus.Entry starlark.Interpreter buildkitd.Client - progresswriter.Writer } func New(ctx context.Context, configFilePath, manifestFilePath, tag string) (Builder, error) { @@ -68,12 +68,6 @@ func New(ctx context.Context, configFilePath, manifestFilePath, tag string) (Bui } b.Client = cli - pw, err := progresswriter.NewPrinter(ctx, os.Stdout, b.progressMode) - if err != nil { - return nil, errors.Wrap(err, "failed to create progress writer") - } - b.Writer = pw - b.Interpreter = starlark.NewInterpreter() return b, nil } @@ -84,20 +78,47 @@ func (b generalBuilder) GPUEnabled() bool { } func (b generalBuilder) Build(ctx context.Context) error { + def, err := b.compile(ctx) + if err != nil { + return errors.Wrap(err, "failed to compile") + } + + pw, err := progresswriter.NewPrinter(ctx, os.Stdout, b.progressMode) + if err != nil { + return errors.Wrap(err, "failed to create progress writer") + } + + if err = b.build(ctx, def, pw); err != nil { + return errors.Wrap(err, "failed to build") + } + return nil +} + +func (b generalBuilder) interpret() error { // Evaluate config first. if _, err := b.ExecFile(b.configFilePath); err != nil { - return err + return errors.Wrap(err, "failed to exec starlark file") } if _, err := b.ExecFile(b.manifestFilePath); err != nil { - return err + return errors.Wrap(err, "failed to exec starlark file") } + return nil +} +func (b generalBuilder) compile(ctx context.Context) (*llb.Definition, error) { + if err := b.interpret(); err != nil { + return nil, errors.Wrap(err, "failed to interpret") + } def, err := ir.Compile(ctx) if err != nil { - return errors.Wrap(err, "failed to compile build.MIDI") + return nil, errors.Wrap(err, "failed to compile build.MIDI") } + b.logger.Debug("compiled build.MIDI") + return def, nil +} +func (b generalBuilder) build(ctx context.Context, def *llb.Definition, pw progresswriter.Writer) error { ctx, cancel := context.WithCancel(ctx) defer cancel() eg, ctx := errgroup.WithContext(ctx) @@ -130,7 +151,7 @@ func (b generalBuilder) Build(ctx context.Context) error { FrontendAttrs: map[string]string{ "build-arg:HTTPS_PROXY": os.Getenv("HTTPS_PROXY"), }, - }, b.Status()) + }, pw.Status()) if err != nil { err = errors.Wrap(err, "failed to solve LLB") b.logger.Error(err) @@ -143,8 +164,8 @@ func (b generalBuilder) Build(ctx context.Context) error { // Watch the progress. eg.Go(func() error { // not using shared context to not disrupt display but let is finish reporting errors - <-b.Done() - return b.Err() + <-pw.Done() + return pw.Err() }) // Load the image to docker host. @@ -164,7 +185,7 @@ func (b generalBuilder) Build(ctx context.Context) error { return nil }) - err = eg.Wait() + err := eg.Wait() if err != nil { if errors.Is(err, context.Canceled) { b.logger.Debug("cancelling the error group") @@ -175,6 +196,5 @@ func (b generalBuilder) Build(ctx context.Context) error { return errors.Wrap(err, "failed to wait error group") } } - return nil } diff --git a/pkg/builder/builder_test.go b/pkg/builder/builder_test.go index e281ae353..0cd9d8564 100644 --- a/pkg/builder/builder_test.go +++ b/pkg/builder/builder_test.go @@ -20,7 +20,7 @@ import ( "os" "github.com/golang/mock/gomock" - "github.com/moby/buildkit/util/progress/progresswriter" + "github.com/moby/buildkit/client/llb" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/sirupsen/logrus" @@ -30,6 +30,10 @@ import ( "github.com/tensorchord/MIDI/pkg/flag" "github.com/tensorchord/MIDI/pkg/home" mockstarlark "github.com/tensorchord/MIDI/pkg/lang/frontend/starlark/mock" + "github.com/tensorchord/MIDI/pkg/lang/ir" + "github.com/tensorchord/MIDI/pkg/progress/compileui" + compileuimock "github.com/tensorchord/MIDI/pkg/progress/compileui/mock" + "github.com/tensorchord/MIDI/pkg/progress/progresswriter" ) var _ = Describe("Builder", func() { @@ -58,6 +62,7 @@ var _ = Describe("Builder", func() { }) When("building the manifest", func() { var b *generalBuilder + var w compileui.Writer BeforeEach(func() { ctrl := gomock.NewController(GinkgoT()) ctrlStarlark := gomock.NewController(GinkgoT()) @@ -72,11 +77,10 @@ var _ = Describe("Builder", func() { } b.Client = mockbuildkitd.NewMockClient(ctrl) b.Interpreter = mockstarlark.NewMockInterpreter(ctrlStarlark) - pw, err := progresswriter.NewPrinter(context.TODO(), os.Stdout, b.progressMode) - if err != nil { - Fail(err.Error()) - } - b.Writer = pw + + ctrlWriter := gomock.NewController(GinkgoT()) + w = compileuimock.NewMockWriter(ctrlWriter) + ir.DefaultGraph.Writer = w }) When("failed to interpret config", func() { @@ -104,15 +108,8 @@ var _ = Describe("Builder", func() { }) }) It("should build successfully", func() { - b.Interpreter.(*mockstarlark.MockInterpreter).EXPECT().ExecFile( - gomock.Eq(configFilePath), - ).Return(nil, nil).Times(1) - b.Interpreter.(*mockstarlark.MockInterpreter).EXPECT().ExecFile( - gomock.Eq(b.manifestFilePath), - ).Return(nil, nil).Times(1) err := home.Initialize() Expect(err).ToNot(HaveOccurred()) - close(b.Writer.Status()) b.Client.(*mockbuildkitd.MockClient).EXPECT().Solve( gomock.Any(), @@ -120,7 +117,14 @@ var _ = Describe("Builder", func() { gomock.Any(), gomock.Any(), ).Return(nil, nil).AnyTimes() - err = b.Build(context.TODO()) + + var def *llb.Definition + pw, err := progresswriter.NewPrinter(context.TODO(), os.Stdout, b.progressMode) + if err != nil { + Fail(err.Error()) + } + close(pw.Status()) + err = b.build(context.TODO(), def, pw) Expect(err).ToNot(HaveOccurred()) }) }) diff --git a/pkg/lang/ir/consts.go b/pkg/lang/ir/consts.go index d0f8fd05e..c57bf438b 100644 --- a/pkg/lang/ir/consts.go +++ b/pkg/lang/ir/consts.go @@ -1,3 +1,17 @@ +// Copyright 2022 The MIDI Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package ir const ( diff --git a/pkg/lang/ir/graph.go b/pkg/lang/ir/graph.go index 31539a335..ca2fe398b 100644 --- a/pkg/lang/ir/graph.go +++ b/pkg/lang/ir/graph.go @@ -17,6 +17,7 @@ package ir import ( "context" "fmt" + "os" "path/filepath" "strings" @@ -27,6 +28,7 @@ import ( "github.com/tensorchord/MIDI/pkg/editor/vscode" "github.com/tensorchord/MIDI/pkg/flag" + "github.com/tensorchord/MIDI/pkg/progress/compileui" "github.com/tensorchord/MIDI/pkg/shell" ) @@ -56,6 +58,11 @@ func GPUEnabled() bool { } func Compile(ctx context.Context) (*llb.Definition, error) { + w, err := compileui.New(ctx, os.Stdout, "auto") + if err != nil { + return nil, errors.Wrap(err, "failed to create compileui") + } + DefaultGraph.Writer = w state, err := DefaultGraph.Compile() if err != nil { return nil, err @@ -105,6 +112,7 @@ func (g Graph) Compile() (llb.State, error) { // TODO(gaocegege): Support order-based exec. run := g.compileRun(merged) + g.Writer.Finish() return run, nil } diff --git a/pkg/lang/ir/interface.go b/pkg/lang/ir/interface.go index 9ba86ed6d..a783410be 100644 --- a/pkg/lang/ir/interface.go +++ b/pkg/lang/ir/interface.go @@ -1,3 +1,17 @@ +// Copyright 2022 The MIDI Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package ir import ( diff --git a/pkg/lang/ir/types.go b/pkg/lang/ir/types.go index bc8a6f5ea..b699077e5 100644 --- a/pkg/lang/ir/types.go +++ b/pkg/lang/ir/types.go @@ -16,6 +16,7 @@ package ir import ( "github.com/tensorchord/MIDI/pkg/editor/vscode" + "github.com/tensorchord/MIDI/pkg/progress/compileui" ) // A Graph contains the state, @@ -37,6 +38,8 @@ type Graph struct { Exec []string *JupyterConfig + + Writer compileui.Writer } type JupyterConfig struct { diff --git a/pkg/progress/color.go b/pkg/progress/color.go deleted file mode 100644 index f70fbb77d..000000000 --- a/pkg/progress/color.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2022 The MIDI Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package progress - -import "github.com/fatih/color" - -var noColor = makeNoColor() -var cachedColor = makeColor(color.FgHiGreen) -var successColor = makeColor(color.FgHiGreen) - -var availablePrefixColors = []*color.Color{ - makeColor(color.FgBlue), - makeColor(color.FgMagenta), - makeColor(color.FgCyan), - makeColor(color.FgRed), - makeColor(color.FgYellow), - makeColor(color.FgGreen), - makeColor(color.FgHiBlue), - makeColor(color.FgHiMagenta), - makeColor(color.FgHiCyan), - makeColor(color.FgHiRed), - makeColor(color.FgHiYellow), - makeColor(color.FgHiWhite), -} - -func makeColor(att color.Attribute) *color.Color { - c := color.New() - c.Add(att) - return c -} - -func makeNoColor() *color.Color { - c := color.New() - c.DisableColor() - return c -} diff --git a/pkg/progress/compileui/display.go b/pkg/progress/compileui/display.go new file mode 100644 index 000000000..7e43ebc26 --- /dev/null +++ b/pkg/progress/compileui/display.go @@ -0,0 +1,120 @@ +// Copyright 2022 The MIDI Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compileui + +import ( + "context" + "fmt" + "time" + + "github.com/cockroachdb/errors" + "github.com/containerd/console" + "github.com/morikuni/aec" + "github.com/sirupsen/logrus" +) + +type Writer interface { + Print(s string) + Finish() +} + +type generalWriter struct { + console console.Console + phase string + trace *trace + doneCh chan bool + repeatd bool +} + +func New(ctx context.Context, out console.File, mode string) (Writer, error) { + var c console.Console + switch mode { + case "auto": + if cons, err := console.ConsoleFromFile(out); err == nil { + c = cons + } else { + return nil, errors.Wrap(err, "failed to get console") + } + case "plain": + default: + return nil, errors.Errorf("invalid progress mode %s", mode) + } + + t := newTrace(out, true) + t.init() + + w := &generalWriter{ + console: c, + phase: "parse build.MIDI and download/cache dependencies", + trace: t, + doneCh: make(chan bool), + repeatd: false, + } + // TODO(gaocegege): Have a result chan + //nolint + go w.run(ctx) + return w, nil +} + +func (w generalWriter) Print(s string) { + fmt.Fprintln(w.console, s) +} + +func (w generalWriter) Finish() { + w.doneCh <- true +} + +func (w *generalWriter) run(ctx context.Context) error { + displayTimeout := 100 * time.Millisecond + ticker := time.NewTicker(displayTimeout) + width, height := w.getSize() + logger := logrus.WithFields(logrus.Fields{ + "console-height": height, + "console-width": width, + }) + logger.Debug("print compile progress") + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-w.doneCh: + return nil + case <-ticker.C: + b := aec.EmptyBuilder.Up(1) + if !w.repeatd { + b = b.Down(1) + } + w.repeatd = true + fmt.Fprint(w.console, b.Column(0).ANSI) + fmt.Fprint(w.console, aec.Hide) + defer fmt.Fprint(w.console, aec.Show) + s := fmt.Sprintf("[+] ⌚ %s %.1fs\n", w.phase, time.Since(*w.trace.startTime).Seconds()) + fmt.Fprint(w.console, s) + } + } +} + +func (w generalWriter) getSize() (int, int) { + width := 80 + height := 10 + if w.console != nil { + size, err := w.console.Size() + if err == nil && size.Width > 0 && size.Height > 0 { + width = int(size.Width) + height = int(size.Height) + } + } + return width, height +} diff --git a/pkg/progress/compileui/mock/mock.go b/pkg/progress/compileui/mock/mock.go new file mode 100644 index 000000000..324791779 --- /dev/null +++ b/pkg/progress/compileui/mock/mock.go @@ -0,0 +1,58 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: pkg/progress/compileui/display.go + +// Package mock is a generated GoMock package. +package mock + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockWriter is a mock of Writer interface. +type MockWriter struct { + ctrl *gomock.Controller + recorder *MockWriterMockRecorder +} + +// MockWriterMockRecorder is the mock recorder for MockWriter. +type MockWriterMockRecorder struct { + mock *MockWriter +} + +// NewMockWriter creates a new mock instance. +func NewMockWriter(ctrl *gomock.Controller) *MockWriter { + mock := &MockWriter{ctrl: ctrl} + mock.recorder = &MockWriterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockWriter) EXPECT() *MockWriterMockRecorder { + return m.recorder +} + +// Finish mocks base method. +func (m *MockWriter) Finish() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Finish") +} + +// Finish indicates an expected call of Finish. +func (mr *MockWriterMockRecorder) Finish() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Finish", reflect.TypeOf((*MockWriter)(nil).Finish)) +} + +// Print mocks base method. +func (m *MockWriter) Print(s string) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Print", s) +} + +// Print indicates an expected call of Print. +func (mr *MockWriterMockRecorder) Print(s interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Print", reflect.TypeOf((*MockWriter)(nil).Print), s) +} diff --git a/pkg/progress/util.go b/pkg/progress/compileui/trace.go similarity index 61% rename from pkg/progress/util.go rename to pkg/progress/compileui/trace.go index 0e2a8241d..e828b588d 100644 --- a/pkg/progress/util.go +++ b/pkg/progress/compileui/trace.go @@ -12,25 +12,27 @@ // See the License for the specific language governing permissions and // limitations under the License. -package progress +package compileui import ( - "strings" - - "github.com/moby/buildkit/client" - "github.com/opencontainers/go-digest" + "io" + "time" ) -func printVertex(vertex *client.Vertex, console consoleLogger) { - out := []string{"-->"} - out = append(out, vertex.Name) - c := console - if vertex.Cached { - c = c.WithCached(true) +type trace struct { + w io.Writer + startTime *time.Time + modeConsole bool +} + +func newTrace(w io.Writer, modeConsole bool) *trace { + return &trace{ + w: w, + modeConsole: modeConsole, } - c.Printf("%s\n", strings.Join(out, " ")) } -func shortDigest(d digest.Digest) string { - return d.Hex()[:12] +func (t *trace) init() { + current := time.Now() + t.startTime = ¤t } diff --git a/pkg/progress/logging.go b/pkg/progress/logging.go deleted file mode 100644 index 1709e6888..000000000 --- a/pkg/progress/logging.go +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright 2022 The MIDI Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package progress - -import ( - "bytes" - "fmt" - "io" - "os" - "strings" - "sync" - - "github.com/fatih/color" -) - -var currentConsoleMutex sync.Mutex - -// consoleLogger is a writer for consoles. -type consoleLogger struct { - prefix string - disableColors bool - isCached bool - - // The following are shared between instances and are protected by the mutex. - mu *sync.Mutex - prefixColors map[string]*color.Color - nextColorIndex *int - w io.Writer -} - -// Current returns the current console. -func Current(disableColors bool) consoleLogger { - return consoleLogger{ - w: os.Stdout, - disableColors: disableColors || color.NoColor, - prefixColors: make(map[string]*color.Color), - nextColorIndex: new(int), - mu: ¤tConsoleMutex, - } -} - -func (cl consoleLogger) clone() consoleLogger { - return consoleLogger{ - w: cl.w, - prefix: cl.prefix, - isCached: cl.isCached, - prefixColors: cl.prefixColors, - disableColors: cl.disableColors, - nextColorIndex: cl.nextColorIndex, - mu: cl.mu, - } -} - -// WithPrefix returns a ConsoleLogger with a prefix added. -func (cl consoleLogger) WithPrefix(prefix string) consoleLogger { - ret := cl.clone() - ret.prefix = prefix - return ret -} - -// Prefix returns the console's prefix. -func (cl consoleLogger) Prefix() string { - return cl.prefix -} - -// WithCached returns a ConsoleLogger with isCached flag set accordingly. -func (cl consoleLogger) WithCached(isCached bool) consoleLogger { - ret := cl.clone() - ret.isCached = isCached - return ret -} - -// PrintSuccess prints the success message. -func (cl consoleLogger) PrintSuccess() { - cl.mu.Lock() - defer cl.mu.Unlock() - successColor.Fprintf(cl.w, "=========================== SUCCESS ===========================\n") -} - -// Printf prints formatted text to the console. -// nolint:errcheck -func (cl consoleLogger) Printf(format string, args ...interface{}) { - cl.mu.Lock() - defer cl.mu.Unlock() - text := fmt.Sprintf(format, args...) - text = strings.TrimSuffix(text, "\n") - for _, line := range strings.Split(text, "\n") { - cl.printPrefix() - cl.w.Write([]byte(line)) - cl.w.Write([]byte("\n")) - } -} - -// PrintBytes prints bytes directly to the console. -// nolint:errcheck -func (cl consoleLogger) PrintBytes(data []byte) { - // TODO: This does not deal well with control characters, because of the prefix. - cl.mu.Lock() - defer cl.mu.Unlock() - if !bytes.Contains(data, []byte("\n")) { - // No prefix when it's not a complete line. - cl.w.Write(data) - } else { - adjustedData := bytes.TrimSuffix(data, []byte("\n")) - for _, line := range bytes.Split(adjustedData, []byte("\n")) { - cl.printPrefix() - cl.w.Write(line) - cl.w.Write([]byte("\n")) - } - } -} - -// nolint:errcheck -func (cl consoleLogger) printPrefix() { - // Assumes mu locked. - if cl.prefix == "" { - return - } - c := noColor - if !cl.disableColors { - var found bool - c, found = cl.prefixColors[cl.prefix] - if !found { - c = availablePrefixColors[*cl.nextColorIndex] - cl.prefixColors[cl.prefix] = c - *cl.nextColorIndex = (*cl.nextColorIndex + 1) % len(availablePrefixColors) - } - } - c.Fprintf(cl.w, "%s", cl.prefix) - cl.w.Write([]byte(" | ")) - if cl.isCached { - cl.w.Write([]byte("*")) - cachedColor.Fprintf(cl.w, "cached") - cl.w.Write([]byte("* ")) - } -} diff --git a/pkg/progress/monitor.go b/pkg/progress/monitor.go deleted file mode 100644 index 8e9deae58..000000000 --- a/pkg/progress/monitor.go +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2022 The MIDI Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package progress - -import ( - "context" - "errors" - - "github.com/moby/buildkit/client" - "github.com/opencontainers/go-digest" - "github.com/sirupsen/logrus" -) - -// Monitor monitors the progress and print the log to the console. -type Monitor interface { - Monitor(ctx context.Context, ch chan *client.SolveStatus) error - Success() -} - -var defaultMonitor Monitor - -type generalMonitor struct { - console consoleLogger -} - -func NewMonitor() Monitor { - if defaultMonitor == nil { - defaultMonitor = generalMonitor{ - console: Current(false), - } - } - return defaultMonitor -} - -func (g generalMonitor) Monitor(ctx context.Context, ch chan *client.SolveStatus) error { - vertexLoggers := make(map[digest.Digest]*logrus.Entry) - vertexConsoles := make(map[digest.Digest]consoleLogger) - vertices := make(map[digest.Digest]*client.Vertex) - introducedVertex := make(map[digest.Digest]bool) - - for { - select { - case ss, ok := <-ch: - if !ok { - return nil - } - for _, vertex := range ss.Vertexes { - logger := logrus.WithContext(ctx). - WithField("name", vertex.Name). - WithField("vertex", shortDigest(vertex.Digest)). - WithField("cached", vertex.Cached). - WithField("error", vertex.Error) - - vertexLoggers[vertex.Digest] = logger - targetConsole := g.console.WithPrefix(vertex.Name) - vertexConsoles[vertex.Digest] = targetConsole - vertices[vertex.Digest] = vertex - if !introducedVertex[vertex.Digest] && (vertex.Cached || vertex.Started != nil) { - introducedVertex[vertex.Digest] = true - printVertex(vertex, targetConsole) - logger.Debug("Vertex started or cached") - } - if vertex.Error != "" { - if !introducedVertex[vertex.Digest] { - introducedVertex[vertex.Digest] = true - printVertex(vertex, targetConsole) - } - targetConsole.Printf("ERROR: (%s) %s\n", vertex.Name, vertex.Error) - logger.Error(errors.New(vertex.Error)) - } - for _, vs := range ss.Statuses { - vertex, found := vertices[vs.Vertex] - if !found { - // No logging for internal operations. - continue - } - logger := vertexLoggers[vs.Vertex] - targetConsole := vertexConsoles[vs.Vertex] - progress := int32(0) - if vs.Total != 0 { - progress = int32(100.0 * float32(vs.Current) / float32(vs.Total)) - } - if vs.Completed != nil { - progress = 100 - } - logger = logger.WithField("progress", progress).WithField("name", vs.Name) - if !introducedVertex[vertex.Digest] { - introducedVertex[vertex.Digest] = true - printVertex(vertex, targetConsole) - } - logger.Debug(vs.ID) - targetConsole.Printf("%s %d%%\n", vs.ID, progress) - } - for _, logLine := range ss.Logs { - vertex, found := vertices[logLine.Vertex] - if !found { - // No logging for internal operations. - continue - } - logger := vertexLoggers[logLine.Vertex] - targetConsole := vertexConsoles[logLine.Vertex] - if !introducedVertex[logLine.Vertex] { - introducedVertex[logLine.Vertex] = true - printVertex(vertex, targetConsole) - } - targetConsole.PrintBytes(logLine.Data) - logger.Debug(string(logLine.Data)) - } - } - case <-ctx.Done(): - return nil - } - } -} - -func (g generalMonitor) Success() { - g.console.PrintSuccess() -} diff --git a/pkg/progress/progressui/display.go b/pkg/progress/progressui/display.go new file mode 100644 index 000000000..3c0af6d45 --- /dev/null +++ b/pkg/progress/progressui/display.go @@ -0,0 +1,932 @@ +// Copyright 2022 The MIDI Authors +// Copyright 2022 The buildkit Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package progressui + +import ( + "bytes" + "container/ring" + "context" + "fmt" + "io" + "os" + "sort" + "strconv" + "strings" + "time" + + "github.com/containerd/console" + "github.com/moby/buildkit/client" + "github.com/morikuni/aec" + digest "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" + "github.com/tonistiigi/units" + "github.com/tonistiigi/vt100" + "golang.org/x/time/rate" +) + +func DisplaySolveStatus(ctx context.Context, phase string, c console.Console, w io.Writer, ch chan *client.SolveStatus) ([]client.VertexWarning, error) { + modeConsole := c != nil + + disp := &display{c: c, phase: phase} + printer := &textMux{w: w} + + if disp.phase == "" { + disp.phase = "Building" + } + + t := newTrace(w, modeConsole) + + tickerTimeout := 150 * time.Millisecond + displayTimeout := 100 * time.Millisecond + + if v := os.Getenv("TTY_DISPLAY_RATE"); v != "" { + if r, err := strconv.ParseInt(v, 10, 64); err == nil { + tickerTimeout = time.Duration(r) * time.Millisecond + displayTimeout = time.Duration(r) * time.Millisecond + } + } + + var done bool + ticker := time.NewTicker(tickerTimeout) + // implemented as closure because "ticker" can change + defer func() { + ticker.Stop() + }() + + displayLimiter := rate.NewLimiter(rate.Every(displayTimeout), 1) + + var height int + width, _ := disp.getSize() + for { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-ticker.C: + case ss, ok := <-ch: + if ok { + t.update(ss, width) + } else { + done = true + } + } + + if modeConsole { + width, height = disp.getSize() + if done { + disp.print(t.displayInfo(), width, height, true) + if err := t.printErrorLogs(c); err != nil { + return nil, err + } + return t.warnings(), nil + } else if displayLimiter.Allow() { + logger := logrus.WithFields(logrus.Fields{ + "console-height": height, + "console-width": width, + }) + logger.Debug("stop ticker and print build progress") + ticker.Stop() + ticker = time.NewTicker(tickerTimeout) + disp.print(t.displayInfo(), width, height, false) + } + } else { + if done || displayLimiter.Allow() { + printer.print(t) + if done { + if err := t.printErrorLogs(c); err != nil { + return nil, err + } + return t.warnings(), nil + } + ticker.Stop() + ticker = time.NewTicker(tickerTimeout) + } + } + } +} + +const termHeight = 6 +const termPad = 10 + +type displayInfo struct { + startTime time.Time + jobs []*job + countTotal int + countCompleted int +} + +type job struct { + intervals []interval + isCompleted bool + name string + status string + hasError bool + isCanceled bool + vertex *vertex + showTerm bool +} + +type trace struct { + w io.Writer + startTime *time.Time + localTimeDiff time.Duration + vertexes []*vertex + byDigest map[digest.Digest]*vertex + updates map[digest.Digest]struct{} + modeConsole bool + groups map[string]*vertexGroup // group id -> group +} + +type vertex struct { + *client.Vertex + + statuses []*status + byID map[string]*status + indent string + index int + + logs [][]byte + logsPartial bool + logsOffset int + logsBuffer *ring.Ring // stores last logs to print them on error + prev *client.Vertex + events []string + lastBlockTime *time.Time + count int + statusUpdates map[string]struct{} + + warnings []client.VertexWarning + warningIdx int + + jobs []*job + jobCached bool + + term *vt100.VT100 + termBytes int + termCount int + + // Interval start time in unix nano -> interval. Using a map ensures + // that updates for the same interval overwrite their previous updates. + intervals map[int64]interval + mergedIntervals []interval + + // whether the vertex should be hidden due to being in a progress group + // that doesn't have any non-weak members that have started + hidden bool +} + +func (v *vertex) update(c int) { + if v.count == 0 { + now := time.Now() + v.lastBlockTime = &now + } + v.count += c +} + +func (v *vertex) mostRecentInterval() *interval { + if v.isStarted() { + ival := v.mergedIntervals[len(v.mergedIntervals)-1] + return &ival + } + return nil +} + +func (v *vertex) isStarted() bool { + return len(v.mergedIntervals) > 0 +} + +func (v *vertex) isCompleted() bool { + if ival := v.mostRecentInterval(); ival != nil { + return ival.stop != nil + } + return false +} + +type vertexGroup struct { + *vertex + subVtxs map[digest.Digest]client.Vertex +} + +func (vg *vertexGroup) refresh() (changed, newlyStarted, newlyRevealed bool) { + newVtx := *vg.Vertex + newVtx.Cached = true + alreadyStarted := vg.isStarted() + wasHidden := vg.hidden + for _, subVtx := range vg.subVtxs { + if subVtx.Started != nil { + newInterval := interval{ + start: subVtx.Started, + stop: subVtx.Completed, + } + prevInterval := vg.intervals[subVtx.Started.UnixNano()] + if !newInterval.isEqual(prevInterval) { + changed = true + } + if !alreadyStarted { + newlyStarted = true + } + vg.intervals[subVtx.Started.UnixNano()] = newInterval + + if !subVtx.ProgressGroup.Weak { + vg.hidden = false + } + } + + // Group is considered cached iff all subvtxs are cached + newVtx.Cached = newVtx.Cached && subVtx.Cached + + // Group error is set to the first error found in subvtxs, if any + if newVtx.Error == "" { + newVtx.Error = subVtx.Error + } else { + vg.hidden = false + } + } + + if vg.Cached != newVtx.Cached { + changed = true + } + if vg.Error != newVtx.Error { + changed = true + } + vg.Vertex = &newVtx + + if !vg.hidden && wasHidden { + changed = true + newlyRevealed = true + } + + var ivals []interval + for _, ival := range vg.intervals { + ivals = append(ivals, ival) + } + vg.mergedIntervals = mergeIntervals(ivals) + + return changed, newlyStarted, newlyRevealed +} + +type interval struct { + start *time.Time + stop *time.Time +} + +func (ival interval) duration() time.Duration { + if ival.start == nil { + return 0 + } + if ival.stop == nil { + return time.Since(*ival.start) + } + return ival.stop.Sub(*ival.start) +} + +func (ival interval) isEqual(other interval) (isEqual bool) { + return equalTimes(ival.start, other.start) && equalTimes(ival.stop, other.stop) +} + +func equalTimes(t1, t2 *time.Time) bool { + if t2 == nil { + return t1 == nil + } + if t1 == nil { + return false + } + return t1.Equal(*t2) +} + +// mergeIntervals takes a slice of (start, stop) pairs and returns a slice where +// any intervals that overlap in time are combined into a single interval. If an +// interval's stop time is nil, it is treated as positive infinity and consumes +// any intervals after it. Intervals with nil start times are ignored and not +// returned. +func mergeIntervals(intervals []interval) []interval { + // remove any intervals that have not started + var filtered []interval + for _, interval := range intervals { + if interval.start != nil { + filtered = append(filtered, interval) + } + } + intervals = filtered + + if len(intervals) == 0 { + return nil + } + + // sort intervals by start time + sort.Slice(intervals, func(i, j int) bool { + return intervals[i].start.Before(*intervals[j].start) + }) + + var merged []interval + cur := intervals[0] + for i := 1; i < len(intervals); i++ { + next := intervals[i] + if cur.stop == nil { + // if cur doesn't stop, all intervals after it will be merged into it + merged = append(merged, cur) + return merged + } + if cur.stop.Before(*next.start) { + // if cur stops before next starts, no intervals after cur will be + // merged into it; cur stands on its own + merged = append(merged, cur) + cur = next + continue + } + if next.stop == nil { + // cur and next partially overlap, but next also never stops, so all + // subsequent intervals will be merged with both cur and next + merged = append(merged, interval{ + start: cur.start, + stop: nil, + }) + return merged + } + if cur.stop.After(*next.stop) || cur.stop.Equal(*next.stop) { + // cur fully subsumes next + continue + } + // cur partially overlaps with next, merge them together into cur + cur = interval{ + start: cur.start, + stop: next.stop, + } + } + // append anything we are left with + merged = append(merged, cur) + return merged +} + +type status struct { + *client.VertexStatus +} + +func newTrace(w io.Writer, modeConsole bool) *trace { + return &trace{ + byDigest: make(map[digest.Digest]*vertex), + updates: make(map[digest.Digest]struct{}), + w: w, + modeConsole: modeConsole, + groups: make(map[string]*vertexGroup), + } +} + +func (t *trace) warnings() []client.VertexWarning { + var out []client.VertexWarning + for _, v := range t.vertexes { + out = append(out, v.warnings...) + } + return out +} + +func (t *trace) triggerVertexEvent(v *client.Vertex) { + if v.Started == nil { + return + } + + var old client.Vertex + vtx := t.byDigest[v.Digest] + if v := vtx.prev; v != nil { + old = *v + } + + changed := false + if v.Digest != old.Digest { + changed = true + } + if v.Name != old.Name { + changed = true + } + if v.Started != old.Started { + if v.Started != nil && old.Started == nil || !v.Started.Equal(*old.Started) { + changed = true + } + } + if v.Completed != old.Completed && v.Completed != nil { + changed = true + } + if v.Cached != old.Cached { + changed = true + } + if v.Error != old.Error { + changed = true + } + + if changed { + vtx.update(1) + t.updates[v.Digest] = struct{}{} + } + + t.byDigest[v.Digest].prev = v +} + +func (t *trace) update(s *client.SolveStatus, termWidth int) { + seenGroups := make(map[string]struct{}) + var groups []string + for _, v := range s.Vertexes { + if t.startTime == nil { + t.startTime = v.Started + } + if v.ProgressGroup != nil { + group, ok := t.groups[v.ProgressGroup.Id] + if !ok { + group = &vertexGroup{ + vertex: &vertex{ + Vertex: &client.Vertex{ + Digest: digest.Digest(v.ProgressGroup.Id), + Name: v.ProgressGroup.Name, + }, + byID: make(map[string]*status), + statusUpdates: make(map[string]struct{}), + intervals: make(map[int64]interval), + hidden: true, + }, + subVtxs: make(map[digest.Digest]client.Vertex), + } + if t.modeConsole { + group.term = vt100.NewVT100(termHeight, termWidth-termPad) + } + t.groups[v.ProgressGroup.Id] = group + t.byDigest[group.Digest] = group.vertex + } + if _, ok := seenGroups[v.ProgressGroup.Id]; !ok { + groups = append(groups, v.ProgressGroup.Id) + seenGroups[v.ProgressGroup.Id] = struct{}{} + } + group.subVtxs[v.Digest] = *v + t.byDigest[v.Digest] = group.vertex + continue + } + prev, ok := t.byDigest[v.Digest] + if !ok { + t.byDigest[v.Digest] = &vertex{ + byID: make(map[string]*status), + statusUpdates: make(map[string]struct{}), + intervals: make(map[int64]interval), + } + if t.modeConsole { + t.byDigest[v.Digest].term = vt100.NewVT100(termHeight, termWidth-termPad) + } + } + t.triggerVertexEvent(v) + if v.Started != nil && (prev == nil || !prev.isStarted()) { + if t.localTimeDiff == 0 { + t.localTimeDiff = time.Since(*v.Started) + } + t.vertexes = append(t.vertexes, t.byDigest[v.Digest]) + } + // allow a duplicate initial vertex that shouldn't reset state + if !(prev != nil && prev.isStarted() && v.Started == nil) { + t.byDigest[v.Digest].Vertex = v + } + if v.Started != nil { + t.byDigest[v.Digest].intervals[v.Started.UnixNano()] = interval{ + start: v.Started, + stop: v.Completed, + } + var ivals []interval + for _, ival := range t.byDigest[v.Digest].intervals { + ivals = append(ivals, ival) + } + t.byDigest[v.Digest].mergedIntervals = mergeIntervals(ivals) + } + t.byDigest[v.Digest].jobCached = false + } + for _, groupID := range groups { + group := t.groups[groupID] + changed, newlyStarted, newlyRevealed := group.refresh() + if newlyStarted { + if t.localTimeDiff == 0 { + t.localTimeDiff = time.Since(*group.mergedIntervals[0].start) + } + } + if group.hidden { + continue + } + if newlyRevealed { + t.vertexes = append(t.vertexes, group.vertex) + } + if changed { + group.update(1) + t.updates[group.Digest] = struct{}{} + } + group.jobCached = false + } + for _, s := range s.Statuses { + v, ok := t.byDigest[s.Vertex] + if !ok { + continue // shouldn't happen + } + v.jobCached = false + prev, ok := v.byID[s.ID] + if !ok { + v.byID[s.ID] = &status{VertexStatus: s} + } + if s.Started != nil && (prev == nil || prev.Started == nil) { + v.statuses = append(v.statuses, v.byID[s.ID]) + } + v.byID[s.ID].VertexStatus = s + v.statusUpdates[s.ID] = struct{}{} + t.updates[v.Digest] = struct{}{} + v.update(1) + } + for _, w := range s.Warnings { + v, ok := t.byDigest[w.Vertex] + if !ok { + continue // shouldn't happen + } + v.warnings = append(v.warnings, *w) + v.update(1) + } + for _, l := range s.Logs { + v, ok := t.byDigest[l.Vertex] + if !ok { + continue // shouldn't happen + } + v.jobCached = false + if v.term != nil { + if v.term.Width != termWidth { + v.term.Resize(termHeight, termWidth-termPad) + } + v.termBytes += len(l.Data) + //nolint + v.term.Write(l.Data) // error unhandled on purpose. don't trust vt100 + } + i := 0 + complete := split(l.Data, byte('\n'), func(dt []byte) { + if v.logsPartial && len(v.logs) != 0 && i == 0 { + v.logs[len(v.logs)-1] = append(v.logs[len(v.logs)-1], dt...) + } else { + ts := time.Duration(0) + if ival := v.mostRecentInterval(); ival != nil { + ts = l.Timestamp.Sub(*ival.start) + } + prec := 1 + sec := ts.Seconds() + if sec < 10 { + prec = 3 + } else if sec < 100 { + prec = 2 + } + v.logs = append(v.logs, []byte(fmt.Sprintf("#%d %s %s", v.index, fmt.Sprintf("%.[2]*[1]f", sec, prec), dt))) + } + i++ + }) + v.logsPartial = !complete + t.updates[v.Digest] = struct{}{} + v.update(1) + } +} + +func (t *trace) printErrorLogs(f io.Writer) error { + for _, v := range t.vertexes { + if v.Error != "" && !strings.HasSuffix(v.Error, context.Canceled.Error()) { + fmt.Fprintln(f, "------") + fmt.Fprintf(f, " > %s:\n", v.Name) + // tty keeps original logs + for _, l := range v.logs { + _, err := f.Write(l) + if err != nil { + return err + } + fmt.Fprintln(f) + } + // printer keeps last logs buffer + if v.logsBuffer != nil { + for i := 0; i < v.logsBuffer.Len(); i++ { + if v.logsBuffer.Value != nil { + fmt.Fprintln(f, string(v.logsBuffer.Value.([]byte))) + } + v.logsBuffer = v.logsBuffer.Next() + } + } + fmt.Fprintln(f, "------") + } + } + return nil +} + +func (t *trace) displayInfo() (d displayInfo) { + d.startTime = time.Now() + if t.startTime != nil { + d.startTime = t.startTime.Add(t.localTimeDiff) + } + d.countTotal = len(t.byDigest) + for _, v := range t.byDigest { + if v.ProgressGroup != nil || v.hidden { + // don't count vtxs in a group, they are merged into a single vtx + d.countTotal-- + continue + } + if v.isCompleted() { + d.countCompleted++ + } + } + + for _, v := range t.vertexes { + if v.jobCached { + d.jobs = append(d.jobs, v.jobs...) + continue + } + var jobs []*job + j := &job{ + name: strings.Replace(v.Name, "\t", " ", -1), + vertex: v, + isCompleted: true, + } + for _, ival := range v.intervals { + j.intervals = append(j.intervals, interval{ + start: addTime(ival.start, t.localTimeDiff), + stop: addTime(ival.stop, t.localTimeDiff), + }) + if ival.stop == nil { + j.isCompleted = false + } + } + j.intervals = mergeIntervals(j.intervals) + if v.Error != "" { + if strings.HasSuffix(v.Error, context.Canceled.Error()) { + j.isCanceled = true + j.name = "CANCELED " + j.name + } else { + j.hasError = true + j.name = "ERROR " + j.name + } + } + if v.Cached { + j.name = "CACHED " + j.name + } + j.name = v.indent + j.name + jobs = append(jobs, j) + for _, s := range v.statuses { + j := &job{ + intervals: []interval{{ + start: addTime(s.Started, t.localTimeDiff), + stop: addTime(s.Completed, t.localTimeDiff), + }}, + isCompleted: s.Completed != nil, + name: v.indent + "=> " + s.ID, + } + if s.Total != 0 { + j.status = fmt.Sprintf("%.2f / %.2f", units.Bytes(s.Current), units.Bytes(s.Total)) + } else if s.Current != 0 { + j.status = fmt.Sprintf("%.2f", units.Bytes(s.Current)) + } + jobs = append(jobs, j) + } + for _, w := range v.warnings { + msg := "WARN: " + string(w.Short) + var mostRecentInterval interval + if ival := v.mostRecentInterval(); ival != nil { + mostRecentInterval = *ival + } + j := &job{ + intervals: []interval{{ + start: addTime(mostRecentInterval.start, t.localTimeDiff), + stop: addTime(mostRecentInterval.stop, t.localTimeDiff), + }}, + name: msg, + isCanceled: true, + } + jobs = append(jobs, j) + } + d.jobs = append(d.jobs, jobs...) + v.jobs = jobs + v.jobCached = true + } + + return d +} + +func split(dt []byte, sep byte, fn func([]byte)) bool { + if len(dt) == 0 { + return false + } + for { + if len(dt) == 0 { + return true + } + idx := bytes.IndexByte(dt, sep) + if idx == -1 { + fn(dt) + return false + } + fn(dt[:idx]) + dt = dt[idx+1:] + } +} + +func addTime(tm *time.Time, d time.Duration) *time.Time { + if tm == nil { + return nil + } + t := (*tm).Add(d) + return &t +} + +type display struct { + c console.Console + phase string + lineCount int + repeated bool +} + +func (disp *display) getSize() (int, int) { + width := 80 + height := 10 + if disp.c != nil { + size, err := disp.c.Size() + if err == nil && size.Width > 0 && size.Height > 0 { + width = int(size.Width) + height = int(size.Height) + } + } + return width, height +} + +func setupTerminals(jobs []*job, height int, all bool) []*job { + var candidates []*job + numInUse := 0 + for _, j := range jobs { + if j.vertex != nil && j.vertex.termBytes > 0 && !j.isCompleted { + candidates = append(candidates, j) + } + if !j.isCompleted { + numInUse++ + } + } + sort.Slice(candidates, func(i, j int) bool { + idxI := candidates[i].vertex.termBytes + candidates[i].vertex.termCount*50 + idxJ := candidates[j].vertex.termBytes + candidates[j].vertex.termCount*50 + return idxI > idxJ + }) + + numFree := height - 2 - numInUse + numToHide := 0 + termLimit := termHeight + 3 + + for i := 0; numFree > termLimit && i < len(candidates); i++ { + candidates[i].showTerm = true + numToHide += candidates[i].vertex.term.UsedHeight() + numFree -= termLimit + } + + if !all { + jobs = wrapHeight(jobs, height-2-numToHide) + } + + return jobs +} + +func (disp *display) print(d displayInfo, width, height int, all bool) { + // this output is inspired by Buck + d.jobs = setupTerminals(d.jobs, height, all) + b := aec.EmptyBuilder + b = b.Up(uint(disp.lineCount) + 1) + if !disp.repeated { + b = b.Down(1) + } + disp.repeated = true + fmt.Fprint(disp.c, b.Column(0).ANSI) + + statusStr := "" + if d.countCompleted > 0 && d.countCompleted == d.countTotal && all { + statusStr = "FINISHED" + } + + fmt.Fprint(disp.c, aec.Hide) + defer fmt.Fprint(disp.c, aec.Show) + + out := fmt.Sprintf("[+] 🐋 %s %.1fs (%d/%d) %s", disp.phase, time.Since(d.startTime).Seconds(), d.countCompleted, d.countTotal, statusStr) + // out = align(out, "", width) + fmt.Fprintln(disp.c, out) + lineCount := 0 + for _, j := range d.jobs { + if len(j.intervals) == 0 { + continue + } + var dt float64 + for _, ival := range j.intervals { + dt += ival.duration().Seconds() + } + if dt < 0.05 { + dt = 0 + } + pfx := " => " + timer := fmt.Sprintf(" %3.1fs\n", dt) + status := j.status + showStatus := false + + left := width - len(pfx) - len(timer) - 1 + if status != "" { + if left+len(status) > 20 { + showStatus = true + left -= len(status) + 1 + } + } + if left < 12 { // too small screen to show progress + continue + } + name := j.name + if len(name) > left { + name = name[:left] + } + + out := pfx + name + if showStatus { + out += " " + status + } + + out = align(out, timer, width) + if j.isCompleted { + color := colorRun + if j.isCanceled { + color = colorCancel + } else if j.hasError { + color = colorError + } + out = aec.Apply(out, color) + } + fmt.Fprint(disp.c, out) + lineCount++ + if j.showTerm { + term := j.vertex.term + term.Resize(termHeight, width-termPad) + for _, l := range term.Content { + if !isEmpty(l) { + out := aec.Apply(fmt.Sprintf(" => => # %s\n", string(l)), aec.Faint) + fmt.Fprint(disp.c, out) + lineCount++ + } + } + j.vertex.termCount++ + j.showTerm = false + } + } + // override previous content + if diff := disp.lineCount - lineCount; diff > 0 { + for i := 0; i < diff; i++ { + fmt.Fprintln(disp.c, strings.Repeat(" ", width)) + } + fmt.Fprint(disp.c, aec.EmptyBuilder.Up(uint(diff)).Column(0).ANSI) + } + disp.lineCount = lineCount +} + +func isEmpty(l []rune) bool { + for _, r := range l { + if r != ' ' { + return false + } + } + return true +} + +func align(l, r string, w int) string { + return fmt.Sprintf("%-[2]*[1]s %[3]s", l, w-len(r)-1, r) +} + +func wrapHeight(j []*job, limit int) []*job { + if limit < 0 { + return nil + } + var wrapped []*job + wrapped = append(wrapped, j...) + if len(j) > limit { + wrapped = wrapped[len(j)-limit:] + + // wrap things around if incomplete jobs were cut + var invisible []*job + for _, j := range j[:len(j)-limit] { + if !j.isCompleted { + invisible = append(invisible, j) + } + } + + if l := len(invisible); l > 0 { + rewrapped := make([]*job, 0, len(wrapped)) + for _, j := range wrapped { + if !j.isCompleted || l <= 0 { + rewrapped = append(rewrapped, j) + } + l-- + } + freespace := len(wrapped) - len(rewrapped) + wrapped = append(invisible[len(invisible)-freespace:], rewrapped...) + } + } + return wrapped +} diff --git a/pkg/progress/progressui/display_test.go b/pkg/progress/progressui/display_test.go new file mode 100644 index 000000000..6a5434049 --- /dev/null +++ b/pkg/progress/progressui/display_test.go @@ -0,0 +1,183 @@ +// Copyright 2022 The MIDI Authors +// Copyright 2022 The buildkit Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package progressui + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func mkinterval(start, stop int64) interval { + unixStart := time.Unix(start, 0) + unixStop := time.Unix(stop, 0) + return interval{start: &unixStart, stop: &unixStop} +} + +func mkOpenInterval(start int64) interval { + unixStart := time.Unix(start, 0) + return interval{start: &unixStart, stop: nil} +} + +func TestMergeIntervals(t *testing.T) { + for _, tc := range []struct { + name string + input []interval + expected []interval + }{ + { + name: "none", + input: nil, + expected: nil, + }, + { + name: "one", + input: []interval{ + mkinterval(0, 1), + }, + expected: []interval{ + mkinterval(0, 1), + }, + }, + { + name: "unstarted", + input: []interval{ + mkinterval(0, 1), + {nil, nil}, + }, + expected: []interval{ + mkinterval(0, 1), + }, + }, + { + name: "equal", + input: []interval{ + mkinterval(2, 4), + mkinterval(2, 4), + }, + expected: []interval{ + mkinterval(2, 4), + }, + }, + { + name: "no overlap", + input: []interval{ + mkinterval(0, 1), + mkinterval(2, 3), + mkinterval(7, 8), + }, + expected: []interval{ + mkinterval(0, 1), + mkinterval(2, 3), + mkinterval(7, 8), + }, + }, + { + name: "subsumed", + input: []interval{ + mkinterval(0, 10), + mkinterval(1, 2), + mkinterval(4, 9), + mkinterval(9, 10), + }, + expected: []interval{ + mkinterval(0, 10), + }, + }, + { + name: "partial overlaps", + input: []interval{ + mkinterval(0, 3), + mkinterval(2, 5), + mkinterval(4, 8), + mkinterval(10, 12), + mkinterval(11, 14), + }, + expected: []interval{ + mkinterval(0, 8), + mkinterval(10, 14), + }, + }, + { + name: "joined", + input: []interval{ + mkinterval(0, 2), + mkinterval(2, 4), + mkinterval(4, 6), + mkinterval(8, 10), + mkinterval(10, 12), + mkinterval(11, 12), + mkinterval(11, 14), + }, + expected: []interval{ + mkinterval(0, 6), + mkinterval(8, 14), + }, + }, + { + name: "open interval", + input: []interval{ + mkinterval(0, 5), + mkOpenInterval(6), + }, + expected: []interval{ + mkinterval(0, 5), + mkOpenInterval(6), + }, + }, + { + name: "open interval with overlaps", + input: []interval{ + mkOpenInterval(1), + mkinterval(3, 5), + }, + expected: []interval{ + mkOpenInterval(1), + }, + }, + { + name: "complex", + input: []interval{ + mkinterval(0, 2), + mkinterval(1, 4), + mkinterval(1, 4), + mkinterval(1, 5), + {nil, nil}, + mkinterval(6, 20), + mkinterval(8, 10), + mkinterval(8, 10), + mkinterval(9, 10), + mkinterval(12, 14), + mkinterval(19, 21), + mkinterval(30, 31), + mkinterval(32, 35), + {nil, nil}, + mkOpenInterval(33), + }, + expected: []interval{ + mkinterval(0, 5), + mkinterval(6, 21), + mkinterval(30, 31), + mkOpenInterval(32), + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.expected, mergeIntervals(tc.input)) + }) + } +} diff --git a/pkg/progress/progressui/printer.go b/pkg/progress/progressui/printer.go new file mode 100644 index 000000000..9c608cd62 --- /dev/null +++ b/pkg/progress/progressui/printer.go @@ -0,0 +1,348 @@ +// Copyright 2022 The MIDI Authors +// Copyright 2022 The buildkit Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package progressui + +import ( + "container/ring" + "context" + "fmt" + "io" + "os" + "sort" + "strings" + "time" + + digest "github.com/opencontainers/go-digest" + "github.com/tonistiigi/units" +) + +const antiFlicker = 5 * time.Second +const maxDelay = 10 * time.Second +const minTimeDelta = 5 * time.Second +const minProgressDelta = 0.05 // % + +const logsBufferSize = 10 + +type lastStatus struct { + Current int64 + Timestamp time.Time +} + +type textMux struct { + w io.Writer + current digest.Digest + last map[string]lastStatus + notFirst bool + nextIndex int +} + +func (p *textMux) printVtx(t *trace, dgst digest.Digest) { + if p.last == nil { + p.last = make(map[string]lastStatus) + } + + v, ok := t.byDigest[dgst] + if !ok { + return + } + + if v.index == 0 { + p.nextIndex++ + v.index = p.nextIndex + } + + if dgst != p.current { + if p.current != "" { + old := t.byDigest[p.current] + if old.logsPartial { + fmt.Fprintln(p.w, "") + } + old.logsOffset = 0 + old.count = 0 + fmt.Fprintf(p.w, "#%d ...\n", old.index) + } + + if p.notFirst { + fmt.Fprintln(p.w, "") + } else { + p.notFirst = true + } + + if os.Getenv("PROGRESS_NO_TRUNC") == "0" { + fmt.Fprintf(p.w, "#%d %s\n", v.index, limitString(v.Name, 72)) + } else { + fmt.Fprintf(p.w, "#%d %s\n", v.index, v.Name) + } + } + + if len(v.events) != 0 { + v.logsOffset = 0 + } + for _, ev := range v.events { + fmt.Fprintf(p.w, "#%d %s\n", v.index, ev) + } + v.events = v.events[:0] + + isOpenStatus := false // remote cache loading can currently produce status updates without active vertex + for _, s := range v.statuses { + if _, ok := v.statusUpdates[s.ID]; ok { + doPrint := true + + if last, ok := p.last[s.ID]; ok && s.Completed == nil { + var progressDelta float64 + if s.Total > 0 { + progressDelta = float64(s.Current-last.Current) / float64(s.Total) + } + timeDelta := s.Timestamp.Sub(last.Timestamp) + if progressDelta < minProgressDelta && timeDelta < minTimeDelta { + doPrint = false + } + } + + if !doPrint { + continue + } + + p.last[s.ID] = lastStatus{ + Timestamp: s.Timestamp, + Current: s.Current, + } + + var bytes string + if s.Total != 0 { + bytes = fmt.Sprintf(" %.2f / %.2f", units.Bytes(s.Current), units.Bytes(s.Total)) + } else if s.Current != 0 { + bytes = fmt.Sprintf(" %.2f", units.Bytes(s.Current)) + } + var tm string + endTime := s.Timestamp + if s.Completed != nil { + endTime = *s.Completed + } + if s.Started != nil { + diff := endTime.Sub(*s.Started).Seconds() + if diff > 0.01 { + tm = fmt.Sprintf(" %.1fs", diff) + } + } + if s.Completed != nil { + tm += " done" + } else { + isOpenStatus = true + } + fmt.Fprintf(p.w, "#%d %s%s%s\n", v.index, s.ID, bytes, tm) + } + } + v.statusUpdates = map[string]struct{}{} + + for _, w := range v.warnings[v.warningIdx:] { + fmt.Fprintf(p.w, "#%d WARN: %s\n", v.index, w.Short) + v.warningIdx++ + } + + for i, l := range v.logs { + if i == 0 { + l = l[v.logsOffset:] + } + fmt.Fprintf(p.w, "%s", []byte(l)) + if i != len(v.logs)-1 || !v.logsPartial { + fmt.Fprintln(p.w, "") + } + if v.logsBuffer == nil { + v.logsBuffer = ring.New(logsBufferSize) + } + v.logsBuffer.Value = l + if !v.logsPartial { + v.logsBuffer = v.logsBuffer.Next() + } + } + + if len(v.logs) > 0 { + if v.logsPartial { + v.logs = v.logs[len(v.logs)-1:] + v.logsOffset = len(v.logs[0]) + } else { + v.logs = nil + v.logsOffset = 0 + } + } + + p.current = dgst + if v.isCompleted() && !isOpenStatus { + p.current = "" + v.count = 0 + + if v.Error != "" { + if v.logsPartial { + fmt.Fprintln(p.w, "") + } + if strings.HasSuffix(v.Error, context.Canceled.Error()) { + fmt.Fprintf(p.w, "#%d CANCELED\n", v.index) + } else { + fmt.Fprintf(p.w, "#%d ERROR: %s\n", v.index, v.Error) + } + } else if v.Cached { + fmt.Fprintf(p.w, "#%d CACHED\n", v.index) + } else { + tm := "" + var ivals []interval + for _, ival := range v.intervals { + ivals = append(ivals, ival) + } + ivals = mergeIntervals(ivals) + if len(ivals) > 0 { + var dt float64 + for _, ival := range ivals { + dt += ival.duration().Seconds() + } + tm = fmt.Sprintf(" %.1fs", dt) + } + fmt.Fprintf(p.w, "#%d DONE%s\n", v.index, tm) + } + } + + delete(t.updates, dgst) +} + +func sortCompleted(t *trace, m map[digest.Digest]struct{}) []digest.Digest { + out := make([]digest.Digest, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Slice(out, func(i, j int) bool { + vtxi := t.byDigest[out[i]] + vtxj := t.byDigest[out[j]] + return vtxi.mostRecentInterval().stop.Before(*vtxj.mostRecentInterval().stop) + }) + return out +} + +func (p *textMux) print(t *trace) { + completed := map[digest.Digest]struct{}{} + rest := map[digest.Digest]struct{}{} + + for dgst := range t.updates { + v, ok := t.byDigest[dgst] + if !ok { + continue + } + if v.ProgressGroup != nil || v.hidden { + // skip vtxs in a group (they are merged into a single vtx) and hidden ones + continue + } + if v.isCompleted() { + completed[dgst] = struct{}{} + } else { + rest[dgst] = struct{}{} + } + } + + current := p.current + + // items that have completed need to be printed first + if _, ok := completed[current]; ok { + p.printVtx(t, current) + } + + for _, dgst := range sortCompleted(t, completed) { + if dgst != current { + p.printVtx(t, dgst) + } + } + + if len(rest) == 0 { + if current != "" { + if v := t.byDigest[current]; v.isStarted() && !v.isCompleted() { + return + } + } + // make any open vertex active + for dgst, v := range t.byDigest { + if v.isStarted() && !v.isCompleted() && v.ProgressGroup == nil && !v.hidden { + p.printVtx(t, dgst) + return + } + } + return + } + + // now print the active one + if _, ok := rest[current]; ok { + p.printVtx(t, current) + } + + stats := map[digest.Digest]*vtxStat{} + now := time.Now() + sum := 0.0 + var max digest.Digest + if current != "" { + rest[current] = struct{}{} + } + for dgst := range rest { + v, ok := t.byDigest[dgst] + if !ok { + continue + } + if v.lastBlockTime == nil { + // shouldn't happen, but not worth crashing over + continue + } + tm := now.Sub(*v.lastBlockTime) + speed := float64(v.count) / tm.Seconds() + overLimit := tm > maxDelay && dgst != current + stats[dgst] = &vtxStat{blockTime: tm, speed: speed, overLimit: overLimit} + sum += speed + if overLimit || max == "" || stats[max].speed < speed { + max = dgst + } + } + for dgst := range stats { + stats[dgst].share = stats[dgst].speed / sum + } + + if _, ok := completed[current]; ok || current == "" { + p.printVtx(t, max) + return + } + + // show items that were hidden + for dgst := range rest { + if stats[dgst].overLimit { + p.printVtx(t, dgst) + return + } + } + + // fair split between vertexes + if 1.0/(1.0-stats[current].share)*antiFlicker.Seconds() < stats[current].blockTime.Seconds() { + p.printVtx(t, max) + return + } +} + +type vtxStat struct { + blockTime time.Duration + speed float64 + share float64 + overLimit bool +} + +func limitString(s string, l int) string { + if len(s) > l { + return s[:l] + "..." + } + return s +} diff --git a/pkg/progress/progressui/term.go b/pkg/progress/progressui/term.go new file mode 100644 index 000000000..9dbc35b7c --- /dev/null +++ b/pkg/progress/progressui/term.go @@ -0,0 +1,27 @@ +// Copyright 2022 The MIDI Authors +// Copyright 2022 The buildkit Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !windows +// +build !windows + +package progressui + +import "github.com/morikuni/aec" + +var ( + colorRun = aec.BlueF + colorCancel = aec.YellowF + colorError = aec.RedF +) diff --git a/pkg/progress/progressui/term_windows.go b/pkg/progress/progressui/term_windows.go new file mode 100644 index 000000000..8ef85bd47 --- /dev/null +++ b/pkg/progress/progressui/term_windows.go @@ -0,0 +1,27 @@ +// Copyright 2022 The MIDI Authors +// Copyright 2022 The buildkit Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build windows +// +build windows + +package progressui + +import "github.com/morikuni/aec" + +var ( + colorRun = aec.CyanF + colorCancel = aec.YellowF + colorError = aec.RedF +) diff --git a/pkg/progress/progresswriter/printer.go b/pkg/progress/progresswriter/printer.go new file mode 100644 index 000000000..9d0329500 --- /dev/null +++ b/pkg/progress/progresswriter/printer.go @@ -0,0 +1,110 @@ +// Copyright 2022 The MIDI Authors +// Copyright 2022 The buildkit Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package progresswriter + +import ( + "context" + "os" + + "github.com/containerd/console" + "github.com/moby/buildkit/client" + "github.com/pkg/errors" + + "github.com/tensorchord/MIDI/pkg/progress/progressui" +) + +type printer struct { + status chan *client.SolveStatus + done <-chan struct{} + err error +} + +func (p *printer) Done() <-chan struct{} { + return p.done +} + +func (p *printer) Err() error { + return p.err +} + +func (p *printer) Status() chan *client.SolveStatus { + if p == nil { + return nil + } + return p.status +} + +type tee struct { + Writer + status chan *client.SolveStatus +} + +func (t *tee) Status() chan *client.SolveStatus { + return t.status +} + +func Tee(w Writer, ch chan *client.SolveStatus) Writer { + st := make(chan *client.SolveStatus) + t := &tee{ + status: st, + Writer: w, + } + go func() { + for v := range st { + w.Status() <- v + ch <- v + } + close(w.Status()) + close(ch) + }() + return t +} + +func NewPrinter(ctx context.Context, out console.File, mode string) (Writer, error) { + statusCh := make(chan *client.SolveStatus) + doneCh := make(chan struct{}) + + pw := &printer{ + status: statusCh, + done: doneCh, + } + + if v := os.Getenv("BUILDKIT_PROGRESS"); v != "" && mode == "auto" { + mode = v + } + + var c console.Console + switch mode { + case "auto", "tty", "": + if cons, err := console.ConsoleFromFile(out); err == nil { + c = cons + } else { + if mode == "tty" { + return nil, errors.Wrap(err, "failed to get console") + } + } + case "plain": + default: + return nil, errors.Errorf("invalid progress mode %s", mode) + } + + go func() { + // not using shared context to not disrupt display but let is finish reporting errors + _, pw.err = progressui.DisplaySolveStatus(ctx, "build MIDI environment", c, out, statusCh) + close(doneCh) + }() + return pw, nil +} diff --git a/pkg/progress/progresswriter/writer.go b/pkg/progress/progresswriter/writer.go new file mode 100644 index 000000000..3e870677b --- /dev/null +++ b/pkg/progress/progresswriter/writer.go @@ -0,0 +1,61 @@ +// Copyright 2022 The MIDI Authors +// Copyright 2022 The buildkit Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package progresswriter + +import ( + "time" + + "github.com/moby/buildkit/client" + "github.com/moby/buildkit/identity" + digest "github.com/opencontainers/go-digest" +) + +type Writer interface { + Done() <-chan struct{} + Err() error + Status() chan *client.SolveStatus +} + +func Write(w Writer, name string, f func() error) { + status := w.Status() + dgst := digest.FromBytes([]byte(identity.NewID())) + tm := time.Now() + + vtx := client.Vertex{ + Digest: dgst, + Name: name, + Started: &tm, + } + + status <- &client.SolveStatus{ + Vertexes: []*client.Vertex{&vtx}, + } + + var err error + if f != nil { + err = f() + } + + tm2 := time.Now() + vtx2 := vtx + vtx2.Completed = &tm2 + if err != nil { + vtx2.Error = err.Error() + } + status <- &client.SolveStatus{ + Vertexes: []*client.Vertex{&vtx2}, + } +}