diff --git a/src/cli/cli.go b/src/cli/cli.go index f109fe4697..932b2cb72c 100644 --- a/src/cli/cli.go +++ b/src/cli/cli.go @@ -15,6 +15,7 @@ import ( "github.com/alcionai/corso/src/cli/print" "github.com/alcionai/corso/src/cli/repo" "github.com/alcionai/corso/src/cli/restore" + "github.com/alcionai/corso/src/internal/observe" "github.com/alcionai/corso/src/pkg/logger" ) @@ -88,6 +89,7 @@ func BuildCommandTree(cmd *cobra.Command) { func Handle() { ctx := config.Seed(context.Background()) ctx = print.SetRootCmd(ctx, corsoCmd) + observe.SeedWriter(print.StderrWriter(ctx)) BuildCommandTree(corsoCmd) diff --git a/src/cli/print/print.go b/src/cli/print/print.go index 777b188ed1..7cbb9c80dc 100644 --- a/src/cli/print/print.go +++ b/src/cli/print/print.go @@ -55,6 +55,12 @@ func JSONFormat() bool { return outputAsJSON || outputAsJSONDebug } +// StderrWriter returns the stderr writer used in the root +// cmd. Returns nil if no root command is seeded. +func StderrWriter(ctx context.Context) io.Writer { + return getRootCmd(ctx).ErrOrStderr() +} + // --------------------------------------------------------------------------------------------------------- // Helper funcs // --------------------------------------------------------------------------------------------------------- diff --git a/src/go.mod b/src/go.mod index 77cbd1673a..68ec4679cd 100644 --- a/src/go.mod +++ b/src/go.mod @@ -16,6 +16,7 @@ require ( github.com/microsoftgraph/msgraph-sdk-go-core v0.28.1 github.com/pkg/errors v0.9.1 github.com/rudderlabs/analytics-go v3.3.3+incompatible + github.com/schollz/progressbar/v3 v3.11.0 github.com/spf13/cobra v1.4.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.12.0 @@ -31,6 +32,7 @@ require ( github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/magiconair/properties v1.8.6 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.1 // indirect @@ -38,6 +40,7 @@ require ( github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.3.0 // indirect + golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/src/go.sum b/src/go.sum index e169a8b4b0..9f006cd20e 100644 --- a/src/go.sum +++ b/src/go.sum @@ -205,6 +205,7 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c= github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= @@ -265,6 +266,8 @@ github.com/minio/minio-go/v7 v7.0.39 h1:upnbu1jCGOqEvrGSpRauSN9ZG7RCHK7VHxXS8Vmg github.com/minio/minio-go/v7 v7.0.39/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -327,6 +330,8 @@ github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rudderlabs/analytics-go v3.3.3+incompatible h1:OG0XlKoXfr539e2t1dXtTB+Gr89uFW+OUNQBVhHIIBY= github.com/rudderlabs/analytics-go v3.3.3+incompatible/go.mod h1:LF8/ty9kUX4PTY3l5c97K3nZZaX5Hwsvt+NBaRL/f30= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/schollz/progressbar/v3 v3.11.0 h1:3nIBUF1Zw/pGUaRHP7PZWmARP7ZQbWQ6vL6hwoQiIvU= +github.com/schollz/progressbar/v3 v3.11.0/go.mod h1:R2djRgv58sn00AGysc4fN0ip4piOGd3z88K+zVBjczs= github.com/segmentio/backo-go v1.0.0 h1:kbOAtGJY2DqOR0jfRkYEorx/b18RgtepGtY3+Cpe6qA= github.com/segmentio/backo-go v1.0.0/go.mod h1:kJ9mm9YmoWSkk+oQ+5Cj8DEoRCX2JT6As4kEtIIOp1M= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -557,10 +562,13 @@ golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI= golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 h1:Q5284mrmYTpACcm+eAKjKJH48BBwSyfJqmmGDTtT8Vc= +golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/src/internal/connector/onedrive/collection.go b/src/internal/connector/onedrive/collection.go index fa836c6285..db758c91d7 100644 --- a/src/internal/connector/onedrive/collection.go +++ b/src/internal/connector/onedrive/collection.go @@ -8,6 +8,7 @@ import ( "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" + "github.com/alcionai/corso/src/internal/observe" "github.com/alcionai/corso/src/pkg/backup/details" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" @@ -138,10 +139,11 @@ func (oc *Collection) populateItems(ctx context.Context) { byteCount += itemInfo.Size itemInfo.ParentPath = parentPathString + progReader := observe.ItemProgress(itemData, itemInfo.ItemName, itemInfo.Size) oc.data <- &Item{ id: itemInfo.ItemName, - data: itemData, + data: progReader, info: itemInfo, } } diff --git a/src/internal/connector/onedrive/item.go b/src/internal/connector/onedrive/item.go index ea16a8c079..1572a76a57 100644 --- a/src/internal/connector/onedrive/item.go +++ b/src/internal/connector/onedrive/item.go @@ -13,6 +13,7 @@ import ( "github.com/pkg/errors" "gopkg.in/resty.v1" + "github.com/alcionai/corso/src/internal/common" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/pkg/backup/details" @@ -40,6 +41,8 @@ func driveItemReader( return nil, nil, errors.Wrapf(err, "failed to get item %s", itemID) } + logger.Ctx(ctx).Debugw("reading item", "name", *item.GetName(), "time", common.Now()) + // Get the download URL - https://docs.microsoft.com/en-us/graph/api/driveitem-get-content // These URLs are pre-authenticated and can be used to download the data using the standard // http client diff --git a/src/internal/observe/observe.go b/src/internal/observe/observe.go new file mode 100644 index 0000000000..5d93fe7742 --- /dev/null +++ b/src/internal/observe/observe.go @@ -0,0 +1,49 @@ +package observe + +import ( + "io" + + "github.com/schollz/progressbar/v3" +) + +var writer io.Writer + +// SeedWriter adds default writer to the observe package. +// Uses a noop writer until seeded. +func SeedWriter(w io.Writer) { + writer = w +} + +// ItemProgress tracks the display of an item by counting the bytes +// read through the provided readcloser, up until the byte count matches +// the totalBytes. +func ItemProgress(rc io.ReadCloser, iname string, totalBytes int64) io.ReadCloser { + if writer == nil { + return rc + } + + opts := progressbar.NewOptions( + int(totalBytes), + progressbar.OptionSetWriter(writer), + progressbar.OptionClearOnFinish(), + progressbar.OptionSetDescription(" | "+iname), + progressbar.OptionShowDescriptionAtLineEnd(), + progressbar.OptionSetRenderBlankState(true), + progressbar.OptionShowCount(), + progressbar.OptionSetPredictTime(false), + progressbar.OptionEnableColorCodes(true), + progressbar.OptionShowBytes(true), + progressbar.OptionSetWidth(20), + progressbar.OptionSetTheme(progressbar.Theme{ + Saucer: "[green]=[reset]", + SaucerHead: "[green]>[reset]", + SaucerPadding: " ", + BarStart: "[", + BarEnd: "]", + }), + ) + + pbr := progressbar.NewReader(rc, opts) + + return &pbr +} diff --git a/src/internal/observe/observe_test.go b/src/internal/observe/observe_test.go new file mode 100644 index 0000000000..7fd6e96c79 --- /dev/null +++ b/src/internal/observe/observe_test.go @@ -0,0 +1,53 @@ +package observe_test + +import ( + "bytes" + "errors" + "io" + "strings" + "testing" + + "github.com/alcionai/corso/src/internal/observe" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type ObserveProgressUnitSuite struct { + suite.Suite +} + +func TestObserveProgressUnitSuite(t *testing.T) { + suite.Run(t, new(ObserveProgressUnitSuite)) +} + +func (suite *ObserveProgressUnitSuite) TestDoesThings() { + t := suite.T() + + recorder := strings.Builder{} + observe.SeedWriter(&recorder) + + from := make([]byte, 100) + prog := observe.ItemProgress( + io.NopCloser(bytes.NewReader(from)), + "test", + 100) + require.NotNil(t, prog) + + for { + to := make([]byte, 25) + n, err := prog.Read(to) + + if errors.Is(err, io.EOF) { + break + } + + assert.NoError(t, err) + assert.Less(t, 0, n) + } + + recorded := recorder.String() + assert.Contains(t, recorded, "25%") + assert.Contains(t, recorded, "50%") + assert.Contains(t, recorded, "75%") +}