Skip to content
This repository has been archived by the owner on Oct 11, 2024. It is now read-only.

Commit

Permalink
add progress display handler and api (#1116)
Browse files Browse the repository at this point in the history
## Description

Adds a new package- Observe- for owning user-
oriented displays like progress bars.  This PR adds
an initial progress bar to onedrive backups as a
proof-of-concept.  The API is more important than
the specific progress bar package at this time.
Future changes may opt for a different pkg.

Display format currently looks like:
```
59% [=============>           ] (6.9/12 kB, 14 MB/s)  |  Item_Name.txt
```

Known Issues:
* the `progressbar` package does not support multiline output, and [the author is not planning to add support](schollz/progressbar#6).  This causes concurrent items to overwrite each other.  We will either need to fork the library, or change to a different one.  

## Type of change

- [x] 🌻 Feature

## Issue(s)

* #1112

## Test Plan

- [x] 💪 Manual
- [x] ⚡ Unit test
  • Loading branch information
ryanfkeepers authored Oct 10, 2022
1 parent ec916bc commit 2cc9b7b
Show file tree
Hide file tree
Showing 8 changed files with 127 additions and 1 deletion.
2 changes: 2 additions & 0 deletions src/cli/cli.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"github.com/alcionai/corso/src/cli/print"
"github.com/alcionai/corso/src/cli/repo"
"github.com/alcionai/corso/src/cli/restore"
"github.com/alcionai/corso/src/internal/observe"
"github.com/alcionai/corso/src/pkg/logger"
)

Expand Down Expand Up @@ -88,6 +89,7 @@ func BuildCommandTree(cmd *cobra.Command) {
func Handle() {
ctx := config.Seed(context.Background())
ctx = print.SetRootCmd(ctx, corsoCmd)
observe.SeedWriter(print.StderrWriter(ctx))

BuildCommandTree(corsoCmd)

Expand Down
6 changes: 6 additions & 0 deletions src/cli/print/print.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,12 @@ func JSONFormat() bool {
return outputAsJSON || outputAsJSONDebug
}

// StderrWriter returns the stderr writer used in the root
// cmd. Returns nil if no root command is seeded.
func StderrWriter(ctx context.Context) io.Writer {
return getRootCmd(ctx).ErrOrStderr()
}

// ---------------------------------------------------------------------------------------------------------
// Helper funcs
// ---------------------------------------------------------------------------------------------------------
Expand Down
3 changes: 3 additions & 0 deletions src/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ require (
github.com/microsoftgraph/msgraph-sdk-go-core v0.28.1
github.com/pkg/errors v0.9.1
github.com/rudderlabs/analytics-go v3.3.3+incompatible
github.com/schollz/progressbar/v3 v3.11.0
github.com/spf13/cobra v1.4.0
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.12.0
Expand All @@ -31,13 +32,15 @@ require (
github.com/fsnotify/fsnotify v1.5.4 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/magiconair/properties v1.8.6 // indirect
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/pelletier/go-toml/v2 v2.0.1 // indirect
github.com/spf13/afero v1.8.2 // indirect
github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/subosito/gotenv v1.3.0 // indirect
golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)

Expand Down
8 changes: 8 additions & 0 deletions src/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,7 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c=
github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
Expand Down Expand Up @@ -265,6 +266,8 @@ github.com/minio/minio-go/v7 v7.0.39 h1:upnbu1jCGOqEvrGSpRauSN9ZG7RCHK7VHxXS8Vmg
github.com/minio/minio-go/v7 v7.0.39/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw=
github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
Expand Down Expand Up @@ -327,6 +330,8 @@ github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rudderlabs/analytics-go v3.3.3+incompatible h1:OG0XlKoXfr539e2t1dXtTB+Gr89uFW+OUNQBVhHIIBY=
github.com/rudderlabs/analytics-go v3.3.3+incompatible/go.mod h1:LF8/ty9kUX4PTY3l5c97K3nZZaX5Hwsvt+NBaRL/f30=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/schollz/progressbar/v3 v3.11.0 h1:3nIBUF1Zw/pGUaRHP7PZWmARP7ZQbWQ6vL6hwoQiIvU=
github.com/schollz/progressbar/v3 v3.11.0/go.mod h1:R2djRgv58sn00AGysc4fN0ip4piOGd3z88K+zVBjczs=
github.com/segmentio/backo-go v1.0.0 h1:kbOAtGJY2DqOR0jfRkYEorx/b18RgtepGtY3+Cpe6qA=
github.com/segmentio/backo-go v1.0.0/go.mod h1:kJ9mm9YmoWSkk+oQ+5Cj8DEoRCX2JT6As4kEtIIOp1M=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
Expand Down Expand Up @@ -557,10 +562,13 @@ golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI=
golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 h1:Q5284mrmYTpACcm+eAKjKJH48BBwSyfJqmmGDTtT8Vc=
golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand Down
4 changes: 3 additions & 1 deletion src/internal/connector/onedrive/collection.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/internal/data"
"github.com/alcionai/corso/src/internal/observe"
"github.com/alcionai/corso/src/pkg/backup/details"
"github.com/alcionai/corso/src/pkg/logger"
"github.com/alcionai/corso/src/pkg/path"
Expand Down Expand Up @@ -138,10 +139,11 @@ func (oc *Collection) populateItems(ctx context.Context) {
byteCount += itemInfo.Size

itemInfo.ParentPath = parentPathString
progReader := observe.ItemProgress(itemData, itemInfo.ItemName, itemInfo.Size)

oc.data <- &Item{
id: itemInfo.ItemName,
data: itemData,
data: progReader,
info: itemInfo,
}
}
Expand Down
3 changes: 3 additions & 0 deletions src/internal/connector/onedrive/item.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"github.com/pkg/errors"
"gopkg.in/resty.v1"

"github.com/alcionai/corso/src/internal/common"
"github.com/alcionai/corso/src/internal/connector/graph"
"github.com/alcionai/corso/src/internal/connector/support"
"github.com/alcionai/corso/src/pkg/backup/details"
Expand Down Expand Up @@ -40,6 +41,8 @@ func driveItemReader(
return nil, nil, errors.Wrapf(err, "failed to get item %s", itemID)
}

logger.Ctx(ctx).Debugw("reading item", "name", *item.GetName(), "time", common.Now())

// Get the download URL - https://docs.microsoft.com/en-us/graph/api/driveitem-get-content
// These URLs are pre-authenticated and can be used to download the data using the standard
// http client
Expand Down
49 changes: 49 additions & 0 deletions src/internal/observe/observe.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package observe

import (
"io"

"github.com/schollz/progressbar/v3"
)

var writer io.Writer

// SeedWriter adds default writer to the observe package.
// Uses a noop writer until seeded.
func SeedWriter(w io.Writer) {
writer = w
}

// ItemProgress tracks the display of an item by counting the bytes
// read through the provided readcloser, up until the byte count matches
// the totalBytes.
func ItemProgress(rc io.ReadCloser, iname string, totalBytes int64) io.ReadCloser {
if writer == nil {
return rc
}

opts := progressbar.NewOptions(
int(totalBytes),
progressbar.OptionSetWriter(writer),
progressbar.OptionClearOnFinish(),
progressbar.OptionSetDescription(" | "+iname),
progressbar.OptionShowDescriptionAtLineEnd(),
progressbar.OptionSetRenderBlankState(true),
progressbar.OptionShowCount(),
progressbar.OptionSetPredictTime(false),
progressbar.OptionEnableColorCodes(true),
progressbar.OptionShowBytes(true),
progressbar.OptionSetWidth(20),
progressbar.OptionSetTheme(progressbar.Theme{
Saucer: "[green]=[reset]",
SaucerHead: "[green]>[reset]",
SaucerPadding: " ",
BarStart: "[",
BarEnd: "]",
}),
)

pbr := progressbar.NewReader(rc, opts)

return &pbr
}
53 changes: 53 additions & 0 deletions src/internal/observe/observe_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package observe_test

import (
"bytes"
"errors"
"io"
"strings"
"testing"

"github.com/alcionai/corso/src/internal/observe"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)

type ObserveProgressUnitSuite struct {
suite.Suite
}

func TestObserveProgressUnitSuite(t *testing.T) {
suite.Run(t, new(ObserveProgressUnitSuite))
}

func (suite *ObserveProgressUnitSuite) TestDoesThings() {
t := suite.T()

recorder := strings.Builder{}
observe.SeedWriter(&recorder)

from := make([]byte, 100)
prog := observe.ItemProgress(
io.NopCloser(bytes.NewReader(from)),
"test",
100)
require.NotNil(t, prog)

for {
to := make([]byte, 25)
n, err := prog.Read(to)

if errors.Is(err, io.EOF) {
break
}

assert.NoError(t, err)
assert.Less(t, 0, n)
}

recorded := recorder.String()
assert.Contains(t, recorded, "25%")
assert.Contains(t, recorded, "50%")
assert.Contains(t, recorded, "75%")
}

0 comments on commit 2cc9b7b

Please sign in to comment.