Skip to content

Commit

Permalink
scyllaclient: rclone, change RcloneListDirOpts to use better defaults
Browse files Browse the repository at this point in the history 
- NoModTime is replaced with ShowModTime
- NoMimeType is removed
- ShowX is removed

Relates to #3039

With this patch

```go
	opts := scyllaclient.RcloneListDirOpts{
		Recurse:     true,
		FilesOnly:   true,
		ShowModTime: true,
	}
	x, err := client.RcloneListDir(ctx, scyllaclienttest.TestHost, "s3:my-foo/backup/meta/cluster/5c3667c9-e8da-4a75-8541-a48b7c2b3db4", &opts)
```

yields

```
$ make pkg-integration-test PKG=./pkg/scyllaclient RUN=TestRcloneS3ListIntegration | grep HEAD
09:49:16.330    DEBUG   rclone  HEAD /my-foo/backup/meta/cluster/5c3667c9-e8da-4a75-8541-a48b7c2b3db4 HTTP/1.1
09:49:16.491    DEBUG   rclone  HEAD /my-foo/backup/meta/cluster/5c3667c9-e8da-4a75-8541-a48b7c2b3db4/dc/dc1/node/139bf9e7-162f-40eb-9b5c-3220a866567b/task_843c83b8-5d2c-4686-b968-815d2689a41a_tag_sm_20220217110728UTC_manifest.json.gz HTTP/1.1
09:49:16.528    DEBUG   rclone  HEAD /my-foo/backup/meta/cluster/5c3667c9-e8da-4a75-8541-a48b7c2b3db4/dc/dc1/node/139bf9e7-162f-40eb-9b5c-3220a866567b/task_843c83b8-5d2c-4686-b968-815d2689a41a_tag_sm_20220217111110UTC_manifest.json.gz HTTP/1.1
09:49:16.555    DEBUG   rclone  HEAD /my-foo/backup/meta/cluster/5c3667c9-e8da-4a75-8541-a48b7c2b3db4/dc/dc1/node/869fbac6-244f-4369-af06-89e9052a33ee/task_843c83b8-5d2c-4686-b968-815d2689a41a_tag_sm_20220217110728UTC_manifest.json.gz HTTP/1.1
09:49:16.586    DEBUG   rclone  HEAD /my-foo/backup/meta/cluster/5c3667c9-e8da-4a75-8541-a48b7c2b3db4/dc/dc1/node/869fbac6-244f-4369-af06-89e9052a33ee/task_843c83b8-5d2c-4686-b968-815d2689a41a_tag_sm_20220217111110UTC_manifest.json.gz HTTP/1.1
09:49:16.614    DEBUG   rclone  HEAD /my-foo/backup/meta/cluster/5c3667c9-e8da-4a75-8541-a48b7c2b3db4/dc/dc1/node/881fbc3d-833b-42cc-ab51-3b66f127c3e8/task_843c83b8-5d2c-4686-b968-815d2689a41a_tag_sm_20220217110728UTC_manifest.json.gz HTTP/1.1
09:49:16.634    DEBUG   rclone  HEAD /my-foo/backup/meta/cluster/5c3667c9-e8da-4a75-8541-a48b7c2b3db4/dc/dc1/node/881fbc3d-833b-42cc-ab51-3b66f127c3e8/task_843c83b8-5d2c-4686-b968-815d2689a41a_tag_sm_20220217111110UTC_manifest.json.gz HTTP/1.1
09:49:16.657    DEBUG   rclone  HEAD /my-foo/backup/meta/cluster/5c3667c9-e8da-4a75-8541-a48b7c2b3db4/dc/dc2/node/59cf508b-91ea-41f1-ba7e-cf0475e65124/task_843c83b8-5d2c-4686-b968-815d2689a41a_tag_sm_20220217110728UTC_manifest.json.gz HTTP/1.1
09:49:16.689    DEBUG   rclone  HEAD /my-foo/backup/meta/cluster/5c3667c9-e8da-4a75-8541-a48b7c2b3db4/dc/dc2/node/59cf508b-91ea-41f1-ba7e-cf0475e65124/task_843c83b8-5d2c-4686-b968-815d2689a41a_tag_sm_20220217111110UTC_manifest.json.gz HTTP/1.1
09:49:16.727    DEBUG   rclone  HEAD /my-foo/backup/meta/cluster/5c3667c9-e8da-4a75-8541-a48b7c2b3db4/dc/dc2/node/92ff550e-52bb-4ac4-866c-576cec82ccd2/task_843c83b8-5d2c-4686-b968-815d2689a41a_tag_sm_20220217110728UTC_manifest.json.gz HTTP/1.1
09:49:16.752    DEBUG   rclone  HEAD /my-foo/backup/meta/cluster/5c3667c9-e8da-4a75-8541-a48b7c2b3db4/dc/dc2/node/92ff550e-52bb-4ac4-866c-576cec82ccd2/task_843c83b8-5d2c-4686-b968-815d2689a41a_tag_sm_20220217111110UTC_manifest.json.gz HTTP/1.1
09:49:16.787    DEBUG   rclone  HEAD /my-foo/backup/meta/cluster/5c3667c9-e8da-4a75-8541-a48b7c2b3db4/dc/dc2/node/a9e62b6f-2823-48e6-a6ae-5b9daae65302/task_843c83b8-5d2c-4686-b968-815d2689a41a_tag_sm_20220217110728UTC_manifest.json.gz HTTP/1.1
09:49:16.810    DEBUG   rclone  HEAD /my-foo/backup/meta/cluster/5c3667c9-e8da-4a75-8541-a48b7c2b3db4/dc/dc2/node/a9e62b6f-2823-48e6-a6ae-5b9daae65302/task_843c83b8-5d2c-4686-b968-815d2689a41a_tag_sm_20220217111110UTC_manifest.json.gz HTTP/1.1
```

and

```go
	opts := scyllaclient.RcloneListDirOpts{
		Recurse:   true,
		FilesOnly: true,
	}
	x, err := client.RcloneListDir(ctx, scyllaclienttest.TestHost, "s3:my-foo/backup/meta/cluster/5c3667c9-e8da-4a75-8541-a48b7c2b3db4", &opts)
```

yields

```
$ make pkg-integration-test PKG=./pkg/scyllaclient RUN=TestRcloneS3ListIntegration | grep HEAD
09:51:32.800    DEBUG   rclone  HEAD /my-foo/backup/meta/cluster/5c3667c9-e8da-4a75-8541-a48b7c2b3db4 HTTP/1.1
```

(cherry picked from commit 062f09f)
  • Loading branch information
@mmatczuk
mmatczuk committed Feb 23, 2022
1 parent c51fffc commit 082cd06
Show file tree
Hide file tree
Showing 3 changed files with 35 additions and 6 deletions.
33 changes: 30 additions & 3 deletions pkg/scyllaclient/client_rclone.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -354,7 +354,34 @@ func (c *Client) RcloneCat(ctx context.Context, host, remotePath string) ([]byte
}

// RcloneListDirOpts specifies options for RcloneListDir.
type RcloneListDirOpts = models.ListOptionsOpt
type RcloneListDirOpts struct {
// Show only directories in the listing
DirsOnly bool
// Show only files in the listing
FilesOnly bool
// Recurse into the listing
Recurse bool
// Read the modification time, for rclone backends declared as SlowModTime setting this to true will send additional
// HEAD request for each listed file.
ShowModTime bool
}

func (opts *RcloneListDirOpts) asModelOpts() *models.ListOptionsOpt {
if opts == nil {
return &models.ListOptionsOpt{
NoModTime: true,
NoMimeType: true,
}
}

return &models.ListOptionsOpt{
DirsOnly: opts.DirsOnly,
FilesOnly: opts.FilesOnly,
Recurse: opts.Recurse,
NoModTime: !opts.ShowModTime,
NoMimeType: true,
}
}

// RcloneListDirItem represents a file in a listing with RcloneListDir.
type RcloneListDirItem = models.ListItem
Expand All @@ -375,7 +402,7 @@ func (c *Client) RcloneListDir(ctx context.Context, host, remotePath string, opt
ListOpts: &models.ListOptions{
Fs: &remotePath,
Remote: pointer.StringPtr(""),
Opt: opts,
Opt: opts.asModelOpts(),
},
}
resp, err := c.agentOps.OperationsList(&p)
Expand All @@ -400,7 +427,7 @@ func (c *Client) RcloneListDirIter(ctx context.Context, host, remotePath string,
listOpts := &models.ListOptions{
Fs: &remotePath,
Remote: pointer.StringPtr(""),
Opt: opts,
Opt: opts.asModelOpts(),
}
b, err := listOpts.MarshalBinary()
if err != nil {
Expand Down
5 changes: 3 additions & 2 deletions pkg/service/backup/purger.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -306,8 +306,9 @@ func (p purger) forEachRemoteFile(ctx context.Context, m *ManifestInfo, f func(*
f(item)
}
opts := scyllaclient.RcloneListDirOpts{
FilesOnly: true,
Recurse: true,
FilesOnly: true,
Recurse: true,
ShowModTime: true,
}
return p.client.RcloneListDirIter(ctx, p.host, m.Location.RemotePath(baseDir), &opts, wrapper)
}
Expand Down
3 changes: 2 additions & 1 deletion pkg/service/backup/worker_index.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,8 @@ func (w *worker) indexSnapshotDirs(ctx context.Context, h hostInfo) ([]snapshotD
size int64
)
opts := &scyllaclient.RcloneListDirOpts{
FilesOnly: true,
FilesOnly: true,
ShowModTime: true,
}
err := w.Client.RcloneListDirIter(ctx, h.IP, d.Path, opts, func(f *scyllaclient.RcloneListDirItem) {
// Filter out Scylla manifest and Schema files, they are not needed.
Expand Down

0 comments on commit 082cd06

Please sign in to comment.