Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make filer.Filer return fs.DirEntry from ReadDir #415

Merged
merged 1 commit into from
May 31, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 18 additions & 7 deletions internal/filer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"io"
"io/fs"
"net/http"
"strings"
"testing"
Expand Down Expand Up @@ -75,6 +76,7 @@ func runFilerReadWriteTest(t *testing.T, ctx context.Context, f filer.Filer) {

func runFilerReadDirTest(t *testing.T, ctx context.Context, f filer.Filer) {
var err error
var info fs.FileInfo

// We start with an empty directory.
entries, err := f.ReadDir(ctx, ".")
Expand Down Expand Up @@ -105,23 +107,32 @@ func runFilerReadDirTest(t *testing.T, ctx context.Context, f filer.Filer) {
entries, err = f.ReadDir(ctx, ".")
require.NoError(t, err)
assert.Len(t, entries, 2)
assert.Equal(t, "dir", entries[0].Name)
assert.Equal(t, "hello.txt", entries[1].Name)
assert.Greater(t, entries[1].ModTime.Unix(), int64(0))
assert.Equal(t, "dir", entries[0].Name())
assert.True(t, entries[0].IsDir())
assert.Equal(t, "hello.txt", entries[1].Name())
assert.False(t, entries[1].IsDir())
info, err = entries[1].Info()
require.NoError(t, err)
assert.Greater(t, info.ModTime().Unix(), int64(0))

// Expect two entries in the directory.
entries, err = f.ReadDir(ctx, "/dir")
require.NoError(t, err)
assert.Len(t, entries, 2)
assert.Equal(t, "a", entries[0].Name)
assert.Equal(t, "world.txt", entries[1].Name)
assert.Greater(t, entries[1].ModTime.Unix(), int64(0))
assert.Equal(t, "a", entries[0].Name())
assert.True(t, entries[0].IsDir())
assert.Equal(t, "world.txt", entries[1].Name())
assert.False(t, entries[1].IsDir())
info, err = entries[1].Info()
require.NoError(t, err)
assert.Greater(t, info.ModTime().Unix(), int64(0))

// Expect a single entry in the nested path.
entries, err = f.ReadDir(ctx, "/dir/a/b")
require.NoError(t, err)
assert.Len(t, entries, 1)
assert.Equal(t, "c", entries[0].Name)
assert.Equal(t, "c", entries[0].Name())
assert.True(t, entries[0].IsDir())
}

func temporaryWorkspaceDir(t *testing.T, w *databricks.WorkspaceClient) string {
Expand Down
59 changes: 51 additions & 8 deletions libs/filer/dbfs_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"errors"
"io"
"io/fs"
"net/http"
"path"
"sort"
Expand All @@ -15,6 +16,52 @@ import (
"golang.org/x/exp/slices"
)

// Type that implements fs.DirEntry for DBFS.
type dbfsDirEntry struct {
dbfsFileInfo
}

func (entry dbfsDirEntry) Type() fs.FileMode {
typ := fs.ModePerm
if entry.fi.IsDir {
typ |= fs.ModeDir
}
return typ
}

func (entry dbfsDirEntry) Info() (fs.FileInfo, error) {
return entry.dbfsFileInfo, nil
}

// Type that implements fs.FileInfo for DBFS.
type dbfsFileInfo struct {
fi files.FileInfo
}

func (info dbfsFileInfo) Name() string {
return path.Base(info.fi.Path)
}

func (info dbfsFileInfo) Size() int64 {
return info.fi.FileSize
}

func (info dbfsFileInfo) Mode() fs.FileMode {
return fs.ModePerm
}

func (info dbfsFileInfo) ModTime() time.Time {
return time.UnixMilli(info.fi.ModificationTime)
}

func (info dbfsFileInfo) IsDir() bool {
return info.fi.IsDir
}

func (info dbfsFileInfo) Sys() any {
return nil
}

// DbfsClient implements the [Filer] interface for the DBFS backend.
type DbfsClient struct {
workspaceClient *databricks.WorkspaceClient
Expand Down Expand Up @@ -152,7 +199,7 @@ func (w *DbfsClient) Delete(ctx context.Context, name string) error {
})
}

func (w *DbfsClient) ReadDir(ctx context.Context, name string) ([]FileInfo, error) {
func (w *DbfsClient) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, error) {
absPath, err := w.root.Join(name)
if err != nil {
return nil, err
Expand All @@ -175,17 +222,13 @@ func (w *DbfsClient) ReadDir(ctx context.Context, name string) ([]FileInfo, erro
return nil, err
}

info := make([]FileInfo, len(res.Files))
info := make([]fs.DirEntry, len(res.Files))
for i, v := range res.Files {
info[i] = FileInfo{
Name: path.Base(v.Path),
Size: v.FileSize,
ModTime: time.UnixMilli(v.ModificationTime),
}
info[i] = dbfsDirEntry{dbfsFileInfo: dbfsFileInfo{fi: v}}
}

// Sort by name for parity with os.ReadDir.
sort.Slice(info, func(i, j int) bool { return info[i].Name < info[j].Name })
sort.Slice(info, func(i, j int) bool { return info[i].Name() < info[j].Name() })
return info, nil
}

Expand Down
20 changes: 2 additions & 18 deletions libs/filer/filer.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
"context"
"fmt"
"io"
"time"
"io/fs"
)

type WriteMode int
Expand All @@ -14,22 +14,6 @@ const (
CreateParentDirectories = iota << 1
)

// FileInfo abstracts over file information from different file systems.
// Inspired by https://pkg.go.dev/io/fs#FileInfo.
type FileInfo struct {
// The type of the file in workspace.
Type string

// Base name.
Name string

// Size in bytes.
Size int64

// Modification time.
ModTime time.Time
}

type FileAlreadyExistsError struct {
path string
}
Expand Down Expand Up @@ -68,7 +52,7 @@ type Filer interface {
Delete(ctx context.Context, path string) error

// Return contents of directory at `path`.
ReadDir(ctx context.Context, path string) ([]FileInfo, error)
ReadDir(ctx context.Context, path string) ([]fs.DirEntry, error)

// Creates directory at `path`, creating any intermediate directories as required.
Mkdir(ctx context.Context, path string) error
Expand Down
61 changes: 52 additions & 9 deletions libs/filer/workspace_files_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"io"
"io/fs"
"net/http"
"net/url"
"path"
Expand All @@ -20,6 +21,53 @@ import (
"golang.org/x/exp/slices"
)

// Type that implements fs.DirEntry for WSFS.
type wsfsDirEntry struct {
wsfsFileInfo
}

func (entry wsfsDirEntry) Type() fs.FileMode {
return entry.wsfsFileInfo.Mode()
}

func (entry wsfsDirEntry) Info() (fs.FileInfo, error) {
return entry.wsfsFileInfo, nil
}

// Type that implements fs.FileInfo for WSFS.
type wsfsFileInfo struct {
oi workspace.ObjectInfo
}

func (info wsfsFileInfo) Name() string {
return path.Base(info.oi.Path)
}

func (info wsfsFileInfo) Size() int64 {
return info.oi.Size
}

func (info wsfsFileInfo) Mode() fs.FileMode {
switch info.oi.ObjectType {
case workspace.ObjectTypeDirectory:
return fs.ModeDir
default:
return fs.ModePerm
}
}

func (info wsfsFileInfo) ModTime() time.Time {
return time.UnixMilli(info.oi.ModifiedAt)
}

func (info wsfsFileInfo) IsDir() bool {
return info.oi.ObjectType == workspace.ObjectTypeDirectory
}

func (info wsfsFileInfo) Sys() any {
return nil
}

// WorkspaceFilesClient implements the files-in-workspace API.

// NOTE: This API is available for files under /Repos if a workspace has files-in-repos enabled.
Expand Down Expand Up @@ -165,7 +213,7 @@ func (w *WorkspaceFilesClient) Delete(ctx context.Context, name string) error {
return err
}

func (w *WorkspaceFilesClient) ReadDir(ctx context.Context, name string) ([]FileInfo, error) {
func (w *WorkspaceFilesClient) ReadDir(ctx context.Context, name string) ([]fs.DirEntry, error) {
absPath, err := w.root.Join(name)
if err != nil {
return nil, err
Expand All @@ -189,18 +237,13 @@ func (w *WorkspaceFilesClient) ReadDir(ctx context.Context, name string) ([]File
return nil, err
}

info := make([]FileInfo, len(objects))
info := make([]fs.DirEntry, len(objects))
for i, v := range objects {
info[i] = FileInfo{
Type: string(v.ObjectType),
Name: path.Base(v.Path),
Size: v.Size,
ModTime: time.UnixMilli(v.ModifiedAt),
}
info[i] = wsfsDirEntry{wsfsFileInfo{oi: v}}
}

// Sort by name for parity with os.ReadDir.
sort.Slice(info, func(i, j int) bool { return info[i].Name < info[j].Name })
sort.Slice(info, func(i, j int) bool { return info[i].Name() < info[j].Name() })
return info, nil
}

Expand Down