From 9d7fb194af640306c384fc6f647ba630e5850f0d Mon Sep 17 00:00:00 2001 From: meganliu Date: Tue, 5 Jul 2022 10:23:00 -0700 Subject: [PATCH 1/5] implemented List() --- component/azstorage/file_share.go | 142 +++++++++++++++++++++++++----- 1 file changed, 118 insertions(+), 24 deletions(-) diff --git a/component/azstorage/file_share.go b/component/azstorage/file_share.go index 398de75d5..19a8afcf5 100644 --- a/component/azstorage/file_share.go +++ b/component/azstorage/file_share.go @@ -41,6 +41,9 @@ import ( "errors" "net/url" "os" + "path/filepath" + "syscall" + "time" "github.com/Azure/azure-storage-file-go/azfile" ) @@ -149,87 +152,178 @@ func (fs *FileShare) TestPipeline() error { return nil } +// TODOOOOOOOOOOOOOOOOO********** func (fs *FileShare) ListContainers() ([]string, error) { - return nil, nil + return nil, syscall.ENOTSUP } // This is just for test, shall not be used otherwise func (fs *FileShare) SetPrefixPath(string) error { - return nil + return syscall.ENOTSUP } +// TODOOOOOOOOOOOOOOOOO********** func (fs *FileShare) Exists(name string) bool { return false } func (fs *FileShare) CreateFile(name string, mode os.FileMode) error { - return nil + return syscall.ENOTSUP } func (fs *FileShare) CreateDirectory(name string) error { - return nil + return syscall.ENOTSUP } func (fs *FileShare) CreateLink(source string, target string) error { - return nil + return syscall.ENOTSUP } func (fs *FileShare) DeleteFile(name string) error { - return nil + return syscall.ENOTSUP } func (fs *FileShare) DeleteDirectory(name string) error { - return nil + return syscall.ENOTSUP } func (fs *FileShare) RenameFile(string, string) error { - return nil + return syscall.ENOTSUP } func (fs *FileShare) RenameDirectory(string, string) error { - return nil + return syscall.ENOTSUP } +// TODOOOOOOOOOOOOOOOOO********** func (fs *FileShare) GetAttr(name string) (attr *internal.ObjAttr, err error) { - return nil, nil + return nil, syscall.ENOTSUP } -// Standard operations to be supported by any account type +// List : Get a list of blobs matching the given prefix +// This fetches the list using a marker so the caller code should handle marker logic +// If count=0 - fetch max entries func (fs *FileShare) List(prefix string, marker *string, count int32) ([]*internal.ObjAttr, *string, error) { - return nil, nil, nil + log.Trace("FileShare::List : prefix %s, marker %s", prefix, func(marker *string) string { + if marker != nil { + return *marker + } else { + return "" + } + }(marker)) + + fileList := make([]*internal.ObjAttr, 0) + + if count == 0 { + count = common.MaxDirListCount + } + + listPath := filepath.Join(fs.Config.prefixPath, prefix) + + // Get a result segment starting with the file indicated by the current Marker. + var listFile *azfile.ListFilesAndDirectoriesSegmentResponse + var err error + if listPath != "" { + listFile, err = fs.Share.NewDirectoryURL(listPath).ListFilesAndDirectoriesSegment(context.Background(), azfile.Marker{Val: marker}, + azfile.ListFilesAndDirectoriesOptions{MaxResults: count}) + } else { + if (prefix != "" && prefix[len(prefix)-1] == '/') || (prefix == "" && fs.Config.prefixPath != "") { + listPath += "/" + } + listFile, err = fs.Share.NewRootDirectoryURL().ListFilesAndDirectoriesSegment(context.Background(), azfile.Marker{Val: marker}, + azfile.ListFilesAndDirectoriesOptions{MaxResults: count}) + } + + if err != nil { + log.Err("File::List : Failed to list the container with the prefix %s", err.Error) + return fileList, nil, err + } + + // Process the files returned in this result segment (if the segment is empty, the loop body won't execute) + for _, fileInfo := range listFile.FileItems { + attr := &internal.ObjAttr{ + Path: split(fs.Config.prefixPath, fileInfo.Name), + Name: filepath.Base(fileInfo.Name), + Size: fileInfo.Properties.ContentLength, + Mode: 0, + // Azure file SDK supports 2019.02.02 but time and metadata are only supported by 2020.x.x onwards + // TODO: support times when Azure SDK is updated + Mtime: time.Now(), + Atime: time.Now(), + Ctime: time.Now(), + Crtime: time.Now(), + Flags: internal.NewFileBitMap(), + } + + // parseMetadata(attr, fileInfo.Metadata) + // attr.Flags.Set(internal.PropFlagMetadataRetrieved) + attr.Flags.Set(internal.PropFlagModeDefault) + fileList = append(fileList, attr) + + if attr.IsDir() { + attr.Size = 4096 + } + } + + for _, dirInfo := range listFile.DirectoryItems { + attr := &internal.ObjAttr{ + Path: split(fs.Config.prefixPath, dirInfo.Name), + Name: filepath.Base(dirInfo.Name), + // Size: dirInfo.Properties.ContentLength, + Mode: os.ModeDir, + // Azure file SDK supports 2019.02.02 but time, metadata, and dir size are only supported by 2020.x.x onwards + // TODO: support times when Azure SDK is updated + Mtime: time.Now(), + Atime: time.Now(), + Ctime: time.Now(), + Crtime: time.Now(), + Flags: internal.NewDirBitMap(), + } + + // parseMetadata(attr, dirInfo.Metadata) + // attr.Flags.Set(internal.PropFlagMetadataRetrieved) + attr.Flags.Set(internal.PropFlagModeDefault) + fileList = append(fileList, attr) + + if attr.IsDir() { + attr.Size = 4096 + } + } + + return fileList, listFile.NextMarker.Val, nil } func (fs *FileShare) ReadToFile(name string, offset int64, count int64, fi *os.File) error { - return nil + return syscall.ENOTSUP } func (fs *FileShare) ReadBuffer(name string, offset int64, len int64) ([]byte, error) { - return nil, nil + return nil, syscall.ENOTSUP } func (fs *FileShare) ReadInBuffer(name string, offset int64, len int64, data []byte) error { - return nil + return syscall.ENOTSUP } func (fs *FileShare) WriteFromFile(name string, metadata map[string]string, fi *os.File) error { - return nil + return syscall.ENOTSUP } func (fs *FileShare) WriteFromBuffer(name string, metadata map[string]string, data []byte) error { - return nil + return syscall.ENOTSUP } func (fs *FileShare) Write(options internal.WriteFileOptions) error { - return nil + return syscall.ENOTSUP } func (fs *FileShare) GetFileBlockOffsets(name string) (*common.BlockOffsetList, error) { - return nil, nil + return nil, syscall.ENOTSUP } func (fs *FileShare) ChangeMod(string, os.FileMode) error { - return nil + return syscall.ENOTSUP } func (fs *FileShare) ChangeOwner(string, int, int) error { - return nil + return syscall.ENOTSUP } func (fs *FileShare) TruncateFile(string, int64) error { - return nil + return syscall.ENOTSUP } func (fs *FileShare) StageAndCommit(name string, bol *common.BlockOffsetList) error { - return nil + return syscall.ENOTSUP } func (fs *FileShare) NewCredentialKey(_, _ string) error { - return nil + return syscall.ENOTSUP } From a17bb3e9c1bb382ee4f71577ba528b10f3119759 Mon Sep 17 00:00:00 2001 From: meganliu Date: Wed, 6 Jul 2022 13:55:44 -0700 Subject: [PATCH 2/5] implemented GetAttr(), Exists(), and ListShares() --- component/azstorage/file_share.go | 97 ++++++++++++++++++++++++++++--- component/azstorage/utils.go | 15 +++++ 2 files changed, 105 insertions(+), 7 deletions(-) diff --git a/component/azstorage/file_share.go b/component/azstorage/file_share.go index 19a8afcf5..dcc875786 100644 --- a/component/azstorage/file_share.go +++ b/component/azstorage/file_share.go @@ -42,6 +42,7 @@ import ( "net/url" "os" "path/filepath" + "strings" "syscall" "time" @@ -152,9 +153,26 @@ func (fs *FileShare) TestPipeline() error { return nil } -// TODOOOOOOOOOOOOOOOOO********** func (fs *FileShare) ListContainers() ([]string, error) { - return nil, syscall.ENOTSUP + log.Trace("FileShare::ListContainers : Listing containers") + cntList := make([]string, 0) + + marker := azfile.Marker{} + for marker.NotDone() { + resp, err := fs.Service.ListSharesSegment(context.Background(), marker, azfile.ListSharesOptions{}) + if err != nil { + log.Err("FileShare::ListContainers : Failed to get container list") + return cntList, err + } + + for _, v := range resp.ShareItems { + cntList = append(cntList, v.Name) + } + + marker = resp.NextMarker + } + + return cntList, nil } // This is just for test, shall not be used otherwise @@ -162,10 +180,14 @@ func (fs *FileShare) SetPrefixPath(string) error { return syscall.ENOTSUP } -// TODOOOOOOOOOOOOOOOOO********** func (fs *FileShare) Exists(name string) bool { - return false + log.Trace("FileShare::Exists : name %s", name) + if _, err := fs.GetAttr(name); err == syscall.ENOENT { + return false + } + return true } + func (fs *FileShare) CreateFile(name string, mode os.FileMode) error { return syscall.ENOTSUP } @@ -190,9 +212,70 @@ func (fs *FileShare) RenameDirectory(string, string) error { return syscall.ENOTSUP } -// TODOOOOOOOOOOOOOOOOO********** +// GetAttr : Retrieve attributes of a file or directory func (fs *FileShare) GetAttr(name string) (attr *internal.ObjAttr, err error) { - return nil, syscall.ENOTSUP + log.Trace("FileShare::GetAttr : name %s", name) + + path := filepath.Join(fs.Config.prefixPath, name) + splitPath := strings.Split(path, "/") + splitPath = splitPath[:len(splitPath)-1] + joinedPath := strings.Join(splitPath, "/") + + fileURL := fs.Share.NewDirectoryURL(joinedPath).NewFileURL(filepath.Base(name)) + log.Trace("FileShare::GetAttr : getting fileeeeeeeeeeee properties for %s, %s, %s", fs.Config.prefixPath, name, fileURL) + prop, err := fileURL.GetProperties(context.Background()) + + if err == nil { // file + attr = &internal.ObjAttr{ + Path: name, // We don't need to strip the prefixPath here since we pass the input name + Name: filepath.Base(name), + Size: prop.ContentLength(), + Mode: 0, + Mtime: prop.LastModified(), + Atime: prop.LastModified(), + Ctime: prop.LastModified(), + Crtime: prop.LastModified(), + Flags: internal.NewFileBitMap(), + } + parseMetadata(attr, prop.NewMetadata()) + attr.Flags.Set(internal.PropFlagMetadataRetrieved) + attr.Flags.Set(internal.PropFlagModeDefault) + + return attr, nil + } else { // directory + dirURL := fs.Share.NewDirectoryURL(filepath.Join(fs.Config.prefixPath, name)) + log.Trace("FileShare::GetAttr : getting directoryyyyyyyyyyyyyy properties for %s, %s, %s", fs.Config.prefixPath, name, dirURL) + prop, err := dirURL.GetProperties(context.Background()) + + if err == nil { + attr = &internal.ObjAttr{ + Path: name, + Name: filepath.Base(name), + Size: 4096, + Mode: 0, + Mtime: prop.LastModified(), + Atime: prop.LastModified(), + Ctime: prop.LastModified(), + Crtime: prop.LastModified(), + Flags: internal.NewDirBitMap(), + } + parseMetadata(attr, prop.NewMetadata()) + attr.Flags.Set(internal.PropFlagMetadataRetrieved) + attr.Flags.Set(internal.PropFlagModeDefault) + + return attr, nil + } else { // error + e := storeFileErrToErr(err) + if e == ErrFileNotFound { + return attr, syscall.ENOENT + } else { + log.Err("FileShare::GetAttr : Failed to get file/directory properties for %s (%s)", name, err.Error()) + return attr, err + } + } + + } + } // List : Get a list of blobs matching the given prefix @@ -264,7 +347,7 @@ func (fs *FileShare) List(prefix string, marker *string, count int32) ([]*intern attr := &internal.ObjAttr{ Path: split(fs.Config.prefixPath, dirInfo.Name), Name: filepath.Base(dirInfo.Name), - // Size: dirInfo.Properties.ContentLength, + Size: 4096, Mode: os.ModeDir, // Azure file SDK supports 2019.02.02 but time, metadata, and dir size are only supported by 2020.x.x onwards // TODO: support times when Azure SDK is updated diff --git a/component/azstorage/utils.go b/component/azstorage/utils.go index 34718e0da..9a3936110 100644 --- a/component/azstorage/utils.go +++ b/component/azstorage/utils.go @@ -354,6 +354,21 @@ func storeDatalakeErrToErr(err error) uint16 { return ErrNoErr } +// Convert datalake storage error to common errors +func storeFileErrToErr(err error) uint16 { + if serr, ok := err.(azfile.StorageError); ok { + switch serr.ServiceCode() { + case azfile.ServiceCodeShareAlreadyExists: + return ErrFileAlreadyExists + case azfile.ServiceCodeShareNotFound: + return ErrFileNotFound + default: + return ErrUnknown + } + } + return ErrNoErr +} + // ----------- Metadata handling --------------- // Converts datalake properties to a metadata map func newMetadata(properties string) map[string]string { From 42e783f3be0221cd3d754e11e89b4ba6b7c046d8 Mon Sep 17 00:00:00 2001 From: meganliu Date: Mon, 11 Jul 2022 09:35:39 -0700 Subject: [PATCH 3/5] address pr comments--cleanup code --- component/azstorage/file_share.go | 78 ++++++++++++++----------------- component/azstorage/utils.go | 6 +-- 2 files changed, 38 insertions(+), 46 deletions(-) diff --git a/component/azstorage/file_share.go b/component/azstorage/file_share.go index dcc875786..b2f2b066b 100644 --- a/component/azstorage/file_share.go +++ b/component/azstorage/file_share.go @@ -194,6 +194,7 @@ func (fs *FileShare) CreateFile(name string, mode os.FileMode) error { func (fs *FileShare) CreateDirectory(name string) error { return syscall.ENOTSUP } + func (fs *FileShare) CreateLink(source string, target string) error { return syscall.ENOTSUP } @@ -216,16 +217,25 @@ func (fs *FileShare) RenameDirectory(string, string) error { func (fs *FileShare) GetAttr(name string) (attr *internal.ObjAttr, err error) { log.Trace("FileShare::GetAttr : name %s", name) + // retrieve DirectoryURL of file/directory path (everything except file name) + // covers case where name param includes subdirectories and not just the file name path := filepath.Join(fs.Config.prefixPath, name) splitPath := strings.Split(path, "/") splitPath = splitPath[:len(splitPath)-1] joinedPath := strings.Join(splitPath, "/") fileURL := fs.Share.NewDirectoryURL(joinedPath).NewFileURL(filepath.Base(name)) - log.Trace("FileShare::GetAttr : getting fileeeeeeeeeeee properties for %s, %s, %s", fs.Config.prefixPath, name, fileURL) - prop, err := fileURL.GetProperties(context.Background()) + prop, fileerr := fileURL.GetProperties(context.Background()) - if err == nil { // file + if fileerr == nil { // file + ctime, err := time.Parse(time.RFC1123, prop.FileChangeTime()) + if err != nil { + ctime = prop.LastModified() + } + crtime, err := time.Parse(time.RFC1123, prop.FileCreationTime()) + if err != nil { + crtime = prop.LastModified() + } attr = &internal.ObjAttr{ Path: name, // We don't need to strip the prefixPath here since we pass the input name Name: filepath.Base(name), @@ -233,8 +243,8 @@ func (fs *FileShare) GetAttr(name string) (attr *internal.ObjAttr, err error) { Mode: 0, Mtime: prop.LastModified(), Atime: prop.LastModified(), - Ctime: prop.LastModified(), - Crtime: prop.LastModified(), + Ctime: ctime, + Crtime: crtime, Flags: internal.NewFileBitMap(), } parseMetadata(attr, prop.NewMetadata()) @@ -242,12 +252,19 @@ func (fs *FileShare) GetAttr(name string) (attr *internal.ObjAttr, err error) { attr.Flags.Set(internal.PropFlagModeDefault) return attr, nil - } else { // directory + } else if storeFileErrToErr(fileerr) == ErrFileNotFound { // directory dirURL := fs.Share.NewDirectoryURL(filepath.Join(fs.Config.prefixPath, name)) - log.Trace("FileShare::GetAttr : getting directoryyyyyyyyyyyyyy properties for %s, %s, %s", fs.Config.prefixPath, name, dirURL) - prop, err := dirURL.GetProperties(context.Background()) + prop, direrr := dirURL.GetProperties(context.Background()) - if err == nil { + if direrr == nil { + ctime, err := time.Parse(time.RFC1123, prop.FileChangeTime()) + if err != nil { + ctime = prop.LastModified() + } + crtime, err := time.Parse(time.RFC1123, prop.FileCreationTime()) + if err != nil { + crtime = prop.LastModified() + } attr = &internal.ObjAttr{ Path: name, Name: filepath.Base(name), @@ -255,8 +272,8 @@ func (fs *FileShare) GetAttr(name string) (attr *internal.ObjAttr, err error) { Mode: 0, Mtime: prop.LastModified(), Atime: prop.LastModified(), - Ctime: prop.LastModified(), - Crtime: prop.LastModified(), + Ctime: ctime, + Crtime: crtime, Flags: internal.NewDirBitMap(), } parseMetadata(attr, prop.NewMetadata()) @@ -264,21 +281,15 @@ func (fs *FileShare) GetAttr(name string) (attr *internal.ObjAttr, err error) { attr.Flags.Set(internal.PropFlagModeDefault) return attr, nil - } else { // error - e := storeFileErrToErr(err) - if e == ErrFileNotFound { - return attr, syscall.ENOENT - } else { - log.Err("FileShare::GetAttr : Failed to get file/directory properties for %s (%s)", name, err.Error()) - return attr, err - } } - + return attr, syscall.ENOENT } - + // error + log.Err("FileShare::GetAttr : Failed to get file/directory properties for %s (%s)", name, err.Error()) + return attr, fileerr } -// List : Get a list of blobs matching the given prefix +// List : Get a list of files/directories matching the given prefix // This fetches the list using a marker so the caller code should handle marker logic // If count=0 - fetch max entries func (fs *FileShare) List(prefix string, marker *string, count int32) ([]*internal.ObjAttr, *string, error) { @@ -298,19 +309,8 @@ func (fs *FileShare) List(prefix string, marker *string, count int32) ([]*intern listPath := filepath.Join(fs.Config.prefixPath, prefix) - // Get a result segment starting with the file indicated by the current Marker. - var listFile *azfile.ListFilesAndDirectoriesSegmentResponse - var err error - if listPath != "" { - listFile, err = fs.Share.NewDirectoryURL(listPath).ListFilesAndDirectoriesSegment(context.Background(), azfile.Marker{Val: marker}, - azfile.ListFilesAndDirectoriesOptions{MaxResults: count}) - } else { - if (prefix != "" && prefix[len(prefix)-1] == '/') || (prefix == "" && fs.Config.prefixPath != "") { - listPath += "/" - } - listFile, err = fs.Share.NewRootDirectoryURL().ListFilesAndDirectoriesSegment(context.Background(), azfile.Marker{Val: marker}, - azfile.ListFilesAndDirectoriesOptions{MaxResults: count}) - } + listFile, err := fs.Share.NewDirectoryURL(listPath).ListFilesAndDirectoriesSegment(context.Background(), azfile.Marker{Val: marker}, + azfile.ListFilesAndDirectoriesOptions{MaxResults: count}) if err != nil { log.Err("File::List : Failed to list the container with the prefix %s", err.Error) @@ -333,8 +333,6 @@ func (fs *FileShare) List(prefix string, marker *string, count int32) ([]*intern Flags: internal.NewFileBitMap(), } - // parseMetadata(attr, fileInfo.Metadata) - // attr.Flags.Set(internal.PropFlagMetadataRetrieved) attr.Flags.Set(internal.PropFlagModeDefault) fileList = append(fileList, attr) @@ -358,14 +356,8 @@ func (fs *FileShare) List(prefix string, marker *string, count int32) ([]*intern Flags: internal.NewDirBitMap(), } - // parseMetadata(attr, dirInfo.Metadata) - // attr.Flags.Set(internal.PropFlagMetadataRetrieved) attr.Flags.Set(internal.PropFlagModeDefault) fileList = append(fileList, attr) - - if attr.IsDir() { - attr.Size = 4096 - } } return fileList, listFile.NextMarker.Val, nil diff --git a/component/azstorage/utils.go b/component/azstorage/utils.go index 9a3936110..477d163d5 100644 --- a/component/azstorage/utils.go +++ b/component/azstorage/utils.go @@ -354,13 +354,13 @@ func storeDatalakeErrToErr(err error) uint16 { return ErrNoErr } -// Convert datalake storage error to common errors +// Convert file storage error to common errors func storeFileErrToErr(err error) uint16 { if serr, ok := err.(azfile.StorageError); ok { switch serr.ServiceCode() { - case azfile.ServiceCodeShareAlreadyExists: + case azfile.ServiceCodeResourceAlreadyExists: return ErrFileAlreadyExists - case azfile.ServiceCodeShareNotFound: + case azfile.ServiceCodeResourceNotFound: return ErrFileNotFound default: return ErrUnknown From 3041a0903800cda9edd98f6c4636b4557dcaac06 Mon Sep 17 00:00:00 2001 From: meganliu Date: Mon, 11 Jul 2022 09:38:53 -0700 Subject: [PATCH 4/5] implement SetPrefixPath() --- component/azstorage/file_share.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/component/azstorage/file_share.go b/component/azstorage/file_share.go index b2f2b066b..4a9485ed8 100644 --- a/component/azstorage/file_share.go +++ b/component/azstorage/file_share.go @@ -176,8 +176,10 @@ func (fs *FileShare) ListContainers() ([]string, error) { } // This is just for test, shall not be used otherwise -func (fs *FileShare) SetPrefixPath(string) error { - return syscall.ENOTSUP +func (fs *FileShare) SetPrefixPath(path string) error { + log.Trace("FileShare::SetPrefixPath : path %s", path) + fs.Config.prefixPath = path + return nil } func (fs *FileShare) Exists(name string) bool { From 56ffcd95dcd4af041f39031424e68c66499c9c6b Mon Sep 17 00:00:00 2001 From: meganliu-msft <107150240+meganliu-msft@users.noreply.github.com> Date: Mon, 11 Jul 2022 13:11:51 -0700 Subject: [PATCH 5/5] update log err Co-authored-by: Sourav Gupta <98318303+souravgupta-msft@users.noreply.github.com> --- component/azstorage/file_share.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/component/azstorage/file_share.go b/component/azstorage/file_share.go index 4a9485ed8..3183e7217 100644 --- a/component/azstorage/file_share.go +++ b/component/azstorage/file_share.go @@ -287,7 +287,7 @@ func (fs *FileShare) GetAttr(name string) (attr *internal.ObjAttr, err error) { return attr, syscall.ENOENT } // error - log.Err("FileShare::GetAttr : Failed to get file/directory properties for %s (%s)", name, err.Error()) + log.Err("FileShare::GetAttr : Failed to get file/directory properties for %s (%s)", name, fileerr.Error()) return attr, fileerr }