Skip to content

Commit

Permalink
GCS fs: move all gcsfs related implementations to its own package
Browse files Browse the repository at this point in the history 
this way we don't force any application that import afero to include
gcfs deps in its binary
  • Loading branch information
@drakkan
drakkan committed Dec 27, 2021
1 parent d70f944 commit 165e3dc
Show file tree
Hide file tree
Showing 8 changed files with 65 additions and 68 deletions.
2 changes: 1 addition & 1 deletion gcsfs/file.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ type GcsFile struct {

func NewGcsFile(
ctx context.Context,
fs *GcsFs,
fs *Fs,
obj stiface.ObjectHandle,
openFlags int,
// Unused: there is no use to the file mode in GCloud just yet - but we keep it here, just in case we need it
Expand Down
2 changes: 1 addition & 1 deletion gcsfs/file_info.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ type FileInfo struct {
fileMode os.FileMode
}

func newFileInfo(name string, fs *GcsFs, fileMode os.FileMode) (*FileInfo, error) {
func newFileInfo(name string, fs *Fs, fileMode os.FileMode) (*FileInfo, error) {
res := &FileInfo{
name: name,
size: folderSize,
Expand Down
2 changes: 1 addition & 1 deletion gcsfs/file_resource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ const (
type gcsFileResource struct {
ctx context.Context

fs *GcsFs
fs *Fs

obj stiface.ObjectHandle
name string
Expand Down
48 changes: 24 additions & 24 deletions gcsfs/fs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ const (
gsPrefix = "gs://"
)

// GcsFs is a Fs implementation that uses functions provided by google cloud storage
type GcsFs struct {
// Fs is a Fs implementation that uses functions provided by google cloud storage
type Fs struct {
ctx context.Context
client stiface.Client
separator string
Expand All @@ -45,12 +45,12 @@ type GcsFs struct {
autoRemoveEmptyFolders bool //trigger for creating "virtual folders" (not required by GCSs)
}

func NewGcsFs(ctx context.Context, client stiface.Client) *GcsFs {
func NewGcsFs(ctx context.Context, client stiface.Client) *Fs {
return NewGcsFsWithSeparator(ctx, client, "/")
}

func NewGcsFsWithSeparator(ctx context.Context, client stiface.Client, folderSep string) *GcsFs {
return &GcsFs{
func NewGcsFsWithSeparator(ctx context.Context, client stiface.Client, folderSep string) *Fs {
return &Fs{
ctx: ctx,
client: client,
separator: folderSep,
Expand All @@ -61,17 +61,17 @@ func NewGcsFsWithSeparator(ctx context.Context, client stiface.Client, folderSep
}

// normSeparators will normalize all "\\" and "/" to the provided separator
func (fs *GcsFs) normSeparators(s string) string {
func (fs *Fs) normSeparators(s string) string {
return strings.Replace(strings.Replace(s, "\\", fs.separator, -1), "/", fs.separator, -1)
}

func (fs *GcsFs) ensureTrailingSeparator(s string) string {
func (fs *Fs) ensureTrailingSeparator(s string) string {
if len(s) > 0 && !strings.HasSuffix(s, fs.separator) {
return s + fs.separator
}
return s
}
func (fs *GcsFs) ensureNoLeadingSeparator(s string) string {
func (fs *Fs) ensureNoLeadingSeparator(s string) string {
if len(s) > 0 && strings.HasPrefix(s, fs.separator) {
s = s[len(fs.separator):]
}
Expand All @@ -94,13 +94,13 @@ func validateName(s string) error {
}

// Splits provided name into bucket name and path
func (fs *GcsFs) splitName(name string) (bucketName string, path string) {
func (fs *Fs) splitName(name string) (bucketName string, path string) {
splitName := strings.Split(name, fs.separator)

return splitName[0], strings.Join(splitName[1:], fs.separator)
}

func (fs *GcsFs) getBucket(name string) (stiface.BucketHandle, error) {
func (fs *Fs) getBucket(name string) (stiface.BucketHandle, error) {
bucket := fs.buckets[name]
if bucket == nil {
bucket = fs.client.Bucket(name)
Expand All @@ -112,7 +112,7 @@ func (fs *GcsFs) getBucket(name string) (stiface.BucketHandle, error) {
return bucket, nil
}

func (fs *GcsFs) getObj(name string) (stiface.ObjectHandle, error) {
func (fs *Fs) getObj(name string) (stiface.ObjectHandle, error) {
bucketName, path := fs.splitName(name)

bucket, err := fs.getBucket(bucketName)
Expand All @@ -123,9 +123,9 @@ func (fs *GcsFs) getObj(name string) (stiface.ObjectHandle, error) {
return bucket.Object(path), nil
}

func (fs *GcsFs) Name() string { return "GcsFs" }
func (fs *Fs) Name() string { return "GcsFs" }

func (fs *GcsFs) Create(name string) (*GcsFile, error) {
func (fs *Fs) Create(name string) (*GcsFile, error) {
name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
if err := validateName(name); err != nil {
return nil, err
Expand Down Expand Up @@ -156,7 +156,7 @@ func (fs *GcsFs) Create(name string) (*GcsFile, error) {
return file, nil
}

func (fs *GcsFs) Mkdir(name string, _ os.FileMode) error {
func (fs *Fs) Mkdir(name string, _ os.FileMode) error {
name = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(name))))
if err := validateName(name); err != nil {
return err
Expand All @@ -179,7 +179,7 @@ func (fs *GcsFs) Mkdir(name string, _ os.FileMode) error {
return w.Close()
}

func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error {
func (fs *Fs) MkdirAll(path string, perm os.FileMode) error {
path = fs.ensureNoLeadingSeparator(fs.ensureTrailingSeparator(fs.normSeparators(ensureNoPrefix(path))))
if err := validateName(path); err != nil {
return err
Expand Down Expand Up @@ -216,11 +216,11 @@ func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error {
return nil
}

func (fs *GcsFs) Open(name string) (*GcsFile, error) {
func (fs *Fs) Open(name string) (*GcsFile, error) {
return fs.OpenFile(name, os.O_RDONLY, 0)
}

func (fs *GcsFs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile, error) {
func (fs *Fs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile, error) {
var file *GcsFile
var err error

Expand Down Expand Up @@ -277,7 +277,7 @@ func (fs *GcsFs) OpenFile(name string, flag int, fileMode os.FileMode) (*GcsFile
return file, nil
}

func (fs *GcsFs) Remove(name string) error {
func (fs *Fs) Remove(name string) error {
name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
if err := validateName(name); err != nil {
return err
Expand Down Expand Up @@ -318,7 +318,7 @@ func (fs *GcsFs) Remove(name string) error {
return obj.Delete(fs.ctx)
}

func (fs *GcsFs) RemoveAll(path string) error {
func (fs *Fs) RemoveAll(path string) error {
path = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(path)))
if err := validateName(path); err != nil {
return err
Expand Down Expand Up @@ -351,7 +351,7 @@ func (fs *GcsFs) RemoveAll(path string) error {
return fs.Remove(path)
}

func (fs *GcsFs) Rename(oldName, newName string) error {
func (fs *Fs) Rename(oldName, newName string) error {
oldName = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(oldName)))
if err := validateName(oldName); err != nil {
return err
Expand All @@ -378,7 +378,7 @@ func (fs *GcsFs) Rename(oldName, newName string) error {
return src.Delete(fs.ctx)
}

func (fs *GcsFs) Stat(name string) (os.FileInfo, error) {
func (fs *Fs) Stat(name string) (os.FileInfo, error) {
name = fs.ensureNoLeadingSeparator(fs.normSeparators(ensureNoPrefix(name)))
if err := validateName(name); err != nil {
return nil, err
Expand All @@ -387,14 +387,14 @@ func (fs *GcsFs) Stat(name string) (os.FileInfo, error) {
return newFileInfo(name, fs, defaultFileMode)
}

func (fs *GcsFs) Chmod(_ string, _ os.FileMode) error {
func (fs *Fs) Chmod(_ string, _ os.FileMode) error {
return errors.New("method Chmod is not implemented in GCS")
}

func (fs *GcsFs) Chtimes(_ string, _, _ time.Time) error {
func (fs *Fs) Chtimes(_ string, _, _ time.Time) error {
return errors.New("method Chtimes is not implemented. Create, Delete, Updated times are read only fields in GCS and set implicitly")
}

func (fs *GcsFs) Chown(_ string, _, _ int) error {
func (fs *Fs) Chown(_ string, _, _ int) error {
return errors.New("method Chown is not implemented for GCS")
}
0 gcs-fake-service-account.json → gcsfs/gcs-fake-service-account.json
View file
File renamed without changes.
25 changes: 12 additions & 13 deletions gcs.go → gcsfs/gcs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,29 +14,28 @@
// See the License for the specific language governing permissions and
// limitations under the License.

package afero
package gcsfs

import (
"context"
"os"
"time"

"github.com/spf13/afero/gcsfs"

"cloud.google.com/go/storage"
"github.com/googleapis/google-cloud-go-testing/storage/stiface"
"github.com/spf13/afero"

"google.golang.org/api/option"
)

type GcsFs struct {
source *gcsfs.GcsFs
source *Fs
}

// NewGcsFS creates a GCS file system, automatically instantiating and decorating the storage client.
// You can provide additional options to be passed to the client creation, as per
// cloud.google.com/go/storage documentation
func NewGcsFS(ctx context.Context, opts ...option.ClientOption) (Fs, error) {
func NewGcsFS(ctx context.Context, opts ...option.ClientOption) (afero.Fs, error) {
if json := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON"); json != "" {
opts = append(opts, option.WithCredentialsJSON([]byte(json)))
}
Expand All @@ -49,7 +48,7 @@ func NewGcsFS(ctx context.Context, opts ...option.ClientOption) (Fs, error) {
}

// NewGcsFSWithSeparator is the same as NewGcsFS, but the files system will use the provided folder separator.
func NewGcsFSWithSeparator(ctx context.Context, folderSeparator string, opts ...option.ClientOption) (Fs, error) {
func NewGcsFSWithSeparator(ctx context.Context, folderSeparator string, opts ...option.ClientOption) (afero.Fs, error) {
client, err := storage.NewClient(ctx, opts...)
if err != nil {
return nil, err
Expand All @@ -59,25 +58,25 @@ func NewGcsFSWithSeparator(ctx context.Context, folderSeparator string, opts ...
}

// NewGcsFSFromClient creates a GCS file system from a given storage client
func NewGcsFSFromClient(ctx context.Context, client *storage.Client) (Fs, error) {
func NewGcsFSFromClient(ctx context.Context, client *storage.Client) (afero.Fs, error) {
c := stiface.AdaptClient(client)

return &GcsFs{gcsfs.NewGcsFs(ctx, c)}, nil
return &GcsFs{NewGcsFs(ctx, c)}, nil
}

// NewGcsFSFromClientWithSeparator is the same as NewGcsFSFromClient, but the file system will use the provided folder separator.
func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client, folderSeparator string) (Fs, error) {
func NewGcsFSFromClientWithSeparator(ctx context.Context, client *storage.Client, folderSeparator string) (afero.Fs, error) {
c := stiface.AdaptClient(client)

return &GcsFs{gcsfs.NewGcsFsWithSeparator(ctx, c, folderSeparator)}, nil
return &GcsFs{NewGcsFsWithSeparator(ctx, c, folderSeparator)}, nil
}

// Wraps gcs.GcsFs and convert some return types to afero interfaces.

func (fs *GcsFs) Name() string {
return fs.source.Name()
}
func (fs *GcsFs) Create(name string) (File, error) {
func (fs *GcsFs) Create(name string) (afero.File, error) {
return fs.source.Create(name)
}
func (fs *GcsFs) Mkdir(name string, perm os.FileMode) error {
Expand All @@ -86,10 +85,10 @@ func (fs *GcsFs) Mkdir(name string, perm os.FileMode) error {
func (fs *GcsFs) MkdirAll(path string, perm os.FileMode) error {
return fs.source.MkdirAll(path, perm)
}
func (fs *GcsFs) Open(name string) (File, error) {
func (fs *GcsFs) Open(name string) (afero.File, error) {
return fs.source.Open(name)
}
func (fs *GcsFs) OpenFile(name string, flag int, perm os.FileMode) (File, error) {
func (fs *GcsFs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
return fs.source.OpenFile(name, flag, perm)
}
func (fs *GcsFs) Remove(name string) error {
Expand Down
Loading

0 comments on commit 165e3dc

Please sign in to comment.