Merge pull request #26 from wolfeidau/chore_added_support_for_remove_…

…file

feat(remove): added support for removing files via an extension

README.md

@@ -6,7 +6,7 @@ This package provides an S3 implementation for [Go1.16 filesystem interface](htt
 
 This package provides an S3 implementation for the Go1.16 filesystem interface using the [AWS SDK for Go v2](https://github.com/aws/aws-sdk-go-v2).
 
-The `S3FS` is currently read-only, and implements the following interfaces:
+The `S3FS` implements the following interfaces:
 
 - `fs.FS`
 - `fs.StatFS`
@@ -19,6 +19,10 @@ The `s3File` implements the following interfaces:
 - `io.ReaderAt`
 - `io.Seeker`
 
+In addition to this the `S3FS` also implements the following interfaces:
+
+- `RemoveFS`, which provides a `Remove(name string) error` method.
+
 This enables libraries such as [apache arrow](https://arrow.apache.org/) to read parts of a parquet file from S3, without downloading the entire file.
 # Usage 
 

integration/s3fs_test.go

@@ -94,6 +94,8 @@ func TestSeek(t *testing.T) {
  s3fs := s3iofs.NewWithClient(testBucketName, client)
 
  t.Run("seek to start", func(t *testing.T) {
+ assert := require.New(t)
+
  f, err := s3fs.Open("test_seek.txt")
  assert.NoError(err)
 
@@ -112,6 +114,8 @@ func TestSeek(t *testing.T) {
  })
 
  t.Run("seek to end", func(t *testing.T) {
+ assert := require.New(t)
+
  f, err := s3fs.Open("test_seek.txt")
  assert.NoError(err)
  defer f.Close()
@@ -124,6 +128,8 @@ func TestSeek(t *testing.T) {
  })
 
  t.Run("seek to current", func(t *testing.T) {
+ assert := require.New(t)
+
  f, err := s3fs.Open("test_seek.txt")
  assert.NoError(err)
  defer f.Close()
@@ -237,3 +243,49 @@ func TestReadBigEOF(t *testing.T) {
  assert.ErrorIs(err, io.ErrUnexpectedEOF)
  assert.Equal(oneMegabyte, n)
 }
+
+func TestRemove(t *testing.T) {
+
+ t.Run("create and remove", func(t *testing.T) {
+ assert := require.New(t)
+
+ _, err := client.PutObject(context.Background(), &s3.PutObjectInput{
+ Bucket: aws.String(testBucketName),
+ Key: aws.String("test_remove.txt"),
+ Body: bytes.NewReader(generateData(oneMegabyte)),
+ })
+ assert.NoError(err)
+
+ s3fs := s3iofs.NewWithClient(testBucketName, client)
+
+ err = s3fs.Remove("test_remove.txt")
+ assert.NoError(err)
+ })
+
+ t.Run("removing non existent file should not error", func(t *testing.T) {
+ assert := require.New(t)
+
+ s3fs := s3iofs.NewWithClient(testBucketName, client)
+
+ err := s3fs.Remove("test_remove_not_exists.txt")
+ assert.NoError(err)
+ })
+
+ t.Run("bad bucket name should error", func(t *testing.T) {
+ assert := require.New(t)
+
+ s3fs := s3iofs.NewWithClient("badbucket", client)
+
+ err := s3fs.Remove("test_remove_not_exists.txt")
+ assert.Error(err)
+ })
+
+ t.Run("invalid name should error", func(t *testing.T) {
+ assert := require.New(t)
+
+ s3fs := s3iofs.NewWithClient(testBucketName, client)
+
+ err := s3fs.Remove("")
+ assert.Error(err)
+ })
+}

s3.go

@@ -11,4 +11,5 @@ type S3API interface {
  GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error)
  ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error)
  HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error)
+ DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error)
 }

s3file_test.go

@@ -37,6 +37,11 @@ func (m *mockS3Client) ListObjectsV2(ctx context.Context, params *s3.ListObjects
  return args.Get(0).(*s3.ListObjectsV2Output), args.Error(1)
 }
 
+func (m *mockS3Client) DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) {
+ args := m.Called(ctx, params, optFns)
+ return args.Get(0).(*s3.DeleteObjectOutput), args.Error(1)
+}
+
 func TestReadFile(t *testing.T) {
  type args struct {
  bucket string

s3iofs.go

@@ -16,8 +16,15 @@ var (
  _ fs.FS = (*S3FS)(nil)
  _ fs.StatFS = (*S3FS)(nil)
  _ fs.ReadDirFS = (*S3FS)(nil)
+ _ RemoveFS = (*S3FS)(nil)
 )
 
+// RemoveFS extend the fs.FS interface to add the Remove method.
+type RemoveFS interface {
+ fs.FS
+ Remove(name string) error
+}
+
 // S3FS is a filesystem implementation using S3.
 type S3FS struct {
  bucket string
@@ -155,6 +162,25 @@ func (s3fs *S3FS) ReadDir(name string) ([]fs.DirEntry, error) {
  return entries, nil
 }
 
+func (s3fs *S3FS) Remove(name string) error {
+ if name == "." {
+ return &fs.PathError{Op: "remove", Path: name, Err: fs.ErrInvalid}
+ }
+ if name == "" {
+ return &fs.PathError{Op: "remove", Path: name, Err: fs.ErrInvalid}
+ }
+
+ _, err := s3fs.s3client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
+ Bucket: aws.String(s3fs.bucket),
+ Key: aws.String(name),
+ })
+ if err != nil {
+ return &fs.PathError{Op: "remove", Path: name, Err: err}
+ }
+
+ return nil
+}
+
 func (s3fs *S3FS) stat(name string) (fs.FileInfo, error) {
  if name == "." {
  return &s3File{