Adding support for S3 prefix

config.toml.example

@@ -26,6 +26,7 @@ path = "test"
   endpoint = "https://s3.us-west-2.amazonaws.com"
   region = "us-west-2"
   bucket = "example-bucket-name"
+  prefix = "example/prefix"
 
 # API keys to be used to access Youtube and Vimeo.
 # These can be either specified as string parameter or array of string (so those will be rotated).

pkg/fs/s3.go

@@ -24,13 +24,24 @@ type S3Config struct {
 	Region string `toml:"region"`
 	// EndpointURL is an HTTP endpoint of the S3 API
 	EndpointURL string `toml:"endpoint_url"`
+	// Prefix is a prefix (subfolder) to use to build key names
+	Prefix string `toml:"prefix"`
 }
 
 // S3 implements file storage for S3-compatible providers.
 type S3 struct {
 	api      s3iface.S3API
 	uploader *s3manager.Uploader
 	bucket   string
+	prefix   string
+}
+
+func buildKey(name string, s *S3) string {
+	if len(s.prefix) == 0 {
+		return name
+	}
+	result := s.prefix + "/" + name
+	return result
 }
 
 func NewS3(c S3Config) (*S3, error) {
@@ -47,6 +58,7 @@ func NewS3(c S3Config) (*S3, error) {
 		api:      s3.New(sess),
 		uploader: s3manager.NewUploader(sess),
 		bucket:   c.Bucket,
+		prefix:   c.Prefix,
 	}, nil
 }
 
@@ -55,21 +67,23 @@ func (s *S3) Open(_name string) (http.File, error) {
 }
 
 func (s *S3) Delete(ctx context.Context, name string) error {
+	key := buildKey(name, s)
 	_, err := s.api.DeleteObjectWithContext(ctx, &s3.DeleteObjectInput{
 		Bucket: &s.bucket,
-		Key:    &name,
+		Key:    &key,
 	})
 	return err
 }
 
 func (s *S3) Create(ctx context.Context, name string, reader io.Reader) (int64, error) {
-	logger := log.WithField("name", name)
+	key := buildKey(name, s)
+	logger := log.WithField("key", key)
 
 	logger.Infof("uploading file to %s", s.bucket)
 	r := &readerWithN{Reader: reader}
 	_, err := s.uploader.UploadWithContext(ctx, &s3manager.UploadInput{
 		Bucket: &s.bucket,
-		Key:    &name,
+		Key:    &key,
 		Body:   r,
 	})
 	if err != nil {
@@ -81,12 +95,13 @@ func (s *S3) Create(ctx context.Context, name string, reader io.Reader) (int64,
 }
 
 func (s *S3) Size(ctx context.Context, name string) (int64, error) {
-	logger := log.WithField("name", name)
+	key := buildKey(name, s)
+	logger := log.WithField("key", key)
 
 	logger.Debugf("getting file size from %s", s.bucket)
 	resp, err := s.api.HeadObjectWithContext(ctx, &s3.HeadObjectInput{
 		Bucket: &s.bucket,
-		Key:    &name,
+		Key:    &key,
 	})
 	if err != nil {
 		if awsErr, ok := err.(awserr.Error); ok {

pkg/fs/s3_test.go

@@ -18,7 +18,7 @@ import (
 
 func TestS3_Create(t *testing.T) {
 	files := make(map[string][]byte)
-	stor, err := newMockS3(files)
+	stor, err := newMockS3(files, "")
 	assert.NoError(t, err)
 
 	written, err := stor.Create(testCtx, "1/test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3}))
@@ -32,7 +32,7 @@ func TestS3_Create(t *testing.T) {
 
 func TestS3_Size(t *testing.T) {
 	files := make(map[string][]byte)
-	stor, err := newMockS3(files)
+	stor, err := newMockS3(files, "")
 	assert.NoError(t, err)
 
 	_, err = stor.Create(testCtx, "1/test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3}))
@@ -45,7 +45,7 @@ func TestS3_Size(t *testing.T) {
 
 func TestS3_NoSize(t *testing.T) {
 	files := make(map[string][]byte)
-	stor, err := newMockS3(files)
+	stor, err := newMockS3(files, "")
 	assert.NoError(t, err)
 
 	_, err = stor.Size(testCtx, "1/test")
@@ -54,7 +54,7 @@ func TestS3_NoSize(t *testing.T) {
 
 func TestS3_Delete(t *testing.T) {
 	files := make(map[string][]byte)
-	stor, err := newMockS3(files)
+	stor, err := newMockS3(files, "")
 	assert.NoError(t, err)
 
 	_, err = stor.Create(testCtx, "1/test", bytes.NewBuffer([]byte{1, 5, 7, 8, 3}))
@@ -70,17 +70,30 @@ func TestS3_Delete(t *testing.T) {
 	assert.False(t, ok)
 }
 
+func TestS3_BuildKey(t *testing.T) {
+	files := make(map[string][]byte)
+
+	stor, _ := newMockS3(files, "")
+	key := buildKey("test-fn", stor)
+	assert.EqualValues(t, "test-fn", key)
+
+	stor, _ = newMockS3(files, "mock-prefix")
+	key = buildKey("test-fn", stor)
+	assert.EqualValues(t, "mock-prefix/test-fn", key)
+}
+
 type mockS3API struct {
 	s3iface.S3API
 	files map[string][]byte
 }
 
-func newMockS3(files map[string][]byte) (*S3, error) {
+func newMockS3(files map[string][]byte, prefix string) (*S3, error) {
 	api := &mockS3API{files: files}
 	return &S3{
 		api:      api,
 		uploader: s3manager.NewUploaderWithClient(api),
 		bucket:   "mock-bucket",
+		prefix:   prefix,
 	}, nil
 }