Add s3 artifact type and refactor common methods

api/pkg/artifact/artifact.go

@@ -1,13 +1,17 @@
 package artifact
 
 import (
+	"bytes"
 	"context"
 	"fmt"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"io"
 	"net/url"
 	"strings"
 
 	"cloud.google.com/go/storage"
+	"github.com/aws/aws-sdk-go-v2/config"
 	"google.golang.org/api/iterator"
 )
 
@@ -23,33 +27,26 @@ type URL struct {
 	Object string
 }
 
-type Service interface {
-	ParseURL(gsURL string) (*URL, error)
-
-	ReadArtifact(ctx context.Context, url string) ([]byte, error)
-	WriteArtifact(ctx context.Context, url string, content []byte) error
-	DeleteArtifact(ctx context.Context, url string) error
-}
+type URLScheme string
 
-type GcsArtifactClient struct {
-	API *storage.Client
+type SchemeInterface interface {
+	GetURLScheme() string
+	ParseURL(gsURL string) (*URL, error)
 }
 
-func NewGcsArtifactClient(api *storage.Client) Service {
-	return &GcsArtifactClient{
-		API: api,
-	}
+func (urlScheme URLScheme) GetURLScheme() string {
+	return string(urlScheme)
 }
 
-// Parse parses a Google Cloud Storage string into a URL struct. The expected
-// format of the string is gs://[bucket-name]/[object-path]. If the provided
+// ParseURL parses an artifact storage string into a URL struct. The expected
+// format of the string is [url-scheme]://[bucket-name]/[object-path]. If the provided
 // URL is formatted incorrectly an error will be returned.
-func (gac *GcsArtifactClient) ParseURL(gsURL string) (*URL, error) {
+func (urlScheme URLScheme) ParseURL(gsURL string) (*URL, error) {
 	u, err := url.Parse(gsURL)
 	if err != nil {
 		return nil, err
 	}
-	if u.Scheme != "gs" {
+	if u.Scheme != string(urlScheme) {
 		return nil, err
 	}
 
@@ -69,6 +66,28 @@ func (gac *GcsArtifactClient) ParseURL(gsURL string) (*URL, error) {
 	}, nil
 }
 
+type Service interface {
+	ReadArtifact(ctx context.Context, url string) ([]byte, error)
+	WriteArtifact(ctx context.Context, url string, content []byte) error
+	DeleteArtifact(ctx context.Context, url string) error
+}
+
+type GcsArtifactClient struct {
+	URLScheme
+	API *storage.Client
+}
+
+func NewGcsArtifactClient() (Service, error) {
+	api, err := storage.NewClient(context.Background())
+	if err != nil {
+		return nil, fmt.Errorf("%s,failed initializing gcs for the artifact client", err.Error())
+	}
+	return &GcsArtifactClient{
+		URLScheme: "gs",
+		API:       api,
+	}, nil
+}
+
 func (gac *GcsArtifactClient) ReadArtifact(ctx context.Context, url string) ([]byte, error) {
 	u, err := gac.ParseURL(url)
 	if err != nil {
@@ -133,24 +152,96 @@ func (gac *GcsArtifactClient) DeleteArtifact(ctx context.Context, url string) er
 	return nil
 }
 
+type S3ArtifactClient struct {
+	URLScheme
+	client *s3.Client
+}
+
+func NewS3ArtifactClient() (Service, error) {
+	cfg, err := config.LoadDefaultConfig(context.Background())
+	if err != nil {
+		return nil, fmt.Errorf("%s,failed loading s3 config for the artifact client", err.Error())
+	}
+	client := s3.NewFromConfig(cfg)
+	return &S3ArtifactClient{
+		URLScheme: "s3",
+		client:    client,
+	}, nil
+}
+
+func (s3c *S3ArtifactClient) ReadArtifact(ctx context.Context, url string) ([]byte, error) {
+	u, err := s3c.ParseURL(url)
+	if err != nil {
+		return nil, err
+	}
+
+	reader, err := s3c.client.GetObject(ctx, &s3.GetObjectInput{
+		Bucket: aws.String(u.Bucket),
+		Key:    aws.String(u.Object),
+	})
+	if err != nil {
+		return nil, err
+	}
+	defer reader.Body.Close() //nolint:errcheck
+
+	bytes, err := io.ReadAll(reader.Body)
+	if err != nil {
+		return nil, err
+	}
+	return bytes, nil
+}
+
+func (s3c *S3ArtifactClient) WriteArtifact(ctx context.Context, url string, content []byte) error {
+	u, err := s3c.ParseURL(url)
+	if err != nil {
+		return err
+	}
+
+	_, err = s3c.client.PutObject(ctx, &s3.PutObjectInput{
+		Bucket: aws.String(u.Bucket),
+		Key:    aws.String(u.Object),
+		Body:   bytes.NewReader(content),
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s3c *S3ArtifactClient) DeleteArtifact(ctx context.Context, url string) error {
+	u, err := s3c.ParseURL(url)
+	if err != nil {
+		return err
+	}
+
+	// TODO: To confirm and refactor if versioning is enabled on S3 and to specify the versionId to be deleted
+	input := &s3.DeleteObjectInput{
+		Bucket: aws.String(u.Bucket),
+		Key:    aws.String(u.Object),
+	}
+
+	_, err = s3c.client.DeleteObject(ctx, input)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 type NopArtifactClient struct{}
 
 func NewNopArtifactClient() Service {
 	return &NopArtifactClient{}
 }
 
-func (nac *NopArtifactClient) ParseURL(gsURL string) (*URL, error) {
-	return nil, nil
-}
-
-func (nac *NopArtifactClient) ReadArtifact(ctx context.Context, url string) ([]byte, error) {
+func (nac *NopArtifactClient) ReadArtifact(_ context.Context, _ string) ([]byte, error) {
 	return nil, nil
 }
 
-func (nac *NopArtifactClient) WriteArtifact(ctx context.Context, url string, content []byte) error {
+func (nac *NopArtifactClient) WriteArtifact(_ context.Context, _ string, _ []byte) error {
 	return nil
 }
 
-func (nac *NopArtifactClient) DeleteArtifact(ctx context.Context, url string) error {
+func (nac *NopArtifactClient) DeleteArtifact(_ context.Context, _ string) error {
 	return nil
 }

api/pkg/artifact/artifact_test.go

@@ -3,29 +3,53 @@ package artifact
 import (
 	"reflect"
 	"testing"
-
-	"cloud.google.com/go/storage"
 )
 
-func TestGcsArtifactClient_ParseURL(t *testing.T) {
-	type fields struct {
-		API *storage.Client
+func TestArtifactClient_GetURLScheme(t *testing.T) {
+	tests := []struct {
+		name           string
+		artifactClient SchemeInterface
+		want           string
+	}{
+		{
+			name: "gcs client",
+			artifactClient: &GcsArtifactClient{
+				URLScheme: "gs",
+				API:       nil,
+			},
+			want: "gs",
+		},
+		{
+			name: "s3 client",
+			artifactClient: &S3ArtifactClient{
+				URLScheme: "gs",
+			},
+			want: "gs",
+		},
 	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.artifactClient.GetURLScheme(); got != tt.want {
+				t.Errorf("GcsArtifactClient.GetScheme() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestURLScheme_ParseURL(t *testing.T) {
 	type args struct {
 		gsURL string
 	}
 	tests := []struct {
-		name    string
-		fields  fields
-		args    args
-		want    *URL
-		wantErr bool
+		name      string
+		urlScheme URLScheme
+		args      args
+		want      *URL
+		wantErr   bool
 	}{
 		{
-			name: "valid short url",
-			fields: fields{
-				API: nil,
-			},
+			name:      "valid short url",
+			urlScheme: "gs",
 			args: args{
 				gsURL: "gs://bucket-name/object-path",
 			},
@@ -36,10 +60,8 @@ func TestGcsArtifactClient_ParseURL(t *testing.T) {
 			wantErr: false,
 		},
 		{
-			name: "valid url",
-			fields: fields{
-				API: nil,
-			},
+			name:      "valid url",
+			urlScheme: "gs",
 			args: args{
 				gsURL: "gs://bucket-name/object-path/object-path-2/object-path-3/file-1.txt",
 			},
@@ -52,16 +74,13 @@ func TestGcsArtifactClient_ParseURL(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			gac := &GcsArtifactClient{
-				API: tt.fields.API,
-			}
-			got, err := gac.ParseURL(tt.args.gsURL)
+			got, err := tt.urlScheme.ParseURL(tt.args.gsURL)
 			if (err != nil) != tt.wantErr {
-				t.Errorf("GcsArtifactClient.ParseURL() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("URLScheme.ParseURL() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
 			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("GcsArtifactClient.ParseURL() = %v, want %v", got, tt.want)
+				t.Errorf("URLScheme.ParseURL() = %v, want %v", got, tt.want)
 			}
 		})
 	}

go.mod

@@ -1,12 +1,16 @@
 module github.com/caraml-dev/mlp
 
-go 1.20
+go 1.21
+
+toolchain go1.22.5
 
 require (
 	cloud.google.com/go/storage v1.29.0
 	github.com/antihax/optional v1.0.0
 	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a
 	github.com/avast/retry-go/v4 v4.6.0
+	github.com/aws/aws-sdk-go-v2/config v1.8.3
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.61.2
 	github.com/gavv/httpexpect/v2 v2.15.0
 	github.com/getsentry/raven-go v0.2.0
 	github.com/go-playground/locales v0.14.0
@@ -54,6 +58,21 @@ require (
 	cloud.google.com/go/iam v0.8.0 // indirect
 	github.com/ajg/form v1.5.1 // indirect
 	github.com/andybalholm/brotli v1.0.4 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.30.6-0.20240906182417-827d25db0048 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.4.3 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.6.0 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.2.4 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.4.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.7.2 // indirect
+	github.com/aws/smithy-go v1.20.4 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cenkalti/backoff/v3 v3.0.0 // indirect
 	github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40 // indirect
@@ -87,6 +106,7 @@ require (
 	github.com/imkira/go-interpol v1.1.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.15.0 // indirect
 	github.com/leodido/go-urn v1.2.1 // indirect