From aa4ca63ff9c6396eac0c78d16ac1b9d10f109c9e Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Thu, 25 Jul 2024 11:27:49 -0500 Subject: [PATCH] Update AWS/S3 configs --- backends/rapidpro/backend.go | 17 ++++++----------- backends/rapidpro/backend_test.go | 4 ++-- config.go | 30 ++++++++++++++---------------- 3 files changed, 22 insertions(+), 29 deletions(-) diff --git a/backends/rapidpro/backend.go b/backends/rapidpro/backend.go index e9b77329e..0b08a4265 100644 --- a/backends/rapidpro/backend.go +++ b/backends/rapidpro/backend.go @@ -174,22 +174,17 @@ func (b *backend) Start() error { s3config := &storage.S3Options{ AWSAccessKeyID: b.config.AWSAccessKeyID, AWSSecretAccessKey: b.config.AWSSecretAccessKey, + Region: b.config.AWSRegion, Endpoint: b.config.S3Endpoint, - Region: b.config.S3Region, - DisableSSL: b.config.S3DisableSSL, ForcePathStyle: b.config.S3ForcePathStyle, MaxRetries: 3, } - if b.config.AWSAccessKeyID != "" && !b.config.AWSUseCredChain { - s3config.AWSAccessKeyID = b.config.AWSAccessKeyID - s3config.AWSSecretAccessKey = b.config.AWSSecretAccessKey - } s3Client, err := storage.NewS3Client(s3config) if err != nil { return err } - b.attachmentStorage = storage.NewS3(s3Client, b.config.S3AttachmentsBucket, b.config.S3Region, s3.BucketCannedACLPublicRead, 32) - b.logStorage = storage.NewS3(s3Client, b.config.S3LogsBucket, b.config.S3Region, s3.BucketCannedACLPrivate, 32) + b.attachmentStorage = storage.NewS3(s3Client, b.config.S3AttachmentsBucket, b.config.AWSRegion, s3.BucketCannedACLPublicRead, 32) + b.logStorage = storage.NewS3(s3Client, b.config.S3LogsBucket, b.config.AWSRegion, s3.BucketCannedACLPrivate, 32) } else { b.attachmentStorage = storage.NewFS(storageDir+"/attachments", 0766) b.logStorage = storage.NewFS(storageDir+"/logs", 0766) @@ -655,7 +650,7 @@ func (b *backend) SaveAttachment(ctx context.Context, ch courier.Channel, conten orgID := ch.(*Channel).OrgID() - path := filepath.Join(b.config.S3AttachmentsPrefix, strconv.FormatInt(int64(orgID), 10), filename[:4], filename[4:8], filename) + path := filepath.Join("attachments", strconv.FormatInt(int64(orgID), 10), filename[:4], filename[4:8], filename) storageURL, err := b.attachmentStorage.Put(ctx, path, contentType, data) if err != nil { @@ -675,7 +670,7 @@ func (b *backend) ResolveMedia(ctx context.Context, mediaUrl string) (courier.Me mediaUUID := uuidRegex.FindString(u.Path) // if hostname isn't our media domain, or path doesn't contain a UUID, don't try to resolve - if strings.Replace(u.Hostname(), fmt.Sprintf("%s.", b.config.S3Region), "", -1) != b.config.MediaDomain || mediaUUID == "" { + if strings.Replace(u.Hostname(), fmt.Sprintf("%s.", b.config.AWSRegion), "", -1) != b.config.MediaDomain || mediaUUID == "" { return nil, nil } @@ -704,7 +699,7 @@ func (b *backend) ResolveMedia(ctx context.Context, mediaUrl string) (courier.Me } // if we found a media record but it doesn't match the URL, don't use it - if media == nil || (media.URL() != mediaUrl && media.URL() != strings.Replace(mediaUrl, fmt.Sprintf("%s.", b.config.S3Region), "", -1)) { + if media == nil || (media.URL() != mediaUrl && media.URL() != strings.Replace(mediaUrl, fmt.Sprintf("%s.", b.config.AWSRegion), "", -1)) { return nil, nil } diff --git a/backends/rapidpro/backend_test.go b/backends/rapidpro/backend_test.go index a22ee3dea..968873bb5 100644 --- a/backends/rapidpro/backend_test.go +++ b/backends/rapidpro/backend_test.go @@ -1086,7 +1086,7 @@ func (ts *BackendTestSuite) TestSaveAttachment() { newURL, err := ts.b.SaveAttachment(ctx, knChannel, "image/jpeg", testJPG, "jpg") ts.NoError(err) - ts.Equal("_test_storage/attachments/media/1/c00e/5d67/c00e5d67-c275-4389-aded-7d8b151cbd5b.jpg", newURL) + ts.Equal("_test_storage/attachments/attachments/1/c00e/5d67/c00e5d67-c275-4389-aded-7d8b151cbd5b.jpg", newURL) } func (ts *BackendTestSuite) TestWriteMsg() { @@ -1215,7 +1215,7 @@ func (ts *BackendTestSuite) TestWriteMsgWithAttachments() { // should have actually fetched and saved it to storage, with the correct content type err = ts.b.WriteMsg(ctx, msg, clog) ts.NoError(err) - ts.Equal([]string{"image/jpeg:_test_storage/attachments/media/1/9b95/5e36/9b955e36-ac16-4c6b-8ab6-9b9af5cd042a.jpg"}, msg.Attachments()) + ts.Equal([]string{"image/jpeg:_test_storage/attachments/attachments/1/9b95/5e36/9b955e36-ac16-4c6b-8ab6-9b9af5cd042a.jpg"}, msg.Attachments()) // try an invalid embedded attachment msg = ts.b.NewIncomingMsg(knChannel, urn, "invalid embedded attachment data", "", clog).(*Msg) diff --git a/config.go b/config.go index 1436481bd..7789ee1b5 100644 --- a/config.go +++ b/config.go @@ -25,16 +25,15 @@ type Config struct { Redis string `validate:"url,startswith=redis:" help:"URL for your Redis instance"` SpoolDir string `help:"the local directory where courier will write statuses or msgs that need to be retried (needs to be writable)"` - AWSAccessKeyID string `help:"the access key id to use when authenticating S3"` - AWSSecretAccessKey string `help:"the secret access key id to use when authenticating S3"` - AWSUseCredChain bool `help:"whether to use the AWS credentials chain. Defaults to false."` - S3Endpoint string `help:"the S3 endpoint we will write attachments to"` - S3Region string `help:"the S3 region we will write attachments to"` - S3AttachmentsBucket string `help:"the S3 bucket we will write attachments to"` - S3AttachmentsPrefix string `help:"the prefix that will be added to attachment filenames"` - S3LogsBucket string `help:"the S3 bucket we will write channel logs to"` - S3DisableSSL bool `help:"whether we disable SSL when accessing S3. Should always be set to False unless you're hosting an S3 compatible service within a secure internal network"` - S3ForcePathStyle bool `help:"whether we force S3 path style. Should generally need to default to False unless you're hosting an S3 compatible service"` + AWSAccessKeyID string `help:"access key ID to use for AWS services"` + AWSSecretAccessKey string `help:"secret access key to use for AWS services"` + AWSUseCredChain bool `help:"using the default AWS credential provider chain instead of values above"` + AWSRegion string `help:"region to use for AWS services, e.g. us-east-1"` + + S3Endpoint string `help:"S3 service endpoint, e.g. https://s3.amazonaws.com"` + S3AttachmentsBucket string `help:"S3 bucket to write attachments to"` + S3LogsBucket string `help:"S3 bucket to write channel logs to"` + S3ForcePathStyle bool `help:"S3 should used /bucket/path style URLs"` FacebookApplicationSecret string `help:"the Facebook app secret"` FacebookWebhookSecret string `help:"the secret for Facebook webhook URL verification"` @@ -69,15 +68,14 @@ func NewDefaultConfig() *Config { Redis: "redis://localhost:6379/15", SpoolDir: "/var/spool/courier", - AWSAccessKeyID: "", - AWSSecretAccessKey: "", - AWSUseCredChain: false, + AWSAccessKeyID: "", + AWSSecretAccessKey: "", + AWSUseCredChain: false, + AWSRegion: "us-east-1", + S3Endpoint: "https://s3.amazonaws.com", - S3Region: "us-east-1", S3AttachmentsBucket: "courier-media", - S3AttachmentsPrefix: "media/", S3LogsBucket: "courier-logs", - S3DisableSSL: false, S3ForcePathStyle: false, FacebookApplicationSecret: "missing_facebook_app_secret",