Skip to content

Commit

Permalink
Merge pull request #764 from nyaruka/update_config
Browse files Browse the repository at this point in the history
Update AWS/S3 config
  • Loading branch information
rowanseymour authored Jul 25, 2024
2 parents 5d72fe3 + aa4ca63 commit 39691f2
Show file tree
Hide file tree
Showing 3 changed files with 22 additions and 29 deletions.
17 changes: 6 additions & 11 deletions backends/rapidpro/backend.go
Original file line number Diff line number Diff line change
Expand Up @@ -174,22 +174,17 @@ func (b *backend) Start() error {
s3config := &storage.S3Options{
AWSAccessKeyID: b.config.AWSAccessKeyID,
AWSSecretAccessKey: b.config.AWSSecretAccessKey,
Region: b.config.AWSRegion,
Endpoint: b.config.S3Endpoint,
Region: b.config.S3Region,
DisableSSL: b.config.S3DisableSSL,
ForcePathStyle: b.config.S3ForcePathStyle,
MaxRetries: 3,
}
if b.config.AWSAccessKeyID != "" && !b.config.AWSUseCredChain {
s3config.AWSAccessKeyID = b.config.AWSAccessKeyID
s3config.AWSSecretAccessKey = b.config.AWSSecretAccessKey
}
s3Client, err := storage.NewS3Client(s3config)
if err != nil {
return err
}
b.attachmentStorage = storage.NewS3(s3Client, b.config.S3AttachmentsBucket, b.config.S3Region, s3.BucketCannedACLPublicRead, 32)
b.logStorage = storage.NewS3(s3Client, b.config.S3LogsBucket, b.config.S3Region, s3.BucketCannedACLPrivate, 32)
b.attachmentStorage = storage.NewS3(s3Client, b.config.S3AttachmentsBucket, b.config.AWSRegion, s3.BucketCannedACLPublicRead, 32)
b.logStorage = storage.NewS3(s3Client, b.config.S3LogsBucket, b.config.AWSRegion, s3.BucketCannedACLPrivate, 32)
} else {
b.attachmentStorage = storage.NewFS(storageDir+"/attachments", 0766)
b.logStorage = storage.NewFS(storageDir+"/logs", 0766)
Expand Down Expand Up @@ -655,7 +650,7 @@ func (b *backend) SaveAttachment(ctx context.Context, ch courier.Channel, conten

orgID := ch.(*Channel).OrgID()

path := filepath.Join(b.config.S3AttachmentsPrefix, strconv.FormatInt(int64(orgID), 10), filename[:4], filename[4:8], filename)
path := filepath.Join("attachments", strconv.FormatInt(int64(orgID), 10), filename[:4], filename[4:8], filename)

storageURL, err := b.attachmentStorage.Put(ctx, path, contentType, data)
if err != nil {
Expand All @@ -675,7 +670,7 @@ func (b *backend) ResolveMedia(ctx context.Context, mediaUrl string) (courier.Me
mediaUUID := uuidRegex.FindString(u.Path)

// if hostname isn't our media domain, or path doesn't contain a UUID, don't try to resolve
if strings.Replace(u.Hostname(), fmt.Sprintf("%s.", b.config.S3Region), "", -1) != b.config.MediaDomain || mediaUUID == "" {
if strings.Replace(u.Hostname(), fmt.Sprintf("%s.", b.config.AWSRegion), "", -1) != b.config.MediaDomain || mediaUUID == "" {
return nil, nil
}

Expand Down Expand Up @@ -704,7 +699,7 @@ func (b *backend) ResolveMedia(ctx context.Context, mediaUrl string) (courier.Me
}

// if we found a media record but it doesn't match the URL, don't use it
if media == nil || (media.URL() != mediaUrl && media.URL() != strings.Replace(mediaUrl, fmt.Sprintf("%s.", b.config.S3Region), "", -1)) {
if media == nil || (media.URL() != mediaUrl && media.URL() != strings.Replace(mediaUrl, fmt.Sprintf("%s.", b.config.AWSRegion), "", -1)) {
return nil, nil
}

Expand Down
4 changes: 2 additions & 2 deletions backends/rapidpro/backend_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1086,7 +1086,7 @@ func (ts *BackendTestSuite) TestSaveAttachment() {

newURL, err := ts.b.SaveAttachment(ctx, knChannel, "image/jpeg", testJPG, "jpg")
ts.NoError(err)
ts.Equal("_test_storage/attachments/media/1/c00e/5d67/c00e5d67-c275-4389-aded-7d8b151cbd5b.jpg", newURL)
ts.Equal("_test_storage/attachments/attachments/1/c00e/5d67/c00e5d67-c275-4389-aded-7d8b151cbd5b.jpg", newURL)
}

func (ts *BackendTestSuite) TestWriteMsg() {
Expand Down Expand Up @@ -1215,7 +1215,7 @@ func (ts *BackendTestSuite) TestWriteMsgWithAttachments() {
// should have actually fetched and saved it to storage, with the correct content type
err = ts.b.WriteMsg(ctx, msg, clog)
ts.NoError(err)
ts.Equal([]string{"image/jpeg:_test_storage/attachments/media/1/9b95/5e36/9b955e36-ac16-4c6b-8ab6-9b9af5cd042a.jpg"}, msg.Attachments())
ts.Equal([]string{"image/jpeg:_test_storage/attachments/attachments/1/9b95/5e36/9b955e36-ac16-4c6b-8ab6-9b9af5cd042a.jpg"}, msg.Attachments())

// try an invalid embedded attachment
msg = ts.b.NewIncomingMsg(knChannel, urn, "invalid embedded attachment data", "", clog).(*Msg)
Expand Down
30 changes: 14 additions & 16 deletions config.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,15 @@ type Config struct {
Redis string `validate:"url,startswith=redis:" help:"URL for your Redis instance"`
SpoolDir string `help:"the local directory where courier will write statuses or msgs that need to be retried (needs to be writable)"`

AWSAccessKeyID string `help:"the access key id to use when authenticating S3"`
AWSSecretAccessKey string `help:"the secret access key id to use when authenticating S3"`
AWSUseCredChain bool `help:"whether to use the AWS credentials chain. Defaults to false."`
S3Endpoint string `help:"the S3 endpoint we will write attachments to"`
S3Region string `help:"the S3 region we will write attachments to"`
S3AttachmentsBucket string `help:"the S3 bucket we will write attachments to"`
S3AttachmentsPrefix string `help:"the prefix that will be added to attachment filenames"`
S3LogsBucket string `help:"the S3 bucket we will write channel logs to"`
S3DisableSSL bool `help:"whether we disable SSL when accessing S3. Should always be set to False unless you're hosting an S3 compatible service within a secure internal network"`
S3ForcePathStyle bool `help:"whether we force S3 path style. Should generally need to default to False unless you're hosting an S3 compatible service"`
AWSAccessKeyID string `help:"access key ID to use for AWS services"`
AWSSecretAccessKey string `help:"secret access key to use for AWS services"`
AWSUseCredChain bool `help:"using the default AWS credential provider chain instead of values above"`
AWSRegion string `help:"region to use for AWS services, e.g. us-east-1"`

S3Endpoint string `help:"S3 service endpoint, e.g. https://s3.amazonaws.com"`
S3AttachmentsBucket string `help:"S3 bucket to write attachments to"`
S3LogsBucket string `help:"S3 bucket to write channel logs to"`
S3ForcePathStyle bool `help:"S3 should used /bucket/path style URLs"`

FacebookApplicationSecret string `help:"the Facebook app secret"`
FacebookWebhookSecret string `help:"the secret for Facebook webhook URL verification"`
Expand Down Expand Up @@ -69,15 +68,14 @@ func NewDefaultConfig() *Config {
Redis: "redis://localhost:6379/15",
SpoolDir: "/var/spool/courier",

AWSAccessKeyID: "",
AWSSecretAccessKey: "",
AWSUseCredChain: false,
AWSAccessKeyID: "",
AWSSecretAccessKey: "",
AWSUseCredChain: false,
AWSRegion: "us-east-1",

S3Endpoint: "https://s3.amazonaws.com",
S3Region: "us-east-1",
S3AttachmentsBucket: "courier-media",
S3AttachmentsPrefix: "media/",
S3LogsBucket: "courier-logs",
S3DisableSSL: false,
S3ForcePathStyle: false,

FacebookApplicationSecret: "missing_facebook_app_secret",
Expand Down

0 comments on commit 39691f2

Please sign in to comment.