diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9024f71ba..0a6175efe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,16 @@ jobs: ports: - 5432:5432 options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 - + minio: + image: bitnami/minio:latest + env: + MINIO_ROOT_USER: root + MINIO_ROOT_PASSWORD: tembatemba + MINIO_DEFAULT_BUCKETS: temba-attachments,temba-logs + ports: + - 9000:9000 + options: --health-cmd "mc ready local" --health-interval 10s --health-timeout 5s --health-retries 5 + steps: - name: Checkout code uses: actions/checkout@v4 diff --git a/backends/rapidpro/backend.go b/backends/rapidpro/backend.go index 0b08a4265..0da68eb81 100644 --- a/backends/rapidpro/backend.go +++ b/backends/rapidpro/backend.go @@ -45,9 +45,6 @@ const sentSetName = "msgs_sent_%s" // our timeout for backend operations const backendTimeout = time.Second * 20 -// storage directory (only used with file system storage) -var storageDir = "_storage" - var uuidRegex = regexp.MustCompile(`[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`) func init() { @@ -169,31 +166,25 @@ func (b *backend) Start() error { queue.StartDethrottler(b.redisPool, b.stopChan, b.waitGroup, msgQueueName) } - // create our storage (S3 or file system) - if b.config.AWSAccessKeyID != "" || b.config.AWSUseCredChain { - s3config := &storage.S3Options{ - AWSAccessKeyID: b.config.AWSAccessKeyID, - AWSSecretAccessKey: b.config.AWSSecretAccessKey, - Region: b.config.AWSRegion, - Endpoint: b.config.S3Endpoint, - ForcePathStyle: b.config.S3ForcePathStyle, - MaxRetries: 3, - } - s3Client, err := storage.NewS3Client(s3config) - if err != nil { - return err - } - b.attachmentStorage = storage.NewS3(s3Client, b.config.S3AttachmentsBucket, b.config.AWSRegion, s3.BucketCannedACLPublicRead, 32) - b.logStorage = storage.NewS3(s3Client, b.config.S3LogsBucket, b.config.AWSRegion, s3.BucketCannedACLPrivate, 32) - } else { - b.attachmentStorage = storage.NewFS(storageDir+"/attachments", 0766) - b.logStorage = storage.NewFS(storageDir+"/logs", 0766) + s3config := &storage.S3Options{ + AWSAccessKeyID: b.config.AWSAccessKeyID, + AWSSecretAccessKey: b.config.AWSSecretAccessKey, + Region: b.config.AWSRegion, + Endpoint: b.config.S3Endpoint, + ForcePathStyle: b.config.S3ForcePathStyle, + MaxRetries: 3, + } + s3Client, err := storage.NewS3Client(s3config) + if err != nil { + return err } + b.attachmentStorage = storage.NewS3(s3Client, b.config.S3AttachmentsBucket, b.config.AWSRegion, s3.BucketCannedACLPublicRead, 32) + b.logStorage = storage.NewS3(s3Client, b.config.S3LogsBucket, b.config.AWSRegion, s3.BucketCannedACLPrivate, 32) // create and start channel caches... - b.channelsByUUID = cache.NewLocal[courier.ChannelUUID, *Channel](b.loadChannelByUUID, time.Minute) + b.channelsByUUID = cache.NewLocal(b.loadChannelByUUID, time.Minute) b.channelsByUUID.Start() - b.channelsByAddr = cache.NewLocal[courier.ChannelAddress, *Channel](b.loadChannelByAddress, time.Minute) + b.channelsByAddr = cache.NewLocal(b.loadChannelByAddress, time.Minute) b.channelsByAddr.Start() // check our storages diff --git a/backends/rapidpro/backend_test.go b/backends/rapidpro/backend_test.go index 968873bb5..98c95123f 100644 --- a/backends/rapidpro/backend_test.go +++ b/backends/rapidpro/backend_test.go @@ -42,12 +42,17 @@ func testConfig() *courier.Config { config.DB = "postgres://courier_test:temba@localhost:5432/courier_test?sslmode=disable" config.Redis = "redis://localhost:6379/0" config.MediaDomain = "nyaruka.s3.com" + + // configure S3 to use a local minio instance + config.AWSAccessKeyID = "root" + config.AWSSecretAccessKey = "tembatemba" + config.S3Endpoint = "http://localhost:9000" + config.S3ForcePathStyle = true + return config } func (ts *BackendTestSuite) SetupSuite() { - storageDir = "_test_storage" - // turn off logging log.SetOutput(io.Discard) @@ -82,10 +87,6 @@ func (ts *BackendTestSuite) SetupSuite() { func (ts *BackendTestSuite) TearDownSuite() { ts.b.Stop() ts.b.Cleanup() - - if err := os.RemoveAll(storageDir); err != nil { - panic(err) - } } func (ts *BackendTestSuite) clearRedis() { @@ -1086,7 +1087,7 @@ func (ts *BackendTestSuite) TestSaveAttachment() { newURL, err := ts.b.SaveAttachment(ctx, knChannel, "image/jpeg", testJPG, "jpg") ts.NoError(err) - ts.Equal("_test_storage/attachments/attachments/1/c00e/5d67/c00e5d67-c275-4389-aded-7d8b151cbd5b.jpg", newURL) + ts.Equal("https://temba-attachments.s3.us-east-1.amazonaws.com/attachments/1/c00e/5d67/c00e5d67-c275-4389-aded-7d8b151cbd5b.jpg", newURL) } func (ts *BackendTestSuite) TestWriteMsg() { @@ -1215,7 +1216,7 @@ func (ts *BackendTestSuite) TestWriteMsgWithAttachments() { // should have actually fetched and saved it to storage, with the correct content type err = ts.b.WriteMsg(ctx, msg, clog) ts.NoError(err) - ts.Equal([]string{"image/jpeg:_test_storage/attachments/attachments/1/9b95/5e36/9b955e36-ac16-4c6b-8ab6-9b9af5cd042a.jpg"}, msg.Attachments()) + ts.Equal([]string{"image/jpeg:https://temba-attachments.s3.us-east-1.amazonaws.com/attachments/1/9b95/5e36/9b955e36-ac16-4c6b-8ab6-9b9af5cd042a.jpg"}, msg.Attachments()) // try an invalid embedded attachment msg = ts.b.NewIncomingMsg(knChannel, urn, "invalid embedded attachment data", "", clog).(*Msg) diff --git a/config.go b/config.go index 7789ee1b5..1eb80012f 100644 --- a/config.go +++ b/config.go @@ -27,7 +27,6 @@ type Config struct { AWSAccessKeyID string `help:"access key ID to use for AWS services"` AWSSecretAccessKey string `help:"secret access key to use for AWS services"` - AWSUseCredChain bool `help:"using the default AWS credential provider chain instead of values above"` AWSRegion string `help:"region to use for AWS services, e.g. us-east-1"` S3Endpoint string `help:"S3 service endpoint, e.g. https://s3.amazonaws.com"` @@ -70,12 +69,11 @@ func NewDefaultConfig() *Config { AWSAccessKeyID: "", AWSSecretAccessKey: "", - AWSUseCredChain: false, AWSRegion: "us-east-1", S3Endpoint: "https://s3.amazonaws.com", - S3AttachmentsBucket: "courier-media", - S3LogsBucket: "courier-logs", + S3AttachmentsBucket: "temba-attachments", + S3LogsBucket: "temba-logs", S3ForcePathStyle: false, FacebookApplicationSecret: "missing_facebook_app_secret",