S3: add config flag to proxy S3 media

docs/configuration/storage.md

@@ -32,6 +32,11 @@ storage-local-base-path: "/gotosocial/storage"
 # Default: ""
 storage-s3-endpoint: ""
 
+# Bool. If data stored in S3 should be proxied through GoToSocial instead of redirecting to a presigned URL.
+#
+# Default: false
+storage-s3-proxy: false
+
 # String. Access key part of the S3 credentials.
 # Consider setting this value using environment variables to avoid leaking it via the config file
 # Only required when running with the s3 storage backend.

example/config.yaml

@@ -320,6 +320,11 @@ storage-local-base-path: "/gotosocial/storage"
 # Default: ""
 storage-s3-endpoint: ""
 
+# Bool. If data stored in S3 should be proxied through GoToSocial instead of redirecting to a presigned URL.
+#
+# Default: false
+storage-s3-proxy: false
+
 # String. Access key part of the S3 credentials.
 # Consider setting this value using environment variables to avoid leaking it via the config file
 # Only required when running with the s3 storage backend.

internal/config/config.go

@@ -93,6 +93,7 @@ type Configuration struct {
 	StorageS3SecretKey   string `name:"storage-s3-secret-key" usage:"S3 Secret Key"`
 	StorageS3UseSSL      bool   `name:"storage-s3-use-ssl" usage:"Use SSL for S3 connections. Only set this to 'false' when testing locally"`
 	StorageS3BucketName  string `name:"storage-s3-bucket" usage:"Place blobs in this bucket"`
+	StorageS3Proxy       bool   `name:"storage-s3-proxy" usage:"Proxy S3 contents through GoToSocial instead of redirecting to a presigned URL"`
 
 	StatusesMaxChars           int `name:"statuses-max-chars" usage:"Max permitted characters for posted statuses"`
 	StatusesCWMaxChars         int `name:"statuses-cw-max-chars" usage:"Max permitted characters for content/spoiler warnings on statuses"`

internal/config/defaults.go

@@ -67,6 +67,7 @@ var Defaults = Configuration{
 	StorageBackend:       "local",
 	StorageLocalBasePath: "/gotosocial/storage",
 	StorageS3UseSSL:      true,
+	StorageS3Proxy:       false,
 
 	StatusesMaxChars:           5000,
 	StatusesCWMaxChars:         100,

internal/config/helpers.gen.go

@@ -1095,6 +1095,31 @@ func GetStorageS3BucketName() string { return global.GetStorageS3BucketName() }
 // SetStorageS3BucketName safely sets the value for global configuration 'StorageS3BucketName' field
 func SetStorageS3BucketName(v string) { global.SetStorageS3BucketName(v) }
 
+// GetStorageS3Proxy safely fetches the Configuration value for state's 'StorageS3Proxy' field
+func (st *ConfigState) GetStorageS3Proxy() (v bool) {
+	st.mutex.Lock()
+	v = st.config.StorageS3Proxy
+	st.mutex.Unlock()
+	return
+}
+
+// SetStorageS3Proxy safely sets the Configuration value for state's 'StorageS3Proxy' field
+func (st *ConfigState) SetStorageS3Proxy(v bool) {
+	st.mutex.Lock()
+	defer st.mutex.Unlock()
+	st.config.StorageS3Proxy = v
+	st.reloadToViper()
+}
+
+// StorageS3ProxyFlag returns the flag name for the 'StorageS3Proxy' field
+func StorageS3ProxyFlag() string { return "storage-s3-proxy" }
+
+// GetStorageS3Proxy safely fetches the value for global configuration 'StorageS3Proxy' field
+func GetStorageS3Proxy() bool { return global.GetStorageS3Proxy() }
+
+// SetStorageS3Proxy safely sets the value for global configuration 'StorageS3Proxy' field
+func SetStorageS3Proxy(v bool) { global.SetStorageS3Proxy(v) }
+
 // GetStatusesMaxChars safely fetches the Configuration value for state's 'StatusesMaxChars' field
 func (st *ConfigState) GetStatusesMaxChars() (v int) {
 	st.mutex.Lock()
@@ -1844,3 +1869,4 @@ func GetAdvancedRateLimitRequests() int { return global.GetAdvancedRateLimitRequ
 
 // SetAdvancedRateLimitRequests safely sets the value for global configuration 'AdvancedRateLimitRequests' field
 func SetAdvancedRateLimitRequests(v int) { global.SetAdvancedRateLimitRequests(v) }
+

internal/storage/s3.go

@@ -34,12 +34,14 @@ import (
 type S3 struct {
 	mc     *minio.Client
 	bucket string
+	proxy  bool
 }
 
-func NewS3(mc *minio.Client, bucket string) *S3 {
+func NewS3(mc *minio.Client, bucket string, proxy bool) *S3 {
 	return &S3{
 		mc:     mc,
 		bucket: bucket,
+		proxy:  proxy,
 	}
 }
 
@@ -83,6 +85,10 @@ func (s *S3) Delete(ctx context.Context, key string) error {
 }
 
 func (s *S3) URL(ctx context.Context, key string) *url.URL {
+	if s.proxy {
+		return nil
+	}
+
 	// it's safe to ignore the error here, as we just fall back to fetching the
 	// file if the url request fails
 	url, _ := s.mc.PresignedGetObject(ctx, s.bucket, key, time.Hour, url.Values{

internal/storage/storage.go

@@ -59,7 +59,11 @@ func AutoConfig() (Driver, error) {
 		if err != nil {
 			return nil, fmt.Errorf("creating minio client: %w", err)
 		}
-		return NewS3(mc, config.GetStorageS3BucketName()), nil
+		return NewS3(
+			mc,
+			config.GetStorageS3BucketName(),
+			config.GetStorageS3Proxy(),
+		), nil
 	case "local":
 		basePath := config.GetStorageLocalBasePath()
 		disk, err := storage.OpenDisk(basePath, &storage.DiskConfig{

test/envparsing.sh

@@ -2,7 +2,7 @@
 
 set -eu
 
-EXPECT='{"account-domain":"peepee","accounts-allow-custom-css":true,"accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","advanced-rate-limit-requests":6969,"application-name":"gts","bind-address":"127.0.0.1","config-path":"./test/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-password":"hunter2","db-port":6969,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","email":"","host":"example.com","instance-deliver-to-shared-inboxes":false,"instance-expose-peers":true,"instance-expose-suspended":true,"landing-page-user":"admin","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-emoji-local-max-size":420,"media-emoji-remote-max-size":420,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","smtp-from":"queen.rip.in.piss@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","storage-s3-access-key":"minio","storage-s3-bucket":"gts","storage-s3-endpoint":"localhost:9000","storage-s3-secret-key":"miniostorage","storage-s3-use-ssl":false,"syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","docker.host.local"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}'
+EXPECT='{"account-domain":"peepee","accounts-allow-custom-css":true,"accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","advanced-rate-limit-requests":6969,"application-name":"gts","bind-address":"127.0.0.1","config-path":"./test/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-password":"hunter2","db-port":6969,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","email":"","host":"example.com","instance-deliver-to-shared-inboxes":false,"instance-expose-peers":true,"instance-expose-suspended":true,"landing-page-user":"admin","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-emoji-local-max-size":420,"media-emoji-remote-max-size":420,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","smtp-from":"queen.rip.in.piss@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","storage-s3-access-key":"minio","storage-s3-bucket":"gts","storage-s3-endpoint":"localhost:9000","storage-s3-proxy":true,"storage-s3-secret-key":"miniostorage","storage-s3-use-ssl":false,"syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","docker.host.local"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}'
 
 # Set all the environment variables to 
 # ensure that these are parsed without panic
@@ -46,6 +46,7 @@ GTS_STORAGE_S3_ACCESS_KEY='minio' \
 GTS_STORAGE_S3_SECRET_KEY='miniostorage' \
 GTS_STORAGE_S3_ENDPOINT='localhost:9000' \
 GTS_STORAGE_S3_USE_SSL='false' \
+GTS_STORAGE_S3_PROXY='true' \
 GTS_STORAGE_S3_BUCKET='gts' \
 GTS_STATUSES_MAX_CHARS=69 \
 GTS_STATUSES_CW_MAX_CHARS=420 \

testrig/storage.go

@@ -48,7 +48,7 @@ func NewS3Storage() gtsstorage.Driver {
 	if err != nil {
 		panic(err)
 	}
-	return gtsstorage.NewS3(mc, os.Getenv("GTS_STORAGE_S3_BUCKET"))
+	return gtsstorage.NewS3(mc, os.Getenv("GTS_STORAGE_S3_BUCKET"), false)
 }
 
 // StandardStorageSetup populates the storage with standard test entries from the given directory.