Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow fine-grained configuration of S3 server setup #27

Merged
merged 1 commit into from
Jun 12, 2022
Merged

Allow fine-grained configuration of S3 server setup #27

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,15 +115,18 @@ All settings from the configuration file can also be set via environment variabl
| :-- | :-- | :-- |
| `GOSE_LISTEN` | `":8080"` | Listen address and port of Gose |
| `GOSE_BASE_URL` | `"http://localhost:8080"` | Base URL at which Gose is accessible |
| `GOSE_STATIC` | `"./dist"` | Directory of frontend assets if not bundled into the binary |
| `GOSE_STATIC` | `"./dist"` | Directory of frontend assets (pre-compiled binaries of GoSƐ come with assets embedded into binary.) |
| `GOSE_BUCKET` | `gose-uploads` | Name of S3 bucket |
| `GOSE_ENDPOINT` | (without `http(s)://` prefix, but with port number) | Hostname:Port of S3 server |
| `GOSE_REGION` | `us-east-1` | Region of S3 server |
| `GOSE_PATH_STYLE` | `false` | Prepend bucket name to path |
| `GOSE_NO_SSL` | `false` | Disable SSL encryption for S3 |
| `GOSE_ACCESS_KEY` | | S3 Access Key |
| `GOSE_SECRET_KEY` | | S3 Secret Key |
| `GOSE_CREATE_BUCKET` | `true` | Create S3 bucket if non-existant |
| `GOSE_SETUP_BUCKET` | `true` | Create S3 bucket if do not exists |
| `GOSE_SETUP_CORS` | `true` (if supported by S3 implementation) | Setup S3 bucket CORS rules |
| `GOSE_SETUP_LIFECYCLE` | `true` (if supported by S3 implementation) | Setup S3 bucket lifecycle rules |
| `GOSE_SETUP_ABORT_INCOMPLETE_UPLOADS` | `31` | Number of days after which incomplete uploads are cleaned-up (set to 0 to disable) |
| `GOSE_MAX_UPLOAD_SIZE` | `1TB` | Maximum upload size |
| `GOSE_PART_SIZE` | `16MB` | Part-size for multi-part uploads |
| `AWS_ACCESS_KEY_ID` | | alias for `GOSE_ACCESS_KEY` |
Expand Down
22 changes: 19 additions & 3 deletions config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,28 @@ servers:
access_key: ""
secret_key: ""

# Create the bucket if it does not exist
create_bucket: true

max_upload_size: 5TB
part_size: 16MB

# Manual configuration of S3 implementation
# Usually its auto-detected so the is only required in case
# a proxy or CDN manipulates the "Server" HTTP-response header
# implementation: MinIO

setup:
# Create the bucket if it does not exist
bucket: true

# Setup CORS rules for S3 bucket
cors: true

# Setup lifecycle rules for object expiration
# The rules are defined by the following expiration setting
lifecycle: true

# Number of days after which incomplete uploads are cleaned-up (set to 0 to disable)
abort_incomplete_uploads: 31

# A list of expiration/rentention classes
# The first class is selected by default
expiration:
Expand Down
37 changes: 26 additions & 11 deletions pkg/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,23 +81,34 @@ type S3ServerConfig struct {
ID string `json:"id" yaml:"id"`
Title string `json:"title" yaml:"title"`

MaxUploadSize size `json:"max_upload_size" yaml:"max_upload_size"`
PartSize size `json:"part_size" yaml:"part_size"`
Expiration []Expiration `json:"expiration" yaml:"expiration"`
Implementation string `json:"implementation" yaml:"implementation"`
MaxUploadSize size `json:"max_upload_size" yaml:"max_upload_size"`
PartSize size `json:"part_size" yaml:"part_size"`
Expiration []Expiration `json:"expiration" yaml:"expiration"`
}

// S3ServerSetup describes initial configuration for an S3 server/bucket
type S3ServerSetup struct {
CreateBucket bool `json:"create_bucket" yaml:"create_bucket"`
CORS bool `json:"cors" yaml:"cors"`
Lifecycle bool `json:"lifecycle" yaml:"ifecycle"`
AbortIncompleteUploads int `json:"abort_incomplete_uploads" yaml:"abort_incomplete_uploads"`
}

// S3Server describes an S3 server
type S3Server struct {
// S3ServerConfig is the public info about an S3 server shared with the frontend
S3ServerConfig `json:",squash"`

Endpoint string `json:"endpoint" yaml:"endpoint"`
Bucket string `json:"bucket" yaml:"bucket"`
Region string `json:"region" yaml:"region"`
PathStyle bool `json:"path_style" yaml:"path_style"`
NoSSL bool `json:"no_ssl" yaml:"no_ssl"`
AccessKey string `json:"access_key" yaml:"access_key"`
SecretKey string `json:"secret_key" yaml:"secret_key"`
CreateBucket bool `json:"create_bucket" yaml:"create_bucket"`
Endpoint string `json:"endpoint" yaml:"endpoint"`
Bucket string `json:"bucket" yaml:"bucket"`
Region string `json:"region" yaml:"region"`
PathStyle bool `json:"path_style" yaml:"path_style"`
NoSSL bool `json:"no_ssl" yaml:"no_ssl"`
AccessKey string `json:"access_key" yaml:"access_key"`
SecretKey string `json:"secret_key" yaml:"secret_key"`

Setup S3ServerSetup `json:"setup" yaml:"setup"`
}

// ShortenerConfig contains Link-shortener specific configuration
Expand Down Expand Up @@ -167,6 +178,10 @@ func NewConfig(configFile string) (*Config, error) {
cfg.SetDefault("access_key", "")
cfg.SetDefault("secret_key", "")
cfg.SetDefault("create_bucket", true)
cfg.SetDefault("implementation", "")
cfg.SetDefault("setup.cors", true)
cfg.SetDefault("setup.lifecycle", true)
cfg.SetDefault("setup.abort_incomplete_uploads", 31)

cfg.BindEnv("access_key", "AWS_ACCESS_KEY_ID")
cfg.BindEnv("secret_key", "AWS_SECRET_ACCESS_KEY")
Expand Down
8 changes: 2 additions & 6 deletions pkg/server/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@ import (
"github.com/stv0g/gose/pkg/config"
)

type Implementation string

const (
ImplementationAWS = "AmazonS3"
ImplementationMinio = "MinIO"
Expand All @@ -25,8 +23,6 @@ type Server struct {
*s3.S3

Config *config.S3Server

Implementation Implementation
}

// GetURL returns the full endpoint URL of the S3 server
Expand Down Expand Up @@ -69,7 +65,7 @@ func (s *Server) GetExpirationClass(cls string) *config.Expiration {
return nil
}

func (s *Server) DetectImplementation() Implementation {
func (s *Server) DetectImplementation() string {
if strings.Contains(s.Config.Endpoint, "digitaloceanspaces.com") {
return ImplementationDigitalOceanSpaces
} else if strings.Contains(s.Config.Endpoint, "storage.googleapis.com") {
Expand All @@ -83,7 +79,7 @@ func (s *Server) DetectImplementation() Implementation {
}
if err := req.Send(); err == nil {
if svr := req.HTTPResponse.Header.Get("Server"); svr != "" {
return Implementation(svr)
return svr
}

return ImplementationUnknown
Expand Down
100 changes: 52 additions & 48 deletions pkg/server/setup.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,24 @@ import (

// Setup initializes the S3 bucket (life-cycle rules & CORS)
func (s *Server) Setup() error {
s.Implementation = s.DetectImplementation()
log.Printf("Detected %s S3 implementation for server %s\n", s.Implementation, s.GetURL())
if s.Config.Implementation == "" {
s.Config.Implementation = s.DetectImplementation()
log.Printf("Detected %s S3 implementation for server %s", s.Config.Implementation, s.GetURL())
} else {
log.Printf("Using %s S3 implementation for server %s", s.Config.Implementation, s.GetURL())
}

// MinIO does not support the setup of bucket CORS rules and MPU abortion lifecycle
if s.Config.Implementation == ImplementationMinio {
s.Config.Setup.CORS = false
s.Config.Setup.AbortIncompleteUploads = 0
}

// Create bucket if it does not exist yet
if _, err := s.GetBucketPolicy(&s3.GetBucketPolicyInput{
Bucket: aws.String(s.Config.Bucket),
}); err != nil {
if aerr, ok := err.(awserr.Error); ok && aerr.Code() == s3.ErrCodeNoSuchBucket && s.Config.CreateBucket {
if aerr, ok := err.(awserr.Error); ok && aerr.Code() == s3.ErrCodeNoSuchBucket && s.Config.Setup.CreateBucket {
if _, err := s.CreateBucket(&s3.CreateBucketInput{
Bucket: aws.String(s.Config.Bucket),
}); err != nil {
Expand All @@ -28,7 +38,7 @@ func (s *Server) Setup() error {
}

// Set CORS configuration for bucket
if s.Implementation != ImplementationMinio {
if s.Config.Setup.CORS {
corsRule := &s3.CORSRule{
AllowedHeaders: aws.StringSlice([]string{"Authorization"}),
AllowedOrigins: aws.StringSlice([]string{"*"}),
Expand All @@ -49,56 +59,50 @@ func (s *Server) Setup() error {
}
}

// Create lifecycle policies
lcRules := []*s3.LifecycleRule{}
if s.Config.Setup.Lifecycle {
// Create lifecycle policies
lcRules := []*s3.LifecycleRule{}

if s.Implementation != ImplementationMinio {
lcRules = append(lcRules, &s3.LifecycleRule{
ID: aws.String("Abort Multipart Uploads"),
Status: aws.String("Enabled"),
AbortIncompleteMultipartUpload: &s3.AbortIncompleteMultipartUpload{
DaysAfterInitiation: aws.Int64(31),
},
Filter: &s3.LifecycleRuleFilter{
Prefix: aws.String("/"),
},
})
}
if s.Config.Setup.AbortIncompleteUploads > 0 {
lcRules = append(lcRules, &s3.LifecycleRule{
ID: aws.String("Abort Multipart Uploads"),
Status: aws.String("Enabled"),
AbortIncompleteMultipartUpload: &s3.AbortIncompleteMultipartUpload{
DaysAfterInitiation: aws.Int64(31),
},
Filter: &s3.LifecycleRuleFilter{
Prefix: aws.String("/"),
},
})
}

for _, cls := range s.Config.Expiration {
lcRules = append(lcRules, &s3.LifecycleRule{
ID: aws.String(fmt.Sprintf("Expiration after %s", cls.Title)),
Status: aws.String("Enabled"),
Filter: &s3.LifecycleRuleFilter{
Tag: &s3.Tag{
Key: aws.String("expiration"),
Value: aws.String(cls.ID),
for _, cls := range s.Config.Expiration {
lcRules = append(lcRules, &s3.LifecycleRule{
ID: aws.String(fmt.Sprintf("Expiration after %s", cls.Title)),
Status: aws.String("Enabled"),
Filter: &s3.LifecycleRuleFilter{
Tag: &s3.Tag{
Key: aws.String("expiration"),
Value: aws.String(cls.ID),
},
},
},
Expiration: &s3.LifecycleExpiration{
Days: aws.Int64(cls.Days),
},
})
}
Expiration: &s3.LifecycleExpiration{
Days: aws.Int64(cls.Days),
},
})
}

if len(lcRules) > 0 {
if _, err := s.PutBucketLifecycleConfiguration(&s3.PutBucketLifecycleConfigurationInput{
Bucket: aws.String(s.Config.Bucket),
LifecycleConfiguration: &s3.BucketLifecycleConfiguration{
Rules: lcRules,
},
}); err != nil {
return fmt.Errorf("failed to set bucket %s's lifecycle rules: %w", s.Config.Bucket, err)
if len(lcRules) > 0 {
if _, err := s.PutBucketLifecycleConfiguration(&s3.PutBucketLifecycleConfigurationInput{
Bucket: aws.String(s.Config.Bucket),
LifecycleConfiguration: &s3.BucketLifecycleConfiguration{
Rules: lcRules,
},
}); err != nil {
return fmt.Errorf("failed to set bucket %s's lifecycle rules: %w", s.Config.Bucket, err)
}
}
}

// lc, err := svc.GetBucketLifecycleConfiguration(&s3.GetBucketLifecycleConfigurationInput{
// Bucket: aws.String(.Bucket),
// })
// if err != nil {
// return fmt.Errorf("failed get life-cycle rules: %w", err)
// }
// log.Printf("Life-cycle rules: %+#v\n", lc)

return nil
}