implement maxisam#131: create bucket if it doesn't exist yet, mc only…

… yet (maxisam#132)

Co-authored-by: Thomas von Dein <tom@vondein.org>
Co-authored-by: Sam Lin <456807+maxisam@users.noreply.github.com>

.document/BACKUP_PLAN.md

@@ -56,6 +56,8 @@ s3:
   #storageClass: STANDARD
   # For Minio and AWS use S3v4 for GCP use S3v2
   api: "S3v4"
+  # optional, automatically create the bucket if it does not exist yet
+  #createbucketifneeded: false
 # GCloud upload (optional)
 gcloud:
   bucket: "backup"

charts/mgob/values.yaml

@@ -106,6 +106,7 @@ config: {}
 #        bucket: "backup"
 #        accessKey: "Q3AM3UQ867SPQQA43P2F"
 #        secretKey: "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
+#        createbucketifneeded: false
 #        # For Minio and AWS use S3v4 for GCP use S3v2
 #        api: "S3v4"
 #      # GCloud upload (optional)

pkg/backup/s3.go

@@ -2,10 +2,10 @@ package backup
 
 import (
 	"fmt"
+	"net/url"
 	"path/filepath"
 	"strings"
 	"time"
-	"net/url"
 
 	"github.com/codeskyblue/go-sh"
 	"github.com/pkg/errors"
@@ -89,6 +89,13 @@ func minioUpload(file string, plan config.Plan) (string, error) {
 		return "", errors.Wrapf(err, "mc config host for plan %v failed %s", plan.Name, output)
 	}
 
+	if plan.S3.CreateBucketIfNeeded {
+		err := minioCreateBucket(plan)
+		if err != nil {
+			return "", err
+		}
+	}
+
 	fileName := filepath.Base(file)
 
 	upload := fmt.Sprintf("mc --quiet cp %v %v/%v/%v",
@@ -110,3 +117,30 @@ func minioUpload(file string, plan config.Plan) (string, error) {
 
 	return strings.Replace(output, "\n", " ", -1), nil
 }
+
+func minioCreateBucket(plan config.Plan) error {
+	listbucket := fmt.Sprintf("mc --quiet ls %v/%v", plan.Name, plan.S3.Bucket)
+
+	_, err := sh.Command("/bin/sh", "-c", listbucket).CombinedOutput()
+
+	if err == nil {
+		// nothing to do
+		return nil
+	}
+
+	// bucket does not seem to exist, try to create it
+	createbucket := fmt.Sprintf("mc --quiet mb %v/%v", plan.Name, plan.S3.Bucket)
+
+	result, err := sh.Command("/bin/sh", "-c", createbucket).CombinedOutput()
+
+	if err != nil {
+		output := ""
+		if len(result) > 0 {
+			output = strings.ReplaceAll(string(result), "\n", " ")
+		}
+
+		return errors.Wrapf(err, "S3 creation of bucket %v/%v failed %v", plan.Name, plan.S3.Bucket, output)
+	}
+
+	return nil
+}

pkg/config/plan.go

@@ -65,13 +65,14 @@ type Gpg struct {
 }
 
 type S3 struct {
-	Bucket       string `yaml:"bucket"`
-	AccessKey    string `yaml:"accessKey"`
-	API          string `yaml:"api"`
-	SecretKey    string `yaml:"secretKey"`
-	URL          string `yaml:"url"`
-	KmsKeyId     string `yaml:"kmsKeyId"`
-	StorageClass string `yaml:"storageClass" validate:"omitempty,oneof=STANDARD REDUCED_REDUNDANCY STANDARD_IA ONE-ZONE_IA INTELLIGENT_TIERING GLACIER DEEP_ARCHIVE"`
+	Bucket               string `yaml:"bucket"`
+	AccessKey            string `yaml:"accessKey"`
+	API                  string `yaml:"api"`
+	SecretKey            string `yaml:"secretKey"`
+	URL                  string `yaml:"url"`
+	KmsKeyId             string `yaml:"kmsKeyId"`
+	StorageClass         string `yaml:"storageClass" validate:"omitempty,oneof=STANDARD REDUCED_REDUNDANCY STANDARD_IA ONE-ZONE_IA INTELLIGENT_TIERING GLACIER DEEP_ARCHIVE"`
+	CreateBucketIfNeeded bool   `yaml:"createbucketifneeded"`
 }
 
 type GCloud struct {