Merge pull request #135 from kubefirst/feat-restore-certs

feat-restore-certs

cmd/backupSsl.go

@@ -0,0 +1,29 @@
+package cmd
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/kubefirst/kubefirst/internal/ssl"
+	"github.com/spf13/cobra"
+)
+
+// backupSslCmd represents the backupSsl command
+var backupSslCmd = &cobra.Command{
+	Use:   "backupSSL",
+	Short: "Backup Secrets (cert-manager/certificates) to bucket kubefirst-<DOMAIN>",
+	Long: `This command create a backupt of secrets from certmanager certificates to bucket named kubefirst-<DOMAIN> 
+where can be used on provisioning phase with the flag --recycle-ssl`,
+
+	Run: func(cmd *cobra.Command, args []string) {
+		_, err := ssl.GetBackupCertificates()
+		if err != nil {
+			log.Panic(err)
+		}
+		fmt.Println("Backup certificates finished successfully")
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(backupSslCmd)
+}

cmd/create.go

@@ -73,6 +73,8 @@ to quickly create a Cobra application.`,
 		terraform.ApplyBaseTerraform(dryRun, directory)
 		progressPrinter.IncrementTracker("step-softserve", 1)
 
+		restoreSSLCmd.Run(cmd, args)
+
 		//! soft-serve was just applied
 
 		softserve.CreateSoftServe(dryRun, config.KubeConfigPath)
@@ -111,6 +113,7 @@ to quickly create a Cobra application.`,
 
 		//! argocd was just helm installed
 		waitArgoCDToBeReady(dryRun)
+
 		informUser("ArgoCD Ready")
 		progressPrinter.IncrementTracker("step-argo", 1)
 
@@ -153,7 +156,6 @@ to quickly create a Cobra application.`,
 		informUser("Syncing the registry application")
 		argocd.SyncArgocdApplication(dryRun, "registry", token)
 		progressPrinter.IncrementTracker("step-argo", 1)
-
 		// todo, need to stall until the registry has synced, then get to ui asap
 
 		//! skip this if syncing from argocd and not helm installing

cmd/restoreSSL.go

@@ -0,0 +1,31 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/kubefirst/kubefirst/internal/ssl"
+	"github.com/spf13/cobra"
+)
+
+// restoreSSLCmd represents the restoreSSL command
+var restoreSSLCmd = &cobra.Command{
+	Use:   "restoreSSL",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("restoreSSL called")
+		err := ssl.RestoreSSL()
+		if err != nil {
+			fmt.Println("Bucket not found, missing SSL backup, assuming first installation")
+		}
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(restoreSSLCmd)
+}

configs/config.go

@@ -36,6 +36,8 @@ type Config struct {
 
 	// todo: move it back
 	KubefirstVersion string
+
+	CertsPath string
 }
 
 func ReadConfig() *Config {
@@ -65,7 +67,7 @@ func ReadConfig() *Config {
 	config.KubeConfigPath = fmt.Sprintf("%s/gitops/terraform/base/kubeconfig", config.K1FolderPath)
 	config.TerraformPath = fmt.Sprintf("%s/tools/terraform", config.K1FolderPath)
 	config.HelmClientPath = fmt.Sprintf("%s/tools/helm", config.K1FolderPath)
-
+	config.CertsPath = fmt.Sprintf("%s/ssl", config.K1FolderPath)
 	config.TerraformVersion = "1.0.11"
 
 	// todo adopt latest helmVersion := "v3.9.0"

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/go-git/go-git/v5 v5.4.2
 	github.com/google/uuid v1.1.2
 	github.com/hashicorp/vault/api v1.6.0
+	github.com/itchyny/gojq v0.12.8
 	github.com/jedib0t/go-pretty/v6 v6.3.1
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/viper v1.11.0
@@ -81,6 +82,7 @@ require (
 	github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/itchyny/timefmt-go v0.1.3 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
@@ -118,7 +120,7 @@ require (
 	go.uber.org/atomic v1.9.0 // indirect
 	golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 // indirect
 	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
-	golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect
+	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect
@@ -131,7 +133,7 @@ require (
 	gopkg.in/square/go-jose.v2 v2.5.1 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/klog/v2 v2.60.1 // indirect
 	k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect

go.sum

@@ -407,6 +407,10 @@ github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
 github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/itchyny/gojq v0.12.8 h1:Zxcwq8w4IeR8JJYEtoG2MWJZUv0RGY6QqJcO1cqV8+A=
+github.com/itchyny/gojq v0.12.8/go.mod h1:gE2kZ9fVRU0+JAksaTzjIlgnCa2akU+a1V0WXgJQN5c=
+github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU=
+github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
 github.com/jedib0t/go-pretty/v6 v6.3.1 h1:aOXiD9oqiuLH8btPQW6SfgtQN5zwhyfzZls8a6sPJ/I=
@@ -856,8 +860,9 @@ golang.org/x/sys v0.0.0-20220204135822-1c1b9b1eba6a/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0=
 golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1141,8 +1146,9 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

internal/aws/aws.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"net"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -18,6 +19,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"github.com/cip8/autoname"
 	"github.com/kubefirst/kubefirst/pkg"
 	"github.com/spf13/viper"
@@ -330,3 +332,98 @@ func DestroyBucketsInUse(destroyBuckets bool) {
 		log.Println("Skip: DestroyBucketsInUse")
 	}
 }
+
+func CreateBucket(dryRun bool, name string) {
+	log.Println("createBucketCalled")
+
+	s3Client := s3.New(GetAWSSession())
+
+	log.Println("creating", "bucket", name)
+
+	regionName := viper.GetString("aws.region")
+	log.Println("region is ", regionName)
+	if !dryRun {
+		_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
+			Bucket: &name,
+			CreateBucketConfiguration: &s3.CreateBucketConfiguration{
+				LocationConstraint: aws.String(regionName),
+			},
+		})
+		if err != nil {
+			if awsErr, ok := err.(awserr.Error); ok {
+				switch awsErr.Code() {
+				case s3.ErrCodeBucketAlreadyExists:
+					log.Println("Bucket already exists " + name)
+					os.Exit(1)
+				case s3.ErrCodeBucketAlreadyOwnedByYou:
+					log.Println("Bucket already exists but OwnedByYou, the process will continue: " + name)
+				}
+			} else {
+				log.Println("failed to create bucket "+name, err.Error())
+				os.Exit(1)
+			}
+		}
+	} else {
+		log.Printf("[#99] Dry-run mode, bucket creation skipped:  %s", name)
+	}
+	viper.Set(fmt.Sprintf("bucket.%s.created", name), true)
+	viper.Set(fmt.Sprintf("bucket.%s.name", name), name)
+	viper.WriteConfig()
+}
+
+func UploadFile(bucket, key, fileName string) error {
+	uploader := s3manager.NewUploader(GetAWSSession())
+
+	f, err := os.Open(fileName)
+	if err != nil {
+		return fmt.Errorf("failed to open file %q, %v", fileName, err)
+	}
+
+	// Upload the file to S3.
+	result, err := uploader.Upload(&s3manager.UploadInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+		Body:   f,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to upload file, %v", err)
+	}
+	log.Printf("file uploaded to, %s\n", result.Location)
+	return nil
+}
+
+func DownloadBucket(bucket string, destFolder string) error {
+	s3Client := s3.New(GetAWSSession())
+	downloader := s3manager.NewDownloader(GetAWSSession())
+
+	log.Println("Listing the objects in the bucket:")
+	listObjsResponse, err := s3Client.ListObjectsV2(&s3.ListObjectsV2Input{
+		Bucket: aws.String(bucket),
+		Prefix: aws.String(""),
+	})
+
+	if err != nil {
+		log.Printf("Couldn't list bucket contents")
+		return fmt.Errorf("Couldn't list bucket contents")
+	}
+
+	for _, object := range listObjsResponse.Contents {
+		log.Printf("%s (%d bytes, class %v) \n", *object.Key, object.Size, object.StorageClass)
+
+		f, err := pkg.CreateFullPath(filepath.Join(destFolder, *object.Key))
+		if err != nil {
+			return fmt.Errorf("failed to create file %q, %v", *object.Key, err)
+		}
+
+		// Write the contents of S3 Object to the file
+		_, err = downloader.Download(f, &s3.GetObjectInput{
+			Bucket: aws.String(bucket),
+			Key:    aws.String(*object.Key),
+		})
+		if err != nil {
+			return fmt.Errorf("failed to download file, %v", err)
+		}
+		f.Close()
+	}
+	return nil
+}

internal/k8s/kubernetes.go

@@ -11,10 +11,15 @@ import (
 	"log"
 	"time"
 
+	"github.com/itchyny/gojq"
 	"github.com/kubefirst/kubefirst/internal/argocd"
 	"github.com/kubefirst/kubefirst/pkg"
 	"github.com/spf13/viper"
 	metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/dynamic"
 	coreV1Types "k8s.io/client-go/kubernetes/typed/core/v1"
 )
 
@@ -100,3 +105,75 @@ func DeleteRegistryApplication(skipDeleteRegistryApplication bool) {
 		log.Println("skip:  deleteRegistryApplication")
 	}
 }
+
+func GetResourcesDynamically(dynamic dynamic.Interface,
+	ctx context.Context,
+	group string,
+	version string,
+	resource string,
+	namespace string) (
+	[]unstructured.Unstructured, error) {
+
+	resourceId := schema.GroupVersionResource{
+		Group:    group,
+		Version:  version,
+		Resource: resource,
+	}
+	list, err := dynamic.Resource(resourceId).Namespace(namespace).
+		List(ctx, metaV1.ListOptions{})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return list.Items, nil
+}
+
+func GetResourcesByJq(dynamic dynamic.Interface, ctx context.Context, group string,
+	version string, resource string, namespace string, jq string) (
+	[]unstructured.Unstructured, error) {
+
+	resources := make([]unstructured.Unstructured, 0)
+
+	query, err := gojq.Parse(jq)
+	if err != nil {
+		return nil, err
+	}
+
+	items, err := GetResourcesDynamically(dynamic, ctx, group, version, resource, namespace)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, item := range items {
+
+		// Convert object to raw JSON
+		var rawJson interface{}
+		err = runtime.DefaultUnstructuredConverter.FromUnstructured(item.Object, &rawJson)
+		if err != nil {
+			return nil, err
+		}
+
+		// Evaluate jq against JSON
+		iter := query.Run(rawJson)
+		for {
+			result, ok := iter.Next()
+			if !ok {
+				break
+			}
+			if err, ok := result.(error); ok {
+				if err != nil {
+					return nil, err
+				}
+			} else {
+				boolResult, ok := result.(bool)
+				if !ok {
+					fmt.Println("Query returned non-boolean value")
+				} else if boolResult {
+					resources = append(resources, item)
+				}
+			}
+		}
+	}
+	return resources, nil
+}