refactor: improve template download logic (#791)

* refactor: improve template download logic (for local only)

Signed-off-by: João Vanzuita <joao@kubeshop.io>

cmd/destroyAwsGithub.go

@@ -95,7 +95,7 @@ var destroyAwsGithubCmd = &cobra.Command{
 			githubTfApplied := viper.GetBool("terraform.github.apply.complete")
 			if githubTfApplied {
 				informUser("terraform destroying github resources", globalFlags.SilentMode)
-				tfEntrypoint := config.GitOpsRepoPath + "/terraform/github"
+				tfEntrypoint := config.GitOpsLocalRepoPath + "/terraform/github"
 				terraform.InitDestroyAutoApprove(globalFlags.DryRun, tfEntrypoint)
 				informUser("successfully destroyed github resources", globalFlags.SilentMode)
 			}

cmd/destroyLocalGithub.go

@@ -99,7 +99,7 @@ var destroyLocalGithubCmd = &cobra.Command{
 		githubTfApplied := viper.GetBool("terraform.github.apply.complete")
 		if githubTfApplied {
 			informUser("terraform destroying github resources", globalFlags.SilentMode)
-			tfEntrypoint := config.GitOpsRepoPath + "/terraform/github"
+			tfEntrypoint := config.GitOpsLocalRepoPath + "/terraform/github"
 			terraform.InitReconfigureDestroyAutoApprove(globalFlags.DryRun, tfEntrypoint)
 			informUser("successfully destroyed github resources", globalFlags.SilentMode)
 		}

cmd/local/local.go

@@ -120,7 +120,7 @@ func runLocal(cmd *cobra.Command, args []string) error {
 	if !executionControl {
 		pkg.InformUser("Creating github resources with terraform", silentMode)
 
-		tfEntrypoint := config.GitOpsRepoPath + "/terraform/github"
+		tfEntrypoint := config.GitOpsLocalRepoPath + "/terraform/github"
 		terraform.InitApplyAutoApprove(dryRun, tfEntrypoint)
 
 		pkg.InformUser(fmt.Sprintf("Created gitops Repo in github.com/%s", viper.GetString("github.owner")), silentMode)
@@ -284,7 +284,7 @@ func runLocal(cmd *cobra.Command, args []string) error {
 
 		//* run vault terraform
 		pkg.InformUser("configuring vault with terraform", silentMode)
-		tfEntrypoint := config.GitOpsRepoPath + "/terraform/vault"
+		tfEntrypoint := config.GitOpsLocalRepoPath + "/terraform/vault"
 		terraform.InitApplyAutoApprove(dryRun, tfEntrypoint)
 
 		pkg.InformUser("vault terraform executed successfully", silentMode)
@@ -303,7 +303,7 @@ func runLocal(cmd *cobra.Command, args []string) error {
 	if !executionControl {
 		pkg.InformUser("applying users terraform", silentMode)
 
-		tfEntrypoint := config.GitOpsRepoPath + "/terraform/users"
+		tfEntrypoint := config.GitOpsLocalRepoPath + "/terraform/users"
 		terraform.InitApplyAutoApprove(dryRun, tfEntrypoint)
 
 		pkg.InformUser("executed users terraform successfully", silentMode)
@@ -371,7 +371,6 @@ func runLocal(cmd *cobra.Command, args []string) error {
 
 	// create a PR, atlantis will identify it's a Terraform change/file update and trigger atlantis plan
 	// it's a goroutine since it can run in background
-	k8s.OpenAtlantisPortForward()
 	var wg sync.WaitGroup
 	wg.Add(1)
 	go func() {
@@ -398,7 +397,7 @@ func runLocal(cmd *cobra.Command, args []string) error {
 
 		ok, err := gitHubClient.RetrySearchPullRequestComment(
 			githubOwner,
-			gitOpsRepo,
+			pkg.KubefirstGitOpsRepository,
 			"To **apply** all unapplied plans from this pull request, comment",
 			`waiting "atlantis plan" finish to proceed...`,
 		)
@@ -414,6 +413,7 @@ func runLocal(cmd *cobra.Command, args []string) error {
 
 		err = gitHubClient.CommentPR(1, "atlantis apply")
 		if err != nil {
+			log.Println(err)
 		}
 		wg.Done()
 	}()

cmd/local/prerun.go

@@ -11,6 +11,7 @@ import (
 	"github.com/kubefirst/kubefirst/configs"
 	"github.com/kubefirst/kubefirst/internal/addon"
 	"github.com/kubefirst/kubefirst/internal/downloadManager"
+	"github.com/kubefirst/kubefirst/internal/gitClient"
 	"github.com/kubefirst/kubefirst/internal/handlers"
 	"github.com/kubefirst/kubefirst/internal/progressPrinter"
 	"github.com/kubefirst/kubefirst/internal/repo"
@@ -52,21 +53,8 @@ func validateLocal(cmd *cobra.Command, args []string) error {
 		)
 	}
 
-	// if non-development/built/released version, set template tag version to clone tagged templates, in that way
-	// the current built version, uses the same template version.
-	// example: kubefirst version 1.10.3, has template repositories (gitops and metaphor's) tags set as 1.10.3
-	// when Kubefirst download the templates, it will download the tag version that matches Kubefirst version
-	if configs.K1Version != configs.DefaultK1Version {
-		log.Println("loading tag values for built version")
-		log.Printf("Kubefirst version %q, tags %q", configs.K1Version, config.K3dVersion)
-		// in order to make the fallback tags work, set gitops branch as empty
-		gitOpsBranch = ""
-		templateTag = configs.K1Version
-		viper.Set("template.tag", templateTag)
-	}
-
 	// set default values to kubefirst file
-	viper.Set("gitops.repo", gitOpsRepo)
+	viper.Set("gitops.repo", pkg.KubefirstGitOpsRepository)
 	viper.Set("gitops.owner", "kubefirst")
 	viper.Set("gitprovider", pkg.GitHubProviderName)
 	viper.Set("metaphor.branch", metaphorBranch)
@@ -148,15 +136,73 @@ func validateLocal(cmd *cobra.Command, args []string) error {
 	log.Println("ssh key pair creation complete")
 	progressPrinter.IncrementTracker("step-ssh", 1)
 
-	repo.PrepareKubefirstTemplateRepo(
-		dryRun,
-		config,
-		viper.GetString("github.owner"),
-		viper.GetString("gitops.repo"),
-		viper.GetString("gitops.branch"),
-		viper.GetString("template.tag"),
-	)
-	log.Println("clone and detokenization of gitops-template repository complete")
+	//
+	// clone gitops template
+	//
+	// todo: add wrapper
+	if configs.K1Version == configs.DefaultK1Version {
+
+		gitHubOrg := "kubefirst"
+		repoName := "gitops"
+
+		repoURL := fmt.Sprintf("https://github.com/%s/%s-template", gitHubOrg, repoName)
+
+		repository, err := gitClient.CloneBranch(repoURL, config.GitOpsLocalRepoPath, gitOpsBranch)
+		if err != nil {
+			return err
+		}
+
+		err = gitClient.CheckoutBranch(repository, gitOpsBranch)
+		if err != nil {
+			return err
+		}
+		viper.Set("init.repos.gitops.cloned", true)
+		viper.Set(fmt.Sprintf("git.clone.%s.branch", repoName), gitOpsBranch)
+		if err = viper.WriteConfig(); err != nil {
+			log.Println(err)
+		}
+
+	} else {
+		// use tag
+		gitHubOrg := "kubefirst"
+		repoName := "gitops"
+
+		tag := configs.K1Version
+		repository, err := gitClient.CloneTag(config.GitOpsLocalRepoPath, gitHubOrg, repoName, tag)
+		if err != nil {
+			return err
+		}
+
+		err = gitClient.CheckoutTag(repository, tag)
+		if err != nil {
+			return err
+		}
+
+		viper.Set(fmt.Sprintf("git.clone.%s.tag", repoName), tag)
+		viper.Set("init.repos.gitops.cloned", true)
+		if err = viper.WriteConfig(); err != nil {
+			log.Println(err)
+		}
+	}
+
+	if !viper.GetBool("github.gitops.hydrated") {
+		err = repo.UpdateForLocalMode(config.GitOpsLocalRepoPath)
+		if err != nil {
+			return err
+		}
+	}
+
+	pkg.Detokenize(config.GitOpsLocalRepoPath)
+	viper.Set(fmt.Sprintf("init.repos.%s.detokenized", pkg.KubefirstGitOpsRepository), true)
+	if err = viper.WriteConfig(); err != nil {
+		log.Println(err)
+	}
+
+	err = gitClient.CreateGitHubRemote(config.GitOpsLocalRepoPath, githubUser, pkg.KubefirstGitOpsRepository)
+	if err != nil {
+		return err
+	}
+
 	progressPrinter.IncrementTracker("step-gitops", 1)
 
 	log.Println("sending init completed metric")

configs/config.go

@@ -34,9 +34,9 @@ type Config struct {
 	KubectlClientPath       string
 	KubeConfigPath          string
 	KubeConfigFolder        string
-	HelmClientPath          string
-	GitOpsRepoPath          string
-	NgrokVersion            string
+	HelmClientPath      string
+	GitOpsLocalRepoPath string
+	NgrokVersion        string
 	NgrokClientPath         string
 	TerraformClientPath     string
 	K3dPath                 string
@@ -111,7 +111,7 @@ func ReadConfig() *Config {
 	config.KubectlClientPath = fmt.Sprintf("%s/tools/kubectl", config.K1FolderPath)
 	config.KubeConfigPath = fmt.Sprintf("%s/gitops/terraform/base/kubeconfig", config.K1FolderPath)
 	config.KubeConfigFolder = fmt.Sprintf("%s/gitops/terraform/base", config.K1FolderPath)
-	config.GitOpsRepoPath = fmt.Sprintf("%s/gitops", config.K1FolderPath)
+	config.GitOpsLocalRepoPath = fmt.Sprintf("%s/gitops", config.K1FolderPath)
 	config.NgrokClientPath = fmt.Sprintf("%s/tools/ngrok", config.K1FolderPath)
 	config.TerraformClientPath = fmt.Sprintf("%s/tools/terraform", config.K1FolderPath)
 	config.HelmClientPath = fmt.Sprintf("%s/tools/helm", config.K1FolderPath)

internal/gitClient/git.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/config"
+	gitConfig "github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/plumbing/object"
 	"github.com/go-git/go-git/v5/plumbing/transport/http"
@@ -318,8 +319,7 @@ func PushLocalRepoToEmptyRemote(githubHost, githubOwner, localRepo, remoteName s
 		log.Println("error getting worktree status", err)
 	}
 
-	for file, s := range status {
-		log.Printf("the file is %s the status is %v", file, s.Worktree)
+	for file, _ := range status {
 		_, err = w.Add(file)
 		if err != nil {
 			log.Println("error getting worktree status", err)
@@ -488,3 +488,102 @@ func UpdateLocalTerraformFilesAndPush(githubHost, githubOwner, localRepo, remote
 
 	return nil
 }
+
+// CloneBranch clone a branch and returns a pointer to git.Repository
+func CloneBranch(repoURL string, repoLocalPath string, branch string) (*git.Repository, error) {
+
+	log.Printf("git cloning by branch, branch: %s", configs.K1Version)
+
+	repo, err := git.PlainClone(repoLocalPath, false, &git.CloneOptions{
+		URL:           repoURL,
+		ReferenceName: plumbing.NewBranchReferenceName(branch),
+		SingleBranch:  true,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return repo, nil
+}
+
+// CheckoutBranch checkout a branch
+func CheckoutBranch(repo *git.Repository, branch string) error {
+
+	tree, err := repo.Worktree()
+	if err != nil {
+		return err
+	}
+
+	err = tree.Checkout(&git.CheckoutOptions{
+		Branch: plumbing.NewBranchReferenceName(branch),
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// CloneTag clone a repository using a tag value, and returns a pointer to *git.Repository
+func CloneTag(repoLocalPath string, githubOrg string, repoName string, tag string) (*git.Repository, error) {
+
+	// todo: repoURL como param
+	repoURL := fmt.Sprintf("https://github.com/%s/%s-template", githubOrg, repoName)
+
+	log.Printf("git cloning by tag, tag: %s", configs.K1Version)
+
+	repo, err := git.PlainClone(repoLocalPath, false, &git.CloneOptions{
+		URL:           repoURL,
+		ReferenceName: plumbing.NewTagReferenceName(tag),
+		SingleBranch:  true,
+	})
+	if err != nil {
+		log.Printf("error cloning %s-template repository from GitHub using tag %s", repoName, configs.K1Version)
+		return nil, err
+	}
+
+	return repo, nil
+}
+
+// CheckoutTag repository checkout based on a tag
+func CheckoutTag(repo *git.Repository, tag string) error {
+
+	tree, err := repo.Worktree()
+	if err != nil {
+		return err
+	}
+
+	err = tree.Checkout(&git.CheckoutOptions{
+		Branch: plumbing.ReferenceName("refs/tags/" + tag),
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// CreateGitHubRemote create a remote repository entry
+func CreateGitHubRemote(gitOpsLocalRepoPath string, gitHubUser string, repoName string) error {
+
+	log.Println("creating git remote (github)...")
+
+	repo, err := git.PlainOpen(gitOpsLocalRepoPath)
+	if err != nil {
+		return err
+	}
+
+	repoURL := fmt.Sprintf("https://github.com/%s/%s", gitHubUser, repoName)
+
+	_, err = repo.CreateRemote(&gitConfig.RemoteConfig{
+		Name: "github",
+		URLs: []string{repoURL},
+	})
+	if err != nil {
+		return err
+	}
+
+	log.Println("creating git remote (github) done")
+
+	return nil
+}

internal/metaphor/metaphor.go

@@ -75,7 +75,7 @@ func DeployMetaphorGithub(globalFlags flagset.GlobalFlags) error {
 	}
 	config := configs.ReadConfig()
 
-	tfEntrypoint := config.GitOpsRepoPath + "/terraform/github"
+	tfEntrypoint := config.GitOpsLocalRepoPath + "/terraform/github"
 	err := os.Rename(fmt.Sprintf("%s/%s", tfEntrypoint, "metaphor-repos.md"), fmt.Sprintf("%s/%s", tfEntrypoint, "metaphor-repos.tf"))
 	if err != nil {
 		log.Println("error renaming metaphor-repos.md to metaphor-repos.tf", err)
@@ -114,7 +114,7 @@ func DeployMetaphorGithubLocal(dryRun bool, gitHubOwner string, metaphorBranch s
 
 	config := configs.ReadConfig()
 
-	tfEntrypoint := config.GitOpsRepoPath + "/terraform/github"
+	tfEntrypoint := config.GitOpsLocalRepoPath + "/terraform/github"
 	err := os.Rename(fmt.Sprintf("%s/%s", tfEntrypoint, "metaphor-repos.md"), fmt.Sprintf("%s/%s", tfEntrypoint, "metaphor-repos.tf"))
 	if err != nil {
 		log.Println("error renaming metaphor-repos.md to metaphor-repos.tf", err)

internal/repo/kubefirstTemplate.go

@@ -58,11 +58,11 @@ func PrepareKubefirstTemplateRepo(dryRun bool, config *configs.Config, githubOrg
 
 			},
 		}
-		err := cp.Copy(config.GitOpsRepoPath+"/argo-workflows/.argo", directory+"/.argo", opt)
+		err := cp.Copy(config.GitOpsLocalRepoPath+"/argo-workflows/.argo", directory+"/.argo", opt)
 		if err != nil {
 			log.Println("Error populating argo-workflows .argo/ with local setup:", err)
 		}
-		err = cp.Copy(config.GitOpsRepoPath+"/argo-workflows/.github", directory+"/.github", opt)
+		err = cp.Copy(config.GitOpsLocalRepoPath+"/argo-workflows/.github", directory+"/.github", opt)
 		if err != nil {
 			log.Println("Error populating argo-workflows with .github/ with local setup:", err)
 		}

pkg/constants.go

@@ -1,15 +1,17 @@
 package pkg
 
 const (
-	JSONContentType          = "application/json"
-	SoftServerURI            = "ssh://127.0.0.1:8022/config"
-	GitHubOAuthClientId      = "2ced340927e0a6c49a45"
-	CloudK3d                 = "k3d"
-	GitHubProviderName       = "github"
-	GitHubHost               = "github.com"
-	LocalClusterName         = "kubefirst"
-	MinimumAvailableDiskSize = 10 // 10 GB
-	LocalDNS                 = "localdev.me"
+	JSONContentType              = "application/json"
+	SoftServerURI                = "ssh://127.0.0.1:8022/config"
+	GitHubOAuthClientId          = "2ced340927e0a6c49a45"
+	CloudK3d                     = "k3d"
+	GitHubProviderName           = "github"
+	GitHubHost                   = "github.com"
+	LocalClusterName             = "kubefirst"
+	MinimumAvailableDiskSize     = 10 // 10 GB
+	KubefirstGitOpsRepository    = "gitops"
+	KubefirstGitOpsRepositoryURL = "https://github.com/kubefirst/gitops-template"
+	LocalDNS                     = "localdev.me"
 )
 
 // SegmentIO constants