From a7681bdffb64374d379959e47d1b1accb3d2f371 Mon Sep 17 00:00:00 2001 From: Lucas Marques Date: Fri, 1 Mar 2024 17:18:53 +0100 Subject: [PATCH 1/4] feat(runner): add runnerBinaryPath to runner config --- cmd/runner/start.go | 1 + go.sum | 3 +++ internal/burrito/config/config.go | 1 + 3 files changed, 5 insertions(+) diff --git a/cmd/runner/start.go b/cmd/runner/start.go index 2fb9312b..32543871 100644 --- a/cmd/runner/start.go +++ b/cmd/runner/start.go @@ -20,5 +20,6 @@ func buildRunnerStartCmd(app *burrito.App) *cobra.Command { } cmd.Flags().StringVar(&app.Config.Runner.SSHKnownHostsConfigMapName, "ssh-known-hosts-cm-name", "burrito-ssh-known-hosts", "configmap name to get known hosts file from") + cmd.Flags().StringVar(&app.Config.Runner.RunnerBinaryPath, "runner-binary-path", "/runner/bin", "binary path where the runner can expect to find terraform or terragrunt binaries") return cmd } diff --git a/go.sum b/go.sum index 69b272ae..a0f9698d 100644 --- a/go.sum +++ b/go.sum @@ -85,6 +85,7 @@ github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxG github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= @@ -280,6 +281,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/labstack/echo v3.3.10+incompatible h1:pGRcYk231ExFAyoAjAfD85kQzRJCRI8bbnE7CX5OEgg= +github.com/labstack/echo v3.3.10+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s= github.com/labstack/echo/v4 v4.11.2 h1:T+cTLQxWCDfqDEoydYm5kCobjmHwOwcv4OJAPHilmdE= github.com/labstack/echo/v4 v4.11.2/go.mod h1:UcGuQ8V6ZNRmSweBIJkPvGfwCMIlFmiqrPqiEBfPYws= github.com/labstack/echo/v4 v4.11.4 h1:vDZmA+qNeh1pd/cCkEicDMrjtrnMGQ1QFI9gWN1zGq8= diff --git a/internal/burrito/config/config.go b/internal/burrito/config/config.go index 1b2871a6..d163419c 100644 --- a/internal/burrito/config/config.go +++ b/internal/burrito/config/config.go @@ -79,6 +79,7 @@ type RunnerConfig struct { Layer Layer `mapstructure:"layer"` Repository RepositoryConfig `mapstructure:"repository"` SSHKnownHostsConfigMapName string `mapstructure:"sshKnownHostsConfigMapName"` + RunnerBinaryPath string `mapstructure:"runnerBinaryPath"` } type Layer struct { From 89c4df51b55a4e2230f926d381bf7bd2a6785166 Mon Sep 17 00:00:00 2001 From: Lucas Marques Date: Fri, 1 Mar 2024 17:19:34 +0100 Subject: [PATCH 2/4] feat(runner): ensure terraform installation --- internal/runner/runner.go | 2 +- internal/runner/terraform/terraform.go | 25 +++++++++++++++++++++---- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index c8230c76..abf6c66c 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -146,7 +146,7 @@ func newK8SClient() (client.Client, error) { func (r *Runner) install() error { terraformVersion := configv1alpha1.GetTerraformVersion(r.repository, r.layer) - terraformExec := terraform.NewTerraform(terraformVersion, PlanArtifact) + terraformExec := terraform.NewTerraform(terraformVersion, PlanArtifact, r.config.Runner.RunnerBinaryPath) terraformRuntime := "terraform" if configv1alpha1.GetTerragruntEnabled(r.repository, r.layer) { terraformRuntime = "terragrunt" diff --git a/internal/runner/terraform/terraform.go b/internal/runner/terraform/terraform.go index 9b7531d3..0ae322ec 100644 --- a/internal/runner/terraform/terraform.go +++ b/internal/runner/terraform/terraform.go @@ -8,8 +8,11 @@ import ( "os" "github.com/hashicorp/go-version" + install "github.com/hashicorp/hc-install" + "github.com/hashicorp/hc-install/fs" "github.com/hashicorp/hc-install/product" "github.com/hashicorp/hc-install/releases" + "github.com/hashicorp/hc-install/src" "github.com/hashicorp/terraform-exec/tfexec" ) @@ -18,12 +21,14 @@ type Terraform struct { version string ExecPath string planArtifactPath string + runnerBinaryPath string } -func NewTerraform(version, planArtifactPath string) *Terraform { +func NewTerraform(version, planArtifactPath string, runnerBinaryPath string) *Terraform { return &Terraform{ version: version, planArtifactPath: planArtifactPath, + runnerBinaryPath: runnerBinaryPath, } } @@ -32,11 +37,23 @@ func (t *Terraform) Install() error { if err != nil { return err } - installer := &releases.ExactVersion{ + i := install.NewInstaller() + version := version.Must(terraformVersion, nil) + fs := fs.ExactVersion{ Product: product.Terraform, - Version: version.Must(terraformVersion, nil), + Version: version, + ExtraPaths: []string{ + t.runnerBinaryPath, + }, } - execPath, err := installer.Install(context.Background()) + releases := releases.ExactVersion{ + Product: product.Terraform, + Version: version, + } + execPath, err := i.Ensure(context.Background(), []src.Source{ + &fs, + &releases, + }) if err != nil { return err } From e7fac2a6fcd48507bd4a1e1e3be4b4f7071b5328 Mon Sep 17 00:00:00 2001 From: Lucas Marques Date: Fri, 1 Mar 2024 17:20:02 +0100 Subject: [PATCH 3/4] feat(runner): ensure terragrunt installation --- internal/runner/runner.go | 2 +- internal/runner/terragrunt/terragrunt.go | 97 ++++++++++++++++++++++-- 2 files changed, 90 insertions(+), 9 deletions(-) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index abf6c66c..ec0a4fd1 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -157,7 +157,7 @@ func (r *Runner) install() error { r.exec = terraformExec case "terragrunt": log.Infof("using terragrunt") - r.exec = terragrunt.NewTerragrunt(terraformExec, configv1alpha1.GetTerragruntVersion(r.repository, r.layer), PlanArtifact) + r.exec = terragrunt.NewTerragrunt(terraformExec, configv1alpha1.GetTerragruntVersion(r.repository, r.layer), PlanArtifact, r.config.Runner.RunnerBinaryPath) } err := r.exec.Install() if err != nil { diff --git a/internal/runner/terragrunt/terragrunt.go b/internal/runner/terragrunt/terragrunt.go index f49dbfda..aa101860 100644 --- a/internal/runner/terragrunt/terragrunt.go +++ b/internal/runner/terragrunt/terragrunt.go @@ -1,6 +1,7 @@ package terragrunt import ( + "crypto/sha256" "errors" "fmt" "io" @@ -9,12 +10,10 @@ import ( "os/exec" "path/filepath" "runtime" + "strings" "github.com/padok-team/burrito/internal/runner/terraform" -) - -const ( - BinWorkDir = "/runner/bin" + log "github.com/sirupsen/logrus" ) type Terragrunt struct { @@ -23,13 +22,15 @@ type Terragrunt struct { version string workingDir string terraform *terraform.Terraform + runnerBinaryPath string } -func NewTerragrunt(terraformExec *terraform.Terraform, terragruntVersion, planArtifactPath string) *Terragrunt { +func NewTerragrunt(terraformExec *terraform.Terraform, terragruntVersion, planArtifactPath string, runnerBinaryPath string) *Terragrunt { return &Terragrunt{ version: terragruntVersion, terraform: terraformExec, planArtifactPath: planArtifactPath, + runnerBinaryPath: runnerBinaryPath, } } @@ -43,7 +44,8 @@ func (t *Terragrunt) Install() error { if err != nil { return err } - path, err := downloadTerragrunt(t.version) + + path, err := ensureTerragrunt(t.version, t.runnerBinaryPath) if err != nil { return err } @@ -115,7 +117,86 @@ func (t *Terragrunt) Show(mode string) ([]byte, error) { return output, nil } -func downloadTerragrunt(version string) (string, error) { +func ensureTerragrunt(version string, runnerBinaryPath string) (string, error) { + runnerBinary := filepath.Join(runnerBinaryPath, "terragrunt") + info, err := os.Stat(runnerBinary) + if !os.IsNotExist(err) && !info.IsDir() { + hash, err := calculateFileSHA256(runnerBinary) + if err != nil { + return "", err + } + + trustedHash, err := getTerragruntSHA256(version) + if err != nil { + return "", err + } + + if hash == trustedHash { + err = os.Chmod(runnerBinaryPath, 0755) + if err != nil { + return "", err + } + log.Infof("Terragrunt binary found at %s, using it", runnerBinaryPath) + return filepath.Abs(runnerBinaryPath) + } + } + + log.Infof("Terragrunt binary not found, downloading it. (Consider packaging binaries within your runner image to mitigate eventual network expenses)") + path, err := downloadTerragrunt(version, runnerBinaryPath) + if err != nil { + return "", err + } + + return path, nil +} + +func calculateFileSHA256(filename string) (string, error) { + file, err := os.Open(filename) + if err != nil { + return "", err + } + defer file.Close() + + hash := sha256.New() + + if _, err := io.Copy(hash, file); err != nil { + return "", err + } + + return fmt.Sprintf("%x", hash.Sum(nil)), nil +} + +func getTerragruntSHA256(version string) (string, error) { + cpuArch := runtime.GOARCH + response, err := http.Get(fmt.Sprintf("https://github.com/gruntwork-io/terragrunt/releases/download/v%s/SHA256SUMS", version)) + if err != nil { + return "", err + } + defer response.Body.Close() + + body, err := io.ReadAll(response.Body) + if err != nil { + return "", err + } + + lines := strings.Split(string(body), "\n") + for _, line := range lines { + parts := strings.Fields(line) + if len(parts) != 2 { + continue + } + sha := parts[0] + filename := parts[1] + + if strings.Contains(filename, fmt.Sprintf("linux_%s", cpuArch)) { + return sha, nil + } + } + + return "", errors.New("could not find a hash for this architecture in SHA256SUMS file") +} + +func downloadTerragrunt(version string, runnerBinaryPath string) (string, error) { cpuArch := runtime.GOARCH url := fmt.Sprintf("https://github.com/gruntwork-io/terragrunt/releases/download/v%s/terragrunt_linux_%s", version, cpuArch) @@ -126,7 +207,7 @@ func downloadTerragrunt(version string) (string, error) { } defer response.Body.Close() - filename := fmt.Sprintf("%s/terragrunt_%s", BinWorkDir, cpuArch) + filename := fmt.Sprintf("%s/terragrunt_%s", runnerBinaryPath, cpuArch) file, err := os.Create(filename) if err != nil { return "", err From f84014e854c32a3186c90a1a95604191197409f9 Mon Sep 17 00:00:00 2001 From: Lucas Marques Date: Fri, 1 Mar 2024 20:27:32 +0100 Subject: [PATCH 4/4] feat(runner): loop through all binaries to find Terragrunt --- internal/runner/terragrunt/terragrunt.go | 40 ++++++++++++++---------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/internal/runner/terragrunt/terragrunt.go b/internal/runner/terragrunt/terragrunt.go index aa101860..48d38d6a 100644 --- a/internal/runner/terragrunt/terragrunt.go +++ b/internal/runner/terragrunt/terragrunt.go @@ -118,31 +118,39 @@ func (t *Terragrunt) Show(mode string) ([]byte, error) { } func ensureTerragrunt(version string, runnerBinaryPath string) (string, error) { - runnerBinary := filepath.Join(runnerBinaryPath, "terragrunt") - info, err := os.Stat(runnerBinary) - if !os.IsNotExist(err) && !info.IsDir() { - hash, err := calculateFileSHA256(runnerBinary) - if err != nil { - return "", err - } + files, err := os.ReadDir(runnerBinaryPath) + if err != nil { + return "", err + } - trustedHash, err := getTerragruntSHA256(version) - if err != nil { - return "", err - } + trustedHash, err := getTerragruntSHA256(version) + if err != nil { + return "", err + } - if hash == trustedHash { - err = os.Chmod(runnerBinaryPath, 0755) + for _, file := range files { + if !file.IsDir() { + runnerBinaryFullPath := filepath.Join(runnerBinaryPath, file.Name()) + hash, err := calculateFileSHA256(runnerBinaryFullPath) if err != nil { return "", err } - log.Infof("Terragrunt binary found at %s, using it", runnerBinaryPath) - return filepath.Abs(runnerBinaryPath) + + if hash == trustedHash { + err = os.Chmod(runnerBinaryFullPath, 0755) + if err != nil { + return "", err + } + log.Infof("Terragrunt binary found at %s, using it", runnerBinaryFullPath) + return filepath.Abs(runnerBinaryFullPath) + } + } } - log.Infof("Terragrunt binary not found, downloading it. (Consider packaging binaries within your runner image to mitigate eventual network expenses)") + log.Infof("Terragrunt binary not found, downloading it... (Consider packaging binaries within your runner image to mitigate eventual network expenses)") path, err := downloadTerragrunt(version, runnerBinaryPath) + log.Infof("Downloaded terragrunt binaries to %s", path) if err != nil { return "", err }