chore: speedup pak command in tests

added caching which can be turned on by setting an enviornment variable on local
to speedup your local dev loops while running pak

[#181404983]

Signed-off-by: James Norman <normanja@vmware.com>

cmd/pak.go

@@ -19,11 +19,19 @@ import (
 	"log"
 	"os"
 
+	"github.com/spf13/viper"
+
 	"github.com/cloudfoundry/cloud-service-broker/pkg/brokerpak"
 	"github.com/spf13/cobra"
 )
 
+const (
+	pakCachePath = "pak.cache_path"
+)
+
 func init() {
+	viper.BindEnv(pakCachePath, "PAK_BUILD_CACHE_PATH")
+
 	pakCmd := &cobra.Command{
 		Use:   "pak",
 		Short: "interact with user-defined service definition bundles",
@@ -98,7 +106,7 @@ dependencies, services it provides, and the contents.
 				directory = args[0]
 			}
 
-			pakPath, err := brokerpak.Pack(directory)
+			pakPath, err := brokerpak.Pack(directory, viper.GetString(pakCachePath))
 			if err != nil {
 				log.Fatalf("error while packing %q: %v", directory, err)
 			}
@@ -171,7 +179,7 @@ dependencies, services it provides, and the contents.
 			}
 
 			// Edit the manifest to point to our local server
-			packname, err := brokerpak.Pack(td)
+			packname, err := brokerpak.Pack(td, "")
 			defer os.Remove(packname)
 			if err != nil {
 				log.Fatalf("couldn't pack brokerpak: %v", err)

internal/brokerpak/packer/cache.go

@@ -0,0 +1,56 @@
+package packer
+
+import (
+	"crypto/md5"
+	"fmt"
+	"log"
+	"os"
+	"path"
+
+	cp "github.com/otiai10/copy"
+)
+
+func copyFromCache(cachePath string, source string, destination string) bool {
+	if cachePath == "" {
+		return false
+	}
+	cacheKey := buildCacheKey(cachePath, source)
+	if _, err := os.Stat(cacheKey); err == nil {
+		cp.Copy(cacheKey, destination)
+		log.Println("\t", source, "found in cache at", cacheKey)
+		return true
+	} else {
+		return false
+	}
+}
+
+func populateCache(cachePath string, source string, destination string) {
+	if cachePath == "" {
+		return
+	}
+	cacheKey := buildCacheKey(cachePath, source)
+	err := cp.Copy(destination, cacheKey)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func buildCacheKey(cachePath string, source string) string {
+	return path.Join(cachePath, fmt.Sprintf("%x", md5.Sum([]byte(source))))
+}
+
+func cachedFetchFile(getter func(source string, destination string) error, source, destination, cachePath string) error {
+	if cachePath == "" {
+		return getter(source, destination)
+	}
+
+	if copyFromCache(cachePath, source, destination) {
+		return nil
+	}
+	err := getter(source, destination)
+	if err != nil {
+		return err
+	}
+	populateCache(cachePath, source, destination)
+	return nil
+}

internal/brokerpak/packer/pack.go

@@ -8,19 +8,20 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/cloudfoundry/cloud-service-broker/internal/brokerpak/brokerpakurl"
 	"github.com/cloudfoundry/cloud-service-broker/internal/brokerpak/fetcher"
+	"github.com/hashicorp/go-getter"
+
+	"github.com/cloudfoundry/cloud-service-broker/internal/brokerpak/brokerpakurl"
 	"github.com/cloudfoundry/cloud-service-broker/internal/brokerpak/manifest"
 	"github.com/cloudfoundry/cloud-service-broker/internal/zippy"
 	"github.com/cloudfoundry/cloud-service-broker/pkg/providers/tf"
 	"github.com/cloudfoundry/cloud-service-broker/utils"
 	"github.com/cloudfoundry/cloud-service-broker/utils/stream"
-	"github.com/hashicorp/go-getter"
 )
 
 const manifestName = "manifest.yml"
 
-func Pack(m *manifest.Manifest, base, dest string) error {
+func Pack(m *manifest.Manifest, base, dest, cachePath string) error {
 	// NOTE: we use "log" rather than Lager because this is used by the CLI and
 	// needs to be human-readable rather than JSON.
 	switch base {
@@ -38,12 +39,12 @@ func Pack(m *manifest.Manifest, base, dest string) error {
 	log.Println("Using temp directory:", dir)
 
 	log.Println("Packing sources...")
-	if err := packSources(m, dir); err != nil {
+	if err := packSources(m, dir, cachePath); err != nil {
 		return err
 	}
 
 	log.Println("Packing binaries...")
-	if err := packBinaries(m, dir); err != nil {
+	if err := packBinaries(m, dir, cachePath); err != nil {
 		return err
 	}
 
@@ -56,15 +57,15 @@ func Pack(m *manifest.Manifest, base, dest string) error {
 	return zippy.Archive(dir, dest)
 }
 
-func packSources(m *manifest.Manifest, tmp string) error {
+func packSources(m *manifest.Manifest, tmp string, cachePath string) error {
 	packSource := func(source, name string) error {
 		if source == "" {
 			return nil
 		}
 		destination := filepath.Join(tmp, "src", name+".zip")
 
 		log.Println("\t", source, "->", destination)
-		return fetcher.FetchArchive(source, destination)
+		return cachedFetchFile(fetcher.FetchArchive, source, destination, cachePath)
 	}
 
 	for _, resource := range m.TerraformVersions {
@@ -86,25 +87,29 @@ func packSources(m *manifest.Manifest, tmp string) error {
 	return nil
 }
 
-func packBinaries(m *manifest.Manifest, tmp string) error {
+func getAny(source, destination string) error {
+	return getter.GetAny(source, destination)
+}
+
+func packBinaries(m *manifest.Manifest, tmp string, cachePath string) error {
 	for _, platform := range m.Platforms {
 		p := filepath.Join(tmp, "bin", platform.Os, platform.Arch)
 
 		for _, resource := range m.TerraformVersions {
 			log.Println("\t", brokerpakurl.URL("terraform", resource.Version.String(), resource.URLTemplate, platform), "->", filepath.Join(p, resource.Version.String()))
-			if err := getter.GetAny(filepath.Join(p, resource.Version.String()), brokerpakurl.URL("terraform", resource.Version.String(), resource.URLTemplate, platform)); err != nil {
+			if err := cachedFetchFile(getAny, filepath.Join(p, resource.Version.String()), brokerpakurl.URL("terraform", resource.Version.String(), resource.URLTemplate, platform), cachePath); err != nil {
 				return err
 			}
 		}
 		for _, resource := range m.TerraformProviders {
 			log.Println("\t", brokerpakurl.URL(resource.Name, resource.Version.String(), resource.URLTemplate, platform), "->", p)
-			if err := getter.GetAny(p, brokerpakurl.URL(resource.Name, resource.Version.String(), resource.URLTemplate, platform)); err != nil {
+			if err := cachedFetchFile(getAny, p, brokerpakurl.URL(resource.Name, resource.Version.String(), resource.URLTemplate, platform), cachePath); err != nil {
 				return err
 			}
 		}
 		for _, resource := range m.Binaries {
 			log.Println("\t", brokerpakurl.URL(resource.Name, resource.Version, resource.URLTemplate, platform), "->", p)
-			if err := getter.GetAny(p, brokerpakurl.URL(resource.Name, resource.Version, resource.URLTemplate, platform)); err != nil {
+			if err := cachedFetchFile(getAny, p, brokerpakurl.URL(resource.Name, resource.Version, resource.URLTemplate, platform), cachePath); err != nil {
 				return err
 			}
 		}

internal/brokerpak/reader/reader_test.go

@@ -190,7 +190,7 @@ func fakeBrokerpak(opts ...option) string {
 	}
 
 	packName := path.Join(GinkgoT().TempDir(), "fake.brokerpak")
-	Expect(packer.Pack(m, dir, packName)).NotTo(HaveOccurred())
+	Expect(packer.Pack(m, dir, packName, "")).NotTo(HaveOccurred())
 	return packName
 }
 

pkg/brokerpak/cmd.go

@@ -55,7 +55,7 @@ func Init(directory string) error {
 // Pack creates a new brokerpak from the given directory which MUST contain a
 // manifest.yml file. If the pack was successful, the returned string will be
 // the path to the created brokerpak.
-func Pack(directory string) (string, error) {
+func Pack(directory string, cachePath string) (string, error) {
 	data, err := os.ReadFile(filepath.Join(directory, manifestName))
 	if err != nil {
 		return "", err
@@ -72,7 +72,7 @@ func Pack(directory string) (string, error) {
 		version = m.Version
 	}
 	packname := fmt.Sprintf("%s-%s.brokerpak", m.Name, version)
-	return packname, packer.Pack(m, directory, packname)
+	return packname, packer.Pack(m, directory, packname, cachePath)
 }
 
 // Info writes out human-readable information about the brokerpak.

pkg/brokerpak/cmd_test.go

@@ -94,7 +94,7 @@ func fakeBrokerpak() (string, error) {
 		}
 	}
 
-	return Pack(dir)
+	return Pack(dir, "")
 }
 
 func ExampleValidate() {