Merge pull request #48 from buildpacks/pack-volume-key

When creating volume caches, incorporate "pack volume key" to avoid name collisions

acceptance/acceptance_test.go

@@ -38,6 +38,7 @@ import (
  "github.com/buildpacks/pack/internal/style"
  "github.com/buildpacks/pack/pkg/archive"
  "github.com/buildpacks/pack/pkg/cache"
+ "github.com/buildpacks/pack/pkg/logging"
  h "github.com/buildpacks/pack/testhelpers"
 )
 
@@ -1162,8 +1163,9 @@ func testAcceptance(
  ref, err := name.ParseReference(repoName, name.WeakValidation)
  assert.Nil(err)
  cacheImage := cache.NewImageCache(ref, dockerCli)
- buildCacheVolume := cache.NewVolumeCache(ref, cache.CacheInfo{}, "build", dockerCli)
- launchCacheVolume := cache.NewVolumeCache(ref, cache.CacheInfo{}, "launch", dockerCli)
+ logger := logging.NewSimpleLogger(&bytes.Buffer{})
+ buildCacheVolume, _ := cache.NewVolumeCache(ref, cache.CacheInfo{}, "build", dockerCli, logger)
+ launchCacheVolume, _ := cache.NewVolumeCache(ref, cache.CacheInfo{}, "launch", dockerCli, logger)
  cacheImage.Clear(context.TODO())
  buildCacheVolume.Clear(context.TODO())
  launchCacheVolume.Clear(context.TODO())
@@ -1282,8 +1284,9 @@ func testAcceptance(
  ref, err := name.ParseReference(repoName, name.WeakValidation)
  assert.Nil(err)
  cacheImage := cache.NewImageCache(ref, dockerCli)
- buildCacheVolume := cache.NewVolumeCache(ref, cache.CacheInfo{}, "build", dockerCli)
- launchCacheVolume := cache.NewVolumeCache(ref, cache.CacheInfo{}, "launch", dockerCli)
+ logger := logging.NewSimpleLogger(&bytes.Buffer{})
+ buildCacheVolume, _ := cache.NewVolumeCache(ref, cache.CacheInfo{}, "build", dockerCli, logger)
+ launchCacheVolume, _ := cache.NewVolumeCache(ref, cache.CacheInfo{}, "launch", dockerCli, logger)
  cacheImage.Clear(context.TODO())
  buildCacheVolume.Clear(context.TODO())
  launchCacheVolume.Clear(context.TODO())
@@ -3168,8 +3171,9 @@ include = [ "*.jar", "media/mountain.jpg", "/media/person.png", ]
  imageManager.CleanupImages(origID, repoName, runBefore)
  ref, err := name.ParseReference(repoName, name.WeakValidation)
  assert.Nil(err)
- buildCacheVolume := cache.NewVolumeCache(ref, cache.CacheInfo{}, "build", dockerCli)
- launchCacheVolume := cache.NewVolumeCache(ref, cache.CacheInfo{}, "launch", dockerCli)
+ logger := logging.NewSimpleLogger(&bytes.Buffer{})
+ buildCacheVolume, _ := cache.NewVolumeCache(ref, cache.CacheInfo{}, "build", dockerCli, logger)
+ launchCacheVolume, _ := cache.NewVolumeCache(ref, cache.CacheInfo{}, "launch", dockerCli, logger)
  assert.Succeeds(buildCacheVolume.Clear(context.TODO()))
  assert.Succeeds(launchCacheVolume.Clear(context.TODO()))
  })

internal/build/lifecycle_execution.go

@@ -181,7 +181,11 @@ func (l *LifecycleExecution) Run(ctx context.Context, phaseFactoryCreator PhaseF
  } else {
  switch l.opts.Cache.Build.Format {
  case cache.CacheVolume:
- buildCache = cache.NewVolumeCache(l.opts.Image, l.opts.Cache.Build, "build", l.docker)
+ var err error
+ buildCache, err = cache.NewVolumeCache(l.opts.Image, l.opts.Cache.Build, "build", l.docker, l.logger)
+ if err != nil {
+ return err
+ }
  l.logger.Debugf("Using build cache volume %s", style.Symbol(buildCache.Name()))
  case cache.CacheBind:
  buildCache = cache.NewBindCache(l.opts.Cache.Build, l.docker)
@@ -196,7 +200,10 @@ func (l *LifecycleExecution) Run(ctx context.Context, phaseFactoryCreator PhaseF
  l.logger.Debugf("Build cache %s cleared", style.Symbol(buildCache.Name()))
  }
 
- launchCache := cache.NewVolumeCache(l.opts.Image, l.opts.Cache.Launch, "launch", l.docker)
+ launchCache, err := cache.NewVolumeCache(l.opts.Image, l.opts.Cache.Launch, "launch", l.docker, l.logger)
+ if err != nil {
+ return err
+ }
 
  if l.opts.Network == "" {
  // start an ephemeral bridge network
@@ -249,7 +256,10 @@ func (l *LifecycleExecution) Run(ctx context.Context, phaseFactoryCreator PhaseF
  // lifecycle 0.17.0 (introduces support for Platform API 0.12) and above will ensure that
  // this volume is owned by the CNB user,
  // and hence the restorer (after dropping privileges) will be able to write to it.
- kanikoCache = cache.NewVolumeCache(l.opts.Image, l.opts.Cache.Kaniko, "kaniko", l.docker)
+ kanikoCache, err = cache.NewVolumeCache(l.opts.Image, l.opts.Cache.Kaniko, "kaniko", l.docker, l.logger)
+ if err != nil {
+ return err
+ }
  } else {
  switch {
  case buildCache.Type() == cache.Volume:
@@ -261,7 +271,10 @@ func (l *LifecycleExecution) Run(ctx context.Context, phaseFactoryCreator PhaseF
  return fmt.Errorf("build cache must be volume cache when building with extensions")
  default:
  // The kaniko cache is unused, so it doesn't matter that it's not usable.
- kanikoCache = cache.NewVolumeCache(l.opts.Image, l.opts.Cache.Kaniko, "kaniko", l.docker)
+ kanikoCache, err = cache.NewVolumeCache(l.opts.Image, l.opts.Cache.Kaniko, "kaniko", l.docker, l.logger)
+ if err != nil {
+ return err
+ }
  }
  }
 

internal/config/config.go

@@ -25,6 +25,10 @@ type Config struct {
  LayoutRepositoryDir string `toml:"layout-repo-dir,omitempty"`
 }
 
+type VolumeConfig struct {
+ VolumeKeys map[string]string `toml:"volume-keys,omitempty"`
+}
+
 type Registry struct {
  Name string `toml:"name"`
  Type string `toml:"type"`
@@ -58,6 +62,14 @@ func DefaultConfigPath() (string, error) {
  return filepath.Join(home, "config.toml"), nil
 }
 
+func DefaultVolumeKeysPath() (string, error) {
+ home, err := PackHome()
+ if err != nil {
+ return "", errors.Wrap(err, "getting pack home")
+ }
+ return filepath.Join(home, "volume-keys.toml"), nil
+}
+
 func PackHome() (string, error) {
  packHome := os.Getenv("PACK_HOME")
  if packHome == "" {
@@ -79,7 +91,16 @@ func Read(path string) (Config, error) {
  return cfg, nil
 }
 
-func Write(cfg Config, path string) error {
+func ReadVolumeKeys(path string) (VolumeConfig, error) {
+ cfg := VolumeConfig{}
+ _, err := toml.DecodeFile(path, &cfg)
+ if err != nil && !os.IsNotExist(err) {
+ return VolumeConfig{}, errors.Wrapf(err, "failed to read config file at path %s", path)
+ }
+ return cfg, nil
+}
+
+func Write(cfg interface{}, path string) error {
  if err := MkdirAll(filepath.Dir(path)); err != nil {
  return err
  }

pkg/cache/volume_cache.go

@@ -2,25 +2,36 @@ package cache
 
 import (
  "context"
+ "crypto/rand"
  "crypto/sha256"
  "fmt"
+ "os"
  "strings"
 
+ "github.com/GoogleContainerTools/kaniko/pkg/util/proc"
  "github.com/docker/docker/client"
  "github.com/google/go-containerregistry/pkg/name"
 
+ "github.com/buildpacks/pack/internal/config"
  "github.com/buildpacks/pack/internal/paths"
+ "github.com/buildpacks/pack/pkg/logging"
 )
 
+const EnvVolumeKey = "PACK_VOLUME_KEY"
+
 type VolumeCache struct {
  docker DockerClient
  volume string
 }
 
-func NewVolumeCache(imageRef name.Reference, cacheType CacheInfo, suffix string, dockerClient DockerClient) *VolumeCache {
+func NewVolumeCache(imageRef name.Reference, cacheType CacheInfo, suffix string, dockerClient DockerClient, logger logging.Logger) (*VolumeCache, error) {
  var volumeName string
  if cacheType.Source == "" {
- sum := sha256.Sum256([]byte(imageRef.Name()))
+ volumeKey, err := getVolumeKey(imageRef, logger)
+ if err != nil {
+ return nil, err
+ }
+ sum := sha256.Sum256([]byte(imageRef.Name() + volumeKey))
  vol := paths.FilterReservedNames(fmt.Sprintf("%s-%x", sanitizedRef(imageRef), sum[:6]))
  volumeName = fmt.Sprintf("pack-cache-%s.%s", vol, suffix)
  } else {
@@ -30,7 +41,66 @@ func NewVolumeCache(imageRef name.Reference, cacheType CacheInfo, suffix string,
  return &VolumeCache{
  volume: volumeName,
  docker: dockerClient,
+ }, nil
+}
+
+func getVolumeKey(imageRef name.Reference, logger logging.Logger) (string, error) {
+ var foundKey string
+
+ // first, look for key in env
+
+ foundKey = os.Getenv(EnvVolumeKey)
+ if foundKey != "" {
+ return foundKey, nil
+ }
+
+ // then, look for key in existing config
+
+ volumeKeysPath, err := config.DefaultVolumeKeysPath()
+ if err != nil {
+ return "", err
+ }
+ cfg, err := config.ReadVolumeKeys(volumeKeysPath)
+ if err != nil {
+ return "", err
+ }
+
+ foundKey = cfg.VolumeKeys[imageRef.Name()]
+ if foundKey != "" {
+ return foundKey, nil
+ }
+
+ // finally, create new key and store it in config
+
+ // if we're running in a container, we should log a warning
+ // so that we don't always re-create the cache
+ if RunningInContainer() {
+ logger.Warnf("%s is unset; set this environment variable to a secret value to avoid creating a new volume cache on every build", EnvVolumeKey)
+ }
+
+ newKey := randString(20)
+ if cfg.VolumeKeys == nil {
+ cfg.VolumeKeys = make(map[string]string)
+ }
+ cfg.VolumeKeys[imageRef.Name()] = newKey
+ if err = config.Write(cfg, volumeKeysPath); err != nil {
+ return "", err
+ }
+
+ return newKey, nil
+}
+
+// Returns a string iwith lowercase a-z, of length n
+func randString(n int) string {
+ b := make([]byte, n)
+ _, err := rand.Read(b)
+ if err != nil {
+ panic(err)
  }
+ for i := range b {
+ b[i] = 'a' + (b[i] % 26)
+ }
+ return string(b)
 }
 
 func (c *VolumeCache) Name() string {
@@ -56,3 +126,7 @@ func sanitizedRef(ref name.Reference) string {
  result = strings.ReplaceAll(result, "/", "_")
  return fmt.Sprintf("%s_%s", result, ref.Identifier())
 }
+
+var RunningInContainer = func() bool {
+ return proc.GetContainerRuntime(0, 0) != proc.RuntimeNotFound
+}