Add layer caching to kaniko

cmd/executor/cmd/root.go

@@ -92,6 +92,8 @@ func addKanikoOptionsFlags(cmd *cobra.Command) {
  RootCmd.PersistentFlags().BoolVarP(&opts.Reproducible, "reproducible", "", false, "Strip timestamps out of the image to make it reproducible")
  RootCmd.PersistentFlags().StringVarP(&opts.Target, "target", "", "", "Set the target build stage to build")
  RootCmd.PersistentFlags().BoolVarP(&opts.NoPush, "no-push", "", false, "Do not push the image to the registry")
+ RootCmd.PersistentFlags().StringVarP(&opts.Cache, "cache", "", "", "Specify a registry to use as a chace, otherwise one will be inferred from the destination provided")
+ RootCmd.PersistentFlags().BoolVarP(&opts.UseCache, "use-cache", "", true, "Use cache when building image")
 }
 
 // addHiddenFlags marks certain flags as hidden from the executor help text

integration/dockerfiles/Dockerfile_test_cache

@@ -0,0 +1,22 @@
+# Copyright 2018 Google, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Test to make sure the cache works properly
+# /date should be the same regardless of when this image is built
+# if the cache is implemented correctly
+
+FROM gcr.io/google-appengine/debian9@sha256:1d6a9a6d106bd795098f60f4abb7083626354fa6735e81743c7f8cfca11259f0
+RUN date > /date
+COPY context/foo /foo
+RUN echo hey

integration/dockerfiles/Dockerfile_test_cache_install

@@ -0,0 +1,20 @@
+# Copyright 2018 Google, Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Test to make sure the cache works properly
+# /date should be the same regardless of when this image is built
+# if the cache is implemented correctly
+
+FROM gcr.io/google-appengine/debian9@sha256:1d6a9a6d106bd795098f60f4abb7083626354fa6735e81743c7f8cfca11259f0
+RUN apt-get update && apt-get install -y make

integration/images.go

@@ -23,6 +23,7 @@ import (
  "path"
  "path/filepath"
  "runtime"
+ "strconv"
  "strings"
 )
 
@@ -77,12 +78,16 @@ func GetKanikoImage(imageRepo, dockerfile string) string {
  return strings.ToLower(imageRepo + kanikoPrefix + dockerfile)
 }
 
+// GetVersionedKanikoImage versions constructs the name of the kaniko image that would be built
+// with the dockerfile and versions it for cache testing
+func GetVersionedKanikoImage(imageRepo, dockerfile string, version int) string {
+ return strings.ToLower(imageRepo + kanikoPrefix + dockerfile + strconv.Itoa(version))
+}
+
 // FindDockerFiles will look for test docker files in the directory dockerfilesPath.
 // These files must start with `Dockerfile_test`. If the file is one we are intentionally
 // skipping, it will not be included in the returned list.
 func FindDockerFiles(dockerfilesPath string) ([]string, error) {
- // TODO: remove test_user_run from this when https://github.com/GoogleContainerTools/container-diff/issues/237 is fixed
- testsToIgnore := map[string]bool{"Dockerfile_test_user_run": true}
  allDockerfiles, err := filepath.Glob(path.Join(dockerfilesPath, "Dockerfile_test*"))
  if err != nil {
  return []string{}, fmt.Errorf("Failed to find docker files at %s: %s", dockerfilesPath, err)
@@ -92,9 +97,8 @@ func FindDockerFiles(dockerfilesPath string) ([]string, error) {
  for _, dockerfile := range allDockerfiles {
  // Remove the leading directory from the path
  dockerfile = dockerfile[len("dockerfiles/"):]
- if !testsToIgnore[dockerfile] {
- dockerfiles = append(dockerfiles, dockerfile)
- }
+ dockerfiles = append(dockerfiles, dockerfile)
+
  }
  return dockerfiles, err
 }
@@ -103,7 +107,9 @@ func FindDockerFiles(dockerfilesPath string) ([]string, error) {
 // keeps track of which files have been built.
 type DockerFileBuilder struct {
  // Holds all available docker files and whether or not they've been built
- FilesBuilt map[string]bool
+ FilesBuilt map[string]bool
+ DockerfilesToIgnore map[string]struct{}
+ TestCacheDockerfiles map[string]struct{}
 }
 
 // NewDockerFileBuilder will create a DockerFileBuilder initialized with dockerfiles, which
@@ -113,6 +119,14 @@ func NewDockerFileBuilder(dockerfiles []string) *DockerFileBuilder {
  for _, f := range dockerfiles {
  d.FilesBuilt[f] = false
  }
+ d.DockerfilesToIgnore = map[string]struct{}{
+ // TODO: remove test_user_run from this when https://github.com/GoogleContainerTools/container-diff/issues/237 is fixed
+ "Dockerfile_test_user_run": {},
+ }
+ d.TestCacheDockerfiles = map[string]struct{}{
+ "Dockerfile_test_cache": {},
+ "Dockerfile_test_cache_install": {},
+ }
  return &d
 }
 
@@ -164,6 +178,7 @@ func (d *DockerFileBuilder) BuildImage(imageRepo, gcsBucket, dockerfilesPath, do
  }
  }
 
+ cacheFlag := "--use-cache=false"
  // build kaniko image
  additionalFlags = append(buildArgs, additionalKanikoFlagsMap[dockerfile]...)
  kanikoImage := GetKanikoImage(imageRepo, dockerfile)
@@ -174,6 +189,7 @@ func (d *DockerFileBuilder) BuildImage(imageRepo, gcsBucket, dockerfilesPath, do
  ExecutorImage,
  "-f", path.Join(buildContextPath, dockerfilesPath, dockerfile),
  "-d", kanikoImage, reproducibleFlag,
+ cacheFlag,
  contextFlag, contextPath},
  additionalFlags...)...,
  )
@@ -186,3 +202,28 @@ func (d *DockerFileBuilder) BuildImage(imageRepo, gcsBucket, dockerfilesPath, do
  d.FilesBuilt[dockerfile] = true
  return nil
 }
+
+// buildCachedImages builds the images for testing caching via kaniko where version is the nth time this image has been built
+func (d *DockerFileBuilder) buildCachedImages(imageRepo, cache, dockerfilesPath string, version int) error {
+ _, ex, _, _ := runtime.Caller(0)
+ cwd := filepath.Dir(ex)
+
+ for dockerfile := range d.TestCacheDockerfiles {
+ kanikoImage := GetVersionedKanikoImage(imageRepo, dockerfile, version)
+ kanikoCmd := exec.Command("docker",
+ append([]string{"run",
+ "-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud",
+ "-v", cwd + ":/workspace",
+ ExecutorImage,
+ "-f", path.Join(buildContextPath, dockerfilesPath, dockerfile),
+ "-d", kanikoImage,
+ "-c", buildContextPath,
+ "--cache", cache})...,
+ )
+
+ if _, err := RunCommandWithoutTest(kanikoCmd); err != nil {
+ return fmt.Errorf("Failed to build cached image %s with kaniko command \"%s\": %s", kanikoImage, kanikoCmd.Args, err)
+ }
+ }
+ return nil
+}

integration/integration_test.go

@@ -24,8 +24,10 @@ import (
  "math"
  "os"
  "os/exec"
+ "path/filepath"
  "strings"
  "testing"
+ "time"
 
  "github.com/google/go-containerregistry/pkg/name"
  "github.com/google/go-containerregistry/pkg/v1/daemon"
@@ -148,6 +150,7 @@ func TestMain(m *testing.M) {
  fmt.Printf("error building onbuild base: %v", err)
  os.Exit(1)
  }
+
  pushOnbuildBase := exec.Command("docker", "push", config.onbuildBaseImage)
  if err := pushOnbuildBase.Run(); err != nil {
  fmt.Printf("error pushing onbuild base %s: %v", config.onbuildBaseImage, err)
@@ -165,7 +168,6 @@ func TestMain(m *testing.M) {
  fmt.Printf("error pushing hardlink base %s: %v", config.hardlinkBaseImage, err)
  os.Exit(1)
  }
-
  dockerfiles, err := FindDockerFiles(dockerfilesPath)
  if err != nil {
  fmt.Printf("Coudn't create map of dockerfiles: %s", err)
@@ -177,6 +179,12 @@ func TestMain(m *testing.M) {
 func TestRun(t *testing.T) {
  for dockerfile, built := range imageBuilder.FilesBuilt {
  t.Run("test_"+dockerfile, func(t *testing.T) {
+ if _, ok := imageBuilder.DockerfilesToIgnore[dockerfile]; ok {
+ t.SkipNow()
+ }
+ if _, ok := imageBuilder.TestCacheDockerfiles[dockerfile]; ok {
+ t.SkipNow()
+ }
  if !built {
  err := imageBuilder.BuildImage(config.imageRepo, config.gcsBucket, dockerfilesPath, dockerfile)
  if err != nil {
@@ -195,25 +203,8 @@ func TestRun(t *testing.T) {
  t.Logf("diff = %s", string(diff))
 
  expected := fmt.Sprintf(emptyContainerDiff, dockerImage, kanikoImage, dockerImage, kanikoImage)
+ checkContainerDiffOutput(t, diff, expected)
 
- // Let's compare the json objects themselves instead of strings to avoid
- // issues with spaces and indents
- var diffInt interface{}
- var expectedInt interface{}
-
- err := json.Unmarshal(diff, &diffInt)
- if err != nil {
- t.Error(err)
- t.Fail()
- }
-
- err = json.Unmarshal([]byte(expected), &expectedInt)
- if err != nil {
- t.Error(err)
- t.Fail()
- }
-
- testutil.CheckErrorAndDeepEqual(t, false, nil, expectedInt, diffInt)
  })
  }
 }
@@ -228,6 +219,9 @@ func TestLayers(t *testing.T) {
  }
  for dockerfile, built := range imageBuilder.FilesBuilt {
  t.Run("test_layer_"+dockerfile, func(t *testing.T) {
+ if _, ok := imageBuilder.DockerfilesToIgnore[dockerfile]; ok {
+ t.SkipNow()
+ }
  if !built {
  err := imageBuilder.BuildImage(config.imageRepo, config.gcsBucket, dockerfilesPath, dockerfile)
  if err != nil {
@@ -244,6 +238,58 @@ func TestLayers(t *testing.T) {
  }
 }
 
+// Build each image with kaniko twice, and then make sure they're exactly the same
+func TestCache(t *testing.T) {
+ for dockerfile := range imageBuilder.TestCacheDockerfiles {
+ t.Run("test_cache_"+dockerfile, func(t *testing.T) {
+ cache := filepath.Join(config.imageRepo, "cache", fmt.Sprintf("%v", time.Now().UnixNano()))
+ // Build the initial image which will cache layers
+ if err := imageBuilder.buildCachedImages(config.imageRepo, cache, dockerfilesPath, 0); err != nil {
+ t.Fatalf("error building cached image for the first time: %v", err)
+ }
+ // Build the second image which should pull from the cache
+ if err := imageBuilder.buildCachedImages(config.imageRepo, cache, dockerfilesPath, 1); err != nil {
+ t.Fatalf("error building cached image for the first time: %v", err)
+ }
+ // Make sure both images are the same
+ kanikoVersion0 := GetVersionedKanikoImage(config.imageRepo, dockerfile, 0)
+ kanikoVersion1 := GetVersionedKanikoImage(config.imageRepo, dockerfile, 1)
+
+ // container-diff
+ containerdiffCmd := exec.Command("container-diff", "diff",
+ kanikoVersion0, kanikoVersion1,
+ "-q", "--type=file", "--type=metadata", "--json")
+
+ diff := RunCommand(containerdiffCmd, t)
+ t.Logf("diff = %s", diff)
+
+ expected := fmt.Sprintf(emptyContainerDiff, kanikoVersion0, kanikoVersion1, kanikoVersion0, kanikoVersion1)
+ checkContainerDiffOutput(t, diff, expected)
+ })
+ }
+}
+
+func checkContainerDiffOutput(t *testing.T, diff []byte, expected string) {
+ // Let's compare the json objects themselves instead of strings to avoid
+ // issues with spaces and indents
+ t.Helper()
+
+ var diffInt interface{}
+ var expectedInt interface{}
+
+ err := json.Unmarshal(diff, &diffInt)
+ if err != nil {
+ t.Error(err)
+ }
+
+ err = json.Unmarshal([]byte(expected), &expectedInt)
+ if err != nil {
+ t.Error(err)
+ }
+
+ testutil.CheckErrorAndDeepEqual(t, false, nil, expectedInt, diffInt)
+}
+
 func checkLayers(t *testing.T, image1, image2 string, offset int) {
  t.Helper()
  img1, err := getImageDetails(image1)

pkg/cache/cache.go

@@ -0,0 +1,70 @@
+/*
+Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cache
+
+import (
+ "fmt"
+
+ "github.com/GoogleContainerTools/kaniko/pkg/config"
+ "github.com/google/go-containerregistry/pkg/authn"
+ "github.com/google/go-containerregistry/pkg/authn/k8schain"
+ "github.com/google/go-containerregistry/pkg/name"
+ "github.com/google/go-containerregistry/pkg/v1"
+ "github.com/google/go-containerregistry/pkg/v1/remote"
+ "github.com/pkg/errors"
+ "github.com/sirupsen/logrus"
+)
+
+// RetrieveLayer checks the specified cache for a layer with the tag :cacheKey
+func RetrieveLayer(opts *config.KanikoOptions, cacheKey string) (v1.Image, error) {
+ cache, err := Destination(opts, cacheKey)
+ if err != nil {
+ return nil, errors.Wrap(err, "getting cache destination")
+ }
+ logrus.Infof("Checking for cached layer %s...", cache)
+
+ cacheRef, err := name.NewTag(cache, name.WeakValidation)
+ if err != nil {
+ return nil, errors.Wrap(err, fmt.Sprintf("getting reference for %s", cache))
+ }
+ k8sc, err := k8schain.NewNoClient()
+ if err != nil {
+ return nil, err
+ }
+ kc := authn.NewMultiKeychain(authn.DefaultKeychain, k8sc)
+ img, err := remote.Image(cacheRef, remote.WithAuthFromKeychain(kc))
+ if err != nil {
+ return nil, err
+ }
+ _, err = img.Layers()
+ return img, err
+}
+
+// Destination returns the repo where the layer should be stored
+// If no cache is specified, one is inferred from the destination provided
+func Destination(opts *config.KanikoOptions, cacheKey string) (string, error) {
+ cache := opts.Cache
+ if cache == "" {
+ destination := opts.Destinations[0]
+ destRef, err := name.NewTag(destination, name.WeakValidation)
+ if err != nil {
+ return "", errors.Wrap(err, "getting tag for destination")
+ }
+ return fmt.Sprintf("%s/cache:%s", destRef.Context(), cacheKey), nil
+ }
+ return fmt.Sprintf("%s:%s", cache, cacheKey), nil
+}

pkg/config/options.go

@@ -24,11 +24,13 @@ type KanikoOptions struct {
  Bucket string
  TarPath string
  Target string
+ Cache string
  Destinations multiArg
  BuildArgs multiArg
  InsecurePush bool
  SkipTLSVerify bool
  SingleSnapshot bool
  Reproducible bool
  NoPush bool
+ UseCache bool
 }