diff --git a/hack/preload-images/generate.go b/hack/preload-images/generate.go index 1a22e7404de4..c3e2efa71838 100644 --- a/hack/preload-images/generate.go +++ b/hack/preload-images/generate.go @@ -29,9 +29,13 @@ import ( "k8s.io/minikube/pkg/minikube/bootstrapper/images" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/constants" + "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/driver" + "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/minikube/sysinit" + "k8s.io/minikube/pkg/util" ) func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string) error { @@ -68,17 +72,38 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string if err != nil { return errors.Wrap(err, "kubeadm images") } - if containerRuntime != "docker" { // kic overlay image is only needed by containerd and cri-o https://github.com/kubernetes/minikube/issues/7428 imgs = append(imgs, kic.OverlayImage) } + runner := command.NewKICRunner(profile, driver.OCIBinary) + + // will need to do this to enable the container run-time service + sv, err := util.ParseKubernetesVersion(constants.DefaultKubernetesVersion) + if err != nil { + return errors.Wrap(err, "Failed to parse kubernetes version") + } + + co := cruntime.Config{ + Type: containerRuntime, + Runner: runner, + ImageRepository: "", + KubernetesVersion: sv, // this is just to satisfy cruntime and shouldnt matter what version. + } + cr, err := cruntime.New(co) + if err != nil { + exit.WithError("Failed runtime", err) + } + if err := cr.Enable(true); err != nil { + exit.WithError("enable container runtime ", err) + } + for _, img := range imgs { - cmd := exec.Command("docker", "exec", profile, "docker", "pull", img) + cmd := imagePullCommand(containerRuntime, img) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { - return errors.Wrapf(err, "downloading %s", img) + return errors.Wrapf(err, "pulling image %s", img) } } @@ -86,26 +111,46 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string kcfg := config.KubernetesConfig{ KubernetesVersion: kubernetesVersion, } - runner := command.NewKICRunner(profile, driver.OCIBinary) + sm := sysinit.New(runner) if err := bsutil.TransferBinaries(kcfg, runner, sm); err != nil { return errors.Wrap(err, "transferring k8s binaries") } // Create image tarball - if err := createImageTarball(tarballFilename); err != nil { + if err := createImageTarball(tarballFilename, containerRuntime); err != nil { return errors.Wrap(err, "create tarball") } + return copyTarballToHost(tarballFilename) } -func createImageTarball(tarballFilename string) error { +// returns the right command to pull image for a specific runtime +func imagePullCommand(containerRuntime, img string) *exec.Cmd { + if containerRuntime == "docker" { + return exec.Command("docker", "exec", profile, "docker", "pull", img) + } + + if containerRuntime == "containerd" { + return exec.Command("docker", "exec", profile, "sudo", "crictl", "pull", img) + } + return nil +} + +func createImageTarball(tarballFilename, containerRuntime string) error { // directories to save into tarball dirs := []string{ - fmt.Sprintf("./lib/docker/%s", dockerStorageDriver), - "./lib/docker/image", "./lib/minikube/binaries", } + + if containerRuntime == "docker" { + dirs = append(dirs, fmt.Sprintf("./lib/docker/%s", dockerStorageDriver), "./lib/docker/image") + } + + if containerRuntime == "containerd" { + dirs = append(dirs, fmt.Sprintf("./lib/containerd")) + } + args := []string{"exec", profile, "sudo", "tar", "-I", "lz4", "-C", "/var", "-cvf", tarballFilename} args = append(args, dirs...) cmd := exec.Command("docker", args...) diff --git a/hack/preload-images/preload_images.go b/hack/preload-images/preload_images.go index 37bbd678e7d6..b5fa25698709 100644 --- a/hack/preload-images/preload_images.go +++ b/hack/preload-images/preload_images.go @@ -35,7 +35,7 @@ const ( var ( dockerStorageDriver = "overlay2" - containerRuntimes = []string{"docker"} + containerRuntimes = []string{"docker", "containerd"} k8sVersion string k8sVersions []string ) @@ -65,10 +65,10 @@ func main() { for _, cr := range containerRuntimes { tf := download.TarballName(kv, cr) if download.PreloadExists(kv, cr) { - fmt.Printf("A preloaded tarball for k8s version %s already exists, skipping generation.\n", kv) + fmt.Printf("A preloaded tarball for k8s version %s - runtime %q already exists, skipping generation.\n", kv, cr) continue } - fmt.Printf("A preloaded tarball for k8s version %s doesn't exist, generating now...\n", kv) + fmt.Printf("A preloaded tarball for k8s version %s - runtime %q doesn't exist, generating now...\n", kv, cr) if err := generateTarball(kv, cr, tf); err != nil { exit.WithError(fmt.Sprintf("generating tarball for k8s version %s with %s", kv, cr), err) } diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index 05b1e325ca28..0c1a4e4f487e 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -130,7 +130,7 @@ func (d *Driver) Create() error { return } t := time.Now() - glog.Infof("Starting extracting preloaded images to volume") + glog.Infof("Starting extracting preloaded images to volume ...") // Extract preloaded images to container if err := oci.ExtractTarballToVolume(download.TarballPath(d.NodeConfig.KubernetesVersion, d.NodeConfig.ContainerRuntime), params.Name, BaseImage); err != nil { glog.Infof("Unable to extract preloaded tarball to volume: %v", err) diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index 8eadb57392e2..73261c98190c 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -19,16 +19,20 @@ package cruntime import ( "bytes" "encoding/base64" + "encoding/json" "fmt" "os/exec" "path" "strings" "text/template" + "time" "github.com/blang/semver" "github.com/golang/glog" "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/bootstrapper/images" + "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/out" @@ -310,5 +314,118 @@ func (r *Containerd) Preload(cfg config.KubernetesConfig) error { if !download.PreloadExists(cfg.KubernetesVersion, cfg.ContainerRuntime) { return nil } - return fmt.Errorf("not yet implemented for %s", r.Name()) + + k8sVersion := cfg.KubernetesVersion + cRuntime := cfg.ContainerRuntime + + // If images already exist, return + images, err := images.Kubeadm(cfg.ImageRepository, k8sVersion) + if err != nil { + return errors.Wrap(err, "getting images") + } + if containerdImagesPreloaded(r.Runner, images) { + glog.Info("Images already preloaded, skipping extraction") + return nil + } + + tarballPath := download.TarballPath(k8sVersion, cRuntime) + targetDir := "/" + targetName := "preloaded.tar.lz4" + dest := path.Join(targetDir, targetName) + + c := exec.Command("which", "lz4") + if _, err := r.Runner.RunCmd(c); err != nil { + return NewErrISOFeature("lz4") + } + + // Copy over tarball into host + fa, err := assets.NewFileAsset(tarballPath, targetDir, targetName, "0644") + if err != nil { + return errors.Wrap(err, "getting file asset") + } + t := time.Now() + if err := r.Runner.Copy(fa); err != nil { + return errors.Wrap(err, "copying file") + } + glog.Infof("Took %f seconds to copy over tarball", time.Since(t).Seconds()) + + t = time.Now() + // extract the tarball to /var in the VM + if rr, err := r.Runner.RunCmd(exec.Command("sudo", "tar", "-I", "lz4", "-C", "/var", "-xvf", dest)); err != nil { + return errors.Wrapf(err, "extracting tarball: %s", rr.Output()) + } + glog.Infof("Took %f seconds t extract the tarball", time.Since(t).Seconds()) + + // remove the tarball in the VM + if err := r.Runner.Remove(fa); err != nil { + glog.Infof("error removing tarball: %v", err) + } + + return r.Restart() +} + +// Restart restarts Docker on a host +func (r *Containerd) Restart() error { + return r.Init.Restart("containerd") +} + +// containerdImagesPreloaded returns true if all images have been preloaded +func containerdImagesPreloaded(runner command.Runner, images []string) bool { + rr, err := runner.RunCmd(exec.Command("sudo", "crictl", "images", "--output", "json")) + if err != nil { + return false + } + type containerdImages struct { + Images []struct { + ID string `json:"id"` + RepoTags []string `json:"repoTags"` + RepoDigests []string `json:"repoDigests"` + Size string `json:"size"` + UID interface{} `json:"uid"` + Username string `json:"username"` + } `json:"images"` + } + + var jsonImages containerdImages + err = json.Unmarshal(rr.Stdout.Bytes(), &jsonImages) + if err != nil { + glog.Errorf("failed to unmarshal images, will assume images are not preloaded") + return false + } + + // Make sure images == imgs + for _, i := range images { + found := false + for _, ji := range jsonImages.Images { + for _, rt := range ji.RepoTags { + i = addRepoTagToImageName(i) + if i == rt { + found = true + break + } + } + if found { + break + } + + } + if !found { + glog.Infof("couldn't find preloaded image for %q. assuming images are not preloaded.", i) + return false + } + } + glog.Infof("all images are preloaded for containerd runtime.") + return true +} + +// addRepoTagToImageName makes sure the image name has a repo tag in it. +// in crictl images list have the repo tag prepended to them +// for example "kubernetesui/dashboard:v2.0.0 will show up as "docker.io/kubernetesui/dashboard:v2.0.0" +// warning this is only meant for kuberentes images where we know the GCR addreses have .io in them +// not mean to be used for public images +func addRepoTagToImageName(imgName string) string { + if !strings.Contains(imgName, ".io/") { + return "docker.io/" + imgName + } // else it already has repo name dont add anything + return imgName } diff --git a/pkg/minikube/cruntime/containerd_test.go b/pkg/minikube/cruntime/containerd_test.go new file mode 100644 index 000000000000..cfb1e55f6706 --- /dev/null +++ b/pkg/minikube/cruntime/containerd_test.go @@ -0,0 +1,40 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cruntime + +import ( + "testing" +) + +func TestAddRepoTagToImageName(t *testing.T) { + var tests = []struct { + imgName string + want string + }{ + {"kubernetesui/dashboard:v2.0.0-rc6", "docker.io/kubernetesui/dashboard:v2.0.0-rc6"}, + {"kubernetesui/metrics-scraper:v1.0.2", "docker.io/kubernetesui/metrics-scraper:v1.0.2"}, + {"gcr.io/k8s-minikube/storage-provisioner:v1.8.1", "gcr.io/k8s-minikube/storage-provisioner:v1.8.1"}, + } + for _, tc := range tests { + t.Run(tc.imgName, func(t *testing.T) { + got := addRepoTagToImageName(tc.imgName) + if got != tc.want { + t.Errorf("expected image name to be: %q but got %q", tc.want, got) + } + }) + } +} diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index bbb410be131e..f1866ff1aacc 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -220,3 +220,14 @@ func enableIPForwarding(cr CommandRunner) error { } return nil } + +// ImagesPreloaded returns true if all images have been preloaded +func ImagesPreloaded(containerRuntime string, runner command.Runner, images []string) bool { + if containerRuntime == "docker" { + return dockerImagesPreloaded(runner, images) + } + if containerRuntime == "containerd" { + return containerdImagesPreloaded(runner, images) + } + return false +} diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index 95f855415235..eb11b58e839f 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -290,7 +290,7 @@ func (r *Docker) Preload(cfg config.KubernetesConfig) error { if err != nil { return errors.Wrap(err, "getting images") } - if DockerImagesPreloaded(r.Runner, images) { + if dockerImagesPreloaded(r.Runner, images) { glog.Info("Images already preloaded, skipping extraction") return nil } @@ -342,8 +342,8 @@ func (r *Docker) Preload(cfg config.KubernetesConfig) error { return r.Restart() } -// DockerImagesPreloaded returns true if all images have been preloaded -func DockerImagesPreloaded(runner command.Runner, images []string) bool { +// dockerImagesPreloaded returns true if all images have been preloaded +func dockerImagesPreloaded(runner command.Runner, images []string) bool { rr, err := runner.RunCmd(exec.Command("docker", "images", "--format", "{{.Repository}}:{{.Tag}}")) if err != nil { return false diff --git a/pkg/minikube/download/preload.go b/pkg/minikube/download/preload.go index 59a0c4b7e098..50a12a37bb60 100644 --- a/pkg/minikube/download/preload.go +++ b/pkg/minikube/download/preload.go @@ -78,16 +78,16 @@ func remoteTarballURL(k8sVersion, containerRuntime string) string { // PreloadExists returns true if there is a preloaded tarball that can be used func PreloadExists(k8sVersion, containerRuntime string) bool { + // TODO: debug why this func is being called two times glog.Infof("Checking if preload exists for k8s version %s and runtime %s", k8sVersion, containerRuntime) if !viper.GetBool("preload") { return false } - // See https://github.com/kubernetes/minikube/issues/6933 // and https://github.com/kubernetes/minikube/issues/6934 - // to track status of adding containerd & crio - if containerRuntime != "docker" { - glog.Info("Container runtime isn't docker, skipping preload") + // to track status of adding crio + if containerRuntime == "crio" { + glog.Info("crio is not supported yet, skipping preload") return false } diff --git a/pkg/minikube/machine/cache_images.go b/pkg/minikube/machine/cache_images.go index 93f735467c7b..94702cbc077a 100644 --- a/pkg/minikube/machine/cache_images.go +++ b/pkg/minikube/machine/cache_images.go @@ -65,11 +65,10 @@ func CacheImagesForBootstrapper(imageRepository string, version string, clusterB // LoadImages loads previously cached images into the container runtime func LoadImages(cc *config.ClusterConfig, runner command.Runner, images []string, cacheDir string) error { // Skip loading images if images already exist - if cruntime.DockerImagesPreloaded(runner, images) { + if cruntime.ImagesPreloaded(cc.KubernetesConfig.ContainerRuntime, runner, images) { glog.Infof("Images are preloaded, skipping loading") return nil } - glog.Infof("LoadImages start: %s", images) start := time.Now() diff --git a/test/integration/aaa_download_only_test.go b/test/integration/aaa_download_only_test.go index 7d198b76343c..9d5ec2844373 100644 --- a/test/integration/aaa_download_only_test.go +++ b/test/integration/aaa_download_only_test.go @@ -154,6 +154,7 @@ func TestDownloadOnlyKic(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), Minutes(15)) defer Cleanup(t, profile, cancel) + // TODO: #7795 add containerd to download only too cRuntime := "docker" args := []string{"start", "--download-only", "-p", profile, "--force", "--alsologtostderr"}