Merge pull request #12804 from presztak/kube_binaries_mirror

New flag "--binary-mirror" to override mirror URL downloading (kubectl, kubelet, & kubeadm)

cmd/minikube/cmd/dashboard.go

@@ -98,7 +98,7 @@ var dashboardCmd = &cobra.Command{
 		}
 
 		out.ErrT(style.Launch, "Launching proxy ...")
-		p, hostPort, err := kubectlProxy(kubectlVersion, cname, dashboardExposedPort)
+		p, hostPort, err := kubectlProxy(kubectlVersion, co.Config.BinaryMirror, cname, dashboardExposedPort)
 		if err != nil {
 			exit.Error(reason.HostKubectlProxy, "kubectl proxy", err)
 		}
@@ -132,15 +132,15 @@ var dashboardCmd = &cobra.Command{
 }
 
 // kubectlProxy runs "kubectl proxy", returning host:port
-func kubectlProxy(kubectlVersion string, contextName string, port int) (*exec.Cmd, string, error) {
+func kubectlProxy(kubectlVersion string, binaryURL string, contextName string, port int) (*exec.Cmd, string, error) {
 	// port=0 picks a random system port
 
 	kubectlArgs := []string{"--context", contextName, "proxy", "--port", strconv.Itoa(port)}
 
 	var cmd *exec.Cmd
 	if kubectl, err := exec.LookPath("kubectl"); err == nil {
 		cmd = exec.Command(kubectl, kubectlArgs...)
-	} else if cmd, err = KubectlCommand(kubectlVersion, kubectlArgs...); err != nil {
+	} else if cmd, err = KubectlCommand(kubectlVersion, binaryURL, kubectlArgs...); err != nil {
 		return nil, "", err
 	}
 

cmd/minikube/cmd/kubectl.go

@@ -99,7 +99,7 @@ host. Please be aware that when using --ssh all paths will apply to the remote m
 			args = append(cluster, args...)
 		}
 
-		c, err := KubectlCommand(version, args...)
+		c, err := KubectlCommand(version, cc.BinaryMirror, args...)
 		if err != nil {
 			out.ErrLn("Error caching kubectl: %v", err)
 			os.Exit(1)
@@ -134,12 +134,12 @@ func kubeconfigPath(cfg config.ClusterConfig) string {
 }
 
 // KubectlCommand will return kubectl command with a version matching the cluster
-func KubectlCommand(version string, args ...string) (*exec.Cmd, error) {
+func KubectlCommand(version, binaryURL string, args ...string) (*exec.Cmd, error) {
 	if version == "" {
 		version = constants.DefaultKubernetesVersion
 	}
 
-	path, err := node.CacheKubectlBinary(version)
+	path, err := node.CacheKubectlBinary(version, binaryURL)
 	if err != nil {
 		return nil, err
 	}

cmd/minikube/cmd/start_flags.go

@@ -132,6 +132,7 @@ const (
 	listenAddress           = "listen-address"
 	extraDisks              = "extra-disks"
 	certExpiration          = "cert-expiration"
+	binaryMirror            = "binary-mirror"
 )
 
 var (
@@ -189,6 +190,7 @@ func initMinikubeFlags() {
 	startCmd.Flags().StringP(trace, "", "", "Send trace events. Options include: [gcp]")
 	startCmd.Flags().Int(extraDisks, 0, "Number of extra disks created and attached to the minikube VM (currently only implemented for hyperkit and kvm2 drivers)")
 	startCmd.Flags().Duration(certExpiration, constants.DefaultCertExpiration, "Duration until minikube certificate expiration, defaults to three years (26280h).")
+	startCmd.Flags().String(binaryMirror, "", "Location to fetch kubectl, kubelet, & kubeadm binaries from.")
 }
 
 // initKubernetesFlags inits the commandline flags for Kubernetes related options
@@ -490,6 +492,7 @@ func generateNewConfigFromFlags(cmd *cobra.Command, k8sVersion string, drvName s
 		MountPort:               uint16(viper.GetUint(mountPortFlag)),
 		MountType:               viper.GetString(mountTypeFlag),
 		MountUID:                viper.GetString(mountUID),
+		BinaryMirror:            viper.GetString(binaryMirror),
 		KubernetesConfig: config.KubernetesConfig{
 			KubernetesVersion:      k8sVersion,
 			ClusterName:            ClusterFlagValue(),
@@ -707,6 +710,7 @@ func updateExistingConfigFromFlags(cmd *cobra.Command, existing *config.ClusterC
 	updateUint16FromFlag(cmd, &cc.MountPort, mountPortFlag)
 	updateStringFromFlag(cmd, &cc.MountType, mountTypeFlag)
 	updateStringFromFlag(cmd, &cc.MountUID, mountUID)
+	updateStringFromFlag(cmd, &cc.BinaryMirror, binaryMirror)
 
 	if cmd.Flags().Changed(kubernetesVersion) {
 		cc.KubernetesConfig.KubernetesVersion = getKubernetesVersion(existing)

hack/preload-images/generate.go

@@ -122,7 +122,7 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string
 
 	sm := sysinit.New(runner)
 
-	if err := bsutil.TransferBinaries(kcfg, runner, sm); err != nil {
+	if err := bsutil.TransferBinaries(kcfg, runner, sm, ""); err != nil {
 		return errors.Wrap(err, "transferring k8s binaries")
 	}
 	// Create image tarball

pkg/minikube/bootstrapper/bsutil/binaries.go

@@ -38,7 +38,7 @@ import (
 )
 
 // TransferBinaries transfers all required Kubernetes binaries
-func TransferBinaries(cfg config.KubernetesConfig, c command.Runner, sm sysinit.Manager) error {
+func TransferBinaries(cfg config.KubernetesConfig, c command.Runner, sm sysinit.Manager, binariesURL string) error {
 	ok, err := binariesExist(cfg, c)
 	if err == nil && ok {
 		klog.Info("Found k8s binaries, skipping transfer")
@@ -56,7 +56,7 @@ func TransferBinaries(cfg config.KubernetesConfig, c command.Runner, sm sysinit.
 	for _, name := range constants.KubernetesReleaseBinaries {
 		name := name
 		g.Go(func() error {
-			src, err := download.Binary(name, cfg.KubernetesVersion, "linux", runtime.GOARCH)
+			src, err := download.Binary(name, cfg.KubernetesVersion, "linux", runtime.GOARCH, binariesURL)
 			if err != nil {
 				return errors.Wrapf(err, "downloading %s", name)
 			}

pkg/minikube/bootstrapper/kubeadm/kubeadm.go

@@ -792,7 +792,7 @@ func (k *Bootstrapper) UpdateNode(cfg config.ClusterConfig, n config.Node, r cru
 
 	sm := sysinit.New(k.c)
 
-	if err := bsutil.TransferBinaries(cfg.KubernetesConfig, k.c, sm); err != nil {
+	if err := bsutil.TransferBinaries(cfg.KubernetesConfig, k.c, sm, cfg.BinaryMirror); err != nil {
 		return errors.Wrap(err, "downloading binaries")
 	}
 

pkg/minikube/config/types.go

@@ -95,6 +95,7 @@ type ClusterConfig struct {
 	MountPort               uint16
 	MountType               string
 	MountUID                string
+	BinaryMirror            string // Mirror location for kube binaries (kubectl, kubelet, & kubeadm)
 }
 
 // KubernetesConfig contains the parameters used to configure the VM Kubernetes.

pkg/minikube/download/binary.go

@@ -30,9 +30,18 @@ import (
 	"k8s.io/minikube/pkg/minikube/localpath"
 )
 
+// DefaultKubeBinariesURL returns a URL to kube binaries
+func DefaultKubeBinariesURL() string {
+	return fmt.Sprintf("https://%s/kubernetes-release/release", downloadHost)
+}
+
 // binaryWithChecksumURL gets the location of a Kubernetes binary
-func binaryWithChecksumURL(binaryName, version, osName, archName string) (string, error) {
-	base := fmt.Sprintf("https://%s/kubernetes-release/release/%s/bin/%s/%s/%s", downloadHost, version, osName, archName, binaryName)
+func binaryWithChecksumURL(binaryName, version, osName, archName, binaryURL string) (string, error) {
+	if binaryURL == "" {
+		binaryURL = DefaultKubeBinariesURL()
+	}
+
+	base := fmt.Sprintf("%s/%s/bin/%s/%s/%s", binaryURL, version, osName, archName, binaryName)
 	v, err := semver.Make(version[1:])
 	if err != nil {
 		return "", err
@@ -45,12 +54,12 @@ func binaryWithChecksumURL(binaryName, version, osName, archName string) (string
 }
 
 // Binary will download a binary onto the host
-func Binary(binary, version, osName, archName string) (string, error) {
+func Binary(binary, version, osName, archName, binaryURL string) (string, error) {
 	targetDir := localpath.MakeMiniPath("cache", osName, version)
 	targetFilepath := path.Join(targetDir, binary)
 	targetLock := targetFilepath + ".lock"
 
-	url, err := binaryWithChecksumURL(binary, version, osName, archName)
+	url, err := binaryWithChecksumURL(binary, version, osName, archName, binaryURL)
 	if err != nil {
 		return "", err
 	}

pkg/minikube/download/download_test.go

@@ -61,7 +61,7 @@ func testBinaryDownloadPreventsMultipleDownload(t *testing.T) {
 	var group sync.WaitGroup
 	group.Add(2)
 	dlCall := func() {
-		if _, err := Binary("kubectl", "v1.20.2", "linux", "amd64"); err != nil {
+		if _, err := Binary("kubectl", "v1.20.2", "linux", "amd64", ""); err != nil {
 			t.Errorf("Failed to download binary: %+v", err)
 		}
 		group.Done()

pkg/minikube/machine/cache_binaries.go

@@ -43,7 +43,7 @@ func isExcluded(binary string, excludedBinaries []string) bool {
 }
 
 // CacheBinariesForBootstrapper will cache binaries for a bootstrapper
-func CacheBinariesForBootstrapper(version string, clusterBootstrapper string, excludeBinaries []string) error {
+func CacheBinariesForBootstrapper(version string, clusterBootstrapper string, excludeBinaries []string, binariesURL string) error {
 	binaries := bootstrapper.GetCachedBinaryList(clusterBootstrapper)
 
 	var g errgroup.Group
@@ -53,7 +53,7 @@ func CacheBinariesForBootstrapper(version string, clusterBootstrapper string, ex
 		}
 		bin := bin // https://golang.org/doc/faq#closures_and_goroutines
 		g.Go(func() error {
-			if _, err := download.Binary(bin, version, "linux", detect.EffectiveArch()); err != nil {
+			if _, err := download.Binary(bin, version, "linux", detect.EffectiveArch(), binariesURL); err != nil {
 				return errors.Wrapf(err, "caching binary %s", bin)
 			}
 			return nil

pkg/minikube/machine/cache_binaries_test.go

@@ -121,7 +121,7 @@ func TestCacheBinariesForBootstrapper(t *testing.T) {
 	for _, test := range tc {
 		t.Run(test.version, func(t *testing.T) {
 			os.Setenv("MINIKUBE_HOME", test.minikubeHome)
-			err := CacheBinariesForBootstrapper(test.version, test.clusterBootstrapper, nil)
+			err := CacheBinariesForBootstrapper(test.version, test.clusterBootstrapper, nil, "")
 			if err != nil && !test.err {
 				t.Fatalf("Got unexpected error %v", err)
 			}
@@ -160,7 +160,7 @@ func TestExcludedBinariesNotDownloaded(t *testing.T) {
 		}
 	}()
 
-	if err := CacheBinariesForBootstrapper("v1.16.0", clusterBootstrapper, []string{binaryToExclude}); err != nil {
+	if err := CacheBinariesForBootstrapper("v1.16.0", clusterBootstrapper, []string{binaryToExclude}, ""); err != nil {
 		t.Errorf("Failed to cache binaries: %v", err)
 	}
 }

pkg/minikube/node/cache.go

@@ -78,10 +78,12 @@ func handleDownloadOnly(cacheGroup, kicGroup *errgroup.Group, k8sVersion, contai
 	if !viper.GetBool("download-only") {
 		return
 	}
-	if err := doCacheBinaries(k8sVersion, containerRuntime, driverName); err != nil {
+
+	binariesURL := viper.GetString("binary-mirror")
+	if err := doCacheBinaries(k8sVersion, containerRuntime, driverName, binariesURL); err != nil {
 		exit.Error(reason.InetCacheBinaries, "Failed to cache binaries", err)
 	}
-	if _, err := CacheKubectlBinary(k8sVersion); err != nil {
+	if _, err := CacheKubectlBinary(k8sVersion, binariesURL); err != nil {
 		exit.Error(reason.InetCacheKubectl, "Failed to cache kubectl", err)
 	}
 	waitCacheRequiredImages(cacheGroup)
@@ -94,22 +96,22 @@ func handleDownloadOnly(cacheGroup, kicGroup *errgroup.Group, k8sVersion, contai
 }
 
 // CacheKubectlBinary caches the kubectl binary
-func CacheKubectlBinary(k8sVersion string) (string, error) {
+func CacheKubectlBinary(k8sVersion, binaryURL string) (string, error) {
 	binary := "kubectl"
 	if runtime.GOOS == "windows" {
 		binary = "kubectl.exe"
 	}
 
-	return download.Binary(binary, k8sVersion, runtime.GOOS, detect.EffectiveArch())
+	return download.Binary(binary, k8sVersion, runtime.GOOS, detect.EffectiveArch(), binaryURL)
 }
 
 // doCacheBinaries caches Kubernetes binaries in the foreground
-func doCacheBinaries(k8sVersion, containerRuntime, driverName string) error {
+func doCacheBinaries(k8sVersion, containerRuntime, driverName, binariesURL string) error {
 	existingBinaries := constants.KubernetesReleaseBinaries
 	if !download.PreloadExists(k8sVersion, containerRuntime, driverName) {
 		existingBinaries = nil
 	}
-	return machine.CacheBinariesForBootstrapper(k8sVersion, viper.GetString(cmdcfg.Bootstrapper), existingBinaries)
+	return machine.CacheBinariesForBootstrapper(k8sVersion, viper.GetString(cmdcfg.Bootstrapper), existingBinaries, binariesURL)
 }
 
 // beginDownloadKicBaseImage downloads the kic image

test/integration/aaa_download_only_test.go

@@ -24,8 +24,12 @@ import (
 	"bytes"
 	"context"
 	"crypto/md5"
+	"crypto/sha256"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -247,3 +251,69 @@ func TestDownloadOnlyKic(t *testing.T) {
 		t.Errorf("failed to verify checksum. checksum of %q does not match remote checksum (%q != %q)", tarball, string(remoteChecksum), string(checksum[:]))
 	}
 }
+
+// createSha256File is a helper function which creates sha256 checksum file from given file
+func createSha256File(filePath string) error {
+	dat, _ := os.ReadFile(filePath)
+	sum := sha256.Sum256(dat)
+
+	f, err := os.Create(filePath + ".sha256")
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	_, err = f.WriteString(fmt.Sprintf("%x", sum[:]))
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// TestBinaryMirror tests functionality of --binary-mirror flag
+func TestBinaryMirror(t *testing.T) {
+	profile := UniqueProfileName("binary-mirror")
+	ctx, cancel := context.WithTimeout(context.Background(), Minutes(10))
+	defer Cleanup(t, profile, cancel)
+
+	tmpDir, err := ioutil.TempDir("", "kb_test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	// Start test server which will serve binary files
+	ts := httptest.NewServer(
+		http.FileServer(http.Dir(tmpDir)),
+	)
+	defer ts.Close()
+
+	binaryName := "kubectl"
+	if runtime.GOOS == "windows" {
+		binaryName = "kubectl.exe"
+	}
+	binaryPath, err := download.Binary(binaryName, constants.DefaultKubernetesVersion, runtime.GOOS, runtime.GOARCH, "")
+	if err != nil {
+		t.Errorf("Failed to download binary: %+v", err)
+	}
+
+	newBinaryDir := filepath.Join(tmpDir, constants.DefaultKubernetesVersion, "bin", runtime.GOOS, runtime.GOARCH)
+	if err := os.MkdirAll(newBinaryDir, os.ModePerm); err != nil {
+		t.Errorf("Failed to create %s directories", newBinaryDir)
+	}
+
+	newBinaryPath := filepath.Join(newBinaryDir, binaryName)
+	if err := os.Rename(binaryPath, newBinaryPath); err != nil {
+		t.Errorf("Failed to move binary file: %+v", err)
+	}
+	if err := createSha256File(newBinaryPath); err != nil {
+		t.Errorf("Failed to generate sha256 checksum file: %+v", err)
+	}
+
+	args := []string{"start", "--download-only", "-p", profile, "--alsologtostderr", "--binary-mirror", ts.URL}
+
+	cmd := exec.CommandContext(ctx, Target(), args...)
+	if _, err := Run(t, cmd); err != nil {
+		t.Errorf("start with --binary-mirror failed %q : %v", args, err)
+	}
+}