Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

New flag "--binary-mirror" to override mirror URL downloading (kubectl, kubelet, & kubeadm) #12804

Merged
merged 3 commits into from
Jan 12, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions cmd/minikube/cmd/dashboard.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ var dashboardCmd = &cobra.Command{
}

out.ErrT(style.Launch, "Launching proxy ...")
p, hostPort, err := kubectlProxy(kubectlVersion, cname, dashboardExposedPort)
p, hostPort, err := kubectlProxy(kubectlVersion, co.Config.BinaryMirror, cname, dashboardExposedPort)
if err != nil {
exit.Error(reason.HostKubectlProxy, "kubectl proxy", err)
}
Expand Down Expand Up @@ -132,15 +132,15 @@ var dashboardCmd = &cobra.Command{
}

// kubectlProxy runs "kubectl proxy", returning host:port
func kubectlProxy(kubectlVersion string, contextName string, port int) (*exec.Cmd, string, error) {
func kubectlProxy(kubectlVersion string, binaryURL string, contextName string, port int) (*exec.Cmd, string, error) {
// port=0 picks a random system port

kubectlArgs := []string{"--context", contextName, "proxy", "--port", strconv.Itoa(port)}

var cmd *exec.Cmd
if kubectl, err := exec.LookPath("kubectl"); err == nil {
cmd = exec.Command(kubectl, kubectlArgs...)
} else if cmd, err = KubectlCommand(kubectlVersion, kubectlArgs...); err != nil {
} else if cmd, err = KubectlCommand(kubectlVersion, binaryURL, kubectlArgs...); err != nil {
return nil, "", err
}

Expand Down
6 changes: 3 additions & 3 deletions cmd/minikube/cmd/kubectl.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ host. Please be aware that when using --ssh all paths will apply to the remote m
args = append(cluster, args...)
}

c, err := KubectlCommand(version, args...)
c, err := KubectlCommand(version, cc.BinaryMirror, args...)
if err != nil {
out.ErrLn("Error caching kubectl: %v", err)
os.Exit(1)
Expand Down Expand Up @@ -134,12 +134,12 @@ func kubeconfigPath(cfg config.ClusterConfig) string {
}

// KubectlCommand will return kubectl command with a version matching the cluster
func KubectlCommand(version string, args ...string) (*exec.Cmd, error) {
func KubectlCommand(version, binaryURL string, args ...string) (*exec.Cmd, error) {
if version == "" {
version = constants.DefaultKubernetesVersion
}

path, err := node.CacheKubectlBinary(version)
path, err := node.CacheKubectlBinary(version, binaryURL)
if err != nil {
return nil, err
}
Expand Down
4 changes: 4 additions & 0 deletions cmd/minikube/cmd/start_flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,7 @@ const (
listenAddress = "listen-address"
extraDisks = "extra-disks"
certExpiration = "cert-expiration"
binaryMirror = "binary-mirror"
)

var (
Expand Down Expand Up @@ -189,6 +190,7 @@ func initMinikubeFlags() {
startCmd.Flags().StringP(trace, "", "", "Send trace events. Options include: [gcp]")
startCmd.Flags().Int(extraDisks, 0, "Number of extra disks created and attached to the minikube VM (currently only implemented for hyperkit and kvm2 drivers)")
startCmd.Flags().Duration(certExpiration, constants.DefaultCertExpiration, "Duration until minikube certificate expiration, defaults to three years (26280h).")
startCmd.Flags().String(binaryMirror, "", "Location to fetch kubectl, kubelet, & kubeadm binaries from.")
}

// initKubernetesFlags inits the commandline flags for Kubernetes related options
Expand Down Expand Up @@ -490,6 +492,7 @@ func generateNewConfigFromFlags(cmd *cobra.Command, k8sVersion string, drvName s
MountPort: uint16(viper.GetUint(mountPortFlag)),
MountType: viper.GetString(mountTypeFlag),
MountUID: viper.GetString(mountUID),
BinaryMirror: viper.GetString(binaryMirror),
KubernetesConfig: config.KubernetesConfig{
KubernetesVersion: k8sVersion,
ClusterName: ClusterFlagValue(),
Expand Down Expand Up @@ -707,6 +710,7 @@ func updateExistingConfigFromFlags(cmd *cobra.Command, existing *config.ClusterC
updateUint16FromFlag(cmd, &cc.MountPort, mountPortFlag)
updateStringFromFlag(cmd, &cc.MountType, mountTypeFlag)
updateStringFromFlag(cmd, &cc.MountUID, mountUID)
updateStringFromFlag(cmd, &cc.BinaryMirror, binaryMirror)

if cmd.Flags().Changed(kubernetesVersion) {
cc.KubernetesConfig.KubernetesVersion = getKubernetesVersion(existing)
Expand Down
2 changes: 1 addition & 1 deletion hack/preload-images/generate.go
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string

sm := sysinit.New(runner)

if err := bsutil.TransferBinaries(kcfg, runner, sm); err != nil {
if err := bsutil.TransferBinaries(kcfg, runner, sm, ""); err != nil {
return errors.Wrap(err, "transferring k8s binaries")
}
// Create image tarball
Expand Down
4 changes: 2 additions & 2 deletions pkg/minikube/bootstrapper/bsutil/binaries.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ import (
)

// TransferBinaries transfers all required Kubernetes binaries
func TransferBinaries(cfg config.KubernetesConfig, c command.Runner, sm sysinit.Manager) error {
func TransferBinaries(cfg config.KubernetesConfig, c command.Runner, sm sysinit.Manager, binariesURL string) error {
ok, err := binariesExist(cfg, c)
if err == nil && ok {
klog.Info("Found k8s binaries, skipping transfer")
Expand All @@ -56,7 +56,7 @@ func TransferBinaries(cfg config.KubernetesConfig, c command.Runner, sm sysinit.
for _, name := range constants.KubernetesReleaseBinaries {
name := name
g.Go(func() error {
src, err := download.Binary(name, cfg.KubernetesVersion, "linux", runtime.GOARCH)
src, err := download.Binary(name, cfg.KubernetesVersion, "linux", runtime.GOARCH, binariesURL)
if err != nil {
return errors.Wrapf(err, "downloading %s", name)
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/minikube/bootstrapper/kubeadm/kubeadm.go
Original file line number Diff line number Diff line change
Expand Up @@ -792,7 +792,7 @@ func (k *Bootstrapper) UpdateNode(cfg config.ClusterConfig, n config.Node, r cru

sm := sysinit.New(k.c)

if err := bsutil.TransferBinaries(cfg.KubernetesConfig, k.c, sm); err != nil {
if err := bsutil.TransferBinaries(cfg.KubernetesConfig, k.c, sm, cfg.BinaryMirror); err != nil {
return errors.Wrap(err, "downloading binaries")
}

Expand Down
1 change: 1 addition & 0 deletions pkg/minikube/config/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ type ClusterConfig struct {
MountPort uint16
MountType string
MountUID string
BinaryMirror string // Mirror location for kube binaries (kubectl, kubelet, & kubeadm)
}

// KubernetesConfig contains the parameters used to configure the VM Kubernetes.
Expand Down
17 changes: 13 additions & 4 deletions pkg/minikube/download/binary.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,18 @@ import (
"k8s.io/minikube/pkg/minikube/localpath"
)

// DefaultKubeBinariesURL returns a URL to kube binaries
func DefaultKubeBinariesURL() string {
return fmt.Sprintf("https://%s/kubernetes-release/release", downloadHost)
}

// binaryWithChecksumURL gets the location of a Kubernetes binary
func binaryWithChecksumURL(binaryName, version, osName, archName string) (string, error) {
base := fmt.Sprintf("https://%s/kubernetes-release/release/%s/bin/%s/%s/%s", downloadHost, version, osName, archName, binaryName)
func binaryWithChecksumURL(binaryName, version, osName, archName, binaryURL string) (string, error) {
if binaryURL == "" {
binaryURL = DefaultKubeBinariesURL()
}

base := fmt.Sprintf("%s/%s/bin/%s/%s/%s", binaryURL, version, osName, archName, binaryName)
v, err := semver.Make(version[1:])
if err != nil {
return "", err
Expand All @@ -45,12 +54,12 @@ func binaryWithChecksumURL(binaryName, version, osName, archName string) (string
}

// Binary will download a binary onto the host
func Binary(binary, version, osName, archName string) (string, error) {
func Binary(binary, version, osName, archName, binaryURL string) (string, error) {
targetDir := localpath.MakeMiniPath("cache", osName, version)
targetFilepath := path.Join(targetDir, binary)
targetLock := targetFilepath + ".lock"

url, err := binaryWithChecksumURL(binary, version, osName, archName)
url, err := binaryWithChecksumURL(binary, version, osName, archName, binaryURL)
if err != nil {
return "", err
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/minikube/download/download_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ func testBinaryDownloadPreventsMultipleDownload(t *testing.T) {
var group sync.WaitGroup
group.Add(2)
dlCall := func() {
if _, err := Binary("kubectl", "v1.20.2", "linux", "amd64"); err != nil {
if _, err := Binary("kubectl", "v1.20.2", "linux", "amd64", ""); err != nil {
t.Errorf("Failed to download binary: %+v", err)
}
group.Done()
Expand Down
4 changes: 2 additions & 2 deletions pkg/minikube/machine/cache_binaries.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ func isExcluded(binary string, excludedBinaries []string) bool {
}

// CacheBinariesForBootstrapper will cache binaries for a bootstrapper
func CacheBinariesForBootstrapper(version string, clusterBootstrapper string, excludeBinaries []string) error {
func CacheBinariesForBootstrapper(version string, clusterBootstrapper string, excludeBinaries []string, binariesURL string) error {
binaries := bootstrapper.GetCachedBinaryList(clusterBootstrapper)

var g errgroup.Group
Expand All @@ -53,7 +53,7 @@ func CacheBinariesForBootstrapper(version string, clusterBootstrapper string, ex
}
bin := bin // https://golang.org/doc/faq#closures_and_goroutines
g.Go(func() error {
if _, err := download.Binary(bin, version, "linux", detect.EffectiveArch()); err != nil {
if _, err := download.Binary(bin, version, "linux", detect.EffectiveArch(), binariesURL); err != nil {
return errors.Wrapf(err, "caching binary %s", bin)
}
return nil
Expand Down
4 changes: 2 additions & 2 deletions pkg/minikube/machine/cache_binaries_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ func TestCacheBinariesForBootstrapper(t *testing.T) {
for _, test := range tc {
t.Run(test.version, func(t *testing.T) {
os.Setenv("MINIKUBE_HOME", test.minikubeHome)
err := CacheBinariesForBootstrapper(test.version, test.clusterBootstrapper, nil)
err := CacheBinariesForBootstrapper(test.version, test.clusterBootstrapper, nil, "")
if err != nil && !test.err {
t.Fatalf("Got unexpected error %v", err)
}
Expand Down Expand Up @@ -160,7 +160,7 @@ func TestExcludedBinariesNotDownloaded(t *testing.T) {
}
}()

if err := CacheBinariesForBootstrapper("v1.16.0", clusterBootstrapper, []string{binaryToExclude}); err != nil {
if err := CacheBinariesForBootstrapper("v1.16.0", clusterBootstrapper, []string{binaryToExclude}, ""); err != nil {
t.Errorf("Failed to cache binaries: %v", err)
}
}
14 changes: 8 additions & 6 deletions pkg/minikube/node/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,12 @@ func handleDownloadOnly(cacheGroup, kicGroup *errgroup.Group, k8sVersion, contai
if !viper.GetBool("download-only") {
return
}
if err := doCacheBinaries(k8sVersion, containerRuntime, driverName); err != nil {

binariesURL := viper.GetString("binary-mirror")
if err := doCacheBinaries(k8sVersion, containerRuntime, driverName, binariesURL); err != nil {
exit.Error(reason.InetCacheBinaries, "Failed to cache binaries", err)
}
if _, err := CacheKubectlBinary(k8sVersion); err != nil {
if _, err := CacheKubectlBinary(k8sVersion, binariesURL); err != nil {
exit.Error(reason.InetCacheKubectl, "Failed to cache kubectl", err)
}
waitCacheRequiredImages(cacheGroup)
Expand All @@ -94,22 +96,22 @@ func handleDownloadOnly(cacheGroup, kicGroup *errgroup.Group, k8sVersion, contai
}

// CacheKubectlBinary caches the kubectl binary
func CacheKubectlBinary(k8sVersion string) (string, error) {
func CacheKubectlBinary(k8sVersion, binaryURL string) (string, error) {
binary := "kubectl"
if runtime.GOOS == "windows" {
binary = "kubectl.exe"
}

return download.Binary(binary, k8sVersion, runtime.GOOS, detect.EffectiveArch())
return download.Binary(binary, k8sVersion, runtime.GOOS, detect.EffectiveArch(), binaryURL)
}

// doCacheBinaries caches Kubernetes binaries in the foreground
func doCacheBinaries(k8sVersion, containerRuntime, driverName string) error {
func doCacheBinaries(k8sVersion, containerRuntime, driverName, binariesURL string) error {
existingBinaries := constants.KubernetesReleaseBinaries
if !download.PreloadExists(k8sVersion, containerRuntime, driverName) {
existingBinaries = nil
}
return machine.CacheBinariesForBootstrapper(k8sVersion, viper.GetString(cmdcfg.Bootstrapper), existingBinaries)
return machine.CacheBinariesForBootstrapper(k8sVersion, viper.GetString(cmdcfg.Bootstrapper), existingBinaries, binariesURL)
}

// beginDownloadKicBaseImage downloads the kic image
Expand Down
70 changes: 70 additions & 0 deletions test/integration/aaa_download_only_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,12 @@ import (
"bytes"
"context"
"crypto/md5"
"crypto/sha256"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
Expand Down Expand Up @@ -247,3 +251,69 @@ func TestDownloadOnlyKic(t *testing.T) {
t.Errorf("failed to verify checksum. checksum of %q does not match remote checksum (%q != %q)", tarball, string(remoteChecksum), string(checksum[:]))
}
}

// createSha256File is a helper function which creates sha256 checksum file from given file
func createSha256File(filePath string) error {
dat, _ := os.ReadFile(filePath)
sum := sha256.Sum256(dat)

f, err := os.Create(filePath + ".sha256")
if err != nil {
return err
}
defer f.Close()

_, err = f.WriteString(fmt.Sprintf("%x", sum[:]))
if err != nil {
return err
}
return nil
}

// TestBinaryMirror tests functionality of --binary-mirror flag
func TestBinaryMirror(t *testing.T) {
profile := UniqueProfileName("binary-mirror")
ctx, cancel := context.WithTimeout(context.Background(), Minutes(10))
defer Cleanup(t, profile, cancel)

tmpDir, err := ioutil.TempDir("", "kb_test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpDir)

// Start test server which will serve binary files
ts := httptest.NewServer(
http.FileServer(http.Dir(tmpDir)),
)
defer ts.Close()

binaryName := "kubectl"
if runtime.GOOS == "windows" {
binaryName = "kubectl.exe"
}
binaryPath, err := download.Binary(binaryName, constants.DefaultKubernetesVersion, runtime.GOOS, runtime.GOARCH, "")
if err != nil {
t.Errorf("Failed to download binary: %+v", err)
}

newBinaryDir := filepath.Join(tmpDir, constants.DefaultKubernetesVersion, "bin", runtime.GOOS, runtime.GOARCH)
if err := os.MkdirAll(newBinaryDir, os.ModePerm); err != nil {
t.Errorf("Failed to create %s directories", newBinaryDir)
}

newBinaryPath := filepath.Join(newBinaryDir, binaryName)
if err := os.Rename(binaryPath, newBinaryPath); err != nil {
t.Errorf("Failed to move binary file: %+v", err)
}
if err := createSha256File(newBinaryPath); err != nil {
t.Errorf("Failed to generate sha256 checksum file: %+v", err)
}

args := []string{"start", "--download-only", "-p", profile, "--alsologtostderr", "--binary-mirror", ts.URL}

cmd := exec.CommandContext(ctx, Target(), args...)
if _, err := Run(t, cmd); err != nil {
t.Errorf("start with --binary-mirror failed %q : %v", args, err)
}
}