From def3f067a8ee2ea74817e117dc7420e44f1fba49 Mon Sep 17 00:00:00 2001 From: Jack Zhang Date: Sun, 1 Aug 2021 19:23:39 +0800 Subject: [PATCH] Support Ingress on MacOS, driver docker --- cmd/minikube/cmd/tunnel.go | 2 +- pkg/addons/addons.go | 10 +---- pkg/minikube/out/out.go | 2 +- pkg/minikube/tunnel/kic/ssh_conn.go | 20 ++++----- pkg/minikube/tunnel/kic/ssh_tunnel.go | 58 ++++++++++++++++++++++++++- test/integration/addons_test.go | 5 +-- 6 files changed, 71 insertions(+), 26 deletions(-) diff --git a/cmd/minikube/cmd/tunnel.go b/cmd/minikube/cmd/tunnel.go index ffdd6cd8fb72..8986a89f3311 100644 --- a/cmd/minikube/cmd/tunnel.go +++ b/cmd/minikube/cmd/tunnel.go @@ -86,7 +86,7 @@ var tunnelCmd = &cobra.Command{ sshPort := strconv.Itoa(port) sshKey := filepath.Join(localpath.MiniPath(), "machines", cname, "id_rsa") - kicSSHTunnel := kic.NewSSHTunnel(ctx, sshPort, sshKey, clientset.CoreV1()) + kicSSHTunnel := kic.NewSSHTunnel(ctx, sshPort, sshKey, clientset.CoreV1(), clientset.NetworkingV1()) err = kicSSHTunnel.Start() if err != nil { exit.Error(reason.SvcTunnelStart, "error starting tunnel", err) diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go index 6a00bb5423d0..ab391929328f 100644 --- a/pkg/addons/addons.go +++ b/pkg/addons/addons.go @@ -153,16 +153,8 @@ func EnableOrDisableAddon(cc *config.ClusterConfig, name string, val string) err // to match both ingress and ingress-dns addons if strings.HasPrefix(name, "ingress") && enable { if driver.IsKIC(cc.Driver) { - if runtime.GOOS == "windows" { + if runtime.GOOS == "windows" || runtime.GOOS == "darwin" { out.Styled(style.Tip, `After the addon is enabled, please run "minikube tunnel" and your ingress resources would be available at "127.0.0.1"`) - } else if runtime.GOOS != "linux" { - exit.Message(reason.Usage, `Due to networking limitations of driver {{.driver_name}} on {{.os_name}}, {{.addon_name}} addon is not supported. -Alternatively to use this addon you can use a vm-based driver: - - 'minikube start --vm=true' - -To track the update on this work in progress feature please check: -https://github.com/kubernetes/minikube/issues/7332`, out.V{"driver_name": cc.Driver, "os_name": runtime.GOOS, "addon_name": name}) } else if driver.BareMetal(cc.Driver) { out.WarningT(`Due to networking limitations of driver {{.driver_name}}, {{.addon_name}} addon is not fully supported. Try using a different driver.`, out.V{"driver_name": cc.Driver, "addon_name": name}) diff --git a/pkg/minikube/out/out.go b/pkg/minikube/out/out.go index 738864c068d3..4706f9a7063a 100644 --- a/pkg/minikube/out/out.go +++ b/pkg/minikube/out/out.go @@ -118,7 +118,7 @@ func Styled(st style.Enum, format string, a ...V) { func boxedCommon(printFunc func(format string, a ...interface{}), cfg box.Config, title string, format string, a ...V) { box := box.New(cfg) if !useColor { - box.Config.Color = "" + box.Config.Color = nil } str := Sprintf(style.None, format, a...) printFunc(box.String(title, strings.TrimSpace(str))) diff --git a/pkg/minikube/tunnel/kic/ssh_conn.go b/pkg/minikube/tunnel/kic/ssh_conn.go index 35b1abac0233..2dc1c2131042 100644 --- a/pkg/minikube/tunnel/kic/ssh_conn.go +++ b/pkg/minikube/tunnel/kic/ssh_conn.go @@ -36,7 +36,7 @@ type sshConn struct { activeConn bool } -func createSSHConn(name, sshPort, sshKey string, svc *v1.Service) *sshConn { +func createSSHConn(name, sshPort, sshKey string, resourcePorts []int32, resourceIP string, resourceName string) *sshConn { // extract sshArgs sshArgs := []string{ // TODO: document the options here @@ -50,17 +50,17 @@ func createSSHConn(name, sshPort, sshKey string, svc *v1.Service) *sshConn { askForSudo := false var privilegedPorts []int32 - for _, port := range svc.Spec.Ports { + for _, port := range resourcePorts { arg := fmt.Sprintf( "-L %d:%s:%d", - port.Port, - svc.Spec.ClusterIP, - port.Port, + port, + resourceIP, + port, ) // check if any port is privileged - if port.Port < 1024 { - privilegedPorts = append(privilegedPorts, port.Port) + if port < 1024 { + privilegedPorts = append(privilegedPorts, port) askForSudo = true } @@ -71,8 +71,8 @@ func createSSHConn(name, sshPort, sshKey string, svc *v1.Service) *sshConn { if askForSudo && runtime.GOOS != "windows" { out.Styled( style.Warning, - "The service {{.service}} requires privileged ports to be exposed: {{.ports}}", - out.V{"service": svc.Name, "ports": fmt.Sprintf("%v", privilegedPorts)}, + "The service/ingress {{.resource}} requires privileged ports to be exposed: {{.ports}}", + out.V{"resource": resourceName, "ports": fmt.Sprintf("%v", privilegedPorts)}, ) out.Styled(style.Permissions, "sudo permission will be asked for it.") @@ -89,7 +89,7 @@ func createSSHConn(name, sshPort, sshKey string, svc *v1.Service) *sshConn { return &sshConn{ name: name, - service: svc.Name, + service: resourceName, cmd: cmd, activeConn: false, } diff --git a/pkg/minikube/tunnel/kic/ssh_tunnel.go b/pkg/minikube/tunnel/kic/ssh_tunnel.go index 9fcd6446a2c4..dafa3f94a9d2 100644 --- a/pkg/minikube/tunnel/kic/ssh_tunnel.go +++ b/pkg/minikube/tunnel/kic/ssh_tunnel.go @@ -23,8 +23,10 @@ import ( "time" v1 "k8s.io/api/core/v1" + v1_networking "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" typed_core "k8s.io/client-go/kubernetes/typed/core/v1" + typed_networking "k8s.io/client-go/kubernetes/typed/networking/v1" "k8s.io/klog/v2" "k8s.io/minikube/pkg/minikube/tunnel" @@ -36,19 +38,21 @@ type SSHTunnel struct { sshPort string sshKey string v1Core typed_core.CoreV1Interface + v1Networking typed_networking.NetworkingV1Interface LoadBalancerEmulator tunnel.LoadBalancerEmulator conns map[string]*sshConn connsToStop map[string]*sshConn } // NewSSHTunnel ... -func NewSSHTunnel(ctx context.Context, sshPort, sshKey string, v1Core typed_core.CoreV1Interface) *SSHTunnel { +func NewSSHTunnel(ctx context.Context, sshPort, sshKey string, v1Core typed_core.CoreV1Interface, v1Networking typed_networking.NetworkingV1Interface) *SSHTunnel { return &SSHTunnel{ ctx: ctx, sshPort: sshPort, sshKey: sshKey, v1Core: v1Core, LoadBalancerEmulator: tunnel.NewLoadBalancerEmulator(v1Core), + v1Networking: v1Networking, conns: make(map[string]*sshConn), connsToStop: make(map[string]*sshConn), } @@ -73,6 +77,11 @@ func (t *SSHTunnel) Start() error { klog.Errorf("error listing services: %v", err) } + ingresses, err := t.v1Networking.Ingresses("").List(context.Background(), metav1.ListOptions{}) + if err != nil { + klog.Errorf("error listing ingresses: %v", err) + } + t.markConnectionsToBeStopped() for _, svc := range services.Items { @@ -81,6 +90,10 @@ func (t *SSHTunnel) Start() error { } } + for _, ingress := range ingresses.Items { + t.startConnectionIngress(ingress) + } + t.stopMarkedConnections() // TODO: which time to use? @@ -104,8 +117,14 @@ func (t *SSHTunnel) startConnection(svc v1.Service) { return } + resourcePorts := []int32{} + + for _, port := range svc.Spec.Ports { + resourcePorts = append(resourcePorts, port.Port) + } + // create new ssh conn - newSSHConn := createSSHConn(uniqName, t.sshPort, t.sshKey, &svc) + newSSHConn := createSSHConn(uniqName, t.sshPort, t.sshKey, resourcePorts, svc.Spec.ClusterIP, svc.Name) t.conns[newSSHConn.name] = newSSHConn go func() { @@ -121,6 +140,31 @@ func (t *SSHTunnel) startConnection(svc v1.Service) { } } +func (t *SSHTunnel) startConnectionIngress(ingress v1_networking.Ingress) { + uniqName := sshConnUniqNameIngress(ingress) + existingSSHConn, ok := t.conns[uniqName] + + if ok { + // if the svc still exist we remove the conn from the stopping list + delete(t.connsToStop, existingSSHConn.name) + return + } + + resourcePorts := []int32{80, 443} + resourceIP := "127.0.0.1" + + // create new ssh conn + newSSHConn := createSSHConn(uniqName, t.sshPort, t.sshKey, resourcePorts, resourceIP, ingress.Name) + t.conns[newSSHConn.name] = newSSHConn + + go func() { + err := newSSHConn.startAndWait() + if err != nil { + klog.Errorf("error starting ssh tunnel: %v", err) + } + }() +} + func (t *SSHTunnel) stopActiveConnections() { for _, conn := range t.conns { err := conn.stop() @@ -157,3 +201,13 @@ func sshConnUniqName(service v1.Service) string { return strings.Join(n, "") } + +func sshConnUniqNameIngress(ingress v1_networking.Ingress) string { + n := []string{ingress.Name} + + for _, rule := range ingress.Spec.Rules { + n = append(n, rule.Host) + } + + return strings.Join(n, "") +} diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index 1d3115d2295a..62dfd5339cfa 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -29,7 +29,6 @@ import ( "os/exec" "path/filepath" "reflect" - "runtime" "strings" "testing" "time" @@ -68,7 +67,7 @@ func TestAddons(t *testing.T) { } args := append([]string{"start", "-p", profile, "--wait=true", "--memory=4000", "--alsologtostderr", "--addons=registry", "--addons=metrics-server", "--addons=olm", "--addons=volumesnapshots", "--addons=csi-hostpath-driver"}, StartArgs()...) - if !NoneDriver() && !(runtime.GOOS == "darwin" && KicDriver()) { // none driver and macos docker driver does not support ingress + if !NoneDriver() { // none driver does not support ingress args = append(args, "--addons=ingress") } if !arm64Platform() { @@ -155,7 +154,7 @@ func TestAddons(t *testing.T) { // validateIngressAddon tests the ingress addon by deploying a default nginx pod func validateIngressAddon(ctx context.Context, t *testing.T, profile string) { defer PostMortemLogs(t, profile) - if NoneDriver() || (runtime.GOOS == "darwin" && KicDriver()) { + if NoneDriver() { t.Skipf("skipping: ingress not supported ") }