diff --git a/cmd/minikube/cmd/start_flags.go b/cmd/minikube/cmd/start_flags.go index 5a354854b2e7..8d2cd8178c4f 100644 --- a/cmd/minikube/cmd/start_flags.go +++ b/cmd/minikube/cmd/start_flags.go @@ -62,7 +62,6 @@ const ( hypervUseExternalSwitch = "hyperv-use-external-switch" hypervExternalAdapter = "hyperv-external-adapter" kvmNetwork = "kvm-network" - kvmPrivateNetwork = "kvm-private-network" kvmQemuURI = "kvm-qemu-uri" kvmGPU = "kvm-gpu" kvmHidden = "kvm-hidden" @@ -162,7 +161,7 @@ func initMinikubeFlags() { startCmd.Flags().Bool(preload, true, "If set, download tarball of preloaded images if available to improve start time. Defaults to true.") startCmd.Flags().Bool(deleteOnFailure, false, "If set, delete the current cluster if start fails and try again. Defaults to false.") startCmd.Flags().Bool(forceSystemd, false, "If set, force the container runtime to use sytemd as cgroup manager. Defaults to false.") - startCmd.Flags().StringP(network, "", "", "network to run minikube with. Only available with the docker/podman drivers. If left empty, minikube will create a new network.") + startCmd.Flags().StringP(network, "", "", "network to run minikube with. Now it is used by docker/podman and KVM drivers. If left empty, minikube will create a new network.") startCmd.Flags().StringVarP(&outputFormat, "output", "o", "text", "Format to print stdout in. Options include: [text,json]") startCmd.Flags().StringP(trace, "", "", "Send trace events. Options include: [gcp]") } @@ -193,7 +192,6 @@ func initDriverFlags() { // kvm2 startCmd.Flags().String(kvmNetwork, "default", "The KVM default network name. (kvm2 driver only)") - startCmd.Flags().String(kvmPrivateNetwork, "", "The KVM private network name. (kvm2 driver only) (default: 'mk-')") startCmd.Flags().String(kvmQemuURI, "qemu:///system", "The KVM QEMU connection URI. (kvm2 driver only)") startCmd.Flags().Bool(kvmGPU, false, "Enable experimental NVIDIA GPU support in minikube") startCmd.Flags().Bool(kvmHidden, false, "Hide the hypervisor signature from the guest in minikube (kvm2 driver only)") @@ -313,8 +311,8 @@ func generateClusterConfig(cmd *cobra.Command, existing *config.ClusterConfig, k out.WarningT("With --network-plugin=cni, you will need to provide your own CNI. See --cni flag as a user-friendly alternative") } - if !driver.IsKIC(drvName) && viper.GetString(network) != "" { - out.WarningT("--network flag is only valid with the docker/podman drivers, it will be ignored") + if !(driver.IsKIC(drvName) || driver.IsKVM(drvName)) && viper.GetString(network) != "" { + out.WarningT("--network flag is only valid with the docker/podman and KVM drivers, it will be ignored") } checkNumaCount(k8sVersion) @@ -344,7 +342,6 @@ func generateClusterConfig(cmd *cobra.Command, existing *config.ClusterConfig, k HypervUseExternalSwitch: viper.GetBool(hypervUseExternalSwitch), HypervExternalAdapter: viper.GetString(hypervExternalAdapter), KVMNetwork: viper.GetString(kvmNetwork), - KVMPrivateNetwork: viper.GetString(kvmPrivateNetwork), KVMQemuURI: viper.GetString(kvmQemuURI), KVMGPU: viper.GetBool(kvmGPU), KVMHidden: viper.GetBool(kvmHidden), @@ -383,10 +380,6 @@ func generateClusterConfig(cmd *cobra.Command, existing *config.ClusterConfig, k }, MultiNodeRequested: viper.GetInt(nodes) > 1, } - // if KVMPrivateNetwork is not user-defined, defaults to "mk-" - if cc.KVMPrivateNetwork == "" { - cc.KVMPrivateNetwork = fmt.Sprintf("mk-%s", cc.KubernetesConfig.ClusterName) - } cc.VerifyComponents = interpretWaitFlag(*cmd) if viper.GetBool(createMount) && driver.IsKIC(drvName) { cc.ContainerVolumeMounts = []string{viper.GetString(mountString)} @@ -561,15 +554,7 @@ func updateExistingConfigFromFlags(cmd *cobra.Command, existing *config.ClusterC } if cmd.Flags().Changed(kvmNetwork) { - if cc.KVMNetwork != viper.GetString(kvmNetwork) { - out.WarningT("You cannot change the KVM Default Network name for an exiting minikube cluster. Please first delete the cluster.") - } - } - - if cmd.Flags().Changed(kvmPrivateNetwork) { - if cc.KVMPrivateNetwork != viper.GetString(kvmPrivateNetwork) { - out.WarningT("You cannot change the KVM Private Network name for an exiting minikube cluster. Please first delete the cluster.") - } + cc.KVMNetwork = viper.GetString(kvmNetwork) } if cmd.Flags().Changed(kvmQemuURI) { diff --git a/pkg/drivers/kvm/domain.go b/pkg/drivers/kvm/domain.go index b3b8d0287b79..4a112b37a1c4 100644 --- a/pkg/drivers/kvm/domain.go +++ b/pkg/drivers/kvm/domain.go @@ -20,9 +20,7 @@ package kvm import ( "bytes" - "crypto/rand" "fmt" - "net" "text/template" libvirt "github.com/libvirt/libvirt-go" @@ -68,12 +66,10 @@ const domainTmpl = ` - - @@ -92,25 +88,6 @@ const domainTmpl = ` ` -func randomMAC() (net.HardwareAddr, error) { - buf := make([]byte, 6) - _, err := rand.Read(buf) - if err != nil { - return nil, err - } - // We unset the first and second least significant bits (LSB) of the MAC - // - // The LSB of the first octet - // 0 for unicast - // 1 for multicast - // - // The second LSB of the first octet - // 0 for universally administered addresses - // 1 for locally administered addresses - buf[0] &= 0xfc - return buf, nil -} - func (d *Driver) getDomain() (*libvirt.Domain, *libvirt.Connect, error) { conn, err := getConnection(d.ConnectionURI) if err != nil { @@ -146,22 +123,6 @@ func closeDomain(dom *libvirt.Domain, conn *libvirt.Connect) error { } func (d *Driver) createDomain() (*libvirt.Domain, error) { - // create random MAC addresses first for our NICs - if d.MAC == "" { - mac, err := randomMAC() - if err != nil { - return nil, errors.Wrap(err, "generating mac address") - } - d.MAC = mac.String() - } - - if d.PrivateMAC == "" { - mac, err := randomMAC() - if err != nil { - return nil, errors.Wrap(err, "generating mac address") - } - d.PrivateMAC = mac.String() - } // create the XML for the domain using our domainTmpl template tmpl := template.Must(template.New("domain").Parse(domainTmpl)) var domainXML bytes.Buffer @@ -180,5 +141,17 @@ func (d *Driver) createDomain() (*libvirt.Domain, error) { return nil, errors.Wrapf(err, "error defining domain xml: %s", domainXML.String()) } + // save MAC address + dmac, err := macFromXML(conn, d.MachineName, d.Network) + if err != nil { + return nil, fmt.Errorf("failed saving MAC address: %w", err) + } + d.MAC = dmac + pmac, err := macFromXML(conn, d.MachineName, d.PrivateNetwork) + if err != nil { + return nil, fmt.Errorf("failed saving MAC address: %w", err) + } + d.PrivateMAC = pmac + return dom, nil } diff --git a/pkg/drivers/kvm/kvm.go b/pkg/drivers/kvm/kvm.go index 22b4a9a276b1..0bcb00bdc6d7 100644 --- a/pkg/drivers/kvm/kvm.go +++ b/pkg/drivers/kvm/kvm.go @@ -31,6 +31,7 @@ import ( libvirt "github.com/libvirt/libvirt-go" "github.com/pkg/errors" pkgdrivers "k8s.io/minikube/pkg/drivers" + "k8s.io/minikube/pkg/util/retry" ) // Driver is the machine driver for KVM @@ -209,12 +210,14 @@ func (d *Driver) GetIP() (string, error) { if s != state.Running { return "", errors.New("host is not running") } - ip, err := d.lookupIP() + + conn, err := getConnection(d.ConnectionURI) if err != nil { - return "", errors.Wrap(err, "getting IP") + return "", errors.Wrap(err, "getting libvirt connection") } + defer conn.Close() - return ip, nil + return ipFromXML(conn, d.MachineName, d.PrivateNetwork) } // GetSSHHostname returns hostname for use with ssh @@ -272,32 +275,43 @@ func (d *Driver) Start() (err error) { } log.Info("Waiting to get IP...") - for i := 0; i <= 40; i++ { - ip, err := d.GetIP() + if err := d.waitForStaticIP(conn); err != nil { + return errors.Wrap(err, "IP not available after waiting") + } + + log.Info("Waiting for SSH to be available...") + if err := drivers.WaitForSSH(d); err != nil { + return errors.Wrap(err, "SSH not available after waiting") + } + + return nil +} + +// waitForStaticIP waits for IP address of domain that has been created & starting and then makes that IP static. +func (d *Driver) waitForStaticIP(conn *libvirt.Connect) error { + query := func() error { + sip, err := ipFromAPI(conn, d.MachineName, d.PrivateNetwork) if err != nil { - return errors.Wrap(err, "getting ip during machine start") + return fmt.Errorf("failed getting IP during machine start, will retry: %w", err) } - if ip == "" { - log.Debugf("Waiting for machine to come up %d/%d", i, 40) - time.Sleep(3 * time.Second) - continue + if sip == "" { + return fmt.Errorf("waiting for machine to come up") } - if ip != "" { - log.Infof("Found IP for machine: %s", ip) - d.IPAddress = ip - break - } - } + log.Infof("Found IP for machine: %s", sip) + d.IPAddress = sip - if d.IPAddress == "" { - return errors.New("machine didn't return an IP after 120 seconds") + return nil + } + if err := retry.Local(query, 1*time.Minute); err != nil { + return fmt.Errorf("machine %s didn't return IP after 1 minute", d.MachineName) } - log.Info("Waiting for SSH to be available...") - if err := drivers.WaitForSSH(d); err != nil { - d.IPAddress = "" - return errors.Wrap(err, "SSH not available after waiting") + log.Info("Reserving static IP address...") + if err := addStaticIP(conn, d.PrivateNetwork, d.MachineName, d.PrivateMAC, d.IPAddress); err != nil { + log.Warnf("Failed reserving static IP %s for host %s, will continue anyway: %v", d.IPAddress, d.MachineName, err) + } else { + log.Infof("Reserved static IP address: %s", d.IPAddress) } return nil @@ -385,7 +399,6 @@ func ensureDirPermissions(store string) error { // Stop a host gracefully func (d *Driver) Stop() (err error) { - d.IPAddress = "" s, err := d.GetState() if err != nil { return errors.Wrap(err, "getting state of VM") @@ -458,6 +471,13 @@ func (d *Driver) Remove() error { return errors.Wrap(err, "undefine domain") } + log.Info("Removing static IP address...") + if err := delStaticIP(conn, d.PrivateNetwork, "", "", d.IPAddress); err != nil { + log.Warnf("failed removing static IP %s for host %s, will continue anyway: %v", d.IPAddress, d.MachineName, err) + } else { + log.Info("Removed static IP address") + } + return nil } diff --git a/pkg/drivers/kvm/network.go b/pkg/drivers/kvm/network.go index 52cf8286d620..1fdebd17f2de 100644 --- a/pkg/drivers/kvm/network.go +++ b/pkg/drivers/kvm/network.go @@ -20,11 +20,8 @@ package kvm import ( "bytes" - "encoding/json" "encoding/xml" "fmt" - "io/ioutil" - "strings" "text/template" "time" @@ -56,6 +53,27 @@ type kvmNetwork struct { network.Parameters } +type kvmIface struct { + Type string `xml:"type,attr"` + Mac struct { + Address string `xml:"address,attr"` + } `xml:"mac"` + Source struct { + Network string `xml:"network,attr"` + Portid string `xml:"portid,attr"` + Bridge string `xml:"bridge,attr"` + } `xml:"source"` + Target struct { + Dev string `xml:"dev,attr"` + } `xml:"target"` + Model struct { + Type string `xml:"type,attr"` + } `xml:"model"` + Alias struct { + Name string `xml:"name,attr"` + } `xml:"alias"` +} + // firstSubnetAddr is starting subnet to try for new KVM cluster, // avoiding possible conflict with other local networks by further incrementing it up to 20 times by 10. const firstSubnetAddr = "192.168.39.0" @@ -335,94 +353,195 @@ func (d *Driver) checkDomains(conn *libvirt.Connect) error { return nil } -func (d *Driver) lookupIP() (string, error) { - conn, err := getConnection(d.ConnectionURI) +// Static IP management +// "Update ... existing network definition, with the changes ... taking effect immediately, without needing to destroy and re-start the network." +// ref: https://libvirt.org/manpages/virsh.html#net-update +// ref: https://libvirt.org/html/libvirt-libvirt-network.html#virNetworkUpdate +// ref: https://wiki.libvirt.org/page/Networking#Applying_modifications_to_the_network + +// ref: https://libvirt.org/formatnetwork.html#elementsAddress +// ref: https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainInterfaceAddresses +// ref: https://libvirt.org/manpages/virsh.html#domifaddr + +// addStaticIP appends new host's name, MAC and static IP address record to list of network DHCP leases. +// It will return nil if host record already exists. +func addStaticIP(conn *libvirt.Connect, network, hostname, mac, ip string) error { + l, err := dhcpLease(conn, network, hostname, mac, ip) if err != nil { - return "", errors.Wrap(err, "getting connection and domain") + return fmt.Errorf("failed looking up network %s for host DHCP lease {name: %q, mac: %q, ip: %q}: %w", network, hostname, mac, ip, err) + } + if l != nil { + log.Debugf("skip adding static IP to network %s - found existing host DHCP lease matching {name: %q, mac: %q, ip: %q}", network, hostname, mac, ip) + return nil } - defer conn.Close() - libVersion, err := conn.GetLibVersion() + net, err := conn.LookupNetworkByName(network) if err != nil { - return "", errors.Wrap(err, "getting libversion") + return fmt.Errorf("failed looking up network %s: %w", network, err) + } + defer func() { _ = net.Free() }() + + return net.Update( + libvirt.NETWORK_UPDATE_COMMAND_ADD_LAST, + libvirt.NETWORK_SECTION_IP_DHCP_HOST, + -1, + fmt.Sprintf("", mac, hostname, ip), + libvirt.NETWORK_UPDATE_AFFECT_LIVE+libvirt.NETWORK_UPDATE_AFFECT_CONFIG) +} + +// delStaticIP deletes static IP address record that matches given combination of host's name, MAC and IP from list of network DHCP leases. +// It will return nil if record doesn't exist. +func delStaticIP(conn *libvirt.Connect, network, hostname, mac, ip string) error { + l, err := dhcpLease(conn, network, hostname, mac, ip) + if err != nil { + return fmt.Errorf("failed looking up network %s for host DHCP lease {name: %q, mac: %q, ip: %q}: %w", network, hostname, mac, ip, err) + } + if l == nil { + log.Debugf("skip deleting static IP from network %s - couldn't find host DHCP lease matching {name: %q, mac: %q, ip: %q}", network, hostname, mac, ip) + return nil } - // Earlier versions of libvirt use a lease file instead of a status file - if libVersion < 1002006 { - return d.lookupIPFromLeasesFile() + net, err := conn.LookupNetworkByName(network) + if err != nil { + return fmt.Errorf("failed looking up network %s: %w", network, err) } + defer func() { _ = net.Free() }() - // TODO: for everything > 1002006, there is direct support in the libvirt-go for handling this - return d.lookupIPFromStatusFile(conn) + return net.Update( + libvirt.NETWORK_UPDATE_COMMAND_DELETE, + libvirt.NETWORK_SECTION_IP_DHCP_HOST, + -1, + fmt.Sprintf("", l.Mac, l.Hostname, l.IPaddr), + libvirt.NETWORK_UPDATE_AFFECT_LIVE+libvirt.NETWORK_UPDATE_AFFECT_CONFIG) } -func (d *Driver) lookupIPFromStatusFile(conn *libvirt.Connect) (string, error) { - network, err := conn.LookupNetworkByName(d.PrivateNetwork) +// dhcpLease returns network DHCP lease that matches given combination of host's name, MAC and IP. +func dhcpLease(conn *libvirt.Connect, network, hostname, mac, ip string) (lease *libvirt.NetworkDHCPLease, err error) { + if hostname == "" && mac == "" && ip == "" { + return nil, nil + } + + net, err := conn.LookupNetworkByName(network) if err != nil { - return "", errors.Wrap(err, "looking up network by name") + return nil, fmt.Errorf("failed looking up network %s: %w", network, err) } - defer func() { _ = network.Free() }() + defer func() { _ = net.Free() }() - bridge, err := network.GetBridgeName() + leases, err := net.GetDHCPLeases() if err != nil { - log.Warnf("Failed to get network bridge: %v", err) - return "", err + return nil, fmt.Errorf("failed getting host DHCP leases: %w", err) } - statusFile := fmt.Sprintf("/var/lib/libvirt/dnsmasq/%s.status", bridge) - statuses, err := ioutil.ReadFile(statusFile) + + for _, l := range leases { + if (hostname == "" || hostname == l.Hostname) && (mac == "" || mac == l.Mac) && (ip == "" || ip == l.IPaddr) { + log.Debugf("found host DHCP lease matching {name: %q, mac: %q, ip: %q} in network %s: %+v", hostname, mac, ip, network, l) + return &l, nil + } + } + + log.Debugf("unable to find host DHCP lease matching {name: %q, mac: %q, ip: %q} in network %s", hostname, mac, ip, network) + return nil, nil +} + +// ipFromAPI returns current primary IP address of domain interface in network. +func ipFromAPI(conn *libvirt.Connect, domain, network string) (string, error) { + mac, err := macFromXML(conn, domain, network) if err != nil { - return "", errors.Wrap(err, "reading status file") + return "", fmt.Errorf("failed getting MAC address: %w", err) } - return parseStatusAndReturnIP(d.PrivateMAC, statuses) + ifaces, err := ifListFromAPI(conn, domain) + if err != nil { + return "", fmt.Errorf("failed getting network %s interfaces using API of domain %s: %w", network, domain, err) + } + for _, i := range ifaces { + if i.Hwaddr == mac { + if i.Addrs != nil { + log.Debugf("domain %s has current primary IP address %s and MAC address %s in network %s", domain, i.Addrs[0].Addr, mac, network) + return i.Addrs[0].Addr, nil + } + log.Debugf("domain %s with MAC address %s doesn't have current IP address in network %s: %+v", domain, mac, network, i) + return "", nil + } + } + + log.Debugf("unable to find current IP address of domain %s in network %s", domain, network) + return "", nil } -func parseStatusAndReturnIP(privateMAC string, statuses []byte) (string, error) { - type StatusEntry struct { - IPAddress string `json:"ip-address"` - MacAddress string `json:"mac-address"` +// ifListFromAPI returns current domain interfaces. +func ifListFromAPI(conn *libvirt.Connect, domain string) ([]libvirt.DomainInterface, error) { + dom, err := conn.LookupDomainByName(domain) + if err != nil { + return nil, fmt.Errorf("failed looking up domain %s: %w", domain, err) + } + defer func() { _ = dom.Free() }() + + ifs, err := dom.ListAllInterfaceAddresses(libvirt.DOMAIN_INTERFACE_ADDRESSES_SRC_LEASE) + if err != nil { + return nil, fmt.Errorf("failed listing network interface addresses of domain %s: %w", domain, err) } - var statusEntries []StatusEntry - // empty file return blank - if len(statuses) == 0 { + return ifs, nil +} + +// ipFromXML returns defined IP address of interface in network. +func ipFromXML(conn *libvirt.Connect, domain, network string) (string, error) { + mac, err := macFromXML(conn, domain, network) + if err != nil { + return "", fmt.Errorf("failed getting MAC address: %w", err) + } + + lease, err := dhcpLease(conn, network, "", mac, "") + if err != nil { + return "", fmt.Errorf("failed looking up network %s for host DHCP lease {name: , mac: %q, ip: }: %w", network, mac, err) + } + if lease == nil { + log.Debugf("unable to find defined IP address of network %s interface with MAC address %s", network, mac) return "", nil } - err := json.Unmarshal(statuses, &statusEntries) + log.Debugf("domain %s has defined IP address %s and MAC address %s in network %s", domain, lease.IPaddr, mac, network) + return lease.IPaddr, nil +} + +// macFromXML returns defined MAC address of interface in network from domain XML. +func macFromXML(conn *libvirt.Connect, domain, network string) (string, error) { + domIfs, err := ifListFromXML(conn, domain) if err != nil { - return "", errors.Wrap(err, "reading status file") + return "", fmt.Errorf("failed getting network %s interfaces using XML of domain %s: %w", network, domain, err) } - for _, status := range statusEntries { - if status.MacAddress == privateMAC { - return status.IPAddress, nil + for _, i := range domIfs { + if i.Source.Network == network { + log.Debugf("domain %s has defined MAC address %s in network %s", domain, i.Mac.Address, network) + return i.Mac.Address, nil } } - return "", nil + return "", fmt.Errorf("unable to get defined MAC address of network %s interface using XML of domain %s: network %s not found", network, domain, network) } -func (d *Driver) lookupIPFromLeasesFile() (string, error) { - leasesFile := fmt.Sprintf("/var/lib/libvirt/dnsmasq/%s.leases", d.PrivateNetwork) - leases, err := ioutil.ReadFile(leasesFile) +// ifListFromXML returns defined domain interfaces from domain XML. +func ifListFromXML(conn *libvirt.Connect, domain string) ([]kvmIface, error) { + dom, err := conn.LookupDomainByName(domain) if err != nil { - return "", errors.Wrap(err, "reading leases file") + return nil, fmt.Errorf("failed looking up domain %s: %w", domain, err) } - ipAddress := "" - for _, lease := range strings.Split(string(leases), "\n") { - if len(lease) == 0 { - continue - } - // format for lease entry - // ExpiryTime MAC IP Hostname ExtendedMAC - entry := strings.Split(lease, " ") - if len(entry) != 5 { - return "", fmt.Errorf("malformed leases entry: %s", entry) - } - if entry[1] == d.PrivateMAC { - ipAddress = entry[2] - } + defer func() { _ = dom.Free() }() + + domXML, err := dom.GetXMLDesc(0) + if err != nil { + return nil, fmt.Errorf("failed getting XML of domain %s: %w", domain, err) + } + + var d struct { + Interfaces []kvmIface `xml:"devices>interface"` } - return ipAddress, nil + err = xml.Unmarshal([]byte(domXML), &d) + if err != nil { + return nil, fmt.Errorf("failed parsing XML of domain %s: %w", domain, err) + } + + return d.Interfaces, nil } diff --git a/pkg/drivers/kvm/network_test.go b/pkg/drivers/kvm/network_test.go deleted file mode 100644 index 905816165a0f..000000000000 --- a/pkg/drivers/kvm/network_test.go +++ /dev/null @@ -1,97 +0,0 @@ -// +build linux - -/* -Copyright 2016 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kvm - -import ( - "testing" -) - -var ( - emptyFile = []byte(``) - fileWithInvalidJSON = []byte(`{`) - fileWithNoStatus = []byte(`[ - - ]`) - fileWithStatus = []byte(`[ - { - "ip-address": "1.2.3.5", - "mac-address": "a4:b5:c6:d7:e8:f9", - "hostname": "host2", - "client-id": "01:44:59:e7:fd:f4:d6", - "expiry-time": 1558638717 - }, - { - "ip-address": "1.2.3.4", - "mac-address": "a1:b2:c3:d4:e5:f6", - "hostname": "host1", - "client-id": "01:ec:97:de:a2:86:81", - "expiry-time": 1558639092 - } - ]`) -) - -func TestParseStatusAndReturnIp(t *testing.T) { - type args struct { - mac string - statuses []byte - } - tests := []struct { - name string - args args - want string - wantErr bool - }{ - { - "emptyFile", - args{"a1:b2:c3:d4:e5:f6", emptyFile}, - "", - false, - }, - { - "fileWithStatus", - args{"a1:b2:c3:d4:e5:f6", fileWithStatus}, - "1.2.3.4", - false, - }, - { - "fileWithNoStatus", - args{"a4:b5:c6:d7:e8:f9", fileWithNoStatus}, - "", - false, - }, - { - "fileWithInvalidJSON", - args{"a4:b5:c6:d7:e8:f9", fileWithInvalidJSON}, - "", - true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := parseStatusAndReturnIP(tt.args.mac, tt.args.statuses) - if (err != nil) != tt.wantErr { - t.Errorf("parseStatusAndReturnIP() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != tt.want { - t.Errorf("parseStatusAndReturnIP() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index 44c8c238ba15..672d32bd1203 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -53,7 +53,6 @@ type ClusterConfig struct { HypervUseExternalSwitch bool HypervExternalAdapter string KVMNetwork string // Only used by the KVM2 driver - KVMPrivateNetwork string // Only used by the KVM2 driver KVMQemuURI string // Only used by the KVM2 driver KVMGPU bool // Only used by the KVM2 driver KVMHidden bool // Only used by the KVM2 driver diff --git a/pkg/minikube/driver/driver.go b/pkg/minikube/driver/driver.go index 88a4d2b6658d..47cb5885b693 100644 --- a/pkg/minikube/driver/driver.go +++ b/pkg/minikube/driver/driver.go @@ -141,6 +141,11 @@ func IsMock(name string) bool { return name == Mock } +// IsKVM checks if the driver is a KVM[2] +func IsKVM(name string) bool { + return name == KVM2 || name == AliasKVM +} + // IsVM checks if the driver is a VM func IsVM(name string) bool { if IsKIC(name) || BareMetal(name) { diff --git a/pkg/minikube/registry/drvs/kvm2/kvm2.go b/pkg/minikube/registry/drvs/kvm2/kvm2.go index 2ba3eba3a66d..bebb64a88570 100644 --- a/pkg/minikube/registry/drvs/kvm2/kvm2.go +++ b/pkg/minikube/registry/drvs/kvm2/kvm2.go @@ -83,7 +83,7 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { Memory: cc.Memory, CPU: cc.CPUs, Network: cc.KVMNetwork, - PrivateNetwork: cc.KVMPrivateNetwork, + PrivateNetwork: privateNetwork(cc), Boot2DockerURL: download.LocalISOResource(cc.MinikubeISO), DiskSize: cc.DiskSize, DiskPath: filepath.Join(localpath.MiniPath(), "machines", name, fmt.Sprintf("%s.rawdisk", name)), @@ -95,6 +95,14 @@ func configure(cc config.ClusterConfig, n config.Node) (interface{}, error) { }, nil } +// if network is not user-defined it defaults to "mk-" +func privateNetwork(cc config.ClusterConfig) string { + if cc.Network == "" { + return fmt.Sprintf("mk-%s", cc.KubernetesConfig.ClusterName) + } + return cc.Network +} + // defaultURI returns the QEMU URI to connect to for health checks func defaultURI() string { u := os.Getenv("LIBVIRT_DEFAULT_URI") diff --git a/pkg/network/network.go b/pkg/network/network.go index dbe5c14d5485..0d74ea1181ed 100644 --- a/pkg/network/network.go +++ b/pkg/network/network.go @@ -23,7 +23,6 @@ import ( "sync" "time" - "github.com/pkg/errors" "k8s.io/klog/v2" ) @@ -59,12 +58,13 @@ type reservation struct { // Parameters contains main network parameters. type Parameters struct { - IP string // IP address of the network - Netmask string // form: 4-byte ('a.b.c.d') - CIDR string // form: CIDR - Gateway string // first IP address (assumed, not checked !) + IP string // IP address of network + Netmask string // dotted-decimal format ('a.b.c.d') + Prefix int // network prefix length (number of leading ones in network mask) + CIDR string // CIDR format ('a.b.c.d/n') + Gateway string // taken from network interface address or assumed as first network IP address from given addr ClientMin string // second IP address - ClientMax string // last IP address before broadcastS + ClientMax string // last IP address before broadcast Broadcast string // last IP address Interface } @@ -77,9 +77,9 @@ type Interface struct { IfaceMAC string } -// inspect initialises IPv4 network parameters struct from given address. -// address can be single address (like "192.168.17.42"), network address (like "192.168.17.0"), or in cidr form (like "192.168.17.42/24 or "192.168.17.0/24"). -// If addr is valid existsing interface address, network struct will also contain info about the respective interface. +// inspect initialises IPv4 network parameters struct from given address addr. +// addr can be single address (like "192.168.17.42"), network address (like "192.168.17.0") or in CIDR form (like "192.168.17.42/24 or "192.168.17.0/24"). +// If addr belongs to network of local network interface, parameters will also contain info about that network interface. func inspect(addr string) (*Parameters, error) { n := &Parameters{} @@ -88,21 +88,24 @@ func inspect(addr string) (*Parameters, error) { if err != nil { ip = net.ParseIP(addr) if ip == nil { - return nil, errors.Wrapf(err, "parsing address %q", addr) + return nil, fmt.Errorf("failed parsing address %s: %w", addr, err) } } - // check local interfaces - ifaces, _ := net.Interfaces() + // check local network interfaces + ifaces, err := net.Interfaces() + if err != nil { + return nil, fmt.Errorf("failed listing network interfaces: %w", err) + } for _, iface := range ifaces { ifAddrs, err := iface.Addrs() if err != nil { - return nil, errors.Wrapf(err, "listing addresses of network interface %+v", iface) + return nil, fmt.Errorf("failed listing addresses of network interface %+v: %w", iface, err) } for _, ifAddr := range ifAddrs { ifip, lan, err := net.ParseCIDR(ifAddr.String()) if err != nil { - return nil, errors.Wrapf(err, "parsing address of network iface %+v", ifAddr) + return nil, fmt.Errorf("failed parsing network interface address %+v: %w", ifAddr, err) } if lan.Contains(ip) { n.IfaceName = iface.Name @@ -116,6 +119,7 @@ func inspect(addr string) (*Parameters, error) { } } + // couldn't determine network parameters from addr nor from network interfaces if network == nil { ipnet := &net.IPNet{ IP: ip, @@ -123,15 +127,16 @@ func inspect(addr string) (*Parameters, error) { } _, network, err = net.ParseCIDR(ipnet.String()) if err != nil { - return nil, errors.Wrapf(err, "determining network address from %q", addr) + return nil, fmt.Errorf("failed determining address of network from %s: %w", addr, err) } } n.IP = network.IP.String() - n.Netmask = net.IP(network.Mask).String() // form: 4-byte ('a.b.c.d') + n.Netmask = net.IP(network.Mask).String() // dotted-decimal format ('a.b.c.d') + n.Prefix, _ = network.Mask.Size() n.CIDR = network.String() - networkIP := binary.BigEndian.Uint32(network.IP) // IP address of the network + networkIP := binary.BigEndian.Uint32(network.IP) // IP address of network networkMask := binary.BigEndian.Uint32(network.Mask) // network mask broadcastIP := (networkIP & networkMask) | (networkMask ^ 0xffffffff) // last network IP address @@ -161,14 +166,14 @@ func inspect(addr string) (*Parameters, error) { // isSubnetTaken returns if local network subnet exists and any error occurred. // If will return false in case of an error. func isSubnetTaken(subnet string) (bool, error) { - ips, err := net.InterfaceAddrs() + ifAddrs, err := net.InterfaceAddrs() if err != nil { - return false, errors.Wrap(err, "listing local networks") + return false, fmt.Errorf("failed listing network interface addresses: %w", err) } - for _, ip := range ips { - _, lan, err := net.ParseCIDR(ip.String()) + for _, ifAddr := range ifAddrs { + _, lan, err := net.ParseCIDR(ifAddr.String()) if err != nil { - return false, errors.Wrapf(err, "parsing network iface address %q", ip) + return false, fmt.Errorf("failed parsing network interface address %+v: %w", ifAddr, err) } if lan.Contains(net.ParseIP(subnet)) { return true, nil @@ -177,7 +182,7 @@ func isSubnetTaken(subnet string) (bool, error) { return false, nil } -// isSubnetPrivate returns if subnet is a private network. +// isSubnetPrivate returns if subnet is private network. func isSubnetPrivate(subnet string) bool { for _, ipnet := range privateSubnets { if ipnet.Contains(net.ParseIP(subnet)) { @@ -212,9 +217,9 @@ func FreeSubnet(startSubnet string, step, tries int) (*Parameters, error) { } else { klog.Infof("skipping subnet %s that is not private", n.CIDR) } - ones, _ := net.ParseIP(n.IP).DefaultMask().Size() + prefix, _ := net.ParseIP(n.IP).DefaultMask().Size() nextSubnet := net.ParseIP(startSubnet).To4() - if ones <= 16 { + if prefix <= 16 { nextSubnet[1] += byte(step) } else { nextSubnet[2] += byte(step) @@ -259,3 +264,37 @@ func reserveSubnet(subnet string, period time.Duration) bool { reservedSubnets.Store(subnet, reservation{createdAt: time.Now()}) return true } + +// IncIP returns address calculated by adding (positive or negative) val to addr. +// If addr is not in CIDR format it will use its default mask and set "out-of-network" if calculated address doesn't belong to same network as addr. +func IncIP(addr string, val int) (calc string, oon bool, err error) { + ip, network, err := net.ParseCIDR(addr) + if err != nil { // addr not in CIDR format - use default mask for IP addr + ip = net.ParseIP(addr) + if ip == nil { + return "", false, fmt.Errorf("failed parsing address %s: %w", addr, err) + } + prefix, _ := ip.DefaultMask().Size() // assume default network mask + _, network, err = net.ParseCIDR(fmt.Sprintf("%s/%d", addr, prefix)) + if err != nil { + return "", false, fmt.Errorf("failed parsing CIDR %s/%d: %w", addr, prefix, err) + } + } + + ip0 := ip.To4() // convert to 4-byte representation + n0 := binary.BigEndian.Uint32(ip0) + + n1 := int(n0) + val + if n1 < 0 { + return "", true, fmt.Errorf("failed decrementing IP %s: value %d too big", addr, val) + } + + ip1 := make(net.IP, 4) + binary.BigEndian.PutUint32(ip1, uint32(n1)) + + if !network.Contains(ip1) { + return "", true, fmt.Errorf("failed incrementing IP %s: value %d too big (%s would cross network boundary)", addr, val, ip) + } + + return ip1.String(), false, nil +} diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md index 16be70644418..33b9fb5d16d2 100644 --- a/site/content/en/docs/commands/start.md +++ b/site/content/en/docs/commands/start.md @@ -71,7 +71,6 @@ minikube start [flags] --kvm-hidden Hide the hypervisor signature from the guest in minikube (kvm2 driver only) --kvm-network string The KVM default network name. (kvm2 driver only) (default "default") --kvm-numa-count int Simulate numa node count in minikube, supported numa node count range is 1-8 (kvm2 driver only) (default 1) - --kvm-private-network string The KVM private network name. (kvm2 driver only) (default: 'mk-') --kvm-qemu-uri string The KVM QEMU connection URI. (kvm2 driver only) (default "qemu:///system") --listen-address string IP Address to use to expose ports (docker and podman driver only) --memory string Amount of RAM to allocate to Kubernetes (format: [], where unit = b, k, m or g). @@ -80,7 +79,7 @@ minikube start [flags] --namespace string The named space to activate after start (default "default") --nat-nic-type string NIC Type used for nat network. One of Am79C970A, Am79C973, 82540EM, 82543GC, 82545EM, or virtio (virtualbox driver only) (default "virtio") --native-ssh Use native Golang SSH client (default true). Set to 'false' to use the command line 'ssh' command when accessing the docker machine. Useful for the machine drivers when they will not start with 'Waiting for SSH'. (default true) - --network string network to run minikube with. Only available with the docker/podman drivers. If left empty, minikube will create a new network. + --network string network to run minikube with. Now it is used by docker/podman and KVM drivers. If left empty, minikube will create a new network. --network-plugin string Kubelet network plug-in to use (default: auto) --nfs-share strings Local folders to share with Guest via NFS mounts (hyperkit driver only) --nfs-shares-root string Where to root the NFS Shares, defaults to /nfsshares (hyperkit driver only) (default "/nfsshares") diff --git a/translations/ko.json b/translations/ko.json index d9093b2f1d07..ec07d96c5f3d 100644 --- a/translations/ko.json +++ b/translations/ko.json @@ -20,7 +20,7 @@ "- Ensure your {{.driver_name}} daemon has access to enough CPU/memory resources.": "", "- Prune unused {{.driver_name}} images, volumes, networks and abandoned containers.\n\n\t\t\t\t{{.driver_name}} system prune --volumes": "", "- Restart your {{.driver_name}} service": "{{.driver_name}} 서비스를 다시 시작하세요", - "--network flag is only valid with the docker/podman drivers, it will be ignored": "", + "--network flag is only valid with the docker/podman and KVM drivers, it will be ignored": "", "A set of apiserver IP Addresses which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine": "", "A set of apiserver names which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine": "", "A set of key=value pairs that describe feature gates for alpha/experimental features.": "", @@ -687,7 +687,7 @@ "mount failed": "마운트 실패", "namespaces to pause": "잠시 멈추려는 네임스페이스", "namespaces to unpause": "재개하려는 네임스페이스", - "network to run minikube with. Only available with the docker/podman drivers. If left empty, minikube will create a new network.": "", + "network to run minikube with. Now it is used by docker/podman and KVM drivers. If left empty, minikube will create a new network.": "", "none driver does not support multi-node clusters": "", "not enough arguments ({{.ArgCount}}).\\nusage: minikube config set PROPERTY_NAME PROPERTY_VALUE": "", "output layout (EXPERIMENTAL, JSON only): 'nodes' or 'cluster'": "", diff --git a/translations/strings.txt b/translations/strings.txt index 8a162a6928c2..fdd7706dd41d 100644 --- a/translations/strings.txt +++ b/translations/strings.txt @@ -14,7 +14,7 @@ "- Ensure your {{.driver_name}} daemon has access to enough CPU/memory resources.": "", "- Prune unused {{.driver_name}} images, volumes, networks and abandoned containers.\n\n\t\t\t\t{{.driver_name}} system prune --volumes": "", "- Restart your {{.driver_name}} service": "", - "--network flag is only valid with the docker/podman drivers, it will be ignored": "", + "--network flag is only valid with the docker/podman and KVM drivers, it will be ignored": "", "A set of apiserver IP Addresses which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine": "", "A set of apiserver names which are used in the generated certificate for kubernetes. This can be used if you want to make the apiserver available from outside the machine": "", "A set of key=value pairs that describe feature gates for alpha/experimental features.": "", @@ -613,7 +613,7 @@ "mount failed": "", "namespaces to pause": "", "namespaces to unpause": "", - "network to run minikube with. Only available with the docker/podman drivers. If left empty, minikube will create a new network.": "", + "network to run minikube with. Now it is used by docker/podman and KVM drivers. If left empty, minikube will create a new network.": "", "none driver does not support multi-node clusters": "", "not enough arguments ({{.ArgCount}}).\\nusage: minikube config set PROPERTY_NAME PROPERTY_VALUE": "", "output layout (EXPERIMENTAL, JSON only): 'nodes' or 'cluster'": "",