[WIP] Stabilize CNI connectivity, dns and latent

Makefile

@@ -9,7 +9,7 @@ SHELL:=/bin/bash
 DOCKER := docker
 # Set the first containerd.sock that successfully stats -- fallback to the docker4mac default
 CONTAINERD_SOCK := $(shell \
-	ls 2>/dev/null \
+	sudo ls 2>/dev/null \
 		/run/containerd/containerd.sock \
 		/run/docker/containerd/containerd.sock \
 		/var/run/containerd/containerd.sock \
@@ -110,6 +110,8 @@ ifeq ($(GOARCH),$(GOHOSTARCH))
 	# For dev builds for a clean (non-dirty) environment; "simulate" that
 	# a manifest list has been built by tagging the docker image
 	$(DOCKER) tag $(IMAGE):${IMAGE_TAG}-$(GOARCH) $(IMAGE):${IMAGE_TAG}
+	$(DOCKER) image save $(IMAGE):${IMAGE_TAG} \
+		| $(CTR) -n firecracker image import -
 endif
 endif
 ifeq ($(IS_CI_BUILD),1)

cmd/ignite/cmd/vmcmd/create.go

@@ -66,6 +66,7 @@ func addCreateFlags(fs *pflag.FlagSet, cf *run.CreateFlags) {
 	// Register flags bound to temporary holder values
 	fs.StringSliceVarP(&cf.PortMappings, "ports", "p", cf.PortMappings, "Map host ports to VM ports")
 	fs.StringSliceVarP(&cf.CopyFiles, "copy-files", "f", cf.CopyFiles, "Copy files/directories from the host to the created VM")
+	fs.StringSliceVarP(&cf.DNS, "dns", "", cf.DNS, "Set name servers to the VM /etc/resolv.conf")
 
 	// Register flags for simple types (int, string, etc.)
 	fs.Uint64Var(&cf.VM.Spec.CPUs, "cpus", cf.VM.Spec.CPUs, "VM vCPU count, 1 or even numbers between 1 and 32")

cmd/ignite/cmd/vmcmd/run.go

@@ -31,6 +31,7 @@ func NewCmdRun(out io.Writer) *cobra.Command {
 					--interactive \
 					--name my-vm \
 					--cpus 2 \
+					--dns 8.8.8.8 \
 					--ssh \
 					--memory 2GB \
 					--size 10G

cmd/ignite/run/create.go

@@ -24,6 +24,7 @@ func NewCreateFlags() *CreateFlags {
 type CreateFlags struct {
 	PortMappings []string
 	CopyFiles    []string
+	DNS          []string
 	// This is a placeholder value here for now.
 	// If it was set using flags, it will be copied over to
 	// the API type. TODO: When we later have internal types
@@ -60,6 +61,14 @@ func (cf *CreateFlags) constructVMFromCLI(args []string) error {
 		return err
 	}
 
+	if len(cf.DNS) > 0 {
+		// Assign DNS server to VM spec
+		if cf.VM.Annotations == nil {
+			cf.VM.Annotations = make(map[string]string)
+		}
+		cf.VM.Annotations["ignite.weaveworks.io/dns"] = strings.Join(cf.DNS, ",")
+	}
+
 	// If the SSH flag was set, copy it over to the API type
 	if cf.SSH.Generate || cf.SSH.PublicKey != "" {
 		cf.VM.Spec.SSH = &cf.SSH

cmd/ignite/run/start.go

@@ -2,6 +2,8 @@ package run
 
 import (
 	"fmt"
+	"net"
+	"time"
 
 	"github.com/weaveworks/ignite/pkg/operations"
 	"github.com/weaveworks/ignite/pkg/preflight/checkers"
@@ -47,6 +49,30 @@ func Start(so *startOptions) error {
 		return err
 	}
 
+	// When --ssh is enabled, then wait until SSH service started on port 22
+	ssh := so.vm.Spec.SSH
+	if ssh != nil && ssh.Generate {
+		if len(so.vm.Status.IPAddresses) > 0 {
+			addr := so.vm.Status.IPAddresses[0].String() + ":22"
+			var err error
+			for i := 0; i < 500; i++ {
+				conn, dialErr := net.DialTimeout("tcp", addr, 100*time.Millisecond)
+				if conn != nil {
+					defer conn.Close()
+					err = nil
+					break
+				}
+				err = dialErr
+				time.Sleep(100 * time.Millisecond)
+			}
+			if err != nil {
+				if err, ok := err.(*net.OpError); ok && err.Timeout() {
+					return fmt.Errorf("Tried connecting to SSH but timed out %s", err)
+				}
+			}
+		}
+	}
+
 	// If starting interactively, attach after starting
 	if so.Interactive {
 		return Attach(so.attachOptions)

docs/cli/ignite/ignite_create.md

@@ -37,6 +37,7 @@ ignite create <OCI image> [flags]
       --config string            Specify a path to a file with the API resources you want to pass
   -f, --copy-files strings       Copy files/directories from the host to the created VM
       --cpus uint                VM vCPU count, 1 or even numbers between 1 and 32 (default 1)
+      --dns strings              Set name servers to the VM /etc/resolv.conf
   -h, --help                     help for create
       --kernel-args string       Set the command line for the kernel (default "console=ttyS0 reboot=k panic=1 pci=off ip=dhcp")
   -k, --kernel-image oci-image   Specify an OCI image containing the kernel at /boot/vmlinux and optionally, modules (default weaveworks/ignite-kernel:4.19.47)

docs/cli/ignite/ignite_run.md

@@ -15,6 +15,7 @@ Example usage:
 		--interactive \
 		--name my-vm \
 		--cpus 2 \
+		--dns 8.8.8.8 \
 		--ssh \
 		--memory 2GB \
 		--size 10G
@@ -31,6 +32,7 @@ ignite run <OCI image> [flags]
   -f, --copy-files strings       Copy files/directories from the host to the created VM
       --cpus uint                VM vCPU count, 1 or even numbers between 1 and 32 (default 1)
   -d, --debug                    Debug mode, keep container after VM shutdown
+      --dns strings              Set name servers to the VM /etc/resolv.conf
   -h, --help                     help for run
   -i, --interactive              Attach to the VM after starting
       --kernel-args string       Set the command line for the kernel (default "console=ttyS0 reboot=k panic=1 pci=off ip=dhcp")

docs/cli/ignite/ignite_vm_create.md

@@ -37,6 +37,7 @@ ignite vm create <OCI image> [flags]
       --config string            Specify a path to a file with the API resources you want to pass
   -f, --copy-files strings       Copy files/directories from the host to the created VM
       --cpus uint                VM vCPU count, 1 or even numbers between 1 and 32 (default 1)
+      --dns strings              Set name servers to the VM /etc/resolv.conf
   -h, --help                     help for create
       --kernel-args string       Set the command line for the kernel (default "console=ttyS0 reboot=k panic=1 pci=off ip=dhcp")
   -k, --kernel-image oci-image   Specify an OCI image containing the kernel at /boot/vmlinux and optionally, modules (default weaveworks/ignite-kernel:4.19.47)

docs/cli/ignite/ignite_vm_run.md

@@ -15,6 +15,7 @@ Example usage:
 		--interactive \
 		--name my-vm \
 		--cpus 2 \
+		--dns 8.8.8.8 \
 		--ssh \
 		--memory 2GB \
 		--size 10G
@@ -31,6 +32,7 @@ ignite vm run <OCI image> [flags]
   -f, --copy-files strings       Copy files/directories from the host to the created VM
       --cpus uint                VM vCPU count, 1 or even numbers between 1 and 32 (default 1)
   -d, --debug                    Debug mode, keep container after VM shutdown
+      --dns strings              Set name servers to the VM /etc/resolv.conf
   -h, --help                     help for run
   -i, --interactive              Attach to the VM after starting
       --kernel-args string       Set the command line for the kernel (default "console=ttyS0 reboot=k panic=1 pci=off ip=dhcp")

e2e/run_test.go

@@ -98,6 +98,7 @@ func runCurl(t *testing.T, vmName, runtime, networkPlugin string, sleepDuration
 		"--runtime="+runtime,
 		"--network-plugin="+networkPlugin,
 		"run", "--name="+vmName,
+		"--dns=8.8.8.8", "--dns=8.8.4.4", // override name servers
 		"weaveworks/ignite-ubuntu",
 		"--ssh",
 	)

pkg/container/dhcp.go

@@ -3,6 +3,7 @@ package container
 import (
 	"fmt"
 	"net"
+	"strings"
 	"time"
 
 	dhcp "github.com/krolaw/dhcp4"
@@ -31,6 +32,12 @@ func StartDHCPServers(vm *api.VM, dhcpIfaces []DHCPInterface) error {
 		return fmt.Errorf("failed to get DNS configuration: %v", err)
 	}
 
+	// If there's the dns annotation, override dns
+	if dns, exist := vm.Annotations["ignite.weaveworks.io/dns"]; exist {
+		log.Infof("Overriding DNS with value %q", dns)
+		clientConfig.Servers = strings.Split(dns, ",")
+	}
+
 	for i := range dhcpIfaces {
 		dhcpIface := &dhcpIfaces[i]
 		// Set the VM hostname to the VM ID

pkg/network/cni/cni.go

@@ -37,6 +37,9 @@ const (
 	igniteBridgeName = "ignite-containerd-bridge"
 )
 
+// the default CIDR for ignite bridge network
+var defaultCIDR = "10.42.0.0/16"
+
 // igniteCNIConf is a base CNI configuration that will enable VMs to access the internet connection (docker-bridge style)
 var igniteCNIConf = fmt.Sprintf(`{
 	"cniVersion": "0.4.0",
@@ -51,7 +54,7 @@ var igniteCNIConf = fmt.Sprintf(`{
 			"ipMasq": true,
 			"ipam": {
 				"type": "host-local",
-				"subnet": "172.18.0.0/16"
+				"subnet": "%s"
 			}
 		},
 		{
@@ -62,7 +65,7 @@ var igniteCNIConf = fmt.Sprintf(`{
 		}
 	]
 }
-`, igniteBridgeName)
+`, igniteBridgeName, defaultCIDR)
 
 type cniNetworkPlugin struct {
 	cni     gocni.CNI
@@ -103,12 +106,12 @@ func (plugin *cniNetworkPlugin) PrepareContainerSpec(container *runtime.Containe
 	return nil
 }
 
-func (plugin *cniNetworkPlugin) SetupContainerNetwork(containerid string, portMappings ...meta.PortMapping) (*network.Result, error) {
+func (plugin *cniNetworkPlugin) SetupContainerNetwork(containerId string, portMappings ...meta.PortMapping) (*network.Result, error) {
 	if err := plugin.initialize(); err != nil {
 		return nil, err
 	}
 
-	c, err := plugin.runtime.InspectContainer(containerid)
+	c, err := plugin.runtime.InspectContainer(containerId)
 	if err != nil {
 		return nil, fmt.Errorf("CNI failed to retrieve network namespace path: %v", err)
 	}
@@ -127,10 +130,10 @@ func (plugin *cniNetworkPlugin) SetupContainerNetwork(containerid string, portMa
 		})
 	}
 
-	netnsPath := fmt.Sprintf(netNSPathFmt, c.PID)
-	result, err := plugin.cni.Setup(context.Background(), containerid, netnsPath, gocni.WithCapabilityPortMap(pms))
+	netNsPath := fmt.Sprintf(netNSPathFmt, c.PID)
+	result, err := plugin.cni.Setup(context.Background(), containerId, netNsPath, gocni.WithCapabilityPortMap(pms))
 	if err != nil {
-		log.Errorf("failed to setup network for namespace %q: %v", containerid, err)
+		log.Errorf("failed to setup network for namespace %q: %v", containerId, err)
 		return nil, err
 	}
 
@@ -145,11 +148,24 @@ func (plugin *cniNetworkPlugin) initialize() (err error) {
 		}
 	}
 
+	// setup forward rules once for the defaultCIDR
+	ipt, err := iptables.NewWithProtocol(iptables.ProtocolIPv4)
+	if err != nil {
+		return
+	}
+	if err = ipt.AppendUnique("filter", "FORWARD", "-s", defaultCIDR, "-j", "ACCEPT"); err != nil {
+		return
+	}
+	if err = ipt.AppendUnique("filter", "FORWARD", "-d", defaultCIDR, "-j", "ACCEPT"); err != nil {
+		return
+	}
+
 	plugin.once.Do(func() {
 		if err = plugin.cni.Load(gocni.WithLoNetwork, gocni.WithDefaultConf); err != nil {
 			log.Errorf("failed to load cni configuration: %v", err)
 		}
 	})
+
 	return
 }