Enable multiple non-IP interface to be connected via tc redirect

cmd/ignite-spawn/ignite-spawn.go

@@ -39,8 +39,9 @@ func decodeVM(vmID string) (*api.VM, error) {
 }
 
 func StartVM(vm *api.VM) (err error) {
+
  // Setup networking inside of the container, return the available interfaces
- dhcpIfaces, err := container.SetupContainerNetworking()
+ fcIfaces, dhcpIfaces, err := container.SetupContainerNetworking(vm)
  if err != nil {
  return fmt.Errorf("network setup failed: %v", err)
  }
@@ -66,7 +67,7 @@ func StartVM(vm *api.VM) (err error) {
  defer util.DeferErr(&err, func() error { return os.Remove(metricsSocket) })
 
  // Execute Firecracker
- if err = container.ExecuteFirecracker(vm, dhcpIfaces); err != nil {
+ if err = container.ExecuteFirecracker(vm, fcIfaces); err != nil {
  return fmt.Errorf("runtime error for VM %q: %v", vm.GetUID(), err)
  }
 

cmd/ignite/run/exec.go

@@ -2,6 +2,7 @@ package run
 
 import (
  api "github.com/weaveworks/ignite/pkg/apis/ignite"
+ "github.com/weaveworks/ignite/pkg/constants"
 )
 
 // ExecFlags contains the flags supported by the exec command.
@@ -30,5 +31,8 @@ func (ef *ExecFlags) NewExecOptions(vmMatch string, command ...string) (eo *Exec
 
 // Exec executes command in a VM based on the provided ExecOptions.
 func Exec(eo *ExecOptions) error {
+ if err := waitForSSH(eo.vm, constants.SSH_DEFAULT_TIMEOUT_SECONDS, int(eo.Timeout)); err != nil {
+ return err
+ }
  return runSSH(eo.vm, eo.IdentityFile, eo.command, eo.Tty, eo.Timeout)
 }

e2e/multinet_test.go

@@ -0,0 +1,268 @@
+package e2e
+
+import (
+ "fmt"
+ "strings"
+ "testing"
+
+ "github.com/weaveworks/ignite/e2e/util"
+ api "github.com/weaveworks/ignite/pkg/apis/ignite"
+ meta "github.com/weaveworks/ignite/pkg/apis/meta/v1alpha1"
+ igniteConstants "github.com/weaveworks/ignite/pkg/constants"
+ "github.com/weaveworks/ignite/pkg/dmlegacy"
+ "github.com/weaveworks/ignite/pkg/metadata"
+ "github.com/weaveworks/ignite/pkg/network"
+ "github.com/weaveworks/ignite/pkg/operations"
+ "github.com/weaveworks/ignite/pkg/providers"
+ igniteDocker "github.com/weaveworks/ignite/pkg/providers/docker"
+ "github.com/weaveworks/ignite/pkg/providers/ignite"
+ "github.com/weaveworks/ignite/pkg/runtime"
+ igniteUtil "github.com/weaveworks/ignite/pkg/util"
+ "gotest.tools/assert"
+)
+
+var (
+ multinetVM = "e2e-test-vm-multinet"
+ sanboxImage = "weaveworks/ignite:dev"
+ kernelImage = "weaveworks/ignite-kernel:5.4.108"
+ vmImage = "weaveworks/ignite-ubuntu"
+)
+
+func startAsyncVM(t *testing.T, intfs []string) (*operations.VMChannels, string) {
+
+ assert.Assert(t, e2eHome != "", "IGNITE_E2E_HOME should be set")
+ igniteUtil.GenericCheckErr(providers.Populate(ignite.Preload))
+
+ _ = igniteDocker.SetDockerRuntime()
+ _ = igniteDocker.SetDockerNetwork()
+
+ providers.RuntimeName = runtime.RuntimeDocker
+ providers.NetworkPluginName = network.PluginDockerBridge
+ _ = providers.Populate(ignite.Providers)
+
+ vm := providers.Client.VMs().New()
+ vm.Status.Runtime.Name = runtime.RuntimeDocker
+ vm.Status.Network.Plugin = network.PluginDockerBridge
+
+ ociRef, err := meta.NewOCIImageRef(sanboxImage)
+ if err != nil {
+ t.Fatalf("Failed to parse OCI image ref %s: %s", sanboxImage, err)
+ }
+ vm.Spec.Sandbox.OCI = ociRef
+
+ ociRef, err = meta.NewOCIImageRef(kernelImage)
+ if err != nil {
+ t.Fatalf("Failed to parse OCI image ref %s: %s", kernelImage, err)
+ }
+ vm.Spec.Kernel.OCI = ociRef
+ k, _ := operations.FindOrImportKernel(providers.Client, ociRef)
+ vm.SetKernel(k)
+
+ ociRef, err = meta.NewOCIImageRef(vmImage)
+ if err != nil {
+ t.Fatalf("Failed to parse OCI image ref %s: %s", vmImage, err)
+ }
+ img, err := operations.FindOrImportImage(providers.Client, ociRef)
+ if err != nil {
+ t.Fatalf("Failed to find OCI image ref %s: %s", ociRef, err)
+ }
+ vm.SetImage(img)
+
+ vm.Name = multinetVM
+ vm.Spec.SSH = &api.SSH{Generate: true}
+
+ _ = metadata.SetNameAndUID(vm, providers.Client)
+
+ for _, intf := range intfs {
+ vm.SetAnnotation(igniteConstants.IGNITE_INTERFACE_ANNOTATION+intf, "tc-redirect")
+ }
+
+ _ = providers.Client.VMs().Set(vm)
+
+ err = dmlegacy.AllocateAndPopulateOverlay(vm)
+ if err != nil {
+ t.Fatalf("Error AllocateAndPopulateOverlay: %s", err)
+ }
+
+ vmChans, err := operations.StartVMNonBlocking(vm, false)
+ if err != nil {
+ t.Fatalf("failed to start a VM: \n%q\n", err)
+ }
+
+ return vmChans, vm.GetUID().String()
+}
+
+// TestMultipleInterface tests that a VM's can be configured with more than 1 interface
+func TestOneExtraInterface(t *testing.T) {
+ assert.Assert(t, e2eHome != "", "IGNITE_E2E_HOME should be set")
+
+ igniteCmd := util.NewCommand(t, igniteBin)
+ dockerCmd := util.NewCommand(t, runtime.RuntimeDocker.String())
+
+ vmChans, vmID := startAsyncVM(t, []string{"foo"})
+
+ // Clean-up the following VM.
+ defer igniteCmd.New().
+ With("rm", "-f", multinetVM).
+ Run()
+
+ fooAddr := "aa:ca:e9:12:34:56"
+ dockerCmd.New().
+ With("exec", fmt.Sprintf("ignite-%s", vmID)).
+ With("ip", "link", "add", "foo", "type", "veth").
+ Run()
+
+ dockerCmd.New().
+ With("exec", fmt.Sprintf("ignite-%s", vmID)).
+ With("ip", "link", "set", "foo", "address", fooAddr).
+ Run()
+
+ // check that the VM has started before trying exec
+ if err := <-vmChans.SpawnFinished; err != nil {
+ t.Fatalf("failed to start a VM: \n%q\n", err)
+ }
+
+ eth1Addr := igniteCmd.New().
+ With("exec", multinetVM).
+ With("cat", "/sys/class/net/eth1/address")
+
+ foundEth1Addr, _ := eth1Addr.Cmd.CombinedOutput()
+ gotEth1Addr := strings.TrimSuffix(string(foundEth1Addr), "\n")
+ assert.Check(t, strings.Contains(gotEth1Addr, fooAddr), fmt.Sprintf("unexpected address found:\n\t(WNT): %q\n\t(GOT): %q", fooAddr, gotEth1Addr))
+
+}
+
+func TestMultipleInterface(t *testing.T) {
+ assert.Assert(t, e2eHome != "", "IGNITE_E2E_HOME should be set")
+
+ igniteCmd := util.NewCommand(t, igniteBin)
+ dockerCmd := util.NewCommand(t, runtime.RuntimeDocker.String())
+
+ vmChans, vmID := startAsyncVM(t, []string{"foo", "bar"})
+
+ // Clean-up the following VM.
+ defer igniteCmd.New().
+ With("rm", "-f", multinetVM).
+ Run()
+
+ fooAddr := "aa:ca:e9:12:34:56"
+ dockerCmd.New().
+ With("exec", fmt.Sprintf("ignite-%s", vmID)).
+ With("ip", "link", "add", "foo", "type", "veth").
+ Run()
+
+ dockerCmd.New().
+ With("exec", fmt.Sprintf("ignite-%s", vmID)).
+ With("ip", "link", "set", "foo", "address", fooAddr).
+ Run()
+
+ barAddr := "aa:ca:e9:12:34:78"
+ dockerCmd.New().
+ With("exec", fmt.Sprintf("ignite-%s", vmID)).
+ With("ip", "link", "add", "bar", "type", "veth").
+ Run()
+
+ dockerCmd.New().
+ With("exec", fmt.Sprintf("ignite-%s", vmID)).
+ With("ip", "link", "set", "bar", "address", barAddr).
+ Run()
+
+ // check that the VM has started before trying exec
+ if err := <-vmChans.SpawnFinished; err != nil {
+ t.Fatalf("failed to start a VM: \n%q\n", err)
+ }
+
+ eth1Addr := igniteCmd.New().
+ With("exec", multinetVM).
+ With("cat", "/sys/class/net/eth1/address")
+
+ foundEth1Addr, _ := eth1Addr.Cmd.CombinedOutput()
+ gotEth1Addr := strings.TrimSuffix(string(foundEth1Addr), "\n")
+ assert.Check(t, strings.Contains(gotEth1Addr, barAddr), fmt.Sprintf("unexpected address found:\n\t(WNT): %q\n\t(GOT): %q", barAddr, gotEth1Addr))
+
+ eth2Addr := igniteCmd.New().
+ With("exec", multinetVM).
+ With("cat", "/sys/class/net/eth2/address")
+
+ foundEth2Addr, _ := eth2Addr.Cmd.CombinedOutput()
+ gotEth2Addr := strings.TrimSuffix(string(foundEth2Addr), "\n")
+ assert.Check(t, strings.Contains(gotEth2Addr, fooAddr), fmt.Sprintf("unexpected address found:\n\t(WNT): %q\n\t(GOT): %q", fooAddr, gotEth2Addr))
+
+}
+
+func TestMultipleInterfaceImplicit(t *testing.T) {
+ assert.Assert(t, e2eHome != "", "IGNITE_E2E_HOME should be set")
+
+ igniteCmd := util.NewCommand(t, igniteBin)
+ dockerCmd := util.NewCommand(t, runtime.RuntimeDocker.String())
+
+ vmChans, vmID := startAsyncVM(t, []string{"foo", "bar"})
+
+ // Clean-up the following VM.
+ defer igniteCmd.New().
+ With("rm", "-f", multinetVM).
+ Run()
+
+ fooAddr := "aa:ca:e9:12:34:56"
+ dockerCmd.New().
+ With("exec", fmt.Sprintf("ignite-%s", vmID)).
+ With("ip", "link", "add", "foo", "type", "veth").
+ Run()
+
+ dockerCmd.New().
+ With("exec", fmt.Sprintf("ignite-%s", vmID)).
+ With("ip", "link", "set", "foo", "address", fooAddr).
+ Run()
+
+ barAddr := "aa:ca:e9:12:34:78"
+ dockerCmd.New().
+ With("exec", fmt.Sprintf("ignite-%s", vmID)).
+ With("ip", "link", "add", "bar", "type", "veth").
+ Run()
+
+ dockerCmd.New().
+ With("exec", fmt.Sprintf("ignite-%s", vmID)).
+ With("ip", "link", "set", "bar", "address", barAddr).
+ Run()
+
+ // this interface should never be found inside a VM
+ bazAddr := "aa:ca:e9:12:34:90"
+ dockerCmd.New().
+ With("exec", fmt.Sprintf("ignite-%s", vmID)).
+ With("ip", "link", "add", "baz", "type", "veth").
+ Run()
+
+ dockerCmd.New().
+ With("exec", fmt.Sprintf("ignite-%s", vmID)).
+ With("ip", "link", "set", "baz", "address", bazAddr).
+ Run()
+
+ // check that the VM has started before trying exec
+ if err := <-vmChans.SpawnFinished; err != nil {
+ t.Fatalf("failed to start a VM: \n%q\n", err)
+ }
+
+ eth1Addr := igniteCmd.New().
+ With("exec", multinetVM).
+ With("cat", "/sys/class/net/eth1/address")
+
+ foundEth1Addr, _ := eth1Addr.Cmd.CombinedOutput()
+ gotEth1Addr := strings.TrimSuffix(string(foundEth1Addr), "\n")
+ assert.Check(t, strings.Contains(gotEth1Addr, barAddr), fmt.Sprintf("unexpected address found:\n\t(WNT): %q\n\t(GOT): %q", barAddr, gotEth1Addr))
+
+ eth2Addr := igniteCmd.New().
+ With("exec", multinetVM).
+ With("cat", "/sys/class/net/eth2/address")
+
+ foundEth2Addr, _ := eth2Addr.Cmd.CombinedOutput()
+ gotEth2Addr := strings.TrimSuffix(string(foundEth2Addr), "\n")
+ assert.Check(t, strings.Contains(gotEth2Addr, fooAddr), fmt.Sprintf("unexpected address found:\n\t(WNT): %q\n\t(GOT): %q", fooAddr, gotEth2Addr))
+
+ eth3Addr := igniteCmd.New().
+ With("exec", multinetVM).
+ With("cat", "/sys/class/net/eth3/address")
+
+ _, foundEth3Err := eth3Addr.Cmd.CombinedOutput()
+ assert.Error(t, foundEth3Err, "exit status 1", fmt.Sprintf("unexpected output when looking for eth3 : \n%s", foundEth3Err))
+
+}

pkg/constants/vm.go

@@ -38,4 +38,10 @@ const (
  // DEFAULT_SANDBOX_IMAGE_NAME is the name of the default sandbox container
  // image to be used.
  DEFAULT_SANDBOX_IMAGE_TAG = "dev"
+
+ // IGNITE_INTERFACE_ANNOTATION is the annotation prefix to store a list of extra interfaces
+ IGNITE_INTERFACE_ANNOTATION = "ignite.weave.works/interface/"
+
+ // IGNITE_SANDBOX_ENV_VAR is the annotation prefix to store a list of env variables
+ IGNITE_SANDBOX_ENV_VAR = "ignite.weave.works/sanbox-env/"
 )

pkg/container/dhcp.go

@@ -11,19 +11,13 @@ import (
  log "github.com/sirupsen/logrus"
  api "github.com/weaveworks/ignite/pkg/apis/ignite"
  "github.com/weaveworks/ignite/pkg/constants"
- "github.com/weaveworks/ignite/pkg/util"
 )
 
 var leaseDuration, _ = time.ParseDuration(constants.DHCP_INFINITE_LEASE) // Infinite lease time
 
 // StartDHCPServers starts multiple DHCP servers for the VM, one per interface
 // It returns the IP addresses that the API object may post in .status, and a potential error
 func StartDHCPServers(vm *api.VM, dhcpIfaces []DHCPInterface) error {
- // Generate the MAC addresses for the VM's adapters
- macAddresses := make([]string, 0, len(dhcpIfaces))
- if err := util.NewMAC(&macAddresses); err != nil {
- return fmt.Errorf("failed to generate MAC addresses: %v", err)
- }
 
  // Fetch the DNS servers given to the container
  clientConfig, err := dns.ClientConfigFromFile("/etc/resolv.conf")
@@ -36,9 +30,6 @@ func StartDHCPServers(vm *api.VM, dhcpIfaces []DHCPInterface) error {
  // Set the VM hostname to the VM ID
  dhcpIface.Hostname = vm.GetUID().String()
 
- // Set the MAC address filter for the DHCP server
- dhcpIface.MACFilter = macAddresses[i]
-
  // Add the DNS servers from the container
  dhcpIface.SetDNSServers(clientConfig.Servers)
 

pkg/container/firecracker.go

@@ -20,19 +20,9 @@ import (
 )
 
 // ExecuteFirecracker executes the firecracker process using the Go SDK
-func ExecuteFirecracker(vm *api.VM, dhcpIfaces []DHCPInterface) (err error) {
+func ExecuteFirecracker(vm *api.VM, fcIfaces firecracker.NetworkInterfaces) (err error) {
  drivePath := vm.SnapshotDev()
 
- networkInterfaces := make([]firecracker.NetworkInterface, 0, len(dhcpIfaces))
- for _, dhcpIface := range dhcpIfaces {
- networkInterfaces = append(networkInterfaces, firecracker.NetworkInterface{
- StaticConfiguration: &firecracker.StaticNetworkConfiguration{
- MacAddress: dhcpIface.MACFilter,
- HostDevName: dhcpIface.VMTAP,
- },
- })
- }
-
  vCPUCount := int64(vm.Spec.CPUs)
  memSizeMib := int64(vm.Spec.Memory.MBytes())
 
@@ -67,7 +57,7 @@ func ExecuteFirecracker(vm *api.VM, dhcpIfaces []DHCPInterface) (err error) {
  IsRootDevice: firecracker.Bool(true),
  PathOnHost: &drivePath,
  }},
- NetworkInterfaces: networkInterfaces,
+ NetworkInterfaces: fcIfaces,
  MachineCfg: models.MachineConfiguration{
  VcpuCount: &vCPUCount,
  MemSizeMib: &memSizeMib,