diff --git a/virtcontainers/bridgedmacvlan_endpoint.go b/virtcontainers/bridgedmacvlan_endpoint.go new file mode 100644 index 0000000000..705ff9d2cd --- /dev/null +++ b/virtcontainers/bridgedmacvlan_endpoint.go @@ -0,0 +1,112 @@ +// Copyright (c) 2018 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "fmt" + + "github.com/containernetworking/plugins/pkg/ns" +) + +// BridgedMacvlanEndpoint represents a macvlan endpoint that is bridged to the VM +type BridgedMacvlanEndpoint struct { + NetPair NetworkInterfacePair + EndpointProperties NetworkInfo + EndpointType EndpointType + PCIAddr string +} + +func createBridgedMacvlanNetworkEndpoint(idx int, ifName string, interworkingModel NetInterworkingModel) (*BridgedMacvlanEndpoint, error) { + if idx < 0 { + return &BridgedMacvlanEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx) + } + + netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel) + if err != nil { + return nil, err + } + + endpoint := &BridgedMacvlanEndpoint{ + NetPair: netPair, + EndpointType: BridgedMacvlanEndpointType, + } + if ifName != "" { + endpoint.NetPair.VirtIface.Name = ifName + } + + return endpoint, nil +} + +// Properties returns properties of the interface. +func (endpoint *BridgedMacvlanEndpoint) Properties() NetworkInfo { + return endpoint.EndpointProperties +} + +// Name returns name of the veth interface in the network pair. +func (endpoint *BridgedMacvlanEndpoint) Name() string { + return endpoint.NetPair.VirtIface.Name +} + +// HardwareAddr returns the mac address that is assigned to the tap interface +// in th network pair. +func (endpoint *BridgedMacvlanEndpoint) HardwareAddr() string { + return endpoint.NetPair.TAPIface.HardAddr +} + +// Type identifies the endpoint as a virtual endpoint. +func (endpoint *BridgedMacvlanEndpoint) Type() EndpointType { + return endpoint.EndpointType +} + +// SetProperties sets the properties for the endpoint. +func (endpoint *BridgedMacvlanEndpoint) SetProperties(properties NetworkInfo) { + endpoint.EndpointProperties = properties +} + +// PciAddr returns the PCI address of the endpoint. +func (endpoint *BridgedMacvlanEndpoint) PciAddr() string { + return endpoint.PCIAddr +} + +// NetworkPair returns the network pair of the endpoint. +func (endpoint *BridgedMacvlanEndpoint) NetworkPair() *NetworkInterfacePair { + return &endpoint.NetPair +} + +// Attach for virtual endpoint bridges the network pair and adds the +// tap interface of the network pair to the hypervisor. +func (endpoint *BridgedMacvlanEndpoint) Attach(h hypervisor) error { + if err := xconnectVMNetwork(endpoint, true, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil { + networkLogger().WithError(err).Error("Error bridging virtual ep") + return err + } + + return h.addDevice(endpoint, netDev) +} + +// Detach for the virtual endpoint tears down the tap and bridge +// created for the veth interface. +func (endpoint *BridgedMacvlanEndpoint) Detach(netNsCreated bool, netNsPath string) error { + // The network namespace would have been deleted at this point + // if it has not been created by virtcontainers. + if !netNsCreated { + return nil + } + + return doNetNS(netNsPath, func(_ ns.NetNS) error { + return xconnectVMNetwork(endpoint, false, 0, false) + }) +} + +// HotAttach for physical endpoint not supported yet +func (endpoint *BridgedMacvlanEndpoint) HotAttach(h hypervisor) error { + return fmt.Errorf("BridgedMacvlanEndpoint does not support Hot attach") +} + +// HotDetach for physical endpoint not supported yet +func (endpoint *BridgedMacvlanEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { + return fmt.Errorf("BridgedMacvlanEndpoint does not support Hot detach") +} diff --git a/virtcontainers/bridgedmacvlan_endpoint_test.go b/virtcontainers/bridgedmacvlan_endpoint_test.go new file mode 100644 index 0000000000..5bee07e750 --- /dev/null +++ b/virtcontainers/bridgedmacvlan_endpoint_test.go @@ -0,0 +1,47 @@ +// Copyright (c) 2018 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "net" + "reflect" + "testing" +) + +func TestCreateBridgedMacvlanEndpoint(t *testing.T) { + macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x04} + + expected := &BridgedMacvlanEndpoint{ + NetPair: NetworkInterfacePair{ + ID: "uniqueTestID-4", + Name: "br4_kata", + VirtIface: NetworkInterface{ + Name: "eth4", + HardAddr: macAddr.String(), + }, + TAPIface: NetworkInterface{ + Name: "tap4_kata", + }, + NetInterworkingModel: DefaultNetInterworkingModel, + }, + EndpointType: BridgedMacvlanEndpointType, + } + + result, err := createBridgedMacvlanNetworkEndpoint(4, "", DefaultNetInterworkingModel) + if err != nil { + t.Fatal(err) + } + + // the resulting ID will be random - so let's overwrite to test the rest of the flow + result.NetPair.ID = "uniqueTestID-4" + + // the resulting mac address will be random - so lets overwrite it + result.NetPair.VirtIface.HardAddr = macAddr.String() + + if reflect.DeepEqual(result, expected) == false { + t.Fatalf("\nGot: %+v, \n\nExpected: %+v", result, expected) + } +} diff --git a/virtcontainers/default_network.go b/virtcontainers/default_network.go index 293e20ed6c..31494338d3 100644 --- a/virtcontainers/default_network.go +++ b/virtcontainers/default_network.go @@ -58,6 +58,7 @@ func (n *defNetwork) add(s *Sandbox) error { err = doNetNS(s.config.NetworkConfig.NetNSPath, func(_ ns.NetNS) error { for _, endpoint := range s.networkNS.Endpoints { + n.logger().WithField("endpoint-type", endpoint.Type()).Info("Attaching endpoint") if err := endpoint.Attach(s.hypervisor); err != nil { return err } @@ -83,6 +84,7 @@ func (n *defNetwork) remove(s *Sandbox) error { for _, endpoint := range s.networkNS.Endpoints { // Detach for an endpoint should enter the network namespace // if required. + n.logger().WithField("endpoint-type", endpoint.Type()).Info("Detaching endpoint") if err := endpoint.Detach(s.networkNS.NetNsCreated, s.networkNS.NetNsPath); err != nil { return err } diff --git a/virtcontainers/endpoint.go b/virtcontainers/endpoint.go new file mode 100644 index 0000000000..af5b39cbfb --- /dev/null +++ b/virtcontainers/endpoint.go @@ -0,0 +1,87 @@ +// Copyright (c) 2018 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "fmt" +) + +// Endpoint represents a physical or virtual network interface. +type Endpoint interface { + Properties() NetworkInfo + Name() string + HardwareAddr() string + Type() EndpointType + PciAddr() string + NetworkPair() *NetworkInterfacePair + + SetProperties(NetworkInfo) + Attach(hypervisor) error + Detach(netNsCreated bool, netNsPath string) error + HotAttach(h hypervisor) error + HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error +} + +// EndpointType identifies the type of the network endpoint. +type EndpointType string + +const ( + // PhysicalEndpointType is the physical network interface. + PhysicalEndpointType EndpointType = "physical" + + // VethEndpointType is the virtual network interface. + VethEndpointType EndpointType = "virtual" + + // VhostUserEndpointType is the vhostuser network interface. + VhostUserEndpointType EndpointType = "vhost-user" + + // BridgedMacvlanEndpointType is macvlan network interface. + BridgedMacvlanEndpointType EndpointType = "macvlan" + + // MacvtapEndpointType is macvtap network interface. + MacvtapEndpointType EndpointType = "macvtap" +) + +// Set sets an endpoint type based on the input string. +func (endpointType *EndpointType) Set(value string) error { + switch value { + case "physical": + *endpointType = PhysicalEndpointType + return nil + case "virtual": + *endpointType = VethEndpointType + return nil + case "vhost-user": + *endpointType = VhostUserEndpointType + return nil + case "macvlan": + *endpointType = BridgedMacvlanEndpointType + return nil + case "macvtap": + *endpointType = MacvtapEndpointType + return nil + default: + return fmt.Errorf("Unknown endpoint type %s", value) + } +} + +// String converts an endpoint type to a string. +func (endpointType *EndpointType) String() string { + switch *endpointType { + case PhysicalEndpointType: + return string(PhysicalEndpointType) + case VethEndpointType: + return string(VethEndpointType) + case VhostUserEndpointType: + return string(VhostUserEndpointType) + case BridgedMacvlanEndpointType: + return string(BridgedMacvlanEndpointType) + case MacvtapEndpointType: + return string(MacvtapEndpointType) + default: + return "" + } +} diff --git a/virtcontainers/endpoint_test.go b/virtcontainers/endpoint_test.go new file mode 100644 index 0000000000..251791fcb9 --- /dev/null +++ b/virtcontainers/endpoint_test.go @@ -0,0 +1,89 @@ +// Copyright (c) 2018 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import "testing" + +func testEndpointTypeSet(t *testing.T, value string, expected EndpointType) { + //var netModel NetworkModel + var endpointType EndpointType + + err := endpointType.Set(value) + if err != nil { + t.Fatal(err) + } + + if endpointType != expected { + t.Fatal() + } +} + +func TestPhysicalEndpointTypeSet(t *testing.T) { + testEndpointTypeSet(t, "physical", PhysicalEndpointType) +} + +func TestVethEndpointTypeSet(t *testing.T) { + testEndpointTypeSet(t, "virtual", VethEndpointType) +} + +func TestVhostUserEndpointTypeSet(t *testing.T) { + testEndpointTypeSet(t, "vhost-user", VhostUserEndpointType) +} + +func TestBridgedMacvlanEndpointTypeSet(t *testing.T) { + testEndpointTypeSet(t, "macvlan", BridgedMacvlanEndpointType) +} + +func TestMacvtapEndpointTypeSet(t *testing.T) { + testEndpointTypeSet(t, "macvtap", MacvtapEndpointType) +} + +func TestEndpointTypeSetFailure(t *testing.T) { + var endpointType EndpointType + + err := endpointType.Set("wrong-value") + if err == nil { + t.Fatal(err) + } +} + +func testEndpointTypeString(t *testing.T, endpointType *EndpointType, expected string) { + result := endpointType.String() + + if result != expected { + t.Fatal() + } +} + +func TestPhysicalEndpointTypeString(t *testing.T) { + endpointType := PhysicalEndpointType + testEndpointTypeString(t, &endpointType, string(PhysicalEndpointType)) +} + +func TestVethEndpointTypeString(t *testing.T) { + endpointType := VethEndpointType + testEndpointTypeString(t, &endpointType, string(VethEndpointType)) +} + +func TestVhostUserEndpointTypeString(t *testing.T) { + endpointType := VhostUserEndpointType + testEndpointTypeString(t, &endpointType, string(VhostUserEndpointType)) +} + +func TestBridgedMacvlanEndpointTypeString(t *testing.T) { + endpointType := BridgedMacvlanEndpointType + testEndpointTypeString(t, &endpointType, string(BridgedMacvlanEndpointType)) +} + +func TestMacvtapEndpointTypeString(t *testing.T) { + endpointType := MacvtapEndpointType + testEndpointTypeString(t, &endpointType, string(MacvtapEndpointType)) +} + +func TestIncorrectEndpointTypeString(t *testing.T) { + var endpointType EndpointType + testEndpointTypeString(t, &endpointType, "") +} diff --git a/virtcontainers/macvtap_endpoint.go b/virtcontainers/macvtap_endpoint.go new file mode 100644 index 0000000000..0abba7d582 --- /dev/null +++ b/virtcontainers/macvtap_endpoint.go @@ -0,0 +1,99 @@ +// Copyright (c) 2018 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "fmt" + "os" +) + +// MacvtapEndpoint represents a macvtap endpoint +type MacvtapEndpoint struct { + EndpointProperties NetworkInfo + EndpointType EndpointType + VMFds []*os.File + VhostFds []*os.File + PCIAddr string +} + +func createMacvtapNetworkEndpoint(netInfo NetworkInfo) (*MacvtapEndpoint, error) { + endpoint := &MacvtapEndpoint{ + EndpointType: MacvtapEndpointType, + EndpointProperties: netInfo, + } + + return endpoint, nil +} + +// Properties returns the properties of the macvtap interface. +func (endpoint *MacvtapEndpoint) Properties() NetworkInfo { + return endpoint.EndpointProperties +} + +// HardwareAddr returns the mac address of the macvtap network interface. +func (endpoint *MacvtapEndpoint) HardwareAddr() string { + return endpoint.EndpointProperties.Iface.HardwareAddr.String() +} + +// Name returns name of the macvtap interface. +func (endpoint *MacvtapEndpoint) Name() string { + return endpoint.EndpointProperties.Iface.Name +} + +// Type indentifies the endpoint as a macvtap endpoint. +func (endpoint *MacvtapEndpoint) Type() EndpointType { + return endpoint.EndpointType +} + +// SetProperties sets the properties of the macvtap endpoint. +func (endpoint *MacvtapEndpoint) SetProperties(properties NetworkInfo) { + endpoint.EndpointProperties = properties +} + +// Attach for macvtap endpoint passes macvtap device to the hypervisor. +func (endpoint *MacvtapEndpoint) Attach(h hypervisor) error { + var err error + + endpoint.VMFds, err = createMacvtapFds(endpoint.EndpointProperties.Iface.Index, int(h.hypervisorConfig().NumVCPUs)) + if err != nil { + return fmt.Errorf("Could not setup macvtap fds %s: %s", endpoint.EndpointProperties.Iface.Name, err) + } + + if !h.hypervisorConfig().DisableVhostNet { + vhostFds, err := createVhostFds(int(h.hypervisorConfig().NumVCPUs)) + if err != nil { + return fmt.Errorf("Could not setup vhost fds %s : %s", endpoint.EndpointProperties.Iface.Name, err) + } + endpoint.VhostFds = vhostFds + } + + return h.addDevice(endpoint, netDev) +} + +// Detach for macvtap endpoint does nothing. +func (endpoint *MacvtapEndpoint) Detach(netNsCreated bool, netNsPath string) error { + return nil +} + +// HotAttach for macvtap endpoint not supported yet +func (endpoint *MacvtapEndpoint) HotAttach(h hypervisor) error { + return fmt.Errorf("MacvtapEndpoint does not support Hot attach") +} + +// HotDetach for macvtap endpoint not supported yet +func (endpoint *MacvtapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { + return fmt.Errorf("MacvtapEndpoint does not support Hot detach") +} + +// PciAddr returns the PCI address of the endpoint. +func (endpoint *MacvtapEndpoint) PciAddr() string { + return endpoint.PCIAddr +} + +// NetworkPair returns the network pair of the endpoint. +func (endpoint *MacvtapEndpoint) NetworkPair() *NetworkInterfacePair { + return nil +} diff --git a/virtcontainers/macvtap_endpoint_test.go b/virtcontainers/macvtap_endpoint_test.go new file mode 100644 index 0000000000..fbef06f30e --- /dev/null +++ b/virtcontainers/macvtap_endpoint_test.go @@ -0,0 +1,32 @@ +// Copyright (c) 2018 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "reflect" + "testing" +) + +func TestCreateMacvtapEndpoint(t *testing.T) { + netInfo := NetworkInfo{ + Iface: NetlinkIface{ + Type: "macvtap", + }, + } + expected := &MacvtapEndpoint{ + EndpointType: MacvtapEndpointType, + EndpointProperties: netInfo, + } + + result, err := createMacvtapNetworkEndpoint(netInfo) + if err != nil { + t.Fatal(err) + } + + if reflect.DeepEqual(result, expected) == false { + t.Fatalf("\nGot: %+v, \n\nExpected: %+v", result, expected) + } +} diff --git a/virtcontainers/network.go b/virtcontainers/network.go index d892b34d21..7510dac419 100644 --- a/virtcontainers/network.go +++ b/virtcontainers/network.go @@ -7,29 +7,22 @@ package virtcontainers import ( cryptoRand "crypto/rand" - "encoding/hex" "encoding/json" "fmt" - "io/ioutil" "math/rand" "net" "os" - "path/filepath" "runtime" "sort" - "strings" "time" "github.com/containernetworking/plugins/pkg/ns" - "github.com/safchain/ethtool" "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" "golang.org/x/sys/unix" "github.com/kata-containers/agent/protocols/grpc" - "github.com/kata-containers/runtime/virtcontainers/device/config" - "github.com/kata-containers/runtime/virtcontainers/device/drivers" "github.com/kata-containers/runtime/virtcontainers/pkg/uuid" "github.com/kata-containers/runtime/virtcontainers/utils" ) @@ -148,548 +141,10 @@ type NetworkConfig struct { InterworkingModel NetInterworkingModel } -// Endpoint represents a physical or virtual network interface. -type Endpoint interface { - Properties() NetworkInfo - Name() string - HardwareAddr() string - Type() EndpointType - PciAddr() string - NetworkPair() *NetworkInterfacePair - - SetProperties(NetworkInfo) - Attach(hypervisor) error - Detach(netNsCreated bool, netNsPath string) error - HotAttach(h hypervisor) error - HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error -} - -// VirtualEndpoint gathers a network pair and its properties. -type VirtualEndpoint struct { - NetPair NetworkInterfacePair - EndpointProperties NetworkInfo - Physical bool - EndpointType EndpointType - PCIAddr string -} - -// PhysicalEndpoint gathers a physical network interface and its properties -type PhysicalEndpoint struct { - IfaceName string - HardAddr string - EndpointProperties NetworkInfo - EndpointType EndpointType - BDF string - Driver string - VendorDeviceID string - PCIAddr string -} - -// VhostUserEndpoint represents a vhost-user socket based network interface -type VhostUserEndpoint struct { - // Path to the vhost-user socket on the host system - SocketPath string - // MAC address of the interface - HardAddr string - IfaceName string - EndpointProperties NetworkInfo - EndpointType EndpointType - PCIAddr string -} - -// BridgedMacvlanEndpoint represents a macvlan endpoint that is bridged to the VM -type BridgedMacvlanEndpoint struct { - NetPair NetworkInterfacePair - EndpointProperties NetworkInfo - EndpointType EndpointType - PCIAddr string -} - -// MacvtapEndpoint represents a macvtap endpoint -type MacvtapEndpoint struct { - EndpointProperties NetworkInfo - EndpointType EndpointType - VMFds []*os.File - VhostFds []*os.File - PCIAddr string -} - -// Properties returns properties for the veth interface in the network pair. -func (endpoint *VirtualEndpoint) Properties() NetworkInfo { - return endpoint.EndpointProperties -} - -// Name returns name of the veth interface in the network pair. -func (endpoint *VirtualEndpoint) Name() string { - return endpoint.NetPair.VirtIface.Name -} - -// HardwareAddr returns the mac address that is assigned to the tap interface -// in th network pair. -func (endpoint *VirtualEndpoint) HardwareAddr() string { - return endpoint.NetPair.TAPIface.HardAddr -} - -// Type identifies the endpoint as a virtual endpoint. -func (endpoint *VirtualEndpoint) Type() EndpointType { - return endpoint.EndpointType -} - -// PciAddr returns the PCI address of the endpoint. -func (endpoint *VirtualEndpoint) PciAddr() string { - return endpoint.PCIAddr -} - -// NetworkPair returns the network pair of the endpoint. -func (endpoint *VirtualEndpoint) NetworkPair() *NetworkInterfacePair { - return &endpoint.NetPair -} - -// SetProperties sets the properties for the endpoint. -func (endpoint *VirtualEndpoint) SetProperties(properties NetworkInfo) { - endpoint.EndpointProperties = properties -} - func networkLogger() *logrus.Entry { return virtLog.WithField("subsystem", "network") } -// Attach for virtual endpoint bridges the network pair and adds the -// tap interface of the network pair to the hypervisor. -func (endpoint *VirtualEndpoint) Attach(h hypervisor) error { - networkLogger().WithField("endpoint-type", "virtual").Info("Attaching endpoint") - - if err := xconnectVMNetwork(endpoint, true, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil { - networkLogger().WithError(err).Error("Error bridging virtual endpoint") - return err - } - - return h.addDevice(endpoint, netDev) -} - -// Detach for the virtual endpoint tears down the tap and bridge -// created for the veth interface. -func (endpoint *VirtualEndpoint) Detach(netNsCreated bool, netNsPath string) error { - // The network namespace would have been deleted at this point - // if it has not been created by virtcontainers. - if !netNsCreated { - return nil - } - - networkLogger().WithField("endpoint-type", "virtual").Info("Detaching endpoint") - - return doNetNS(netNsPath, func(_ ns.NetNS) error { - return xconnectVMNetwork(endpoint, false, 0, false) - }) -} - -// HotAttach for the virtual endpoint uses hot plug device -func (endpoint *VirtualEndpoint) HotAttach(h hypervisor) error { - networkLogger().Info("Hot attaching virtual endpoint") - if err := xconnectVMNetwork(endpoint, true, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil { - networkLogger().WithError(err).Error("Error bridging virtual ep") - return err - } - - if _, err := h.hotplugAddDevice(endpoint, netDev); err != nil { - networkLogger().WithError(err).Error("Error attach virtual ep") - return err - } - return nil -} - -// HotDetach for the virtual endpoint uses hot pull device -func (endpoint *VirtualEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { - if !netNsCreated { - return nil - } - networkLogger().Info("Hot detaching virtual endpoint") - if err := doNetNS(netNsPath, func(_ ns.NetNS) error { - return xconnectVMNetwork(endpoint, false, 0, h.hypervisorConfig().DisableVhostNet) - }); err != nil { - networkLogger().WithError(err).Warn("Error un-bridging virtual ep") - } - - if _, err := h.hotplugRemoveDevice(endpoint, netDev); err != nil { - networkLogger().WithError(err).Error("Error detach virtual ep") - return err - } - return nil -} - -// Properties returns the properties of the interface. -func (endpoint *VhostUserEndpoint) Properties() NetworkInfo { - return endpoint.EndpointProperties -} - -// Name returns name of the interface. -func (endpoint *VhostUserEndpoint) Name() string { - return endpoint.IfaceName -} - -// HardwareAddr returns the mac address of the vhostuser network interface -func (endpoint *VhostUserEndpoint) HardwareAddr() string { - return endpoint.HardAddr -} - -// Type indentifies the endpoint as a vhostuser endpoint. -func (endpoint *VhostUserEndpoint) Type() EndpointType { - return endpoint.EndpointType -} - -// SetProperties sets the properties of the endpoint. -func (endpoint *VhostUserEndpoint) SetProperties(properties NetworkInfo) { - endpoint.EndpointProperties = properties -} - -// PciAddr returns the PCI address of the endpoint. -func (endpoint *VhostUserEndpoint) PciAddr() string { - return endpoint.PCIAddr -} - -// NetworkPair returns the network pair of the endpoint. -func (endpoint *VhostUserEndpoint) NetworkPair() *NetworkInterfacePair { - return nil -} - -// Attach for vhostuser endpoint -func (endpoint *VhostUserEndpoint) Attach(h hypervisor) error { - networkLogger().WithField("endpoint-type", "vhostuser").Info("Attaching endpoint") - - // Generate a unique ID to be used for hypervisor commandline fields - randBytes, err := utils.GenerateRandomBytes(8) - if err != nil { - return err - } - id := hex.EncodeToString(randBytes) - - d := config.VhostUserDeviceAttrs{ - DevID: id, - SocketPath: endpoint.SocketPath, - MacAddress: endpoint.HardAddr, - Type: config.VhostUserNet, - } - - return h.addDevice(d, vhostuserDev) -} - -// Detach for vhostuser endpoint -func (endpoint *VhostUserEndpoint) Detach(netNsCreated bool, netNsPath string) error { - networkLogger().WithField("endpoint-type", "vhostuser").Info("Detaching endpoint") - return nil -} - -// HotAttach for vhostuser endpoint not supported yet -func (endpoint *VhostUserEndpoint) HotAttach(h hypervisor) error { - return fmt.Errorf("VhostUserEndpoint does not support Hot attach") -} - -// HotDetach for vhostuser endpoint not supported yet -func (endpoint *VhostUserEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { - return fmt.Errorf("VhostUserEndpoint does not support Hot detach") -} - -// Create a vhostuser endpoint -func createVhostUserEndpoint(netInfo NetworkInfo, socket string) (*VhostUserEndpoint, error) { - - vhostUserEndpoint := &VhostUserEndpoint{ - SocketPath: socket, - HardAddr: netInfo.Iface.HardwareAddr.String(), - IfaceName: netInfo.Iface.Name, - EndpointType: VhostUserEndpointType, - } - return vhostUserEndpoint, nil -} - -// Properties returns the properties of the physical interface. -func (endpoint *PhysicalEndpoint) Properties() NetworkInfo { - return endpoint.EndpointProperties -} - -// HardwareAddr returns the mac address of the physical network interface. -func (endpoint *PhysicalEndpoint) HardwareAddr() string { - return endpoint.HardAddr -} - -// Name returns name of the physical interface. -func (endpoint *PhysicalEndpoint) Name() string { - return endpoint.IfaceName -} - -// Type indentifies the endpoint as a physical endpoint. -func (endpoint *PhysicalEndpoint) Type() EndpointType { - return endpoint.EndpointType -} - -// PciAddr returns the PCI address of the endpoint. -func (endpoint *PhysicalEndpoint) PciAddr() string { - return endpoint.PCIAddr -} - -// SetProperties sets the properties of the physical endpoint. -func (endpoint *PhysicalEndpoint) SetProperties(properties NetworkInfo) { - endpoint.EndpointProperties = properties -} - -// NetworkPair returns the network pair of the endpoint. -func (endpoint *PhysicalEndpoint) NetworkPair() *NetworkInterfacePair { - return nil -} - -// Attach for physical endpoint binds the physical network interface to -// vfio-pci and adds device to the hypervisor with vfio-passthrough. -func (endpoint *PhysicalEndpoint) Attach(h hypervisor) error { - networkLogger().WithField("endpoint-type", "physical").Info("Attaching endpoint") - - // Unbind physical interface from host driver and bind to vfio - // so that it can be passed to qemu. - if err := bindNICToVFIO(endpoint); err != nil { - return err - } - - // TODO: use device manager as general device management entrance - d := config.VFIODev{ - BDF: endpoint.BDF, - } - - return h.addDevice(d, vfioDev) -} - -// Detach for physical endpoint unbinds the physical network interface from vfio-pci -// and binds it back to the saved host driver. -func (endpoint *PhysicalEndpoint) Detach(netNsCreated bool, netNsPath string) error { - // Bind back the physical network interface to host. - // We need to do this even if a new network namespace has not - // been created by virtcontainers. - networkLogger().WithField("endpoint-type", "physical").Info("Detaching endpoint") - - // We do not need to enter the network namespace to bind back the - // physical interface to host driver. - return bindNICToHost(endpoint) -} - -// HotAttach for physical endpoint not supported yet -func (endpoint *PhysicalEndpoint) HotAttach(h hypervisor) error { - return fmt.Errorf("PhysicalEndpoint does not support Hot attach") -} - -// HotDetach for physical endpoint not supported yet -func (endpoint *PhysicalEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { - return fmt.Errorf("PhysicalEndpoint does not support Hot detach") -} - -// Macvlan - -// Properties returns properties of the interface. -func (endpoint *BridgedMacvlanEndpoint) Properties() NetworkInfo { - return endpoint.EndpointProperties -} - -// Name returns name of the veth interface in the network pair. -func (endpoint *BridgedMacvlanEndpoint) Name() string { - return endpoint.NetPair.VirtIface.Name -} - -// HardwareAddr returns the mac address that is assigned to the tap interface -// in th network pair. -func (endpoint *BridgedMacvlanEndpoint) HardwareAddr() string { - return endpoint.NetPair.TAPIface.HardAddr -} - -// Type identifies the endpoint as a virtual endpoint. -func (endpoint *BridgedMacvlanEndpoint) Type() EndpointType { - return endpoint.EndpointType -} - -// SetProperties sets the properties for the endpoint. -func (endpoint *BridgedMacvlanEndpoint) SetProperties(properties NetworkInfo) { - endpoint.EndpointProperties = properties -} - -// PciAddr returns the PCI address of the endpoint. -func (endpoint *BridgedMacvlanEndpoint) PciAddr() string { - return endpoint.PCIAddr -} - -// NetworkPair returns the network pair of the endpoint. -func (endpoint *BridgedMacvlanEndpoint) NetworkPair() *NetworkInterfacePair { - return &endpoint.NetPair -} - -// Attach for virtual endpoint bridges the network pair and adds the -// tap interface of the network pair to the hypervisor. -func (endpoint *BridgedMacvlanEndpoint) Attach(h hypervisor) error { - networkLogger().Info("Attaching macvlan endpoint") - if err := xconnectVMNetwork(endpoint, true, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil { - networkLogger().WithError(err).Error("Error bridging virtual ep") - return err - } - - return h.addDevice(endpoint, netDev) -} - -// Detach for the virtual endpoint tears down the tap and bridge -// created for the veth interface. -func (endpoint *BridgedMacvlanEndpoint) Detach(netNsCreated bool, netNsPath string) error { - // The network namespace would have been deleted at this point - // if it has not been created by virtcontainers. - if !netNsCreated { - return nil - } - - networkLogger().Info("Detaching virtual endpoint") - - return doNetNS(netNsPath, func(_ ns.NetNS) error { - //return xconnectVMNetwork(&(endpoint.NetPair), false, 0, false, endpoint.EndpointType) - return xconnectVMNetwork(endpoint, false, 0, false) - }) -} - -// HotAttach for physical endpoint not supported yet -func (endpoint *BridgedMacvlanEndpoint) HotAttach(h hypervisor) error { - return fmt.Errorf("BridgedMacvlanEndpoint does not support Hot attach") -} - -// HotDetach for physical endpoint not supported yet -func (endpoint *BridgedMacvlanEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { - return fmt.Errorf("BridgedMacvlanEndpoint does not support Hot detach") -} - -//Macvtap - -// Properties returns the properties of the macvtap interface. -func (endpoint *MacvtapEndpoint) Properties() NetworkInfo { - return endpoint.EndpointProperties -} - -// HardwareAddr returns the mac address of the macvtap network interface. -func (endpoint *MacvtapEndpoint) HardwareAddr() string { - return endpoint.EndpointProperties.Iface.HardwareAddr.String() -} - -// Name returns name of the macvtap interface. -func (endpoint *MacvtapEndpoint) Name() string { - return endpoint.EndpointProperties.Iface.Name -} - -// Type indentifies the endpoint as a macvtap endpoint. -func (endpoint *MacvtapEndpoint) Type() EndpointType { - return endpoint.EndpointType -} - -// SetProperties sets the properties of the macvtap endpoint. -func (endpoint *MacvtapEndpoint) SetProperties(properties NetworkInfo) { - endpoint.EndpointProperties = properties -} - -// Attach for macvtap endpoint passes macvtap device to the hypervisor. -func (endpoint *MacvtapEndpoint) Attach(h hypervisor) error { - networkLogger().WithField("endpoint-type", "macvtap").Info("Attaching endpoint") - var err error - - endpoint.VMFds, err = createMacvtapFds(endpoint.EndpointProperties.Iface.Index, int(h.hypervisorConfig().NumVCPUs)) - if err != nil { - return fmt.Errorf("Could not setup macvtap fds %s: %s", endpoint.EndpointProperties.Iface.Name, err) - } - - if !h.hypervisorConfig().DisableVhostNet { - vhostFds, err := createVhostFds(int(h.hypervisorConfig().NumVCPUs)) - if err != nil { - return fmt.Errorf("Could not setup vhost fds %s : %s", endpoint.EndpointProperties.Iface.Name, err) - } - endpoint.VhostFds = vhostFds - } - - return h.addDevice(endpoint, netDev) -} - -// Detach for macvtap endpoint does nothing. -func (endpoint *MacvtapEndpoint) Detach(netNsCreated bool, netNsPath string) error { - networkLogger().WithField("endpoint-type", "macvtap").Info("Detaching endpoint") - return nil -} - -// HotAttach for macvtap endpoint not supported yet -func (endpoint *MacvtapEndpoint) HotAttach(h hypervisor) error { - return fmt.Errorf("MacvtapEndpoint does not support Hot attach") -} - -// HotDetach for macvtap endpoint not supported yet -func (endpoint *MacvtapEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { - return fmt.Errorf("MacvtapEndpoint does not support Hot detach") -} - -// PciAddr returns the PCI address of the endpoint. -func (endpoint *MacvtapEndpoint) PciAddr() string { - return endpoint.PCIAddr -} - -// NetworkPair returns the network pair of the endpoint. -func (endpoint *MacvtapEndpoint) NetworkPair() *NetworkInterfacePair { - return nil -} - -// EndpointType identifies the type of the network endpoint. -type EndpointType string - -const ( - // PhysicalEndpointType is the physical network interface. - PhysicalEndpointType EndpointType = "physical" - - // VirtualEndpointType is the virtual network interface. - VirtualEndpointType EndpointType = "virtual" - - // VhostUserEndpointType is the vhostuser network interface. - VhostUserEndpointType EndpointType = "vhost-user" - - // BridgedMacvlanEndpointType is macvlan network interface. - BridgedMacvlanEndpointType EndpointType = "macvlan" - - // MacvtapEndpointType is macvtap network interface. - MacvtapEndpointType EndpointType = "macvtap" -) - -// Set sets an endpoint type based on the input string. -func (endpointType *EndpointType) Set(value string) error { - switch value { - case "physical": - *endpointType = PhysicalEndpointType - return nil - case "virtual": - *endpointType = VirtualEndpointType - return nil - case "vhost-user": - *endpointType = VhostUserEndpointType - return nil - case "macvlan": - *endpointType = BridgedMacvlanEndpointType - return nil - case "macvtap": - *endpointType = MacvtapEndpointType - return nil - default: - return fmt.Errorf("Unknown endpoint type %s", value) - } -} - -// String converts an endpoint type to a string. -func (endpointType *EndpointType) String() string { - switch *endpointType { - case PhysicalEndpointType: - return string(PhysicalEndpointType) - case VirtualEndpointType: - return string(VirtualEndpointType) - case VhostUserEndpointType: - return string(VhostUserEndpointType) - case BridgedMacvlanEndpointType: - return string(BridgedMacvlanEndpointType) - case MacvtapEndpointType: - return string(MacvtapEndpointType) - default: - return "" - } -} - // NetworkNamespace contains all data related to its network namespace. type NetworkNamespace struct { NetNsPath string @@ -777,8 +232,8 @@ func (n *NetworkNamespace) UnmarshalJSON(b []byte) error { "endpoint-type": "physical", }).Info("endpoint unmarshalled") - case VirtualEndpointType: - var endpoint VirtualEndpoint + case VethEndpointType: + var endpoint VethEndpoint err := json.Unmarshal(e.Data, &endpoint) if err != nil { return err @@ -939,7 +394,7 @@ func getLinkForEndpoint(endpoint Endpoint, netHandle *netlink.Handle) (netlink.L var link netlink.Link switch ep := endpoint.(type) { - case *VirtualEndpoint: + case *VethEndpoint: link = &netlink.Veth{} case *BridgedMacvlanEndpoint: link = &netlink.Macvlan{} @@ -1411,60 +866,6 @@ func deleteNetNS(netNSPath string) error { return nil } -func createVirtualNetworkEndpoint(idx int, ifName string, interworkingModel NetInterworkingModel) (*VirtualEndpoint, error) { - if idx < 0 { - return &VirtualEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx) - } - - netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel) - if err != nil { - return nil, err - } - - endpoint := &VirtualEndpoint{ - // TODO This is too specific. We may need to create multiple - // end point types here and then decide how to connect them - // at the time of hypervisor attach and not here - NetPair: netPair, - EndpointType: VirtualEndpointType, - } - if ifName != "" { - endpoint.NetPair.VirtIface.Name = ifName - } - - return endpoint, nil -} - -func createMacvtapNetworkEndpoint(netInfo NetworkInfo) (*MacvtapEndpoint, error) { - endpoint := &MacvtapEndpoint{ - EndpointType: MacvtapEndpointType, - EndpointProperties: netInfo, - } - - return endpoint, nil -} - -func createBridgedMacvlanNetworkEndpoint(idx int, ifName string, interworkingModel NetInterworkingModel) (*BridgedMacvlanEndpoint, error) { - if idx < 0 { - return &BridgedMacvlanEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx) - } - - netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel) - if err != nil { - return nil, err - } - - endpoint := &BridgedMacvlanEndpoint{ - NetPair: netPair, - EndpointType: BridgedMacvlanEndpointType, - } - if ifName != "" { - endpoint.NetPair.VirtIface.Name = ifName - } - - return endpoint, nil -} - func generateInterfacesAndRoutes(networkNS NetworkNamespace) ([]*grpc.Interface, []*grpc.Route, error) { if networkNS.NetNsPath == "" { @@ -1715,140 +1116,16 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel) (E } else if netInfo.Iface.Type == "macvtap" { networkLogger().Infof("macvtap interface found") endpoint, err = createMacvtapNetworkEndpoint(netInfo) + } else if netInfo.Iface.Type == "veth" { + endpoint, err = createVethNetworkEndpoint(idx, netInfo.Iface.Name, model) } else { - endpoint, err = createVirtualNetworkEndpoint(idx, netInfo.Iface.Name, model) + return nil, fmt.Errorf("Unsupported network interface") } } return endpoint, err } -// isPhysicalIface checks if an interface is a physical device. -// We use ethtool here to not rely on device sysfs inside the network namespace. -func isPhysicalIface(ifaceName string) (bool, error) { - if ifaceName == "lo" { - return false, nil - } - - ethHandle, err := ethtool.NewEthtool() - if err != nil { - return false, err - } - - bus, err := ethHandle.BusInfo(ifaceName) - if err != nil { - return false, nil - } - - // Check for a pci bus format - tokens := strings.Split(bus, ":") - if len(tokens) != 3 { - return false, nil - } - - return true, nil -} - -var sysPCIDevicesPath = "/sys/bus/pci/devices" - -func createPhysicalEndpoint(netInfo NetworkInfo) (*PhysicalEndpoint, error) { - // Get ethtool handle to derive driver and bus - ethHandle, err := ethtool.NewEthtool() - if err != nil { - return nil, err - } - - // Get BDF - bdf, err := ethHandle.BusInfo(netInfo.Iface.Name) - if err != nil { - return nil, err - } - - // Get Driver - driver, err := ethHandle.DriverName(netInfo.Iface.Name) - if err != nil { - return nil, err - } - - // Get vendor and device id from pci space (sys/bus/pci/devices/$bdf) - - ifaceDevicePath := filepath.Join(sysPCIDevicesPath, bdf, "device") - contents, err := ioutil.ReadFile(ifaceDevicePath) - if err != nil { - return nil, err - } - - deviceID := strings.TrimSpace(string(contents)) - - // Vendor id - ifaceVendorPath := filepath.Join(sysPCIDevicesPath, bdf, "vendor") - contents, err = ioutil.ReadFile(ifaceVendorPath) - if err != nil { - return nil, err - } - - vendorID := strings.TrimSpace(string(contents)) - vendorDeviceID := fmt.Sprintf("%s %s", vendorID, deviceID) - vendorDeviceID = strings.TrimSpace(vendorDeviceID) - - physicalEndpoint := &PhysicalEndpoint{ - IfaceName: netInfo.Iface.Name, - HardAddr: netInfo.Iface.HardwareAddr.String(), - VendorDeviceID: vendorDeviceID, - EndpointType: PhysicalEndpointType, - Driver: driver, - BDF: bdf, - } - - return physicalEndpoint, nil -} - -func bindNICToVFIO(endpoint *PhysicalEndpoint) error { - return drivers.BindDevicetoVFIO(endpoint.BDF, endpoint.Driver, endpoint.VendorDeviceID) -} - -func bindNICToHost(endpoint *PhysicalEndpoint) error { - return drivers.BindDevicetoHost(endpoint.BDF, endpoint.Driver, endpoint.VendorDeviceID) -} - -// Long term, this should be made more configurable. For now matching path -// provided by CNM VPP and OVS-DPDK plugins, available at github.com/clearcontainers/vpp and -// github.com/clearcontainers/ovsdpdk. The plugins create the socket on the host system -// using this path. -const hostSocketSearchPath = "/tmp/vhostuser_%s/vhu.sock" - -// findVhostUserNetSocketPath checks if an interface is a dummy placeholder -// for a vhost-user socket, and if it is it returns the path to the socket -func findVhostUserNetSocketPath(netInfo NetworkInfo) (string, error) { - if netInfo.Iface.Name == "lo" { - return "", nil - } - - // check for socket file existence at known location. - for _, addr := range netInfo.Addrs { - socketPath := fmt.Sprintf(hostSocketSearchPath, addr.IPNet.IP) - if _, err := os.Stat(socketPath); err == nil { - return socketPath, nil - } - } - - return "", nil -} - -// vhostUserSocketPath returns the path of the socket discovered. This discovery -// will vary depending on the type of vhost-user socket. -// Today only VhostUserNetDevice is supported. -func vhostUserSocketPath(info interface{}) (string, error) { - - switch v := info.(type) { - case NetworkInfo: - return findVhostUserNetSocketPath(v) - default: - return "", nil - } - -} - // network is the virtcontainers network interface. // Container network plugins are used to setup virtual network // between VM netns and the host network physical interface. diff --git a/virtcontainers/network_test.go b/virtcontainers/network_test.go index a0f276b3c5..209bfbc30d 100644 --- a/virtcontainers/network_test.go +++ b/virtcontainers/network_test.go @@ -6,17 +6,14 @@ package virtcontainers import ( - "fmt" "net" "os" "reflect" "testing" - "github.com/containernetworking/plugins/pkg/ns" "github.com/kata-containers/agent/protocols/grpc" "github.com/stretchr/testify/assert" "github.com/vishvananda/netlink" - "github.com/vishvananda/netns" ) func testNetworkModelSet(t *testing.T, value string, expected NetworkModel) { @@ -118,509 +115,6 @@ func TestCreateDeleteNetNS(t *testing.T) { } } -func testEndpointTypeSet(t *testing.T, value string, expected EndpointType) { - //var netModel NetworkModel - var endpointType EndpointType - - err := endpointType.Set(value) - if err != nil { - t.Fatal(err) - } - - if endpointType != expected { - t.Fatal() - } -} - -func TestPhysicalEndpointTypeSet(t *testing.T) { - testEndpointTypeSet(t, "physical", PhysicalEndpointType) -} - -func TestVirtualEndpointTypeSet(t *testing.T) { - testEndpointTypeSet(t, "virtual", VirtualEndpointType) -} - -func TestVhostUserEndpointTypeSet(t *testing.T) { - testEndpointTypeSet(t, "vhost-user", VhostUserEndpointType) -} - -func TestBridgedMacvlanEndpointTypeSet(t *testing.T) { - testEndpointTypeSet(t, "macvlan", BridgedMacvlanEndpointType) -} - -func TestMacvtapEndpointTypeSet(t *testing.T) { - testEndpointTypeSet(t, "macvtap", MacvtapEndpointType) -} - -func TestEndpointTypeSetFailure(t *testing.T) { - var endpointType EndpointType - - err := endpointType.Set("wrong-value") - if err == nil { - t.Fatal(err) - } -} - -func testEndpointTypeString(t *testing.T, endpointType *EndpointType, expected string) { - result := endpointType.String() - - if result != expected { - t.Fatal() - } -} - -func TestPhysicalEndpointTypeString(t *testing.T) { - endpointType := PhysicalEndpointType - testEndpointTypeString(t, &endpointType, string(PhysicalEndpointType)) -} - -func TestVirtualEndpointTypeString(t *testing.T) { - endpointType := VirtualEndpointType - testEndpointTypeString(t, &endpointType, string(VirtualEndpointType)) -} - -func TestVhostUserEndpointTypeString(t *testing.T) { - endpointType := VhostUserEndpointType - testEndpointTypeString(t, &endpointType, string(VhostUserEndpointType)) -} - -func TestBridgedMacvlanEndpointTypeString(t *testing.T) { - endpointType := BridgedMacvlanEndpointType - testEndpointTypeString(t, &endpointType, string(BridgedMacvlanEndpointType)) -} - -func TestMacvtapEndpointTypeString(t *testing.T) { - endpointType := MacvtapEndpointType - testEndpointTypeString(t, &endpointType, string(MacvtapEndpointType)) -} - -func TestIncorrectEndpointTypeString(t *testing.T) { - var endpointType EndpointType - testEndpointTypeString(t, &endpointType, "") -} - -func TestCreateVhostUserEndpoint(t *testing.T) { - macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x48} - ifcName := "vhost-deadbeef" - socket := "/tmp/vhu_192.168.0.1" - - netinfo := NetworkInfo{ - Iface: NetlinkIface{ - LinkAttrs: netlink.LinkAttrs{ - HardwareAddr: macAddr, - Name: ifcName, - }, - }, - } - - expected := &VhostUserEndpoint{ - SocketPath: socket, - HardAddr: macAddr.String(), - IfaceName: ifcName, - EndpointType: VhostUserEndpointType, - } - - result, err := createVhostUserEndpoint(netinfo, socket) - if err != nil { - t.Fatal(err) - } - - if reflect.DeepEqual(result, expected) == false { - t.Fatalf("\n\tGot %v\n\tExpecting %v", result, expected) - } -} - -func TestCreateVirtualNetworkEndpoint(t *testing.T) { - macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x04} - - expected := &VirtualEndpoint{ - NetPair: NetworkInterfacePair{ - ID: "uniqueTestID-4", - Name: "br4_kata", - VirtIface: NetworkInterface{ - Name: "eth4", - HardAddr: macAddr.String(), - }, - TAPIface: NetworkInterface{ - Name: "tap4_kata", - }, - NetInterworkingModel: DefaultNetInterworkingModel, - }, - EndpointType: VirtualEndpointType, - } - - result, err := createVirtualNetworkEndpoint(4, "", DefaultNetInterworkingModel) - if err != nil { - t.Fatal(err) - } - - // the resulting ID will be random - so let's overwrite to test the rest of the flow - result.NetPair.ID = "uniqueTestID-4" - - // the resulting mac address will be random - so lets overwrite it - result.NetPair.VirtIface.HardAddr = macAddr.String() - - if reflect.DeepEqual(result, expected) == false { - t.Fatalf("\nGot: %+v, \n\nExpected: %+v", result, expected) - } -} - -func TestCreateVirtualNetworkEndpointChooseIfaceName(t *testing.T) { - macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x04} - - expected := &VirtualEndpoint{ - NetPair: NetworkInterfacePair{ - ID: "uniqueTestID-4", - Name: "br4_kata", - VirtIface: NetworkInterface{ - Name: "eth1", - HardAddr: macAddr.String(), - }, - TAPIface: NetworkInterface{ - Name: "tap4_kata", - }, - NetInterworkingModel: DefaultNetInterworkingModel, - }, - EndpointType: VirtualEndpointType, - } - - result, err := createVirtualNetworkEndpoint(4, "eth1", DefaultNetInterworkingModel) - if err != nil { - t.Fatal(err) - } - - // the resulting ID will be random - so let's overwrite to test the rest of the flow - result.NetPair.ID = "uniqueTestID-4" - - // the resulting mac address will be random - so lets overwrite it - result.NetPair.VirtIface.HardAddr = macAddr.String() - - if reflect.DeepEqual(result, expected) == false { - t.Fatalf("\nGot: %+v, \n\nExpected: %+v", result, expected) - } -} - -func TestCreateVirtualNetworkEndpointInvalidArgs(t *testing.T) { - type endpointValues struct { - idx int - ifName string - } - - // all elements are expected to result in failure - failingValues := []endpointValues{ - {-1, "bar"}, - {-1, ""}, - } - - for _, d := range failingValues { - result, err := createVirtualNetworkEndpoint(d.idx, d.ifName, DefaultNetInterworkingModel) - if err == nil { - t.Fatalf("expected invalid endpoint for %v, got %v", d, result) - } - } -} - -func TestCreateBridgedMacvlanEndpoint(t *testing.T) { - macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x04} - - expected := &BridgedMacvlanEndpoint{ - NetPair: NetworkInterfacePair{ - ID: "uniqueTestID-4", - Name: "br4_kata", - VirtIface: NetworkInterface{ - Name: "eth4", - HardAddr: macAddr.String(), - }, - TAPIface: NetworkInterface{ - Name: "tap4_kata", - }, - NetInterworkingModel: DefaultNetInterworkingModel, - }, - EndpointType: BridgedMacvlanEndpointType, - } - - result, err := createBridgedMacvlanNetworkEndpoint(4, "", DefaultNetInterworkingModel) - if err != nil { - t.Fatal(err) - } - - // the resulting ID will be random - so let's overwrite to test the rest of the flow - result.NetPair.ID = "uniqueTestID-4" - - // the resulting mac address will be random - so lets overwrite it - result.NetPair.VirtIface.HardAddr = macAddr.String() - - if reflect.DeepEqual(result, expected) == false { - t.Fatalf("\nGot: %+v, \n\nExpected: %+v", result, expected) - } -} - -func TestCreateMacvtapEndpoint(t *testing.T) { - netInfo := NetworkInfo{ - Iface: NetlinkIface{ - Type: "macvtap", - }, - } - expected := &MacvtapEndpoint{ - EndpointType: MacvtapEndpointType, - EndpointProperties: netInfo, - } - - result, err := createMacvtapNetworkEndpoint(netInfo) - if err != nil { - t.Fatal(err) - } - - if reflect.DeepEqual(result, expected) == false { - t.Fatalf("\nGot: %+v, \n\nExpected: %+v", result, expected) - } -} - -func TestIsPhysicalIface(t *testing.T) { - if os.Geteuid() != 0 { - t.Skip(testDisabledAsNonRoot) - } - - testNetIface := "testIface0" - testMTU := 1500 - testMACAddr := "00:00:00:00:00:01" - - hwAddr, err := net.ParseMAC(testMACAddr) - if err != nil { - t.Fatal(err) - } - - link := &netlink.Bridge{ - LinkAttrs: netlink.LinkAttrs{ - Name: testNetIface, - MTU: testMTU, - HardwareAddr: hwAddr, - TxQLen: -1, - }, - } - - n, err := ns.NewNS() - if err != nil { - t.Fatal(err) - } - defer n.Close() - - netnsHandle, err := netns.GetFromPath(n.Path()) - if err != nil { - t.Fatal(err) - } - defer netnsHandle.Close() - - netlinkHandle, err := netlink.NewHandleAt(netnsHandle) - if err != nil { - t.Fatal(err) - } - defer netlinkHandle.Delete() - - if err := netlinkHandle.LinkAdd(link); err != nil { - t.Fatal(err) - } - - var isPhysical bool - err = doNetNS(n.Path(), func(_ ns.NetNS) error { - var err error - isPhysical, err = isPhysicalIface(testNetIface) - return err - }) - - if err != nil { - t.Fatal(err) - } - - if isPhysical == true { - t.Fatalf("Got %+v\nExpecting %+v", isPhysical, false) - } -} - -func TestNetInterworkingModelIsValid(t *testing.T) { - tests := []struct { - name string - n NetInterworkingModel - want bool - }{ - {"Invalid Model", NetXConnectInvalidModel, false}, - {"Default Model", NetXConnectDefaultModel, true}, - {"Bridged Model", NetXConnectBridgedModel, true}, - {"Macvtap Model", NetXConnectMacVtapModel, true}, - {"Enlightened Model", NetXConnectEnlightenedModel, true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := tt.n.IsValid(); got != tt.want { - t.Errorf("NetInterworkingModel.IsValid() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestNetInterworkingModelSetModel(t *testing.T) { - var n NetInterworkingModel - tests := []struct { - name string - modelName string - wantErr bool - }{ - {"Invalid Model", "Invalid", true}, - {"default Model", "default", false}, - {"bridged Model", "bridged", false}, - {"macvtap Model", "macvtap", false}, - {"enlightened Model", "enlightened", false}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if err := n.SetModel(tt.modelName); (err != nil) != tt.wantErr { - t.Errorf("NetInterworkingModel.SetModel() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func TestVhostUserSocketPath(t *testing.T) { - - // First test case: search for existing: - addresses := []netlink.Addr{ - { - IPNet: &net.IPNet{ - IP: net.IPv4(192, 168, 0, 2), - Mask: net.IPv4Mask(192, 168, 0, 2), - }, - }, - { - IPNet: &net.IPNet{ - IP: net.IPv4(192, 168, 0, 1), - Mask: net.IPv4Mask(192, 168, 0, 1), - }, - }, - } - - expectedPath := "/tmp/vhostuser_192.168.0.1" - expectedFileName := "vhu.sock" - expectedResult := fmt.Sprintf("%s/%s", expectedPath, expectedFileName) - - err := os.Mkdir(expectedPath, 0777) - if err != nil { - t.Fatal(err) - } - - _, err = os.Create(expectedResult) - if err != nil { - t.Fatal(err) - } - netinfo := NetworkInfo{ - Addrs: addresses, - } - - path, _ := vhostUserSocketPath(netinfo) - - if path != expectedResult { - t.Fatalf("Got %+v\nExpecting %+v", path, expectedResult) - } - - // Second test case: search doesn't include matching vsock: - addressesFalse := []netlink.Addr{ - { - IPNet: &net.IPNet{ - IP: net.IPv4(192, 168, 0, 4), - Mask: net.IPv4Mask(192, 168, 0, 4), - }, - }, - } - netinfoFail := NetworkInfo{ - Addrs: addressesFalse, - } - - path, _ = vhostUserSocketPath(netinfoFail) - if path != "" { - t.Fatalf("Got %+v\nExpecting %+v", path, "") - } - - err = os.Remove(expectedResult) - if err != nil { - t.Fatal(err) - } - - err = os.Remove(expectedPath) - if err != nil { - t.Fatal(err) - } - -} - -func TestVhostUserEndpointAttach(t *testing.T) { - v := &VhostUserEndpoint{ - SocketPath: "/tmp/sock", - HardAddr: "mac-addr", - EndpointType: VhostUserEndpointType, - } - - h := &mockHypervisor{} - - err := v.Attach(h) - if err != nil { - t.Fatal(err) - } -} - -func TestVhostUserEndpoint_HotAttach(t *testing.T) { - assert := assert.New(t) - v := &VhostUserEndpoint{ - SocketPath: "/tmp/sock", - HardAddr: "mac-addr", - EndpointType: VhostUserEndpointType, - } - - h := &mockHypervisor{} - - err := v.HotAttach(h) - assert.Error(err) -} - -func TestVhostUserEndpoint_HotDetach(t *testing.T) { - assert := assert.New(t) - v := &VhostUserEndpoint{ - SocketPath: "/tmp/sock", - HardAddr: "mac-addr", - EndpointType: VhostUserEndpointType, - } - - h := &mockHypervisor{} - - err := v.HotDetach(h, true, "") - assert.Error(err) -} - -func TestPhysicalEndpoint_HotAttach(t *testing.T) { - assert := assert.New(t) - v := &PhysicalEndpoint{ - IfaceName: "eth0", - HardAddr: net.HardwareAddr{0x02, 0x00, 0xca, 0xfe, 0x00, 0x04}.String(), - } - - h := &mockHypervisor{} - - err := v.HotAttach(h) - assert.Error(err) -} - -func TestPhysicalEndpoint_HotDetach(t *testing.T) { - assert := assert.New(t) - v := &PhysicalEndpoint{ - IfaceName: "eth0", - HardAddr: net.HardwareAddr{0x02, 0x00, 0xca, 0xfe, 0x00, 0x04}.String(), - } - - h := &mockHypervisor{} - - err := v.HotDetach(h, true, "") - assert.Error(err) -} - func TestGenerateInterfacesAndRoutes(t *testing.T) { // //Create a couple of addresses @@ -689,6 +183,50 @@ func TestGenerateInterfacesAndRoutes(t *testing.T) { } +func TestNetInterworkingModelIsValid(t *testing.T) { + tests := []struct { + name string + n NetInterworkingModel + want bool + }{ + {"Invalid Model", NetXConnectInvalidModel, false}, + {"Default Model", NetXConnectDefaultModel, true}, + {"Bridged Model", NetXConnectBridgedModel, true}, + {"Macvtap Model", NetXConnectMacVtapModel, true}, + {"Enlightened Model", NetXConnectEnlightenedModel, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.n.IsValid(); got != tt.want { + t.Errorf("NetInterworkingModel.IsValid() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNetInterworkingModelSetModel(t *testing.T) { + var n NetInterworkingModel + tests := []struct { + name string + modelName string + wantErr bool + }{ + {"Invalid Model", "Invalid", true}, + {"default Model", "default", false}, + {"bridged Model", "bridged", false}, + {"macvtap Model", "macvtap", false}, + {"enlightened Model", "enlightened", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := n.SetModel(tt.modelName); (err != nil) != tt.wantErr { + t.Errorf("NetInterworkingModel.SetModel() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + func TestGenerateRandomPrivateMacAdd(t *testing.T) { assert := assert.New(t) diff --git a/virtcontainers/physical_endpoint.go b/virtcontainers/physical_endpoint.go new file mode 100644 index 0000000000..57c5f510eb --- /dev/null +++ b/virtcontainers/physical_endpoint.go @@ -0,0 +1,191 @@ +// Copyright (c) 2018 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "fmt" + "io/ioutil" + "path/filepath" + "strings" + + "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/device/drivers" + "github.com/safchain/ethtool" +) + +// PhysicalEndpoint gathers a physical network interface and its properties +type PhysicalEndpoint struct { + IfaceName string + HardAddr string + EndpointProperties NetworkInfo + EndpointType EndpointType + BDF string + Driver string + VendorDeviceID string + PCIAddr string +} + +// Properties returns the properties of the physical interface. +func (endpoint *PhysicalEndpoint) Properties() NetworkInfo { + return endpoint.EndpointProperties +} + +// HardwareAddr returns the mac address of the physical network interface. +func (endpoint *PhysicalEndpoint) HardwareAddr() string { + return endpoint.HardAddr +} + +// Name returns name of the physical interface. +func (endpoint *PhysicalEndpoint) Name() string { + return endpoint.IfaceName +} + +// Type indentifies the endpoint as a physical endpoint. +func (endpoint *PhysicalEndpoint) Type() EndpointType { + return endpoint.EndpointType +} + +// PciAddr returns the PCI address of the endpoint. +func (endpoint *PhysicalEndpoint) PciAddr() string { + return endpoint.PCIAddr +} + +// SetProperties sets the properties of the physical endpoint. +func (endpoint *PhysicalEndpoint) SetProperties(properties NetworkInfo) { + endpoint.EndpointProperties = properties +} + +// NetworkPair returns the network pair of the endpoint. +func (endpoint *PhysicalEndpoint) NetworkPair() *NetworkInterfacePair { + return nil +} + +// Attach for physical endpoint binds the physical network interface to +// vfio-pci and adds device to the hypervisor with vfio-passthrough. +func (endpoint *PhysicalEndpoint) Attach(h hypervisor) error { + // Unbind physical interface from host driver and bind to vfio + // so that it can be passed to qemu. + if err := bindNICToVFIO(endpoint); err != nil { + return err + } + + // TODO: use device manager as general device management entrance + d := config.VFIODev{ + BDF: endpoint.BDF, + } + + return h.addDevice(d, vfioDev) +} + +// Detach for physical endpoint unbinds the physical network interface from vfio-pci +// and binds it back to the saved host driver. +func (endpoint *PhysicalEndpoint) Detach(netNsCreated bool, netNsPath string) error { + // Bind back the physical network interface to host. + // We need to do this even if a new network namespace has not + // been created by virtcontainers. + + // We do not need to enter the network namespace to bind back the + // physical interface to host driver. + return bindNICToHost(endpoint) +} + +// HotAttach for physical endpoint not supported yet +func (endpoint *PhysicalEndpoint) HotAttach(h hypervisor) error { + return fmt.Errorf("PhysicalEndpoint does not support Hot attach") +} + +// HotDetach for physical endpoint not supported yet +func (endpoint *PhysicalEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { + return fmt.Errorf("PhysicalEndpoint does not support Hot detach") +} + +// isPhysicalIface checks if an interface is a physical device. +// We use ethtool here to not rely on device sysfs inside the network namespace. +func isPhysicalIface(ifaceName string) (bool, error) { + if ifaceName == "lo" { + return false, nil + } + + ethHandle, err := ethtool.NewEthtool() + if err != nil { + return false, err + } + + bus, err := ethHandle.BusInfo(ifaceName) + if err != nil { + return false, nil + } + + // Check for a pci bus format + tokens := strings.Split(bus, ":") + if len(tokens) != 3 { + return false, nil + } + + return true, nil +} + +var sysPCIDevicesPath = "/sys/bus/pci/devices" + +func createPhysicalEndpoint(netInfo NetworkInfo) (*PhysicalEndpoint, error) { + // Get ethtool handle to derive driver and bus + ethHandle, err := ethtool.NewEthtool() + if err != nil { + return nil, err + } + + // Get BDF + bdf, err := ethHandle.BusInfo(netInfo.Iface.Name) + if err != nil { + return nil, err + } + + // Get Driver + driver, err := ethHandle.DriverName(netInfo.Iface.Name) + if err != nil { + return nil, err + } + + // Get vendor and device id from pci space (sys/bus/pci/devices/$bdf) + + ifaceDevicePath := filepath.Join(sysPCIDevicesPath, bdf, "device") + contents, err := ioutil.ReadFile(ifaceDevicePath) + if err != nil { + return nil, err + } + + deviceID := strings.TrimSpace(string(contents)) + + // Vendor id + ifaceVendorPath := filepath.Join(sysPCIDevicesPath, bdf, "vendor") + contents, err = ioutil.ReadFile(ifaceVendorPath) + if err != nil { + return nil, err + } + + vendorID := strings.TrimSpace(string(contents)) + vendorDeviceID := fmt.Sprintf("%s %s", vendorID, deviceID) + vendorDeviceID = strings.TrimSpace(vendorDeviceID) + + physicalEndpoint := &PhysicalEndpoint{ + IfaceName: netInfo.Iface.Name, + HardAddr: netInfo.Iface.HardwareAddr.String(), + VendorDeviceID: vendorDeviceID, + EndpointType: PhysicalEndpointType, + Driver: driver, + BDF: bdf, + } + + return physicalEndpoint, nil +} + +func bindNICToVFIO(endpoint *PhysicalEndpoint) error { + return drivers.BindDevicetoVFIO(endpoint.BDF, endpoint.Driver, endpoint.VendorDeviceID) +} + +func bindNICToHost(endpoint *PhysicalEndpoint) error { + return drivers.BindDevicetoHost(endpoint.BDF, endpoint.Driver, endpoint.VendorDeviceID) +} diff --git a/virtcontainers/physical_endpoint_test.go b/virtcontainers/physical_endpoint_test.go new file mode 100644 index 0000000000..f22a8f1520 --- /dev/null +++ b/virtcontainers/physical_endpoint_test.go @@ -0,0 +1,104 @@ +// Copyright (c) 2018 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "net" + "os" + "testing" + + "github.com/containernetworking/plugins/pkg/ns" + "github.com/stretchr/testify/assert" + "github.com/vishvananda/netlink" + "github.com/vishvananda/netns" +) + +func TestPhysicalEndpoint_HotAttach(t *testing.T) { + assert := assert.New(t) + v := &PhysicalEndpoint{ + IfaceName: "eth0", + HardAddr: net.HardwareAddr{0x02, 0x00, 0xca, 0xfe, 0x00, 0x04}.String(), + } + + h := &mockHypervisor{} + + err := v.HotAttach(h) + assert.Error(err) +} + +func TestPhysicalEndpoint_HotDetach(t *testing.T) { + assert := assert.New(t) + v := &PhysicalEndpoint{ + IfaceName: "eth0", + HardAddr: net.HardwareAddr{0x02, 0x00, 0xca, 0xfe, 0x00, 0x04}.String(), + } + + h := &mockHypervisor{} + + err := v.HotDetach(h, true, "") + assert.Error(err) +} + +func TestIsPhysicalIface(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip(testDisabledAsNonRoot) + } + + testNetIface := "testIface0" + testMTU := 1500 + testMACAddr := "00:00:00:00:00:01" + + hwAddr, err := net.ParseMAC(testMACAddr) + if err != nil { + t.Fatal(err) + } + + link := &netlink.Bridge{ + LinkAttrs: netlink.LinkAttrs{ + Name: testNetIface, + MTU: testMTU, + HardwareAddr: hwAddr, + TxQLen: -1, + }, + } + + n, err := ns.NewNS() + if err != nil { + t.Fatal(err) + } + defer n.Close() + + netnsHandle, err := netns.GetFromPath(n.Path()) + if err != nil { + t.Fatal(err) + } + defer netnsHandle.Close() + + netlinkHandle, err := netlink.NewHandleAt(netnsHandle) + if err != nil { + t.Fatal(err) + } + defer netlinkHandle.Delete() + + if err := netlinkHandle.LinkAdd(link); err != nil { + t.Fatal(err) + } + + var isPhysical bool + err = doNetNS(n.Path(), func(_ ns.NetNS) error { + var err error + isPhysical, err = isPhysicalIface(testNetIface) + return err + }) + + if err != nil { + t.Fatal(err) + } + + if isPhysical == true { + t.Fatalf("Got %+v\nExpecting %+v", isPhysical, false) + } +} diff --git a/virtcontainers/qemu.go b/virtcontainers/qemu.go index fcaf70a838..00e5995be5 100644 --- a/virtcontainers/qemu.go +++ b/virtcontainers/qemu.go @@ -833,7 +833,7 @@ func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) error { return nil } -func (q *qemu) hotplugMacvtap(drive *VirtualEndpoint) error { +func (q *qemu) hotplugMacvtap(drive *VethEndpoint) error { var ( VMFdNames []string VhostFdNames []string @@ -857,7 +857,7 @@ func (q *qemu) hotplugMacvtap(drive *VirtualEndpoint) error { return q.qmpMonitorCh.qmp.ExecuteNetdevAddByFds(q.qmpMonitorCh.ctx, "tap", drive.NetPair.Name, VMFdNames, VhostFdNames) } -func (q *qemu) hotplugNetDevice(drive *VirtualEndpoint, op operation) error { +func (q *qemu) hotplugNetDevice(drive *VethEndpoint, op operation) error { err := q.qmpSetup() if err != nil { return err @@ -914,7 +914,7 @@ func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operati memdev := devInfo.(*memoryDevice) return q.hotplugMemory(memdev, op) case netDev: - device := devInfo.(*VirtualEndpoint) + device := devInfo.(*VethEndpoint) return nil, q.hotplugNetDevice(device, op) default: return nil, fmt.Errorf("cannot hotplug device: unsupported device type '%v'", devType) diff --git a/virtcontainers/qemu_arch_base.go b/virtcontainers/qemu_arch_base.go index f0d3cb7146..f8e6a61f90 100644 --- a/virtcontainers/qemu_arch_base.go +++ b/virtcontainers/qemu_arch_base.go @@ -441,7 +441,7 @@ func networkModelToQemuType(model NetInterworkingModel) govmmQemu.NetDeviceType func (q *qemuArchBase) appendNetwork(devices []govmmQemu.Device, endpoint Endpoint) []govmmQemu.Device { switch ep := endpoint.(type) { - case *VirtualEndpoint, *BridgedMacvlanEndpoint: + case *VethEndpoint, *BridgedMacvlanEndpoint: netPair := ep.NetworkPair() devices = append(devices, govmmQemu.NetDevice{ diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 2f6a331847..c207bd1718 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -1116,6 +1116,7 @@ func (s *Sandbox) AddInterface(inf *grpc.Interface) (*grpc.Interface, error) { endpoint.SetProperties(netInfo) if err := doNetNS(s.networkNS.NetNsPath, func(_ ns.NetNS) error { + s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Hot attaching endpoint") return endpoint.HotAttach(s.hypervisor) }); err != nil { return nil, err @@ -1136,6 +1137,7 @@ func (s *Sandbox) AddInterface(inf *grpc.Interface) (*grpc.Interface, error) { func (s *Sandbox) RemoveInterface(inf *grpc.Interface) (*grpc.Interface, error) { for i, endpoint := range s.networkNS.Endpoints { if endpoint.HardwareAddr() == inf.HwAddr { + s.Logger().WithField("endpoint-type", endpoint.Type()).Info("Hot detaching endpoint") if err := endpoint.HotDetach(s.hypervisor, s.networkNS.NetNsCreated, s.networkNS.NetNsPath); err != nil { return inf, err } diff --git a/virtcontainers/veth_endpoint.go b/virtcontainers/veth_endpoint.go new file mode 100644 index 0000000000..aeca9ac16f --- /dev/null +++ b/virtcontainers/veth_endpoint.go @@ -0,0 +1,139 @@ +// Copyright (c) 2018 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "fmt" + + "github.com/containernetworking/plugins/pkg/ns" +) + +// VethEndpoint gathers a network pair and its properties. +type VethEndpoint struct { + NetPair NetworkInterfacePair + EndpointProperties NetworkInfo + Physical bool + EndpointType EndpointType + PCIAddr string +} + +func createVethNetworkEndpoint(idx int, ifName string, interworkingModel NetInterworkingModel) (*VethEndpoint, error) { + if idx < 0 { + return &VethEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx) + } + + netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel) + if err != nil { + return nil, err + } + + endpoint := &VethEndpoint{ + // TODO This is too specific. We may need to create multiple + // end point types here and then decide how to connect them + // at the time of hypervisor attach and not here + NetPair: netPair, + EndpointType: VethEndpointType, + } + if ifName != "" { + endpoint.NetPair.VirtIface.Name = ifName + } + + return endpoint, nil +} + +// Properties returns properties for the veth interface in the network pair. +func (endpoint *VethEndpoint) Properties() NetworkInfo { + return endpoint.EndpointProperties +} + +// Name returns name of the veth interface in the network pair. +func (endpoint *VethEndpoint) Name() string { + return endpoint.NetPair.VirtIface.Name +} + +// HardwareAddr returns the mac address that is assigned to the tap interface +// in th network pair. +func (endpoint *VethEndpoint) HardwareAddr() string { + return endpoint.NetPair.TAPIface.HardAddr +} + +// Type identifies the endpoint as a veth endpoint. +func (endpoint *VethEndpoint) Type() EndpointType { + return endpoint.EndpointType +} + +// PciAddr returns the PCI address of the endpoint. +func (endpoint *VethEndpoint) PciAddr() string { + return endpoint.PCIAddr +} + +// NetworkPair returns the network pair of the endpoint. +func (endpoint *VethEndpoint) NetworkPair() *NetworkInterfacePair { + return &endpoint.NetPair +} + +// SetProperties sets the properties for the endpoint. +func (endpoint *VethEndpoint) SetProperties(properties NetworkInfo) { + endpoint.EndpointProperties = properties +} + +// Attach for veth endpoint bridges the network pair and adds the +// tap interface of the network pair to the hypervisor. +func (endpoint *VethEndpoint) Attach(h hypervisor) error { + if err := xconnectVMNetwork(endpoint, true, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil { + networkLogger().WithError(err).Error("Error bridging virtual endpoint") + return err + } + + return h.addDevice(endpoint, netDev) +} + +// Detach for the veth endpoint tears down the tap and bridge +// created for the veth interface. +func (endpoint *VethEndpoint) Detach(netNsCreated bool, netNsPath string) error { + // The network namespace would have been deleted at this point + // if it has not been created by virtcontainers. + if !netNsCreated { + return nil + } + + return doNetNS(netNsPath, func(_ ns.NetNS) error { + return xconnectVMNetwork(endpoint, false, 0, false) + }) +} + +// HotAttach for the veth endpoint uses hot plug device +func (endpoint *VethEndpoint) HotAttach(h hypervisor) error { + if err := xconnectVMNetwork(endpoint, true, h.hypervisorConfig().NumVCPUs, h.hypervisorConfig().DisableVhostNet); err != nil { + networkLogger().WithError(err).Error("Error bridging virtual ep") + return err + } + + if _, err := h.hotplugAddDevice(endpoint, netDev); err != nil { + networkLogger().WithError(err).Error("Error attach virtual ep") + return err + } + return nil +} + +// HotDetach for the veth endpoint uses hot pull device +func (endpoint *VethEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { + if !netNsCreated { + return nil + } + + if err := doNetNS(netNsPath, func(_ ns.NetNS) error { + return xconnectVMNetwork(endpoint, false, 0, h.hypervisorConfig().DisableVhostNet) + }); err != nil { + networkLogger().WithError(err).Warn("Error un-bridging virtual ep") + } + + if _, err := h.hotplugRemoveDevice(endpoint, netDev); err != nil { + networkLogger().WithError(err).Error("Error detach virtual ep") + return err + } + return nil +} diff --git a/virtcontainers/veth_endpoint_test.go b/virtcontainers/veth_endpoint_test.go new file mode 100644 index 0000000000..211e409aed --- /dev/null +++ b/virtcontainers/veth_endpoint_test.go @@ -0,0 +1,102 @@ +// Copyright (c) 2018 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "net" + "reflect" + "testing" +) + +func TestCreateVethNetworkEndpoint(t *testing.T) { + macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x04} + + expected := &VethEndpoint{ + NetPair: NetworkInterfacePair{ + ID: "uniqueTestID-4", + Name: "br4_kata", + VirtIface: NetworkInterface{ + Name: "eth4", + HardAddr: macAddr.String(), + }, + TAPIface: NetworkInterface{ + Name: "tap4_kata", + }, + NetInterworkingModel: DefaultNetInterworkingModel, + }, + EndpointType: VethEndpointType, + } + + result, err := createVethNetworkEndpoint(4, "", DefaultNetInterworkingModel) + if err != nil { + t.Fatal(err) + } + + // the resulting ID will be random - so let's overwrite to test the rest of the flow + result.NetPair.ID = "uniqueTestID-4" + + // the resulting mac address will be random - so lets overwrite it + result.NetPair.VirtIface.HardAddr = macAddr.String() + + if reflect.DeepEqual(result, expected) == false { + t.Fatalf("\nGot: %+v, \n\nExpected: %+v", result, expected) + } +} + +func TestCreateVethNetworkEndpointChooseIfaceName(t *testing.T) { + macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x04} + + expected := &VethEndpoint{ + NetPair: NetworkInterfacePair{ + ID: "uniqueTestID-4", + Name: "br4_kata", + VirtIface: NetworkInterface{ + Name: "eth1", + HardAddr: macAddr.String(), + }, + TAPIface: NetworkInterface{ + Name: "tap4_kata", + }, + NetInterworkingModel: DefaultNetInterworkingModel, + }, + EndpointType: VethEndpointType, + } + + result, err := createVethNetworkEndpoint(4, "eth1", DefaultNetInterworkingModel) + if err != nil { + t.Fatal(err) + } + + // the resulting ID will be random - so let's overwrite to test the rest of the flow + result.NetPair.ID = "uniqueTestID-4" + + // the resulting mac address will be random - so lets overwrite it + result.NetPair.VirtIface.HardAddr = macAddr.String() + + if reflect.DeepEqual(result, expected) == false { + t.Fatalf("\nGot: %+v, \n\nExpected: %+v", result, expected) + } +} + +func TestCreateVethNetworkEndpointInvalidArgs(t *testing.T) { + type endpointValues struct { + idx int + ifName string + } + + // all elements are expected to result in failure + failingValues := []endpointValues{ + {-1, "bar"}, + {-1, ""}, + } + + for _, d := range failingValues { + result, err := createVethNetworkEndpoint(d.idx, d.ifName, DefaultNetInterworkingModel) + if err == nil { + t.Fatalf("expected invalid endpoint for %v, got %v", d, result) + } + } +} diff --git a/virtcontainers/vhostuser_endpoint.go b/virtcontainers/vhostuser_endpoint.go new file mode 100644 index 0000000000..3207f3ae49 --- /dev/null +++ b/virtcontainers/vhostuser_endpoint.go @@ -0,0 +1,146 @@ +// Copyright (c) 2018 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "encoding/hex" + "fmt" + "os" + + "github.com/kata-containers/runtime/virtcontainers/device/config" + "github.com/kata-containers/runtime/virtcontainers/utils" +) + +// Long term, this should be made more configurable. For now matching path +// provided by CNM VPP and OVS-DPDK plugins, available at github.com/clearcontainers/vpp and +// github.com/clearcontainers/ovsdpdk. The plugins create the socket on the host system +// using this path. +const hostSocketSearchPath = "/tmp/vhostuser_%s/vhu.sock" + +// VhostUserEndpoint represents a vhost-user socket based network interface +type VhostUserEndpoint struct { + // Path to the vhost-user socket on the host system + SocketPath string + // MAC address of the interface + HardAddr string + IfaceName string + EndpointProperties NetworkInfo + EndpointType EndpointType + PCIAddr string +} + +// Properties returns the properties of the interface. +func (endpoint *VhostUserEndpoint) Properties() NetworkInfo { + return endpoint.EndpointProperties +} + +// Name returns name of the interface. +func (endpoint *VhostUserEndpoint) Name() string { + return endpoint.IfaceName +} + +// HardwareAddr returns the mac address of the vhostuser network interface +func (endpoint *VhostUserEndpoint) HardwareAddr() string { + return endpoint.HardAddr +} + +// Type indentifies the endpoint as a vhostuser endpoint. +func (endpoint *VhostUserEndpoint) Type() EndpointType { + return endpoint.EndpointType +} + +// SetProperties sets the properties of the endpoint. +func (endpoint *VhostUserEndpoint) SetProperties(properties NetworkInfo) { + endpoint.EndpointProperties = properties +} + +// PciAddr returns the PCI address of the endpoint. +func (endpoint *VhostUserEndpoint) PciAddr() string { + return endpoint.PCIAddr +} + +// NetworkPair returns the network pair of the endpoint. +func (endpoint *VhostUserEndpoint) NetworkPair() *NetworkInterfacePair { + return nil +} + +// Attach for vhostuser endpoint +func (endpoint *VhostUserEndpoint) Attach(h hypervisor) error { + // Generate a unique ID to be used for hypervisor commandline fields + randBytes, err := utils.GenerateRandomBytes(8) + if err != nil { + return err + } + id := hex.EncodeToString(randBytes) + + d := config.VhostUserDeviceAttrs{ + DevID: id, + SocketPath: endpoint.SocketPath, + MacAddress: endpoint.HardAddr, + Type: config.VhostUserNet, + } + + return h.addDevice(d, vhostuserDev) +} + +// Detach for vhostuser endpoint +func (endpoint *VhostUserEndpoint) Detach(netNsCreated bool, netNsPath string) error { + return nil +} + +// HotAttach for vhostuser endpoint not supported yet +func (endpoint *VhostUserEndpoint) HotAttach(h hypervisor) error { + return fmt.Errorf("VhostUserEndpoint does not support Hot attach") +} + +// HotDetach for vhostuser endpoint not supported yet +func (endpoint *VhostUserEndpoint) HotDetach(h hypervisor, netNsCreated bool, netNsPath string) error { + return fmt.Errorf("VhostUserEndpoint does not support Hot detach") +} + +// Create a vhostuser endpoint +func createVhostUserEndpoint(netInfo NetworkInfo, socket string) (*VhostUserEndpoint, error) { + + vhostUserEndpoint := &VhostUserEndpoint{ + SocketPath: socket, + HardAddr: netInfo.Iface.HardwareAddr.String(), + IfaceName: netInfo.Iface.Name, + EndpointType: VhostUserEndpointType, + } + return vhostUserEndpoint, nil +} + +// findVhostUserNetSocketPath checks if an interface is a dummy placeholder +// for a vhost-user socket, and if it is it returns the path to the socket +func findVhostUserNetSocketPath(netInfo NetworkInfo) (string, error) { + if netInfo.Iface.Name == "lo" { + return "", nil + } + + // check for socket file existence at known location. + for _, addr := range netInfo.Addrs { + socketPath := fmt.Sprintf(hostSocketSearchPath, addr.IPNet.IP) + if _, err := os.Stat(socketPath); err == nil { + return socketPath, nil + } + } + + return "", nil +} + +// vhostUserSocketPath returns the path of the socket discovered. This discovery +// will vary depending on the type of vhost-user socket. +// Today only VhostUserNetDevice is supported. +func vhostUserSocketPath(info interface{}) (string, error) { + + switch v := info.(type) { + case NetworkInfo: + return findVhostUserNetSocketPath(v) + default: + return "", nil + } + +} diff --git a/virtcontainers/vhostuser_endpoint_test.go b/virtcontainers/vhostuser_endpoint_test.go new file mode 100644 index 0000000000..6d1ad91e86 --- /dev/null +++ b/virtcontainers/vhostuser_endpoint_test.go @@ -0,0 +1,162 @@ +// Copyright (c) 2018 Intel Corporation +// +// SPDX-License-Identifier: Apache-2.0 +// + +package virtcontainers + +import ( + "fmt" + "net" + "os" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vishvananda/netlink" +) + +func TestVhostUserSocketPath(t *testing.T) { + + // First test case: search for existing: + addresses := []netlink.Addr{ + { + IPNet: &net.IPNet{ + IP: net.IPv4(192, 168, 0, 2), + Mask: net.IPv4Mask(192, 168, 0, 2), + }, + }, + { + IPNet: &net.IPNet{ + IP: net.IPv4(192, 168, 0, 1), + Mask: net.IPv4Mask(192, 168, 0, 1), + }, + }, + } + + expectedPath := "/tmp/vhostuser_192.168.0.1" + expectedFileName := "vhu.sock" + expectedResult := fmt.Sprintf("%s/%s", expectedPath, expectedFileName) + + err := os.Mkdir(expectedPath, 0777) + if err != nil { + t.Fatal(err) + } + + _, err = os.Create(expectedResult) + if err != nil { + t.Fatal(err) + } + netinfo := NetworkInfo{ + Addrs: addresses, + } + + path, _ := vhostUserSocketPath(netinfo) + + if path != expectedResult { + t.Fatalf("Got %+v\nExpecting %+v", path, expectedResult) + } + + // Second test case: search doesn't include matching vsock: + addressesFalse := []netlink.Addr{ + { + IPNet: &net.IPNet{ + IP: net.IPv4(192, 168, 0, 4), + Mask: net.IPv4Mask(192, 168, 0, 4), + }, + }, + } + netinfoFail := NetworkInfo{ + Addrs: addressesFalse, + } + + path, _ = vhostUserSocketPath(netinfoFail) + if path != "" { + t.Fatalf("Got %+v\nExpecting %+v", path, "") + } + + err = os.Remove(expectedResult) + if err != nil { + t.Fatal(err) + } + + err = os.Remove(expectedPath) + if err != nil { + t.Fatal(err) + } + +} + +func TestVhostUserEndpointAttach(t *testing.T) { + v := &VhostUserEndpoint{ + SocketPath: "/tmp/sock", + HardAddr: "mac-addr", + EndpointType: VhostUserEndpointType, + } + + h := &mockHypervisor{} + + err := v.Attach(h) + if err != nil { + t.Fatal(err) + } +} + +func TestVhostUserEndpoint_HotAttach(t *testing.T) { + assert := assert.New(t) + v := &VhostUserEndpoint{ + SocketPath: "/tmp/sock", + HardAddr: "mac-addr", + EndpointType: VhostUserEndpointType, + } + + h := &mockHypervisor{} + + err := v.HotAttach(h) + assert.Error(err) +} + +func TestVhostUserEndpoint_HotDetach(t *testing.T) { + assert := assert.New(t) + v := &VhostUserEndpoint{ + SocketPath: "/tmp/sock", + HardAddr: "mac-addr", + EndpointType: VhostUserEndpointType, + } + + h := &mockHypervisor{} + + err := v.HotDetach(h, true, "") + assert.Error(err) +} + +func TestCreateVhostUserEndpoint(t *testing.T) { + macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x48} + ifcName := "vhost-deadbeef" + socket := "/tmp/vhu_192.168.0.1" + + netinfo := NetworkInfo{ + Iface: NetlinkIface{ + LinkAttrs: netlink.LinkAttrs{ + HardwareAddr: macAddr, + Name: ifcName, + }, + }, + } + + expected := &VhostUserEndpoint{ + SocketPath: socket, + HardAddr: macAddr.String(), + IfaceName: ifcName, + EndpointType: VhostUserEndpointType, + } + + result, err := createVhostUserEndpoint(netinfo, socket) + if err != nil { + t.Fatal(err) + } + + if reflect.DeepEqual(result, expected) == false { + t.Fatalf("\n\tGot %v\n\tExpecting %v", result, expected) + } +}