Merge pull request #708 from Mirantis/jell/vfvlan

VLan support for SR-IOV

images/image_skel/libvirt.sh

@@ -4,6 +4,10 @@ set -o nounset
 set -o pipefail
 set -o errtrace
 
+if [[ -f /var/lib/virtlet/config.sh ]]; then
+ . /var/lib/virtlet/config.sh
+fi
+
 testmode=
 if [[ ${1:-} == -testmode ]]; then
  testmode=1

pkg/nettools/nettools.go

@@ -54,6 +54,7 @@ import (
 
  "github.com/Mirantis/virtlet/pkg/cni"
  "github.com/Mirantis/virtlet/pkg/network"
+ "github.com/Mirantis/virtlet/pkg/utils"
 )
 
 const (
@@ -673,18 +674,55 @@ func getMasterLinkOfVf(pciAddress string) (netlink.Link, error) {
  return masterLink, nil
 }
 
-// SetMacOnVf uses VF pci address to locate its parent device and uses
-// it to set mac address on VF. It needs to be called from host netns.
-func SetMacOnVf(pciAddress string, mac net.HardwareAddr) error {
+// setMacAndVlanOnVf uses VF pci address to locate its parent device and uses
+// it to set mac address and VLAN id on VF. It needs to be called from the host netns.
+func setMacAndVlanOnVf(pciAddress string, mac net.HardwareAddr, vlanID int) error {
  virtFNNo, err := getVirtFNNo(pciAddress)
  if err != nil {
- return err
+ return fmt.Errorf("cannot find VF number for device with pci address %q: %v", pciAddress, err)
  }
  masterLink, err := getMasterLinkOfVf(pciAddress)
  if err != nil {
- return err
+ return fmt.Errorf("cannot get link for PF of VF with pci address %q: %v", pciAddress, err)
+ }
+ if err := netlink.LinkSetVfHardwareAddr(masterLink, virtFNNo, mac); err != nil {
+ return fmt.Errorf("cannot set mac address of VF with pci address %q: %v", pciAddress, err)
+ }
+ err = netlink.LinkSetVfVlan(masterLink, virtFNNo, vlanID)
+ if err != nil {
+ return fmt.Errorf("cannot set vlan of VF with pci address %q: %v", pciAddress, err)
  }
- return netlink.LinkSetVfHardwareAddr(masterLink, virtFNNo, mac)
+ return nil
+}
+
+func getVfVlanID(pciAddress string) (int, error) {
+ virtFNNo, err := getVirtFNNo(pciAddress)
+ if err != nil {
+ return 0, err
+ }
+ masterLink, err := getMasterLinkOfVf(pciAddress)
+ if err != nil {
+ return 0, err
+ }
+
+ // vfinfos are gathered using `ip link show` because of failure in vishvananda/netlink
+ // which is occuring for bigger netlink queries like one asking for list ov VFs of an interface.
+ iplinkOutput, err := exec.Command("ip", "link", "show", masterLink.Attrs().Name).CombinedOutput()
+ if err != nil {
+ return 0, fmt.Errorf("error during execution of ip link show: %q\nOutput:%s", err, iplinkOutput)
+ }
+ vfinfos, err := utils.ParseIPLinkOutput(iplinkOutput)
+ if err != nil {
+ return 0, fmt.Errorf("error during parsing ip link output for device %q: %v",
+ masterLink.Attrs().Name, err)
+ }
+
+ for _, vfInfo := range vfinfos {
+ if vfInfo.ID == virtFNNo {
+ return int(vfInfo.VLanID), nil
+ }
+ }
+ return 0, fmt.Errorf("vlan info for %d vf on %s not found", virtFNNo, masterLink.Attrs().Name)
 }
 
 // SetupContainerSideNetwork sets up networking in container
@@ -698,7 +736,7 @@ func SetMacOnVf(pciAddress string, mac net.HardwareAddr) error {
 // In case of SR-IOV VFs this function only sets up a device to be passed to VM.
 // The function should be called from within container namespace.
 // Returns container network struct and an error, if any.
-func SetupContainerSideNetwork(info *cnicurrent.Result, nsPath string, allLinks []netlink.Link, enableSriov bool) (*network.ContainerSideNetwork, error) {
+func SetupContainerSideNetwork(info *cnicurrent.Result, nsPath string, allLinks []netlink.Link, enableSriov bool, hostNS ns.NetNS) (*network.ContainerSideNetwork, error) {
  contLinks, err := getContainerLinks(info)
  if err != nil {
  return nil, err
@@ -713,6 +751,7 @@ func SetupContainerSideNetwork(info *cnicurrent.Result, nsPath string, allLinks
  hwAddr := link.Attrs().HardwareAddr
  ifaceName := link.Attrs().Name
  pciAddress := ""
+ vlanID := 0
  var ifaceType network.InterfaceType
  var fo *os.File
 
@@ -742,6 +781,15 @@ func SetupContainerSideNetwork(info *cnicurrent.Result, nsPath string, allLinks
  return nil, err
  }
 
+ // Switch to the host netns to get VLAN ID of the VF using its master device.
+ if err := utils.CallInNetNSWithSysfsRemounted(hostNS, func(ns.NetNS) error {
+ var err error
+ vlanID, err = getVfVlanID(pciAddress)
+ return err
+ }); err != nil {
+ return nil, err
+ }
+
  if err := unbindDriverFromDevice(pciAddress); err != nil {
  return nil, err
  }
@@ -755,6 +803,14 @@ func SetupContainerSideNetwork(info *cnicurrent.Result, nsPath string, allLinks
  return nil, err
  }
 
+ // Switch to the host netns to set mac address and VLAN id
+ // of VF using its master device.
+ if err := utils.CallInNetNSWithSysfsRemounted(hostNS, func(ns.NetNS) error {
+ return setMacAndVlanOnVf(pciAddress, hwAddr, vlanID)
+ }); err != nil {
+ return nil, err
+ }
+
  glog.V(3).Infof("Adding interface %q as VF on %s address", ifaceName, pciAddress)
  } else {
  newHwAddr, err := GenerateMacAddress()
@@ -814,6 +870,7 @@ func SetupContainerSideNetwork(info *cnicurrent.Result, nsPath string, allLinks
  HardwareAddr: hwAddr,
  PCIAddress: pciAddress,
  MTU: uint16(mtu),
+ VlanID: vlanID,
  })
  }
 
@@ -822,7 +879,7 @@ func SetupContainerSideNetwork(info *cnicurrent.Result, nsPath string, allLinks
 
 // RecoverContainerSideNetwork tries to populate ContainerSideNetwork
 // structure based on a network namespace that was already adjusted for Virtlet
-func RecoverContainerSideNetwork(csn *network.ContainerSideNetwork, nsPath string, allLinks []netlink.Link) error {
+func RecoverContainerSideNetwork(csn *network.ContainerSideNetwork, nsPath string, allLinks []netlink.Link, hostNS ns.NetNS) error {
  if len(csn.Result.Interfaces) == 0 {
  return fmt.Errorf("wrong cni configuration: no interfaces defined: %s", spew.Sdump(csn.Result))
  }
@@ -864,7 +921,7 @@ func RecoverContainerSideNetwork(csn *network.ContainerSideNetwork, nsPath strin
  // this can be problematic in case of machine reboot - we are trying to use the same
  // devices as was used before reboot, but in meantime there is small chance that they
  // were used already by sriov cni plugin (for which reboot means it's starting everything
- // from clean situation) to other containers, before even virtlet was started
+ // from clean situation) for some other pods, before even virtlet was started
  // also in case of virtlet pod restart - device can be already bound to vfio-pci, so we
  // are ignoring any error there)
  bindDeviceToVFIO(devIdentifier)
@@ -1029,7 +1086,7 @@ func ReconstructVFs(csn *network.ContainerSideNetwork, netns ns.NetNS, ignoreUnb
  if err != nil {
  return err
  }
- if err := SetMacOnVf(iface.PCIAddress, iface.HardwareAddr); err != nil {
+ if err := setMacAndVlanOnVf(iface.PCIAddress, iface.HardwareAddr, iface.VlanID); err != nil {
  return err
  }
  link, err := netlink.LinkByName(devName)

pkg/nettools/nettools_test.go

@@ -390,15 +390,15 @@ func TestExtractLinkInfo(t *testing.T) {
  })
 }
 
-func verifyContainerSideNetwork(t *testing.T, origContVeth netlink.Link, contNsPath string) {
+func verifyContainerSideNetwork(t *testing.T, origContVeth netlink.Link, contNsPath string, hostNS ns.NetNS) {
  allLinks, err := netlink.LinkList()
  if err != nil {
  log.Panicf("error listing links: %v", err)
  }
 
  origHwAddr := origContVeth.Attrs().HardwareAddr
  expectedInfo := expectedExtractedLinkInfo(contNsPath)
- csn, err := SetupContainerSideNetwork(expectedInfo, contNsPath, allLinks, false)
+ csn, err := SetupContainerSideNetwork(expectedInfo, contNsPath, allLinks, false, hostNS)
  if err != nil {
  log.Panicf("failed to set up container side network: %v", err)
  }
@@ -445,13 +445,13 @@ func TestSetUpContainerSideNetworkWithInfo(t *testing.T) {
  if err := StripLink(origContVeth); err != nil {
  log.Panicf("StripLink() failed: %v", err)
  }
- verifyContainerSideNetwork(t, origContVeth, contNS.Path())
+ verifyContainerSideNetwork(t, origContVeth, contNS.Path(), hostNS)
  })
 }
 
 func TestLoopbackInterface(t *testing.T) {
  withFakeCNIVethAndGateway(t, func(hostNS, contNS ns.NetNS, origHostVeth, origContVeth netlink.Link) {
- verifyContainerSideNetwork(t, origContVeth, contNS.Path())
+ verifyContainerSideNetwork(t, origContVeth, contNS.Path(), hostNS)
  if out, err := exec.Command("ping", "-c", "1", "127.0.0.1").CombinedOutput(); err != nil {
  log.Panicf("ping 127.0.0.1 failed:\n%s", out)
  }
@@ -535,7 +535,7 @@ func TestTeardownContainerSideNetwork(t *testing.T) {
  log.Panicf("error listing links: %v", err)
  }
 
- csn, err := SetupContainerSideNetwork(expectedExtractedLinkInfo(contNS.Path()), contNS.Path(), allLinks, false)
+ csn, err := SetupContainerSideNetwork(expectedExtractedLinkInfo(contNS.Path()), contNS.Path(), allLinks, false, hostNS)
  if err != nil {
  log.Panicf("failed to set up container side network: %v", err)
  }

pkg/network/csn.go

@@ -52,6 +52,8 @@ type InterfaceDescription struct {
  PCIAddress string
  // MTU contains max transfer unit value for interface.
  MTU uint16
+ // VlanID contains vlan id of sr-iov vf interface.
+ VlanID int
 }
 
 // ContainerSideNetwork struct describes the container (VM) network

pkg/tapmanager/tapfdsource.go

@@ -35,6 +35,7 @@ import (
  "github.com/Mirantis/virtlet/pkg/dhcp"
  "github.com/Mirantis/virtlet/pkg/nettools"
  "github.com/Mirantis/virtlet/pkg/network"
+ "github.com/Mirantis/virtlet/pkg/utils"
 )
 
 const (
@@ -113,6 +114,7 @@ func NewTapFDSource(cniClient cni.Client, enableSriov bool, calicoSubnetSize int
  cniClient: cniClient,
  fdMap: make(map[string]*podNetwork),
  calicoSubnetSize: calicoSubnetSize,
+ enableSriov: enableSriov,
  }
 
  return s, nil
@@ -178,7 +180,7 @@ func (s *TapFDSource) GetFDs(key string, data []byte) ([]int, []byte, error) {
  var fds []int
  var respData []byte
  var csn *network.ContainerSideNetwork
- if err := s.setupNetNS(key, pnd, func(netNSPath string, allLinks []netlink.Link) (*network.ContainerSideNetwork, error) {
+ if err := s.setupNetNS(key, pnd, func(netNSPath string, allLinks []netlink.Link, hostNS ns.NetNS) (*network.ContainerSideNetwork, error) {
  if netConfig, err = nettools.ValidateAndFixCNIResult(netConfig, netNSPath, allLinks); err != nil {
  gotError = true
  return nil, fmt.Errorf("error fixing cni configuration: %v", err)
@@ -190,7 +192,7 @@ func (s *TapFDSource) GetFDs(key string, data []byte) ([]int, []byte, error) {
  glog.V(3).Infof("CNI Result after fix:\n%s", spew.Sdump(netConfig))
 
  var err error
- if csn, err = nettools.SetupContainerSideNetwork(netConfig, netNSPath, allLinks, s.enableSriov); err != nil {
+ if csn, err = nettools.SetupContainerSideNetwork(netConfig, netNSPath, allLinks, s.enableSriov, hostNS); err != nil {
  return nil, err
  }
 
@@ -207,15 +209,6 @@ func (s *TapFDSource) GetFDs(key string, data []byte) ([]int, []byte, error) {
  return nil, nil, err
  }
 
- for _, iface := range csn.Interfaces {
- if iface.Type == network.InterfaceTypeVF {
- if err := nettools.SetMacOnVf(iface.PCIAddress, iface.HardwareAddr); err != nil {
- gotError = true
- return nil, nil, err
- }
- }
- }
-
  return fds, respData, nil
 }
 
@@ -340,15 +333,15 @@ func (s *TapFDSource) Recover(key string, data []byte) error {
  return err
  }
  }
- return s.setupNetNS(key, pnd, func(netNSPath string, allLinks []netlink.Link) (*network.ContainerSideNetwork, error) {
- if err := nettools.RecoverContainerSideNetwork(csn, netNSPath, allLinks); err != nil {
+ return s.setupNetNS(key, pnd, func(netNSPath string, allLinks []netlink.Link, hostNS ns.NetNS) (*network.ContainerSideNetwork, error) {
+ if err := nettools.RecoverContainerSideNetwork(csn, netNSPath, allLinks, hostNS); err != nil {
  return nil, err
  }
  return csn, nil
  })
 }
 
-func (s *TapFDSource) setupNetNS(key string, pnd *PodNetworkDesc, initNet func(netNSPath string, allLinks []netlink.Link) (*network.ContainerSideNetwork, error)) error {
+func (s *TapFDSource) setupNetNS(key string, pnd *PodNetworkDesc, initNet func(netNSPath string, allLinks []netlink.Link, hostNS ns.NetNS) (*network.ContainerSideNetwork, error)) error {
  netNSPath := cni.PodNetNSPath(pnd.PodID)
  vmNS, err := ns.GetNS(netNSPath)
  if err != nil {
@@ -358,24 +351,13 @@ func (s *TapFDSource) setupNetNS(key string, pnd *PodNetworkDesc, initNet func(n
  var csn *network.ContainerSideNetwork
  var dhcpServer *dhcp.Server
  doneCh := make(chan error)
- if err := vmNS.Do(func(ns.NetNS) error {
- // switch /sys to corresponding one in netns
- // to have the correct items under /sys/class/net
- if err := mountSysfs(); err != nil {
- return err
- }
- defer func() {
- if err := unmountSysfs(); err != nil {
- glog.V(3).Infof("Warning, error during umount of /sys: %v", err)
- }
- }()
-
+ if err := utils.CallInNetNSWithSysfsRemounted(vmNS, func(hostNS ns.NetNS) error {
  allLinks, err := netlink.LinkList()
  if err != nil {
  return fmt.Errorf("error listing the links: %v", err)
  }
 
- if csn, err = initNet(netNSPath, allLinks); err != nil {
+ if csn, err = initNet(netNSPath, allLinks, hostNS); err != nil {
  return err
  }