From 055f0a1d8d2b7b3905dbe8a232519694b5a03b0e Mon Sep 17 00:00:00 2001 From: Laszlo Kiraly Date: Tue, 19 Oct 2021 16:31:57 +0200 Subject: [PATCH] Add vlan remote mechanism support (see networkservicemesh/sdk-vpp#399) Fixed search for link by PCI address in pci devices subdirectories Added code for procssing ethernet context set by remote vlan mechanism Signed-off-by: Laszlo Kiraly --- pkg/kernel/link.go | 33 ++++++++++- .../ethernetcontext/vf_client.go | 11 ++++ .../ethernetcontext/vf_common.go | 55 +++++++++++++++++++ .../ethernetcontext/vf_server.go | 23 +++++++- 4 files changed, 118 insertions(+), 4 deletions(-) diff --git a/pkg/kernel/link.go b/pkg/kernel/link.go index d5f707ac..6890433f 100644 --- a/pkg/kernel/link.go +++ b/pkg/kernel/link.go @@ -200,7 +200,6 @@ func FindHostDevice(pciAddress, name string, namespaces ...netns.NsHandle) (Link } } } - return nil, errors.Errorf("failed to obtain netlink link matching criteria: name=%s or pciAddress=%s", name, pciAddress) } @@ -211,8 +210,9 @@ func searchByPCIAddress(ns netns.NsHandle, name, pciAddress string) (netlink.Lin return nil, errors.Errorf("failed to enter namespace: %s", err) } - netDir := filepath.Join("/sys/bus/pci/devices", pciAddress, "net") - if _, err = os.Lstat(netDir); err != nil { + pciDevicePath := filepath.Join("/sys/bus/pci/devices", pciAddress) + netDir, err := findNetDir(pciDevicePath) + if err != nil { return nil, errors.Errorf("no net directory under pci device %s: %q", pciAddress, err) } @@ -238,6 +238,33 @@ func searchByPCIAddress(ns netns.NsHandle, name, pciAddress string) (netlink.Lin return link, nil } +func findNetDir(basePath string) (string, error) { + subDir := filepath.Join(basePath, "net") + if _, err := os.Lstat(subDir); err == nil { + return subDir, nil + } + files, err := ioutil.ReadDir(basePath) + if err != nil { + return "", errors.Wrapf(err, "failed to read directory %s", basePath) + } + for _, file := range files { + if file.IsDir() { + subDir = filepath.Join(basePath, file.Name()) + subdirFiles, err := ioutil.ReadDir(subDir) + if err != nil { + return "", errors.Wrapf(err, "failed to read subdirectory %s", subDir) + } + for _, subdirFile := range subdirFiles { + if subdirFile.IsDir() && subdirFile.Name() == "net" { + subDir = filepath.Join(subDir, subdirFile.Name()) + return subDir, nil + } + } + } + } + return "", errors.Errorf("failed to find net directory") +} + func searchByName(ns netns.NsHandle, name, pciAddress string) (netlink.Link, error) { // execute in context of the pod's namespace err := netns.Set(ns) diff --git a/pkg/kernel/networkservice/ethernetcontext/vf_client.go b/pkg/kernel/networkservice/ethernetcontext/vf_client.go index 629b3a7e..26b144fb 100644 --- a/pkg/kernel/networkservice/ethernetcontext/vf_client.go +++ b/pkg/kernel/networkservice/ethernetcontext/vf_client.go @@ -56,6 +56,17 @@ func (i *vfEthernetClient) Request(ctx context.Context, request *networkservice. err = errors.Wrapf(err, "connection closed with error: %s", closeErr.Error()) } + return nil, err + } + } else { + if err := setKernelHwAddress(ctx, conn, true); err != nil { + closeCtx, cancelClose := postponeCtxFunc() + defer cancelClose() + + if _, closeErr := i.Close(closeCtx, conn); closeErr != nil { + err = errors.Wrapf(err, "connection closed with error: %s", closeErr.Error()) + } + return nil, err } } diff --git a/pkg/kernel/networkservice/ethernetcontext/vf_common.go b/pkg/kernel/networkservice/ethernetcontext/vf_common.go index 761d107c..b5748336 100644 --- a/pkg/kernel/networkservice/ethernetcontext/vf_common.go +++ b/pkg/kernel/networkservice/ethernetcontext/vf_common.go @@ -17,15 +17,70 @@ package ethernetcontext import ( + "context" "net" + "time" "github.com/networkservicemesh/api/pkg/api/networkservice" + "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/kernel" + "github.com/networkservicemesh/sdk/pkg/tools/log" "github.com/pkg/errors" "github.com/vishvananda/netlink" + link "github.com/networkservicemesh/sdk-kernel/pkg/kernel" "github.com/networkservicemesh/sdk-kernel/pkg/kernel/networkservice/vfconfig" ) +func setKernelHwAddress(ctx context.Context, conn *networkservice.Connection, isClient bool) error { + if mechanism := kernel.ToMechanism(conn.GetMechanism()); mechanism != nil { + netlinkHandle, err := link.GetNetlinkHandle(mechanism.GetNetNSURL()) + if err != nil { + return errors.WithStack(err) + } + defer netlinkHandle.Delete() + + ifName := mechanism.GetInterfaceName() + + l, err := netlinkHandle.LinkByName(ifName) + if err != nil { + return errors.WithStack(err) + } + + if ethernetContext := conn.GetContext().GetEthernetContext(); ethernetContext != nil { + var macAddrString string + if isClient { + macAddrString = ethernetContext.GetDstMac() + } else { + macAddrString = ethernetContext.GetSrcMac() + } + + if macAddrString != "" { + now := time.Now() + var macAddr net.HardwareAddr + macAddr, err := net.ParseMAC(macAddrString) + if err != nil { + return errors.Wrapf(err, "invalid MAC address: %v", macAddrString) + } + if err = netlinkHandle.LinkSetDown(l); err != nil { + return errors.WithStack(err) + } + if err = netlinkHandle.LinkSetHardwareAddr(l, macAddr); err != nil { + return errors.Wrapf(err, "failed to set MAC address for the VF: %v", macAddr) + } + if err = netlinkHandle.LinkSetUp(l); err != nil { + return errors.WithStack(err) + } + log.FromContext(ctx). + WithField("link.Name", l.Attrs().Name). + WithField("MACAddr", macAddrString). + WithField("duration", time.Since(now)). + WithField("netlink", "LinkSetHardwareAddr").Debug("completed") + } + } + } + return nil +} + func vfCreate(vfConfig *vfconfig.VFConfig, conn *networkservice.Connection, isClient bool) error { pfLink, err := netlink.LinkByName(vfConfig.PFInterfaceName) if err != nil { diff --git a/pkg/kernel/networkservice/ethernetcontext/vf_server.go b/pkg/kernel/networkservice/ethernetcontext/vf_server.go index e65d2bbf..a4efc5e3 100644 --- a/pkg/kernel/networkservice/ethernetcontext/vf_server.go +++ b/pkg/kernel/networkservice/ethernetcontext/vf_server.go @@ -23,9 +23,11 @@ import ( "context" "github.com/golang/protobuf/ptypes/empty" + "github.com/pkg/errors" "github.com/networkservicemesh/api/pkg/api/networkservice" "github.com/networkservicemesh/sdk/pkg/networkservice/core/next" + "github.com/networkservicemesh/sdk/pkg/tools/postpone" "github.com/networkservicemesh/sdk-kernel/pkg/kernel/networkservice/vfconfig" ) @@ -38,13 +40,32 @@ func NewVFServer() networkservice.NetworkServiceServer { } func (s *vfEthernetContextServer) Request(ctx context.Context, request *networkservice.NetworkServiceRequest) (*networkservice.Connection, error) { + postponeCtxFunc := postpone.ContextWithValues(ctx) if vfConfig, ok := vfconfig.Load(ctx, false); ok { err := vfCreate(vfConfig, request.Connection, false) if err != nil { return nil, err } } - return next.Server(ctx).Request(ctx, request) + + conn, err := next.Server(ctx).Request(ctx, request) + if err != nil { + return nil, err + } + + if _, ok := vfconfig.Load(ctx, false); !ok { + if err := setKernelHwAddress(ctx, conn, false); err != nil { + closeCtx, cancelClose := postponeCtxFunc() + defer cancelClose() + + if _, closeErr := s.Close(closeCtx, conn); closeErr != nil { + err = errors.Wrapf(err, "connection closed with error: %s", closeErr.Error()) + } + + return nil, err + } + } + return conn, nil } func (s *vfEthernetContextServer) Close(ctx context.Context, conn *networkservice.Connection) (*empty.Empty, error) {