diff --git a/.golangci.yml b/.golangci.yml index 3a177ded..b92e4ac5 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -178,6 +178,10 @@ issues: linters: - funlen text: "Function 'TestCombinations'" + - path: internal/tests/vppinit_test.go + linters: + - funlen + text: "Function 'Test_VppInit_AfPacket'" - path: internal/tests/suite_setup_test.go linters: - funlen diff --git a/go.mod b/go.mod index 314a27c3..a872413d 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/networkservicemesh/govpp v0.0.0-20231216233431-9c82b2963820 github.com/networkservicemesh/sdk v0.5.1-0.20231221125318-565471f17fe8 github.com/networkservicemesh/sdk-k8s v0.0.0-20231221125810-601e221e34ba + github.com/networkservicemesh/sdk-kernel v0.0.0-20231212103506-9e8fd0d59366 github.com/networkservicemesh/sdk-sriov v0.0.0-20231212103852-8e2faf058288 github.com/networkservicemesh/sdk-vpp v0.0.0-20231211111023-cc2f396d9bcc github.com/networkservicemesh/vpphelper v0.0.0-20230901145133-a14aecebd1cb @@ -58,7 +59,6 @@ require ( github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/lunixbochs/struc v0.0.0-20200521075829-a4cb8d33dbbe // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/networkservicemesh/sdk-kernel v0.0.0-20231212103506-9e8fd0d59366 // indirect github.com/open-policy-agent/opa v0.44.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.17.0 // indirect diff --git a/internal/imports/imports_linux.go b/internal/imports/imports_linux.go index 1f66a73b..2f327ba1 100644 --- a/internal/imports/imports_linux.go +++ b/internal/imports/imports_linux.go @@ -38,6 +38,7 @@ import ( _ "github.com/networkservicemesh/govpp/binapi/ip_types" _ "github.com/networkservicemesh/sdk-k8s/pkg/tools/deviceplugin" _ "github.com/networkservicemesh/sdk-k8s/pkg/tools/podresources" + _ "github.com/networkservicemesh/sdk-kernel/pkg/kernel/tools/nshandle" _ "github.com/networkservicemesh/sdk-sriov/pkg/networkservice/chains/forwarder" _ "github.com/networkservicemesh/sdk-sriov/pkg/networkservice/common/resourcepool" _ "github.com/networkservicemesh/sdk-sriov/pkg/sriov/config" @@ -103,9 +104,11 @@ import ( _ "github.com/thanhpk/randstr" _ "github.com/vishvananda/netlink" _ "github.com/vishvananda/netns" + _ "go.fd.io/govpp/adapter/statsclient" _ "go.fd.io/govpp/api" _ "go.fd.io/govpp/binapi/vlib" _ "go.fd.io/govpp/binapi/vpe" + _ "go.fd.io/govpp/core" _ "golang.org/x/text/cases" _ "golang.org/x/text/language" _ "google.golang.org/grpc" diff --git a/internal/tests/suite_setup_test.go b/internal/tests/suite_setup_test.go index 7ca36451..b0d264bd 100644 --- a/internal/tests/suite_setup_test.go +++ b/internal/tests/suite_setup_test.go @@ -73,13 +73,15 @@ func (f *ForwarderTestSuite) SetupSuite() { // ******************************************************************************** log.FromContext(f.ctx).Infof("Creating test bridge (time since start: %s)", time.Since(starttime)) // ******************************************************************************** - f.Require().NoError(SetupBridge()) + bridgeCancel, err := SetupBridge() + f.Require().NoError(err) + f.bridgeCancel = bridgeCancel // ******************************************************************************** log.FromContext(f.ctx).Infof("Creating test vpp Server (time since start: %s)", time.Since(starttime)) // ******************************************************************************** f.vppServerConn, f.vppServerRoot, f.vppServerErrCh = f.createVpp(f.ctx, "vpp-server") - _, err := vppinit.LinkToSocket(f.ctx, f.vppServerConn, net.ParseIP(serverIP), vppinit.AfPacket) + _, err = vppinit.LinkToSocket(f.ctx, f.vppServerConn, net.ParseIP(serverIP), vppinit.AfPacket) f.Require().NoError(err) // ******************************************************************************** @@ -117,8 +119,7 @@ func (f *ForwarderTestSuite) SetupSuite() { // ******************************************************************************** log.FromContext(f.ctx).Infof("Running system under test (SUT) (time since start: %s)", time.Since(starttime)) // ******************************************************************************** - cmdStr := "forwarder" - f.sutErrCh = exechelper.Start(cmdStr, + f.sutErrCh = exechelper.Start(forwarderName, exechelper.WithContext(f.ctx), exechelper.WithEnvirons(append(os.Environ(), "NSM_REGISTRY_CLIENT_POLICIES=\"\"")...), exechelper.WithStdout(os.Stdout), @@ -221,6 +222,7 @@ func (f *ForwarderTestSuite) createVpp(ctx context.Context, name string) (vppCon func (f *ForwarderTestSuite) TearDownSuite() { f.cancel() + f.bridgeCancel() for { _, ok := <-f.sutErrCh if !ok { diff --git a/internal/tests/suite_test.go b/internal/tests/suite_test.go index 07fb9df6..629cdc8f 100644 --- a/internal/tests/suite_test.go +++ b/internal/tests/suite_test.go @@ -40,6 +40,8 @@ type ForwarderTestSuite struct { cancel context.CancelFunc config config.Config + bridgeCancel func() + sutErrCh <-chan error sutCC grpc.ClientConnInterface diff --git a/internal/tests/suite_util_test.go b/internal/tests/suite_util_test.go index f8b989f7..e078f0a7 100644 --- a/internal/tests/suite_util_test.go +++ b/internal/tests/suite_util_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2022 Cisco and/or its affiliates. +// Copyright (c) 2020-2023 Cisco and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 // @@ -36,6 +36,8 @@ const ( forwarderIP = "10.0.2.1" clientIP = "10.0.2.2" serverIP = "10.0.2.3" + + forwarderName = "forwarder" ) func (f *ForwarderTestSuite) ListenAndServe(ctx context.Context, listenOn *url.URL, server *grpc.Server) <-chan error { @@ -58,16 +60,16 @@ func (f *ForwarderTestSuite) ListenAndServe(ctx context.Context, listenOn *url.U return returnErrCh } -func SetupBridge() error { +func SetupBridge() (cancelFn func(), err error) { la := netlink.NewLinkAttrs() la.Name = "bridge" bridge := &netlink.Bridge{LinkAttrs: la} - err := netlink.LinkAdd(bridge) + err = netlink.LinkAdd(bridge) if err != nil { - return errors.Wrapf(err, "could not add %s: %v", la.Name, err) + return nil, errors.Wrapf(err, "could not add %s: %v", la.Name, err) } if err := netlink.LinkSetUp(bridge); err != nil { - return errors.Wrapf(err, "failure creating bridge") + return nil, errors.Wrapf(err, "failure creating bridge") } ifaceMap := map[string]*net.IPNet{ @@ -75,6 +77,8 @@ func SetupBridge() error { "client": {IP: net.ParseIP(clientIP), Mask: net.CIDRMask(24, 32)}, "server": {IP: net.ParseIP(serverIP), Mask: net.CIDRMask(24, 32)}, } + + var linkList []netlink.Link for ifaceName, netIP := range ifaceMap { la := netlink.NewLinkAttrs() la.Name = ifaceName @@ -83,25 +87,31 @@ func SetupBridge() error { PeerName: la.Name + "-veth", } if err := netlink.LinkAdd(l); err != nil { - return errors.Wrapf(err, "unable to create link %s", l.PeerName) + return nil, errors.Wrapf(err, "unable to create link %s", l.PeerName) } peer, err := netlink.LinkByName(l.PeerName) if err != nil { - return errors.Wrapf(err, "unable to get link %s", l.PeerName) + return nil, errors.Wrapf(err, "unable to get link %s", l.PeerName) } if err := netlink.LinkSetUp(l); err != nil { - return errors.Wrapf(err, "unable to up link %s", l.Attrs().Name) + return nil, errors.Wrapf(err, "unable to up link %s", l.Attrs().Name) } if err := netlink.LinkSetUp(peer); err != nil { - return errors.Wrapf(err, "unable to up link %s", peer.Attrs().Name) + return nil, errors.Wrapf(err, "unable to up link %s", peer.Attrs().Name) } if err := netlink.AddrAdd(l, &netlink.Addr{IPNet: netIP}); err != nil { - return errors.Wrapf(err, "unable to add address %s to link %s", netIP, l.Attrs().Name) + return nil, errors.Wrapf(err, "unable to add address %s to link %s", netIP, l.Attrs().Name) } if err := netlink.LinkSetMaster(peer, bridge); err != nil { - return errors.Wrapf(err, "unable to add link %s to bridge %s", peer.Attrs().Name, bridge.LinkAttrs.Name) + return nil, errors.Wrapf(err, "unable to add link %s to bridge %s", peer.Attrs().Name, bridge.LinkAttrs.Name) } + linkList = append(linkList, l) } - return nil + return func() { + for _, l := range linkList { + _ = netlink.LinkDel(l) + } + _ = netlink.LinkDel(bridge) + }, nil } diff --git a/internal/tests/vppinit_test.go b/internal/tests/vppinit_test.go new file mode 100644 index 00000000..59665721 --- /dev/null +++ b/internal/tests/vppinit_test.go @@ -0,0 +1,244 @@ +// Copyright (c) 2023 Cisco and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tests + +import ( + "context" + "fmt" + "net" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/edwarnicke/exechelper" + "github.com/edwarnicke/grpcfd" + "github.com/pkg/errors" + "github.com/spiffe/go-spiffe/v2/spiffetls/tlsconfig" + "github.com/spiffe/go-spiffe/v2/workloadapi" + "github.com/stretchr/testify/require" + "github.com/vishvananda/netlink" + "github.com/vishvananda/netns" + "go.fd.io/govpp/adapter/statsclient" + "go.fd.io/govpp/api" + "go.fd.io/govpp/core" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + + "github.com/networkservicemesh/api/pkg/api/registry" + "github.com/networkservicemesh/cmd-forwarder-vpp/internal/config" + "github.com/networkservicemesh/sdk-kernel/pkg/kernel/tools/nshandle" + "github.com/networkservicemesh/sdk/pkg/registry/common/memory" + "github.com/networkservicemesh/sdk/pkg/tools/grpcutils" + "github.com/networkservicemesh/sdk/pkg/tools/log" + "github.com/networkservicemesh/sdk/pkg/tools/log/logruslogger" + "github.com/networkservicemesh/sdk/pkg/tools/spire" +) + +// This test shows that the AF_PACKET forwarder interface is capable of receiving data after creation +func Test_VppInit_AfPacket(t *testing.T) { + forwarderIntName := forwarderName + ctx, cancelCtx := context.WithTimeout(context.Background(), time.Second*20) + ctx = log.WithLog(ctx, logruslogger.New(ctx)) + starttime := time.Now() + + // ******************************************************************************** + log.FromContext(ctx).Infof("Creating veth pair and put peer to a different netns (time since start: %s)", time.Since(starttime)) + // ******************************************************************************** + curNSHandle, err := nshandle.Current() + require.NoError(t, err) + + peerNS, err := netns.New() + require.NoError(t, err) + defer func() { _ = peerNS.Close() }() + + err = netns.Set(curNSHandle) + require.NoError(t, err) + + vethCancel, err := setupVeth(forwarderIntName, peerNS) + require.NoError(t, err) + defer vethCancel() + + // ******************************************************************************** + log.FromContext(ctx).Infof("Getting Config from Env (time since start: %s)", time.Since(starttime)) + // ******************************************************************************** + var cfg config.Config + _ = os.Setenv("NSM_TUNNEL_IP", forwarderIP) + _ = os.Setenv("NSM_VPP_INIT", "AF_PACKET") + require.NoError(t, cfg.Process()) + + // ******************************************************************************** + log.FromContext(ctx).Infof("Flooding the interface that will be used by forwarder (time since start: %s)", time.Since(starttime)) + // ******************************************************************************** + defer cancelCtx() + _ = nshandle.RunIn(curNSHandle, peerNS, func() error { + cmdStr := fmt.Sprintf("ping %s -f", forwarderIP) + _ = exechelper.Start(cmdStr, + exechelper.WithContext(ctx), + ) + return nil + }) + + // ******************************************************************************** + log.FromContext(ctx).Infof("Running Spire (time since start: %s)", time.Since(starttime)) + // ******************************************************************************** + executable, err := os.Executable() + require.NoError(t, err) + spireErrCh := spire.Start( + spire.WithContext(ctx), + spire.WithEntry("spiffe://example.org/forwarder", "unix:path:/usr/bin/forwarder"), + spire.WithEntry(fmt.Sprintf("spiffe://example.org/%s", filepath.Base(executable)), + fmt.Sprintf("unix:path:%s", executable), + ), + ) + require.Len(t, spireErrCh, 0) + + // ******************************************************************************** + log.FromContext(ctx).Infof("Running forwarder app (time since start: %s)", time.Since(starttime)) + // ******************************************************************************** + cmdStr := forwarderName + sutErrCh := exechelper.Start(cmdStr, + exechelper.WithContext(ctx), + exechelper.WithEnvirons(append(os.Environ(), "NSM_REGISTRY_CLIENT_POLICIES=\"\"")...), + exechelper.WithStdout(os.Stdout), + exechelper.WithStderr(os.Stderr), + exechelper.WithGracePeriod(30*time.Second), + ) + require.Len(t, sutErrCh, 0) + + source, err := workloadapi.NewX509Source(ctx) + x509source := source + x509bundle := source + require.NoError(t, err) + svid, err := x509source.GetX509SVID() + require.NoError(t, err, "error getting x509 svid") + log.FromContext(ctx).Infof("SVID: %q received (time since start: %s)", svid.ID, time.Since(starttime)) + + registryServer := memory.NewNetworkServiceEndpointRegistryServer() + serverCreds := credentials.NewTLS(tlsconfig.MTLSServerConfig(x509source, x509bundle, tlsconfig.AuthorizeAny())) + serverCreds = grpcfd.TransportCredentials(serverCreds) + server := grpc.NewServer(grpc.Creds(serverCreds)) + registry.RegisterNetworkServiceEndpointRegistryServer(server, registryServer) + + errCh := grpcutils.ListenAndServe(ctx, &cfg.ConnectTo, server) + select { + case err = <-errCh: + require.NoError(t, err) + default: + } + + // ******************************************************************************** + log.FromContext(ctx).Infof("Getting forwarder interface statistic (time since start: %s)", time.Since(starttime)) + // ******************************************************************************** + statsConn, err := core.ConnectStats(statsclient.NewStatsClient("")) + require.NoError(t, err) + defer statsConn.Disconnect() + + packetsReceived := false + defer func() { require.True(t, packetsReceived) }() + + // Сheck until we receive packets or ctx.Done() + for stats := new(api.InterfaceStats); len(stats.Interfaces) < 2 || !packetsReceived; time.Sleep(time.Millisecond * 100) { + select { + case <-ctx.Done(): + return + default: + } + if e := statsConn.GetInterfaceStats(stats); e != nil { + log.FromContext(ctx).Errorf("getting interface stats failed:", e) + continue + } + + for idx := range stats.Interfaces { + iface := &stats.Interfaces[idx] + if !strings.Contains(iface.InterfaceName, forwarderIntName) { + continue + } + if iface.Rx.Packets > 0 { + packetsReceived = true + return + } + break + } + } +} + +func setupVeth(forwarderIntName string, peerNS netns.NsHandle) (cancelVeth func(), err error) { + fwdAddr := &net.IPNet{ + IP: net.ParseIP(forwarderIP), Mask: net.CIDRMask(24, 32), + } + peerAddr := &net.IPNet{ + IP: net.ParseIP(clientIP), Mask: net.CIDRMask(24, 32), + } + + la := netlink.NewLinkAttrs() + la.Name = forwarderIntName + l := &netlink.Veth{ + LinkAttrs: la, + PeerName: la.Name + "-peer", + } + if err = netlink.LinkAdd(l); err != nil { + return nil, errors.Wrapf(err, "unable to create link %s", l.PeerName) + } + + if err = netlink.LinkSetUp(l); err != nil { + return nil, errors.Wrapf(err, "unable to up link %s", l.Attrs().Name) + } + + if err = netlink.AddrAdd(l, &netlink.Addr{IPNet: fwdAddr}); err != nil { + return nil, errors.Wrapf(err, "unable to add address %s to link %s", fwdAddr, l.Attrs().Name) + } + + peer, err := netlink.LinkByName(l.PeerName) + if err != nil { + return nil, errors.Wrapf(err, "unable to get link %s", l.PeerName) + } + + if err = netlink.LinkSetNsFd(peer, int(peerNS)); err != nil { + return nil, errors.Wrapf(err, "unable to set peer netns") + } + + curNSHandle, err := nshandle.Current() + if err != nil { + return nil, err + } + defer func() { _ = curNSHandle.Close() }() + + peerNSHandle, err := netlink.NewHandleAtFrom(peerNS, curNSHandle) + if err != nil { + return nil, err + } + defer peerNSHandle.Close() + + peer, err = peerNSHandle.LinkByName(l.PeerName) + if err != nil { + return nil, errors.Wrapf(err, "unable to get peer %s", peer.Attrs().Name) + } + + if err := peerNSHandle.AddrAdd(peer, &netlink.Addr{IPNet: peerAddr}); err != nil { + return nil, errors.Wrapf(err, "unable to add address %s to peer %s", peerAddr, peer.Attrs().Name) + } + + if err := peerNSHandle.LinkSetUp(peer); err != nil { + return nil, errors.Wrapf(err, "unable to up link %s", peer.Attrs().Name) + } + + return func() { + _ = netlink.LinkDel(l) + }, nil +}