From 7958db6def8cd70ede950e2ce00cd59b0ec29a2f Mon Sep 17 00:00:00 2001 From: Ed Warnicke Date: Mon, 22 Nov 2021 09:09:20 -0600 Subject: [PATCH] Fix for mtu issues with kernelvethpair (#452) This *probably* fixes: https://github.com/networkservicemesh/sdk/issues/1148 The underlying issue was that the end of the vethpair to which VPP was attaching with afpacket was not getting its MTU set correctly. As a result, if an oversized packet was sent over it, it would be fragmented by the kernel to a size that matches the mtu on the end of the veth pair that was in the NSC network namespace. The resulting packet would *still* be to large for the MTU of the end of the vethpair attached to the VPP instance, and would be dropped there. The second fragment, being smaller, would be smaller than the MTU of the end of the vethpair to which VPP was attached with af-packet, and so get through. Signed-off-by: Ed Warnicke --- .../kernel/kernelvethpair/client.go | 3 + .../kernel/kernelvethpair/mtu/client.go | 97 +++++++++++++++++++ .../kernel/kernelvethpair/mtu/common.go | 61 ++++++++++++ .../kernel/kernelvethpair/mtu/doc.go | 20 ++++ .../kernel/kernelvethpair/mtu/server.go | 96 ++++++++++++++++++ .../kernel/kernelvethpair/server.go | 3 + 6 files changed, 280 insertions(+) create mode 100644 pkg/networkservice/mechanisms/kernel/kernelvethpair/mtu/client.go create mode 100644 pkg/networkservice/mechanisms/kernel/kernelvethpair/mtu/common.go create mode 100644 pkg/networkservice/mechanisms/kernel/kernelvethpair/mtu/doc.go create mode 100644 pkg/networkservice/mechanisms/kernel/kernelvethpair/mtu/server.go diff --git a/pkg/networkservice/mechanisms/kernel/kernelvethpair/client.go b/pkg/networkservice/mechanisms/kernel/kernelvethpair/client.go index 453f5350..8fc3271d 100644 --- a/pkg/networkservice/mechanisms/kernel/kernelvethpair/client.go +++ b/pkg/networkservice/mechanisms/kernel/kernelvethpair/client.go @@ -26,6 +26,8 @@ import ( "github.com/pkg/errors" "google.golang.org/grpc" + "github.com/networkservicemesh/sdk-vpp/pkg/networkservice/mechanisms/kernel/kernelvethpair/mtu" + "github.com/networkservicemesh/api/pkg/api/networkservice" "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/cls" "github.com/networkservicemesh/sdk/pkg/networkservice/core/chain" @@ -44,6 +46,7 @@ func NewClient(vppConn api.Connection) networkservice.NetworkServiceClient { return chain.NewNetworkServiceClient( ipneighbor.NewClient(vppConn), afpacket.NewClient(vppConn), + mtu.NewClient(), &kernelVethPairClient{}, ) } diff --git a/pkg/networkservice/mechanisms/kernel/kernelvethpair/mtu/client.go b/pkg/networkservice/mechanisms/kernel/kernelvethpair/mtu/client.go new file mode 100644 index 00000000..5b8df631 --- /dev/null +++ b/pkg/networkservice/mechanisms/kernel/kernelvethpair/mtu/client.go @@ -0,0 +1,97 @@ +// Copyright (c) 2021 Cisco and/or its affiliates. +// +// Copyright (c) 2021 Nordix Foundation. +// +// Copyright (c) 2021 Doc.ai and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build linux + +package mtu + +import ( + "context" + + "github.com/golang/protobuf/ptypes/empty" + "github.com/pkg/errors" + "google.golang.org/grpc" + + "github.com/networkservicemesh/sdk/pkg/networkservice/utils/metadata" + + "github.com/networkservicemesh/api/pkg/api/networkservice" + "github.com/networkservicemesh/sdk/pkg/networkservice/core/next" + "github.com/networkservicemesh/sdk/pkg/tools/log" + "github.com/networkservicemesh/sdk/pkg/tools/postpone" +) + +type mtuClient struct{} + +// NewClient provides a NetworkServiceClient that sets the MTU on a kernel interface +// It sets the MTU on the *kernel* side of an interface leaving the +// Client. Generally only used by privileged Clients like those implementing +// the Cross Connect Network Service for K8s (formerly known as NSM Forwarder). +// Client +// +---------------------------+ +// | | +// | | +// | | +// | | +// | | +// | | +// | | +// | +-------------------+ +// | | mtu.NewClient() +// | | +// | | +// | | +// | | +// | | +// | | +// +---------------------------+ +// +func NewClient() networkservice.NetworkServiceClient { + return &mtuClient{} +} + +func (m *mtuClient) Request(ctx context.Context, request *networkservice.NetworkServiceRequest, opts ...grpc.CallOption) (*networkservice.Connection, error) { + logger := log.FromContext(ctx).WithField("mtuClient", "Request") + + postponeCtxFunc := postpone.ContextWithValues(ctx) + + conn, err := next.Client(ctx).Request(ctx, request, opts...) + if err != nil { + return nil, err + } + + if err := setMTU(ctx, conn, metadata.IsClient(m)); err != nil { + logger.Debugf("about to Close due to error: %s", err.Error()) + + closeCtx, cancelClose := postponeCtxFunc() + defer cancelClose() + + if _, closeErr := m.Close(closeCtx, conn, opts...); closeErr != nil { + err = errors.Wrapf(err, "connection closed with error: %s", closeErr.Error()) + } + + return nil, err + } + + return conn, nil +} + +func (m *mtuClient) Close(ctx context.Context, conn *networkservice.Connection, opts ...grpc.CallOption) (*empty.Empty, error) { + return next.Client(ctx).Close(ctx, conn, opts...) +} diff --git a/pkg/networkservice/mechanisms/kernel/kernelvethpair/mtu/common.go b/pkg/networkservice/mechanisms/kernel/kernelvethpair/mtu/common.go new file mode 100644 index 00000000..b15e1c6e --- /dev/null +++ b/pkg/networkservice/mechanisms/kernel/kernelvethpair/mtu/common.go @@ -0,0 +1,61 @@ +// Copyright (c) 2021 Cisco and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build linux + +package mtu + +import ( + "context" + "time" + + "github.com/pkg/errors" + "github.com/vishvananda/netlink" + + "github.com/networkservicemesh/api/pkg/api/networkservice" + "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/kernel" + "github.com/networkservicemesh/sdk/pkg/tools/log" + + "github.com/networkservicemesh/sdk-vpp/pkg/tools/ifindex" + "github.com/networkservicemesh/sdk-vpp/pkg/tools/peer" +) + +func setMTU(ctx context.Context, conn *networkservice.Connection, isClient bool) error { + if mechanism := kernel.ToMechanism(conn.GetMechanism()); mechanism != nil && mechanism.GetVLAN() == 0 { + mtu := conn.GetContext().GetMTU() + if mtu == 0 { + return nil + } + if _, ok := ifindex.Load(ctx, isClient); ok { + return nil + } + peerLink, ok := peer.Load(ctx, isClient) + if !ok { + return errors.New("peer link not found") + } + now := time.Now() + + if err := netlink.LinkSetMTU(peerLink, int(mtu)); err != nil { + return errors.Wrapf(err, "error attempting to set MTU on link %q to value %q", peerLink.Attrs().Name, mtu) + } + log.FromContext(ctx). + WithField("link.Name", peerLink.Attrs().Name). + WithField("MTU", mtu). + WithField("duration", time.Since(now)). + WithField("netlink", "LinkSetMTU").Debug("completed") + } + return nil +} diff --git a/pkg/networkservice/mechanisms/kernel/kernelvethpair/mtu/doc.go b/pkg/networkservice/mechanisms/kernel/kernelvethpair/mtu/doc.go new file mode 100644 index 00000000..0fbb76ce --- /dev/null +++ b/pkg/networkservice/mechanisms/kernel/kernelvethpair/mtu/doc.go @@ -0,0 +1,20 @@ +// Copyright (c) 2021 Cisco and/or its affiliates. +// +// Copyright (c) 2021 Nordix Foundation. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package mtu provides networkservice chain elements that support setting MTU on the vethpeer connected to VPP interfaces +package mtu diff --git a/pkg/networkservice/mechanisms/kernel/kernelvethpair/mtu/server.go b/pkg/networkservice/mechanisms/kernel/kernelvethpair/mtu/server.go new file mode 100644 index 00000000..5d9ce415 --- /dev/null +++ b/pkg/networkservice/mechanisms/kernel/kernelvethpair/mtu/server.go @@ -0,0 +1,96 @@ +// Copyright (c) 2021 Cisco and/or its affiliates. +// +// Copyright (c) 2021 Nordix Foundation. +// +// Copyright (c) 2021 Doc.ai and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build linux + +package mtu + +import ( + "context" + + "github.com/golang/protobuf/ptypes/empty" + "github.com/pkg/errors" + + "github.com/networkservicemesh/sdk/pkg/networkservice/utils/metadata" + + "github.com/networkservicemesh/api/pkg/api/networkservice" + "github.com/networkservicemesh/sdk/pkg/networkservice/core/next" + "github.com/networkservicemesh/sdk/pkg/tools/log" + "github.com/networkservicemesh/sdk/pkg/tools/postpone" +) + +type mtuServer struct { +} + +// NewServer provides a NetworkServiceServer that sets the MTU on a kernel interface +// It sets the MTU on the *kernel* side of an interface plugged into the +// Endpoint. Generally only used by privileged Endpoints like those implementing +// the Cross Connect Network Service for K8s (formerly known as NSM Forwarder). +// Endpoint +// +---------------------------+ +// | | +// | | +// | | +// | | +// | | +// | | +// | | +// +-------------------+ | +// mtu.NewServer() | | +// | | +// | | +// | | +// | | +// | | +// | | +// +---------------------------+ +// +func NewServer() networkservice.NetworkServiceServer { + return &mtuServer{} +} + +func (m *mtuServer) Request(ctx context.Context, request *networkservice.NetworkServiceRequest) (*networkservice.Connection, error) { + logger := log.FromContext(ctx).WithField("mutServer", "Request") + + postponeCtxFunc := postpone.ContextWithValues(ctx) + + conn, err := next.Server(ctx).Request(ctx, request) + if err != nil { + return nil, err + } + + if err := setMTU(ctx, conn, metadata.IsClient(m)); err != nil { + logger.Debugf("about to Close due to error: %s", err.Error()) + + closeCtx, cancelClose := postponeCtxFunc() + defer cancelClose() + + if _, closeErr := m.Close(closeCtx, conn); closeErr != nil { + err = errors.Wrapf(err, "connection closed with error: %s", closeErr.Error()) + } + + return nil, err + } + return conn, nil +} + +func (m *mtuServer) Close(ctx context.Context, conn *networkservice.Connection) (*empty.Empty, error) { + return next.Server(ctx).Close(ctx, conn) +} diff --git a/pkg/networkservice/mechanisms/kernel/kernelvethpair/server.go b/pkg/networkservice/mechanisms/kernel/kernelvethpair/server.go index db25cf07..b5abd7c8 100644 --- a/pkg/networkservice/mechanisms/kernel/kernelvethpair/server.go +++ b/pkg/networkservice/mechanisms/kernel/kernelvethpair/server.go @@ -24,6 +24,8 @@ import ( "git.fd.io/govpp.git/api" "github.com/golang/protobuf/ptypes/empty" + "github.com/networkservicemesh/sdk-vpp/pkg/networkservice/mechanisms/kernel/kernelvethpair/mtu" + "github.com/networkservicemesh/api/pkg/api/networkservice" "github.com/networkservicemesh/sdk/pkg/networkservice/core/chain" "github.com/networkservicemesh/sdk/pkg/networkservice/core/next" @@ -41,6 +43,7 @@ func NewServer(vppConn api.Connection) networkservice.NetworkServiceServer { return chain.NewNetworkServiceServer( ipneighbor.NewServer(vppConn), afpacket.NewServer(vppConn), + mtu.NewServer(), &kernelVethPairServer{}, ) }