qfix: Add fixes after manual vl3 testing on kind

pkg/networkservice/loopback/client.go

@@ -0,0 +1,68 @@
+// Copyright (c) 2022 Cisco and/or its affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package loopback
+
+import (
+	"context"
+
+	"google.golang.org/grpc"
+
+	"git.fd.io/govpp.git/api"
+	"github.com/golang/protobuf/ptypes/empty"
+
+	"github.com/networkservicemesh/api/pkg/api/networkservice"
+	"github.com/networkservicemesh/sdk/pkg/networkservice/core/next"
+	"github.com/networkservicemesh/sdk/pkg/networkservice/utils/metadata"
+)
+
+type loopbackClient struct {
+	vppConn api.Connection
+
+	loopbacks *Map
+}
+
+// NewClient creates a NetworkServiceClient chain element to create the loopback vpp-interface
+func NewClient(vppConn api.Connection, opts ...Option) networkservice.NetworkServiceClient {
+	o := &options{
+		loopbacks: NewMap(),
+	}
+	for _, opt := range opts {
+		opt(o)
+	}
+
+	return &loopbackClient{
+		vppConn:   vppConn,
+		loopbacks: o.loopbacks,
+	}
+}
+
+func (l *loopbackClient) Request(ctx context.Context, request *networkservice.NetworkServiceRequest, opts ...grpc.CallOption) (*networkservice.Connection, error) {
+	networkService := request.GetConnection().NetworkService
+	if err := createLoopback(ctx, l.vppConn, networkService, l.loopbacks, metadata.IsClient(l)); err != nil {
+		return nil, err
+	}
+	conn, err := next.Client(ctx).Request(ctx, request, opts...)
+	if err != nil {
+		del(ctx, l.vppConn, networkService, l.loopbacks, metadata.IsClient(l))
+	}
+	return conn, err
+}
+
+func (l *loopbackClient) Close(ctx context.Context, conn *networkservice.Connection, opts ...grpc.CallOption) (*empty.Empty, error) {
+	del(ctx, l.vppConn, conn.NetworkService, l.loopbacks, metadata.IsClient(l))
+	return next.Client(ctx).Close(ctx, conn, opts...)
+}

pkg/networkservice/loopback/common.go

@@ -25,25 +25,14 @@ import (
 	"git.fd.io/govpp.git/api"
 	interfaces "github.com/edwarnicke/govpp/binapi/interface"
 	"github.com/edwarnicke/govpp/binapi/interface_types"
-	"github.com/edwarnicke/serialize"
 
 	"github.com/networkservicemesh/sdk/pkg/tools/log"
 )
 
-type loopInfo struct {
-	swIfIndex interface_types.InterfaceIndex
-	count     uint32
-}
-
-type loopbackMap struct {
-	serialize.Executor
-	entries map[string]*loopInfo
-}
-
 /* Create loopback interface and store it in metadata */
-func createLoopback(ctx context.Context, vppConn api.Connection, networkService string, t *loopbackMap, isClient bool) (err error) {
+func createLoopback(ctx context.Context, vppConn api.Connection, networkService string, t *Map, isClient bool) (err error) {
 	if _, ok := Load(ctx, isClient); !ok {
-		<-t.AsyncExec(func() {
+		<-t.exec.AsyncExec(func() {
 			/* Check if we have already created loopback for a given NetworkService previously */
 			info, ok := t.entries[networkService]
 			if !ok {
@@ -77,9 +66,9 @@ func createLoopbackVPP(ctx context.Context, vppConn api.Connection) (interface_t
 	return reply.SwIfIndex, nil
 }
 
-func del(ctx context.Context, vppConn api.Connection, networkService string, t *loopbackMap, isClient bool) {
+func del(ctx context.Context, vppConn api.Connection, networkService string, t *Map, isClient bool) {
 	if swIfIndex, ok := LoadAndDelete(ctx, isClient); ok {
-		t.AsyncExec(func() {
+		t.exec.AsyncExec(func() {
 			t.entries[networkService].count--
 
 			/* If there are no more clients using the loopback - delete it */

pkg/networkservice/loopback/option.go

@@ -0,0 +1,54 @@
+// Copyright (c) 2022 Cisco and/or its affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package loopback
+
+import (
+	"github.com/edwarnicke/govpp/binapi/interface_types"
+	"github.com/edwarnicke/serialize"
+)
+
+type loopInfo struct {
+	swIfIndex interface_types.InterfaceIndex
+	count     uint32
+}
+
+// Map stores loopback swIfIndex by NetworkServiceName
+type Map struct {
+	entries map[string]*loopInfo
+	exec    serialize.Executor
+}
+
+// NewMap creates loopback map
+func NewMap() *Map {
+	return &Map{
+		entries: make(map[string]*loopInfo),
+	}
+}
+
+type options struct {
+	loopbacks *Map
+}
+
+// Option is an option pattern for loopbackClient/Server
+type Option func(o *options)
+
+// WithSharedMap - sets shared loopback map. It may be needed for sharing Map between client and server
+func WithSharedMap(l *Map) Option {
+	return func(o *options) {
+		o.loopbacks = l
+	}
+}

pkg/networkservice/loopback/server.go

@@ -30,16 +30,21 @@ import (
 type loopbackServer struct {
 	vppConn api.Connection
 
-	loopbacks *loopbackMap
+	loopbacks *Map
 }
 
 // NewServer creates a NetworkServiceServer chain element to create the loopback vpp-interface
-func NewServer(vppConn api.Connection) networkservice.NetworkServiceServer {
+func NewServer(vppConn api.Connection, opts ...Option) networkservice.NetworkServiceServer {
+	o := &options{
+		loopbacks: NewMap(),
+	}
+	for _, opt := range opts {
+		opt(o)
+	}
+
 	return &loopbackServer{
-		vppConn: vppConn,
-		loopbacks: &loopbackMap{
-			entries: make(map[string]*loopInfo),
-		},
+		vppConn:   vppConn,
+		loopbacks: o.loopbacks,
 	}
 }
 

pkg/networkservice/vrf/client.go

@@ -0,0 +1,113 @@
+// Copyright (c) 2022 Cisco and/or its affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package vrf
+
+import (
+	"context"
+
+	"github.com/networkservicemesh/sdk/pkg/tools/postpone"
+	"github.com/pkg/errors"
+
+	"git.fd.io/govpp.git/api"
+	"github.com/golang/protobuf/ptypes/empty"
+	"google.golang.org/grpc"
+
+	"github.com/networkservicemesh/api/pkg/api/networkservice"
+
+	"github.com/networkservicemesh/sdk/pkg/networkservice/core/next"
+	"github.com/networkservicemesh/sdk/pkg/networkservice/utils/metadata"
+
+	"github.com/networkservicemesh/sdk-vpp/pkg/tools/ifindex"
+)
+
+type vrfClient struct {
+	vppConn api.Connection
+	loadFn  ifindex.LoadInterfaceFn
+	m       *Map
+}
+
+// NewClient creates a NetworkServiceClient chain element to create the ip table in vpp
+func NewClient(vppConn api.Connection, opts ...Option) networkservice.NetworkServiceClient {
+	o := &options{
+		m:      NewMap(),
+		loadFn: ifindex.Load,
+	}
+
+	for _, opt := range opts {
+		opt(o)
+	}
+
+	return &vrfClient{
+		vppConn: vppConn,
+		m:       o.m,
+		loadFn:  o.loadFn,
+	}
+}
+
+func (v *vrfClient) Request(ctx context.Context, request *networkservice.NetworkServiceRequest, opts ...grpc.CallOption) (*networkservice.Connection, error) {
+	var loadIfaces = []ifindex.LoadInterfaceFn{v.loadFn, ifindex.Load}
+	var networkService = request.GetConnection().GetNetworkService()
+
+	for _, isIPv6 := range []bool{false, true} {
+		t := v.m.ipv4
+		if isIPv6 {
+			t = v.m.ipv6
+		}
+		if _, ok := Load(ctx, metadata.IsClient(v), isIPv6); !ok {
+			vrfID, loaded, err := create(ctx, v.vppConn, networkService, t, isIPv6)
+			if err != nil {
+				return nil, err
+			}
+			Store(ctx, metadata.IsClient(v), isIPv6, vrfID)
+			if loaded {
+				loadIfaces = []ifindex.LoadInterfaceFn{ifindex.Load}
+			}
+		} else {
+			loadIfaces = nil
+		}
+	}
+
+	postponeCtxFunc := postpone.ContextWithValues(ctx)
+
+	conn, err := next.Client(ctx).Request(ctx, request, opts...)
+	if err != nil {
+		delV46(ctx, v.vppConn, v.m, conn.GetNetworkService(), metadata.IsClient(v))
+		return conn, err
+	}
+
+	for _, loadFn := range loadIfaces {
+		if swIfIndex, ok := loadFn(ctx, metadata.IsClient(v)); ok {
+			if attachErr := attach(ctx, v.vppConn, swIfIndex, metadata.IsClient(v)); attachErr != nil {
+				closeCtx, cancelClose := postponeCtxFunc()
+				defer cancelClose()
+
+				if _, closeErr := v.Close(closeCtx, conn, opts...); closeErr != nil {
+					attachErr = errors.Wrapf(attachErr, "connection closed with error: %s", closeErr.Error())
+				}
+				return nil, attachErr
+			}
+		}
+	}
+
+	return conn, err
+}
+
+func (v *vrfClient) Close(ctx context.Context, conn *networkservice.Connection, opts ...grpc.CallOption) (*empty.Empty, error) {
+	_, err := next.Client(ctx).Close(ctx, conn, opts...)
+	delV46(ctx, v.vppConn, v.m, conn.GetNetworkService(), metadata.IsClient(v))
+	return &empty.Empty{}, err
+}

pkg/networkservice/vrf/common.go

@@ -30,7 +30,7 @@ import (
 	"github.com/networkservicemesh/sdk/pkg/tools/log"
 )
 
-func loadOrCreate(ctx context.Context, vppConn api.Connection, networkService string, t *vrfMap, isIPv6 bool) (vrfID uint32, loaded bool, err error) {
+func create(ctx context.Context, vppConn api.Connection, networkService string, t *vrfMap, isIPv6 bool) (vtfID uint32, loaded bool, err error) {
 	t.mut.Lock()
 	defer t.mut.Unlock()
 
@@ -127,3 +127,7 @@ func delVPP(ctx context.Context, vppConn api.Connection, vrfID uint32, isIPv6 bo
 		WithField("vppapi", "IPTableAddDel").Debug("completed")
 	return nil
 }
+func delV46(ctx context.Context, vppConn api.Connection, m *Map, networkService string, isClient bool) {
+	del(ctx, vppConn, networkService, m.ipv6, true, isClient)
+	del(ctx, vppConn, networkService, m.ipv4, false, isClient)
+}

pkg/networkservice/vrf/option.go

@@ -31,29 +31,45 @@ type vrfInfo struct {
 }
 
 type vrfMap struct {
-	/* entries - is a map[NetworkServiceName]{vrfId, count} */
 	entries map[string]*vrfInfo
+	mut     sync.Mutex
+}
 
-	/* mutex for entries */
-	mut sync.Mutex
+// Map contains ipv6 and ipv4 vrf entries.
+type Map struct {
+	ipv6 *vrfMap
+	ipv4 *vrfMap
 }
 
-func newMap() *vrfMap {
-	return &vrfMap{
-		entries: make(map[string]*vrfInfo),
+// NewMap creates a new vrf.Map that can be used together with client and server.
+func NewMap() *Map {
+	return &Map{
+		ipv6: &vrfMap{
+			entries: make(map[string]*vrfInfo),
+		},
+		ipv4: &vrfMap{
+			entries: make(map[string]*vrfInfo),
+		},
 	}
 }
 
 type options struct {
+	m      *Map
 	loadFn ifindex.LoadInterfaceFn
 }
 
 // Option is an option pattern for upClient/Server
 type Option func(o *options)
 
-// WithLoadInterfaceFn allows to attach rtf table to custom interface.
-// This option might be useful for example, for a loopback interface.
-func WithLoadInterfaceFn(loadFn ifindex.LoadInterfaceFn) Option {
+// WithSharedMap - sets shared vrfV4 and vrfV6 map.
+func WithSharedMap(v *Map) Option {
+	return func(o *options) {
+		o.m = v
+	}
+}
+
+// WithLoadInterface replaces for for loading iface to attach the vrf table.
+func WithLoadInterface(loadFn ifindex.LoadInterfaceFn) Option {
 	return func(o *options) {
 		o.loadFn = loadFn
 	}