From 557110e62aa141e2d21ad125117a486ea31e11b5 Mon Sep 17 00:00:00 2001 From: Artem Glazychev Date: Fri, 17 Sep 2021 17:38:40 +0700 Subject: [PATCH 1/5] Add loopback chain-element Signed-off-by: Artem Glazychev --- pkg/networkservice/loopback/client.go | 72 +++++++++++++++++ pkg/networkservice/loopback/common.go | 101 ++++++++++++++++++++++++ pkg/networkservice/loopback/doc.go | 18 +++++ pkg/networkservice/loopback/metadata.go | 72 +++++++++++++++++ pkg/networkservice/loopback/option.go | 61 ++++++++++++++ pkg/networkservice/loopback/server.go | 70 ++++++++++++++++ 6 files changed, 394 insertions(+) create mode 100644 pkg/networkservice/loopback/client.go create mode 100644 pkg/networkservice/loopback/common.go create mode 100644 pkg/networkservice/loopback/doc.go create mode 100644 pkg/networkservice/loopback/metadata.go create mode 100644 pkg/networkservice/loopback/option.go create mode 100644 pkg/networkservice/loopback/server.go diff --git a/pkg/networkservice/loopback/client.go b/pkg/networkservice/loopback/client.go new file mode 100644 index 00000000..9c6dcb97 --- /dev/null +++ b/pkg/networkservice/loopback/client.go @@ -0,0 +1,72 @@ +// Copyright (c) 2021 Doc.ai and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package loopback + +import ( + "context" + + "google.golang.org/grpc" + + "git.fd.io/govpp.git/api" + "github.com/golang/protobuf/ptypes/empty" + + "github.com/networkservicemesh/api/pkg/api/networkservice" + "github.com/networkservicemesh/sdk/pkg/networkservice/core/next" + "github.com/networkservicemesh/sdk/pkg/networkservice/utils/metadata" + "github.com/networkservicemesh/sdk/pkg/tools/log" +) + +type loopbackClient struct { + vppConn api.Connection + + loopbacks *LoopMap +} + +// NewClient creates a NetworkServiceClient chain element to create the loopback vpp-interface +func NewClient(vppConn api.Connection, opts ...Option) networkservice.NetworkServiceClient { + o := &options{ + loopbacks: CreateLoopbackMap(), + } + for _, opt := range opts { + opt(o) + } + + return &loopbackClient{ + vppConn: vppConn, + loopbacks: o.loopbacks, + } +} + +func (l *loopbackClient) Request(ctx context.Context, request *networkservice.NetworkServiceRequest, opts ...grpc.CallOption) (*networkservice.Connection, error) { + if err := createLoopback(ctx, request.GetConnection(), l.vppConn, l.loopbacks, metadata.IsClient(l)); err != nil { + return nil, err + } + conn, err := next.Client(ctx).Request(ctx, request, opts...) + if err != nil { + if e := del(ctx, conn, l.vppConn, l.loopbacks, metadata.IsClient(l)); e != nil { + log.FromContext(ctx).Errorf("unable to delete loopback interface: %v", e) + } + } + return conn, err +} + +func (l *loopbackClient) Close(ctx context.Context, conn *networkservice.Connection, opts ...grpc.CallOption) (*empty.Empty, error) { + if err := del(ctx, conn, l.vppConn, l.loopbacks, metadata.IsClient(l)); err != nil { + log.FromContext(ctx).Errorf("unable to delete loopback interface: %v", err) + } + return next.Client(ctx).Close(ctx, conn, opts...) +} diff --git a/pkg/networkservice/loopback/common.go b/pkg/networkservice/loopback/common.go new file mode 100644 index 00000000..e376e091 --- /dev/null +++ b/pkg/networkservice/loopback/common.go @@ -0,0 +1,101 @@ +// Copyright (c) 2021 Doc.ai and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package loopback + +import ( + "context" + "time" + + "github.com/pkg/errors" + + "git.fd.io/govpp.git/api" + interfaces "github.com/edwarnicke/govpp/binapi/interface" + "github.com/edwarnicke/govpp/binapi/interface_types" + + "github.com/networkservicemesh/api/pkg/api/networkservice" + "github.com/networkservicemesh/sdk/pkg/tools/log" +) + +/* Create loopback interface. Returns new swIfIndex if it doesn't exist otherwise swIfIndex from metadata */ +func createLoopback(ctx context.Context, conn *networkservice.Connection, vppConn api.Connection, t *LoopMap, isClient bool) error { + if _, ok := Load(ctx, isClient); !ok { + /* Check if we have already created loopback for a given NetworkService previously */ + t.mut.Lock() + defer t.mut.Unlock() + + info, ok := t.entries[conn.NetworkService] + if !ok { + var err error + swIfIndex, err := createLoopbackVPP(ctx, vppConn) + if err != nil { + return err + } + info = &loopInfo{ + swIfIndex: swIfIndex, + } + t.entries[conn.NetworkService] = info + } + info.count++ + Store(ctx, isClient, info.swIfIndex) + } + return nil +} + +func createLoopbackVPP(ctx context.Context, vppConn api.Connection) (interface_types.InterfaceIndex, error) { + now := time.Now() + reply, err := interfaces.NewServiceClient(vppConn).CreateLoopback(ctx, &interfaces.CreateLoopback{}) + if err != nil { + return interface_types.InterfaceIndex(^uint32(0)), errors.WithStack(err) + } + log.FromContext(ctx). + WithField("swIfIndex", reply.SwIfIndex). + WithField("duration", time.Since(now)). + WithField("vppapi", "CreateLoopback").Debug("completed") + return reply.SwIfIndex, nil +} + +func del(ctx context.Context, conn *networkservice.Connection, vppConn api.Connection, t *LoopMap, isClient bool) error { + if swIfIndex, ok := LoadAndDelete(ctx, isClient); ok { + t.mut.Lock() + defer t.mut.Unlock() + t.entries[conn.NetworkService].count-- + + /* If there are no more clients using the loopback - delete it */ + if t.entries[conn.NetworkService].count == 0 { + delete(t.entries, conn.NetworkService) + if err := delVPP(ctx, vppConn, swIfIndex); err != nil { + return err + } + } + } + return nil +} + +func delVPP(ctx context.Context, vppConn api.Connection, swIfIndex interface_types.InterfaceIndex) error { + now := time.Now() + _, err := interfaces.NewServiceClient(vppConn).DeleteLoopback(ctx, &interfaces.DeleteLoopback{ + SwIfIndex: swIfIndex, + }) + if err != nil { + return errors.WithStack(err) + } + log.FromContext(ctx). + WithField("swIfIndex", swIfIndex). + WithField("duration", time.Since(now)). + WithField("vppapi", "DeleteLoopback").Debug("completed") + return nil +} diff --git a/pkg/networkservice/loopback/doc.go b/pkg/networkservice/loopback/doc.go new file mode 100644 index 00000000..18becd12 --- /dev/null +++ b/pkg/networkservice/loopback/doc.go @@ -0,0 +1,18 @@ +// Copyright (c) 2021 Doc.ai and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package loopback provides networkservice.NetworkService chain elements for creating loopback interface +package loopback diff --git a/pkg/networkservice/loopback/metadata.go b/pkg/networkservice/loopback/metadata.go new file mode 100644 index 00000000..316306dc --- /dev/null +++ b/pkg/networkservice/loopback/metadata.go @@ -0,0 +1,72 @@ +// Copyright (c) 2021 Doc.ai and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package loopback + +import ( + "context" + + "github.com/edwarnicke/govpp/binapi/interface_types" + + "github.com/networkservicemesh/sdk/pkg/networkservice/utils/metadata" +) + +type key struct{} + +// Store sets the loopback swIfIndex stored in per Connection.Id metadata. +func Store(ctx context.Context, isClient bool, swIfIndex interface_types.InterfaceIndex) { + metadata.Map(ctx, isClient).Store(key{}, swIfIndex) +} + +// Delete deletes the swIfIndex stored in per Connection.Id metadata +func Delete(ctx context.Context, isClient bool) { + metadata.Map(ctx, isClient).Delete(key{}) +} + +// Load returns the swIfIndex stored in per Connection.Id metadata, or nil if no +// value is present. +// The ok result indicates whether value was found in the per Connection.Id metadata. +func Load(ctx context.Context, isClient bool) (value interface_types.InterfaceIndex, ok bool) { + rawValue, ok := metadata.Map(ctx, isClient).Load(key{}) + if !ok { + return + } + value, ok = rawValue.(interface_types.InterfaceIndex) + return value, ok +} + +// LoadOrStore returns the existing swIfIndex stored in per Connection.Id metadata if present. +// Otherwise, it stores and returns the given swIfIndex. +// The loaded result is true if the value was loaded, false if stored. +func LoadOrStore(ctx context.Context, isClient bool, swIfIndex interface_types.InterfaceIndex) (value interface_types.InterfaceIndex, ok bool) { + rawValue, ok := metadata.Map(ctx, isClient).LoadOrStore(key{}, swIfIndex) + if !ok { + return + } + value, ok = rawValue.(interface_types.InterfaceIndex) + return value, ok +} + +// LoadAndDelete deletes the swIfIndex stored in per Connection.Id metadata, +// returning the previous value if any. The loaded result reports whether the key was present. +func LoadAndDelete(ctx context.Context, isClient bool) (value interface_types.InterfaceIndex, ok bool) { + rawValue, ok := metadata.Map(ctx, isClient).LoadAndDelete(key{}) + if !ok { + return + } + value, ok = rawValue.(interface_types.InterfaceIndex) + return value, ok +} diff --git a/pkg/networkservice/loopback/option.go b/pkg/networkservice/loopback/option.go new file mode 100644 index 00000000..be35e349 --- /dev/null +++ b/pkg/networkservice/loopback/option.go @@ -0,0 +1,61 @@ +// Copyright (c) 2021 Doc.ai and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package loopback + +import ( + "sync" + + "github.com/edwarnicke/govpp/binapi/interface_types" +) + +type loopInfo struct { + /* loopback swIfIndex */ + swIfIndex interface_types.InterfaceIndex + + /* count - the number of clients using this loopback */ + count uint32 +} + +// LoopMap stores loopback swIfIndex by NetworkServiceName +type LoopMap struct { + /* entries - is a map[NetworkServiceName]{swIfIndex, count} */ + entries map[string]*loopInfo + + /* mutex for entries */ + mut sync.Mutex +} + +// CreateLoopbackMap creates loopback map +func CreateLoopbackMap() *LoopMap { + return &LoopMap{ + entries: make(map[string]*loopInfo), + } +} + +type options struct { + loopbacks *LoopMap +} + +// Option is an option pattern for loopbackClient/Server +type Option func(o *options) + +// WithSharedLoopbackMap - sets shared loopback map. It may be needed for sharing LoopMap between client and server +func WithSharedLoopbackMap(l *LoopMap) Option { + return func(o *options) { + o.loopbacks = l + } +} diff --git a/pkg/networkservice/loopback/server.go b/pkg/networkservice/loopback/server.go new file mode 100644 index 00000000..b870cb67 --- /dev/null +++ b/pkg/networkservice/loopback/server.go @@ -0,0 +1,70 @@ +// Copyright (c) 2021 Doc.ai and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package loopback + +import ( + "context" + + "git.fd.io/govpp.git/api" + "github.com/golang/protobuf/ptypes/empty" + + "github.com/networkservicemesh/api/pkg/api/networkservice" + "github.com/networkservicemesh/sdk/pkg/networkservice/core/next" + "github.com/networkservicemesh/sdk/pkg/networkservice/utils/metadata" + "github.com/networkservicemesh/sdk/pkg/tools/log" +) + +type loopbackServer struct { + vppConn api.Connection + + loopbacks *LoopMap +} + +// NewServer creates a NetworkServiceServer chain element to create the loopback vpp-interface +func NewServer(vppConn api.Connection, opts ...Option) networkservice.NetworkServiceServer { + o := &options{ + loopbacks: CreateLoopbackMap(), + } + for _, opt := range opts { + opt(o) + } + + return &loopbackServer{ + vppConn: vppConn, + loopbacks: o.loopbacks, + } +} + +func (l *loopbackServer) Request(ctx context.Context, request *networkservice.NetworkServiceRequest) (*networkservice.Connection, error) { + if err := createLoopback(ctx, request.GetConnection(), l.vppConn, l.loopbacks, metadata.IsClient(l)); err != nil { + return nil, err + } + conn, err := next.Server(ctx).Request(ctx, request) + if err != nil { + if e := del(ctx, conn, l.vppConn, l.loopbacks, metadata.IsClient(l)); e != nil { + log.FromContext(ctx).Errorf("unable to delete loopback interface: %v", e) + } + } + return conn, err +} + +func (l *loopbackServer) Close(ctx context.Context, conn *networkservice.Connection) (*empty.Empty, error) { + if err := del(ctx, conn, l.vppConn, l.loopbacks, metadata.IsClient(l)); err != nil { + log.FromContext(ctx).Errorf("unable to delete loopback interface: %v", err) + } + return next.Server(ctx).Close(ctx, conn) +} From 796df167ac23bd77e7a715b2b62b5dc6e558c088 Mon Sep 17 00:00:00 2001 From: Artem Glazychev Date: Mon, 6 Dec 2021 12:27:19 +0700 Subject: [PATCH 2/5] Fix nil connection case Signed-off-by: Artem Glazychev --- pkg/networkservice/loopback/client.go | 7 ++++--- pkg/networkservice/loopback/common.go | 17 ++++++++--------- pkg/networkservice/loopback/server.go | 7 ++++--- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/pkg/networkservice/loopback/client.go b/pkg/networkservice/loopback/client.go index 9c6dcb97..f536badc 100644 --- a/pkg/networkservice/loopback/client.go +++ b/pkg/networkservice/loopback/client.go @@ -52,12 +52,13 @@ func NewClient(vppConn api.Connection, opts ...Option) networkservice.NetworkSer } func (l *loopbackClient) Request(ctx context.Context, request *networkservice.NetworkServiceRequest, opts ...grpc.CallOption) (*networkservice.Connection, error) { - if err := createLoopback(ctx, request.GetConnection(), l.vppConn, l.loopbacks, metadata.IsClient(l)); err != nil { + networkService := request.GetConnection().NetworkService + if err := createLoopback(ctx, l.vppConn, networkService, l.loopbacks, metadata.IsClient(l)); err != nil { return nil, err } conn, err := next.Client(ctx).Request(ctx, request, opts...) if err != nil { - if e := del(ctx, conn, l.vppConn, l.loopbacks, metadata.IsClient(l)); e != nil { + if e := del(ctx, l.vppConn, networkService, l.loopbacks, metadata.IsClient(l)); e != nil { log.FromContext(ctx).Errorf("unable to delete loopback interface: %v", e) } } @@ -65,7 +66,7 @@ func (l *loopbackClient) Request(ctx context.Context, request *networkservice.Ne } func (l *loopbackClient) Close(ctx context.Context, conn *networkservice.Connection, opts ...grpc.CallOption) (*empty.Empty, error) { - if err := del(ctx, conn, l.vppConn, l.loopbacks, metadata.IsClient(l)); err != nil { + if err := del(ctx, l.vppConn, conn.NetworkService, l.loopbacks, metadata.IsClient(l)); err != nil { log.FromContext(ctx).Errorf("unable to delete loopback interface: %v", err) } return next.Client(ctx).Close(ctx, conn, opts...) diff --git a/pkg/networkservice/loopback/common.go b/pkg/networkservice/loopback/common.go index e376e091..8e9cfeff 100644 --- a/pkg/networkservice/loopback/common.go +++ b/pkg/networkservice/loopback/common.go @@ -26,18 +26,17 @@ import ( interfaces "github.com/edwarnicke/govpp/binapi/interface" "github.com/edwarnicke/govpp/binapi/interface_types" - "github.com/networkservicemesh/api/pkg/api/networkservice" "github.com/networkservicemesh/sdk/pkg/tools/log" ) -/* Create loopback interface. Returns new swIfIndex if it doesn't exist otherwise swIfIndex from metadata */ -func createLoopback(ctx context.Context, conn *networkservice.Connection, vppConn api.Connection, t *LoopMap, isClient bool) error { +/* Create loopback interface and store it in metadata */ +func createLoopback(ctx context.Context, vppConn api.Connection, networkService string, t *LoopMap, isClient bool) error { if _, ok := Load(ctx, isClient); !ok { /* Check if we have already created loopback for a given NetworkService previously */ t.mut.Lock() defer t.mut.Unlock() - info, ok := t.entries[conn.NetworkService] + info, ok := t.entries[networkService] if !ok { var err error swIfIndex, err := createLoopbackVPP(ctx, vppConn) @@ -47,7 +46,7 @@ func createLoopback(ctx context.Context, conn *networkservice.Connection, vppCon info = &loopInfo{ swIfIndex: swIfIndex, } - t.entries[conn.NetworkService] = info + t.entries[networkService] = info } info.count++ Store(ctx, isClient, info.swIfIndex) @@ -68,15 +67,15 @@ func createLoopbackVPP(ctx context.Context, vppConn api.Connection) (interface_t return reply.SwIfIndex, nil } -func del(ctx context.Context, conn *networkservice.Connection, vppConn api.Connection, t *LoopMap, isClient bool) error { +func del(ctx context.Context, vppConn api.Connection, networkService string, t *LoopMap, isClient bool) error { if swIfIndex, ok := LoadAndDelete(ctx, isClient); ok { t.mut.Lock() defer t.mut.Unlock() - t.entries[conn.NetworkService].count-- + t.entries[networkService].count-- /* If there are no more clients using the loopback - delete it */ - if t.entries[conn.NetworkService].count == 0 { - delete(t.entries, conn.NetworkService) + if t.entries[networkService].count == 0 { + delete(t.entries, networkService) if err := delVPP(ctx, vppConn, swIfIndex); err != nil { return err } diff --git a/pkg/networkservice/loopback/server.go b/pkg/networkservice/loopback/server.go index b870cb67..e9d7fa7e 100644 --- a/pkg/networkservice/loopback/server.go +++ b/pkg/networkservice/loopback/server.go @@ -50,12 +50,13 @@ func NewServer(vppConn api.Connection, opts ...Option) networkservice.NetworkSer } func (l *loopbackServer) Request(ctx context.Context, request *networkservice.NetworkServiceRequest) (*networkservice.Connection, error) { - if err := createLoopback(ctx, request.GetConnection(), l.vppConn, l.loopbacks, metadata.IsClient(l)); err != nil { + networkService := request.GetConnection().NetworkService + if err := createLoopback(ctx, l.vppConn, networkService, l.loopbacks, metadata.IsClient(l)); err != nil { return nil, err } conn, err := next.Server(ctx).Request(ctx, request) if err != nil { - if e := del(ctx, conn, l.vppConn, l.loopbacks, metadata.IsClient(l)); e != nil { + if e := del(ctx, l.vppConn, networkService, l.loopbacks, metadata.IsClient(l)); e != nil { log.FromContext(ctx).Errorf("unable to delete loopback interface: %v", e) } } @@ -63,7 +64,7 @@ func (l *loopbackServer) Request(ctx context.Context, request *networkservice.Ne } func (l *loopbackServer) Close(ctx context.Context, conn *networkservice.Connection) (*empty.Empty, error) { - if err := del(ctx, conn, l.vppConn, l.loopbacks, metadata.IsClient(l)); err != nil { + if err := del(ctx, l.vppConn, conn.NetworkService, l.loopbacks, metadata.IsClient(l)); err != nil { log.FromContext(ctx).Errorf("unable to delete loopback interface: %v", err) } return next.Server(ctx).Close(ctx, conn) From 9dacb127f038511e39a2db2a2671f4a1fc13e2b8 Mon Sep 17 00:00:00 2001 From: Artem Glazychev Date: Mon, 6 Dec 2021 12:31:55 +0700 Subject: [PATCH 3/5] fix after review Signed-off-by: Artem Glazychev --- pkg/networkservice/loopback/client.go | 4 ++-- pkg/networkservice/loopback/common.go | 4 ++-- pkg/networkservice/loopback/option.go | 16 ++++++++-------- pkg/networkservice/loopback/server.go | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pkg/networkservice/loopback/client.go b/pkg/networkservice/loopback/client.go index f536badc..af96d567 100644 --- a/pkg/networkservice/loopback/client.go +++ b/pkg/networkservice/loopback/client.go @@ -33,13 +33,13 @@ import ( type loopbackClient struct { vppConn api.Connection - loopbacks *LoopMap + loopbacks *Map } // NewClient creates a NetworkServiceClient chain element to create the loopback vpp-interface func NewClient(vppConn api.Connection, opts ...Option) networkservice.NetworkServiceClient { o := &options{ - loopbacks: CreateLoopbackMap(), + loopbacks: NewMap(), } for _, opt := range opts { opt(o) diff --git a/pkg/networkservice/loopback/common.go b/pkg/networkservice/loopback/common.go index 8e9cfeff..2dfe7763 100644 --- a/pkg/networkservice/loopback/common.go +++ b/pkg/networkservice/loopback/common.go @@ -30,7 +30,7 @@ import ( ) /* Create loopback interface and store it in metadata */ -func createLoopback(ctx context.Context, vppConn api.Connection, networkService string, t *LoopMap, isClient bool) error { +func createLoopback(ctx context.Context, vppConn api.Connection, networkService string, t *Map, isClient bool) error { if _, ok := Load(ctx, isClient); !ok { /* Check if we have already created loopback for a given NetworkService previously */ t.mut.Lock() @@ -67,7 +67,7 @@ func createLoopbackVPP(ctx context.Context, vppConn api.Connection) (interface_t return reply.SwIfIndex, nil } -func del(ctx context.Context, vppConn api.Connection, networkService string, t *LoopMap, isClient bool) error { +func del(ctx context.Context, vppConn api.Connection, networkService string, t *Map, isClient bool) error { if swIfIndex, ok := LoadAndDelete(ctx, isClient); ok { t.mut.Lock() defer t.mut.Unlock() diff --git a/pkg/networkservice/loopback/option.go b/pkg/networkservice/loopback/option.go index be35e349..537da798 100644 --- a/pkg/networkservice/loopback/option.go +++ b/pkg/networkservice/loopback/option.go @@ -30,8 +30,8 @@ type loopInfo struct { count uint32 } -// LoopMap stores loopback swIfIndex by NetworkServiceName -type LoopMap struct { +// Map stores loopback swIfIndex by NetworkServiceName +type Map struct { /* entries - is a map[NetworkServiceName]{swIfIndex, count} */ entries map[string]*loopInfo @@ -39,22 +39,22 @@ type LoopMap struct { mut sync.Mutex } -// CreateLoopbackMap creates loopback map -func CreateLoopbackMap() *LoopMap { - return &LoopMap{ +// NewMap creates loopback map +func NewMap() *Map { + return &Map{ entries: make(map[string]*loopInfo), } } type options struct { - loopbacks *LoopMap + loopbacks *Map } // Option is an option pattern for loopbackClient/Server type Option func(o *options) -// WithSharedLoopbackMap - sets shared loopback map. It may be needed for sharing LoopMap between client and server -func WithSharedLoopbackMap(l *LoopMap) Option { +// WithSharedMap - sets shared loopback map. It may be needed for sharing Map between client and server +func WithSharedMap(l *Map) Option { return func(o *options) { o.loopbacks = l } diff --git a/pkg/networkservice/loopback/server.go b/pkg/networkservice/loopback/server.go index e9d7fa7e..2666b1fd 100644 --- a/pkg/networkservice/loopback/server.go +++ b/pkg/networkservice/loopback/server.go @@ -31,13 +31,13 @@ import ( type loopbackServer struct { vppConn api.Connection - loopbacks *LoopMap + loopbacks *Map } // NewServer creates a NetworkServiceServer chain element to create the loopback vpp-interface func NewServer(vppConn api.Connection, opts ...Option) networkservice.NetworkServiceServer { o := &options{ - loopbacks: CreateLoopbackMap(), + loopbacks: NewMap(), } for _, opt := range opts { opt(o) From 555c912777523fe2f433db0b025bf1c4f06ca943 Mon Sep 17 00:00:00 2001 From: Artem Glazychev Date: Tue, 21 Dec 2021 20:23:10 +0700 Subject: [PATCH 4/5] Replace mutex with executor Signed-off-by: Artem Glazychev --- go.mod | 1 + pkg/networkservice/loopback/client.go | 9 +---- pkg/networkservice/loopback/common.go | 58 +++++++++++++-------------- pkg/networkservice/loopback/option.go | 7 ++-- pkg/networkservice/loopback/server.go | 9 +---- 5 files changed, 36 insertions(+), 48 deletions(-) diff --git a/go.mod b/go.mod index 918437ab..7d2330fd 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.16 require ( git.fd.io/govpp.git v0.3.6-0.20210927044411-385ccc0d8ba9 github.com/edwarnicke/govpp v0.0.0-20220311182453-f32f292e0e91 + github.com/edwarnicke/serialize v1.0.7 github.com/golang/protobuf v1.5.2 github.com/hashicorp/go-multierror v1.1.1 github.com/networkservicemesh/api v1.2.1-0.20220315001249-f33f8c3f2feb diff --git a/pkg/networkservice/loopback/client.go b/pkg/networkservice/loopback/client.go index af96d567..6d523d53 100644 --- a/pkg/networkservice/loopback/client.go +++ b/pkg/networkservice/loopback/client.go @@ -27,7 +27,6 @@ import ( "github.com/networkservicemesh/api/pkg/api/networkservice" "github.com/networkservicemesh/sdk/pkg/networkservice/core/next" "github.com/networkservicemesh/sdk/pkg/networkservice/utils/metadata" - "github.com/networkservicemesh/sdk/pkg/tools/log" ) type loopbackClient struct { @@ -58,16 +57,12 @@ func (l *loopbackClient) Request(ctx context.Context, request *networkservice.Ne } conn, err := next.Client(ctx).Request(ctx, request, opts...) if err != nil { - if e := del(ctx, l.vppConn, networkService, l.loopbacks, metadata.IsClient(l)); e != nil { - log.FromContext(ctx).Errorf("unable to delete loopback interface: %v", e) - } + del(ctx, l.vppConn, networkService, l.loopbacks, metadata.IsClient(l)) } return conn, err } func (l *loopbackClient) Close(ctx context.Context, conn *networkservice.Connection, opts ...grpc.CallOption) (*empty.Empty, error) { - if err := del(ctx, l.vppConn, conn.NetworkService, l.loopbacks, metadata.IsClient(l)); err != nil { - log.FromContext(ctx).Errorf("unable to delete loopback interface: %v", err) - } + del(ctx, l.vppConn, conn.NetworkService, l.loopbacks, metadata.IsClient(l)) return next.Client(ctx).Close(ctx, conn, opts...) } diff --git a/pkg/networkservice/loopback/common.go b/pkg/networkservice/loopback/common.go index 2dfe7763..d818e7ee 100644 --- a/pkg/networkservice/loopback/common.go +++ b/pkg/networkservice/loopback/common.go @@ -30,28 +30,27 @@ import ( ) /* Create loopback interface and store it in metadata */ -func createLoopback(ctx context.Context, vppConn api.Connection, networkService string, t *Map, isClient bool) error { +func createLoopback(ctx context.Context, vppConn api.Connection, networkService string, t *Map, isClient bool) (err error) { if _, ok := Load(ctx, isClient); !ok { - /* Check if we have already created loopback for a given NetworkService previously */ - t.mut.Lock() - defer t.mut.Unlock() - - info, ok := t.entries[networkService] - if !ok { - var err error - swIfIndex, err := createLoopbackVPP(ctx, vppConn) - if err != nil { - return err - } - info = &loopInfo{ - swIfIndex: swIfIndex, + <-t.exec.AsyncExec(func() { + /* Check if we have already created loopback for a given NetworkService previously */ + info, ok := t.entries[networkService] + if !ok { + var swIfIndex interface_types.InterfaceIndex + swIfIndex, err = createLoopbackVPP(ctx, vppConn) + if err != nil { + return + } + info = &loopInfo{ + swIfIndex: swIfIndex, + } + t.entries[networkService] = info } - t.entries[networkService] = info - } - info.count++ - Store(ctx, isClient, info.swIfIndex) + info.count++ + Store(ctx, isClient, info.swIfIndex) + }) } - return nil + return } func createLoopbackVPP(ctx context.Context, vppConn api.Connection) (interface_types.InterfaceIndex, error) { @@ -67,21 +66,20 @@ func createLoopbackVPP(ctx context.Context, vppConn api.Connection) (interface_t return reply.SwIfIndex, nil } -func del(ctx context.Context, vppConn api.Connection, networkService string, t *Map, isClient bool) error { +func del(ctx context.Context, vppConn api.Connection, networkService string, t *Map, isClient bool) { if swIfIndex, ok := LoadAndDelete(ctx, isClient); ok { - t.mut.Lock() - defer t.mut.Unlock() - t.entries[networkService].count-- + t.exec.AsyncExec(func() { + t.entries[networkService].count-- - /* If there are no more clients using the loopback - delete it */ - if t.entries[networkService].count == 0 { - delete(t.entries, networkService) - if err := delVPP(ctx, vppConn, swIfIndex); err != nil { - return err + /* If there are no more clients using the loopback - delete it */ + if t.entries[networkService].count == 0 { + delete(t.entries, networkService) + if err := delVPP(ctx, vppConn, swIfIndex); err != nil { + log.FromContext(ctx).Errorf("unable to delete loopback interface: %v", err) + } } - } + }) } - return nil } func delVPP(ctx context.Context, vppConn api.Connection, swIfIndex interface_types.InterfaceIndex) error { diff --git a/pkg/networkservice/loopback/option.go b/pkg/networkservice/loopback/option.go index 537da798..4ac5e6bb 100644 --- a/pkg/networkservice/loopback/option.go +++ b/pkg/networkservice/loopback/option.go @@ -17,9 +17,8 @@ package loopback import ( - "sync" - "github.com/edwarnicke/govpp/binapi/interface_types" + "github.com/edwarnicke/serialize" ) type loopInfo struct { @@ -35,8 +34,8 @@ type Map struct { /* entries - is a map[NetworkServiceName]{swIfIndex, count} */ entries map[string]*loopInfo - /* mutex for entries */ - mut sync.Mutex + /* executor */ + exec serialize.Executor } // NewMap creates loopback map diff --git a/pkg/networkservice/loopback/server.go b/pkg/networkservice/loopback/server.go index 2666b1fd..fd7afcc4 100644 --- a/pkg/networkservice/loopback/server.go +++ b/pkg/networkservice/loopback/server.go @@ -25,7 +25,6 @@ import ( "github.com/networkservicemesh/api/pkg/api/networkservice" "github.com/networkservicemesh/sdk/pkg/networkservice/core/next" "github.com/networkservicemesh/sdk/pkg/networkservice/utils/metadata" - "github.com/networkservicemesh/sdk/pkg/tools/log" ) type loopbackServer struct { @@ -56,16 +55,12 @@ func (l *loopbackServer) Request(ctx context.Context, request *networkservice.Ne } conn, err := next.Server(ctx).Request(ctx, request) if err != nil { - if e := del(ctx, l.vppConn, networkService, l.loopbacks, metadata.IsClient(l)); e != nil { - log.FromContext(ctx).Errorf("unable to delete loopback interface: %v", e) - } + del(ctx, l.vppConn, networkService, l.loopbacks, metadata.IsClient(l)) } return conn, err } func (l *loopbackServer) Close(ctx context.Context, conn *networkservice.Connection) (*empty.Empty, error) { - if err := del(ctx, l.vppConn, conn.NetworkService, l.loopbacks, metadata.IsClient(l)); err != nil { - log.FromContext(ctx).Errorf("unable to delete loopback interface: %v", err) - } + del(ctx, l.vppConn, conn.NetworkService, l.loopbacks, metadata.IsClient(l)) return next.Server(ctx).Close(ctx, conn) } From c1fdf1440cf20ddc2fe99e02f998242ef009afea Mon Sep 17 00:00:00 2001 From: denis-tingaikin Date: Fri, 18 Mar 2022 02:39:32 +0300 Subject: [PATCH 5/5] simplify loopback chain element Signed-off-by: denis-tingaikin --- pkg/networkservice/loopback/client.go | 68 ------------------------- pkg/networkservice/loopback/common.go | 21 ++++++-- pkg/networkservice/loopback/doc.go | 2 +- pkg/networkservice/loopback/metadata.go | 2 +- pkg/networkservice/loopback/option.go | 60 ---------------------- pkg/networkservice/loopback/server.go | 19 +++---- 6 files changed, 25 insertions(+), 147 deletions(-) delete mode 100644 pkg/networkservice/loopback/client.go delete mode 100644 pkg/networkservice/loopback/option.go diff --git a/pkg/networkservice/loopback/client.go b/pkg/networkservice/loopback/client.go deleted file mode 100644 index 6d523d53..00000000 --- a/pkg/networkservice/loopback/client.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright (c) 2021 Doc.ai and/or its affiliates. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at: -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package loopback - -import ( - "context" - - "google.golang.org/grpc" - - "git.fd.io/govpp.git/api" - "github.com/golang/protobuf/ptypes/empty" - - "github.com/networkservicemesh/api/pkg/api/networkservice" - "github.com/networkservicemesh/sdk/pkg/networkservice/core/next" - "github.com/networkservicemesh/sdk/pkg/networkservice/utils/metadata" -) - -type loopbackClient struct { - vppConn api.Connection - - loopbacks *Map -} - -// NewClient creates a NetworkServiceClient chain element to create the loopback vpp-interface -func NewClient(vppConn api.Connection, opts ...Option) networkservice.NetworkServiceClient { - o := &options{ - loopbacks: NewMap(), - } - for _, opt := range opts { - opt(o) - } - - return &loopbackClient{ - vppConn: vppConn, - loopbacks: o.loopbacks, - } -} - -func (l *loopbackClient) Request(ctx context.Context, request *networkservice.NetworkServiceRequest, opts ...grpc.CallOption) (*networkservice.Connection, error) { - networkService := request.GetConnection().NetworkService - if err := createLoopback(ctx, l.vppConn, networkService, l.loopbacks, metadata.IsClient(l)); err != nil { - return nil, err - } - conn, err := next.Client(ctx).Request(ctx, request, opts...) - if err != nil { - del(ctx, l.vppConn, networkService, l.loopbacks, metadata.IsClient(l)) - } - return conn, err -} - -func (l *loopbackClient) Close(ctx context.Context, conn *networkservice.Connection, opts ...grpc.CallOption) (*empty.Empty, error) { - del(ctx, l.vppConn, conn.NetworkService, l.loopbacks, metadata.IsClient(l)) - return next.Client(ctx).Close(ctx, conn, opts...) -} diff --git a/pkg/networkservice/loopback/common.go b/pkg/networkservice/loopback/common.go index d818e7ee..d3300851 100644 --- a/pkg/networkservice/loopback/common.go +++ b/pkg/networkservice/loopback/common.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021 Doc.ai and/or its affiliates. +// Copyright (c) 2022 Cisco and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 // @@ -25,14 +25,25 @@ import ( "git.fd.io/govpp.git/api" interfaces "github.com/edwarnicke/govpp/binapi/interface" "github.com/edwarnicke/govpp/binapi/interface_types" + "github.com/edwarnicke/serialize" "github.com/networkservicemesh/sdk/pkg/tools/log" ) +type loopInfo struct { + swIfIndex interface_types.InterfaceIndex + count uint32 +} + +type loopbackMap struct { + serialize.Executor + entries map[string]*loopInfo +} + /* Create loopback interface and store it in metadata */ -func createLoopback(ctx context.Context, vppConn api.Connection, networkService string, t *Map, isClient bool) (err error) { +func createLoopback(ctx context.Context, vppConn api.Connection, networkService string, t *loopbackMap, isClient bool) (err error) { if _, ok := Load(ctx, isClient); !ok { - <-t.exec.AsyncExec(func() { + <-t.AsyncExec(func() { /* Check if we have already created loopback for a given NetworkService previously */ info, ok := t.entries[networkService] if !ok { @@ -66,9 +77,9 @@ func createLoopbackVPP(ctx context.Context, vppConn api.Connection) (interface_t return reply.SwIfIndex, nil } -func del(ctx context.Context, vppConn api.Connection, networkService string, t *Map, isClient bool) { +func del(ctx context.Context, vppConn api.Connection, networkService string, t *loopbackMap, isClient bool) { if swIfIndex, ok := LoadAndDelete(ctx, isClient); ok { - t.exec.AsyncExec(func() { + t.AsyncExec(func() { t.entries[networkService].count-- /* If there are no more clients using the loopback - delete it */ diff --git a/pkg/networkservice/loopback/doc.go b/pkg/networkservice/loopback/doc.go index 18becd12..2ee342a4 100644 --- a/pkg/networkservice/loopback/doc.go +++ b/pkg/networkservice/loopback/doc.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021 Doc.ai and/or its affiliates. +// Copyright (c) 2022 Cisco and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 // diff --git a/pkg/networkservice/loopback/metadata.go b/pkg/networkservice/loopback/metadata.go index 316306dc..d32cd410 100644 --- a/pkg/networkservice/loopback/metadata.go +++ b/pkg/networkservice/loopback/metadata.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021 Doc.ai and/or its affiliates. +// Copyright (c) 2022 Cisco and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 // diff --git a/pkg/networkservice/loopback/option.go b/pkg/networkservice/loopback/option.go deleted file mode 100644 index 4ac5e6bb..00000000 --- a/pkg/networkservice/loopback/option.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (c) 2021 Doc.ai and/or its affiliates. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at: -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package loopback - -import ( - "github.com/edwarnicke/govpp/binapi/interface_types" - "github.com/edwarnicke/serialize" -) - -type loopInfo struct { - /* loopback swIfIndex */ - swIfIndex interface_types.InterfaceIndex - - /* count - the number of clients using this loopback */ - count uint32 -} - -// Map stores loopback swIfIndex by NetworkServiceName -type Map struct { - /* entries - is a map[NetworkServiceName]{swIfIndex, count} */ - entries map[string]*loopInfo - - /* executor */ - exec serialize.Executor -} - -// NewMap creates loopback map -func NewMap() *Map { - return &Map{ - entries: make(map[string]*loopInfo), - } -} - -type options struct { - loopbacks *Map -} - -// Option is an option pattern for loopbackClient/Server -type Option func(o *options) - -// WithSharedMap - sets shared loopback map. It may be needed for sharing Map between client and server -func WithSharedMap(l *Map) Option { - return func(o *options) { - o.loopbacks = l - } -} diff --git a/pkg/networkservice/loopback/server.go b/pkg/networkservice/loopback/server.go index fd7afcc4..e93e5769 100644 --- a/pkg/networkservice/loopback/server.go +++ b/pkg/networkservice/loopback/server.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021 Doc.ai and/or its affiliates. +// Copyright (c) 2022 Cisco and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 // @@ -30,21 +30,16 @@ import ( type loopbackServer struct { vppConn api.Connection - loopbacks *Map + loopbacks *loopbackMap } // NewServer creates a NetworkServiceServer chain element to create the loopback vpp-interface -func NewServer(vppConn api.Connection, opts ...Option) networkservice.NetworkServiceServer { - o := &options{ - loopbacks: NewMap(), - } - for _, opt := range opts { - opt(o) - } - +func NewServer(vppConn api.Connection) networkservice.NetworkServiceServer { return &loopbackServer{ - vppConn: vppConn, - loopbacks: o.loopbacks, + vppConn: vppConn, + loopbacks: &loopbackMap{ + entries: make(map[string]*loopInfo), + }, } }