From 3d2c0ccf06d0ca2a40c89fdd33d5966a440d7bb4 Mon Sep 17 00:00:00 2001 From: Vladimir Popov Date: Mon, 25 Jan 2021 22:25:39 +0700 Subject: [PATCH 01/10] Rework sandbox to node API Signed-off-by: Vladimir Popov --- pkg/registry/chains/memory/server.go | 6 +- pkg/tools/sandbox/README.md | 19 ++- pkg/tools/sandbox/builder.go | 83 ++--------- pkg/tools/sandbox/node.go | 207 +++++++++++++++++++++++++++ pkg/tools/sandbox/types.go | 14 +- pkg/tools/sandbox/utils.go | 69 --------- 6 files changed, 237 insertions(+), 161 deletions(-) create mode 100644 pkg/tools/sandbox/node.go diff --git a/pkg/registry/chains/memory/server.go b/pkg/registry/chains/memory/server.go index 6419b4c9f..e5c2fda76 100644 --- a/pkg/registry/chains/memory/server.go +++ b/pkg/registry/chains/memory/server.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 Doc.ai and/or its affiliates. +// Copyright (c) 2020-2021 Doc.ai and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 // @@ -36,10 +36,10 @@ import ( ) // NewServer creates new registry server based on memory storage -func NewServer(ctx context.Context, proxyRegistryURL *url.URL, options ...grpc.DialOption) registryserver.Registry { +func NewServer(ctx context.Context, expiryDuration time.Duration, proxyRegistryURL *url.URL, options ...grpc.DialOption) registryserver.Registry { nseChain := chain.NewNetworkServiceEndpointRegistryServer( setid.NewNetworkServiceEndpointRegistryServer(), - expire.NewNetworkServiceEndpointRegistryServer(time.Minute), + expire.NewNetworkServiceEndpointRegistryServer(expiryDuration), memory.NewNetworkServiceEndpointRegistryServer(), proxy.NewNetworkServiceEndpointRegistryServer(proxyRegistryURL), connect.NewNetworkServiceEndpointRegistryServer(ctx, func(ctx context.Context, cc grpc.ClientConnInterface) registry.NetworkServiceEndpointRegistryClient { diff --git a/pkg/tools/sandbox/README.md b/pkg/tools/sandbox/README.md index 98fd5a344..d9d2c53c4 100644 --- a/pkg/tools/sandbox/README.md +++ b/pkg/tools/sandbox/README.md @@ -28,8 +28,10 @@ Solution: SetRegistryProxySupplier(nil). Build() defer localDomain.Cleanup() - registerMyNewEndpoint(localDomain.Nodes[0].NSMgr.URL) - ... + localDomain.Nodes[0].NewForwarder(ctx, new(registry.Endpoint), sandbox.GenerateTestToken) + localDomain.Nodes[0].NewEndpoint(ctx, ®istry.Endpoint{...}, sandbox.GenerateTestToken, ...myNewEndpointChain) + client := localDomain.Nodes[0].NewClient(ctx, sandbox.GenerateTestToken, ...clientChain) + ... ``` Problem: setup my external NSMgr and Forwarder to checking my external NSMgr.\ @@ -43,7 +45,7 @@ Solution: SetNSMgrSupplier(myExternalNSMgrFunc) Build() defer localDomain.Cleanup() - ... + ... ``` Problem: setup my NSMgr and new Forwarder to checking my Forwarder chain.\ @@ -54,9 +56,9 @@ Solution: SetNodesCount(1). SetNSMgrProxySupplier(nil). SetRegistryProxySupplier(nil). - SetForwarderSupplier(myNewForwarderFunc) Build() defer localDomain.Cleanup() + localDomain.Nodes[0].NewForwarder(ctx, new(registry.Endpoint), sandbox.GenerateTestToken, ...myNewForwarderChain) ... ``` @@ -70,8 +72,11 @@ Solution: SetNodesCount(2). Build() defer localDomain.Cleanup() - urlForNSERegistration := localDomain.Nodes[1].NSMgr.URL - ... + for _, node := range localDomain.Nodes { + node.NewForwarder(ctx, new(registry.Endpoint), sandbox.GenerateTestToken) + } + localDomain.Nodes[1].NewEndpoint(ctx, ®istry.Endpoint{...}, sandbox.GenerateTestToken, ...remoteEndpointChain) + ... ``` ### Setup only local registry @@ -97,7 +102,7 @@ Problem: setup NSMgrs, Forwarders, Registries to checking interdomain use-case v Solution: ```go ... - fakeServer := new(sandbox.FakeDNSResolver) + fakeServer := new(sandbox.FakeDNSResolver) domain1 := sandbox.NewBuilder(t). SetContext(ctx). SetNodesCount(1). diff --git a/pkg/tools/sandbox/builder.go b/pkg/tools/sandbox/builder.go index 32884e530..556159897 100644 --- a/pkg/tools/sandbox/builder.go +++ b/pkg/tools/sandbox/builder.go @@ -23,35 +23,28 @@ import ( "testing" "time" - "github.com/networkservicemesh/sdk/pkg/tools/logger" - "github.com/google/uuid" "github.com/stretchr/testify/require" "google.golang.org/grpc" registryapi "github.com/networkservicemesh/api/pkg/api/registry" - "github.com/networkservicemesh/sdk/pkg/networkservice/chains/client" - "github.com/networkservicemesh/sdk/pkg/networkservice/chains/endpoint" "github.com/networkservicemesh/sdk/pkg/networkservice/chains/nsmgr" "github.com/networkservicemesh/sdk/pkg/networkservice/chains/nsmgrproxy" "github.com/networkservicemesh/sdk/pkg/networkservice/common/authorize" - "github.com/networkservicemesh/sdk/pkg/networkservice/common/clienturl" - "github.com/networkservicemesh/sdk/pkg/networkservice/common/connect" - "github.com/networkservicemesh/sdk/pkg/networkservice/core/adapters" "github.com/networkservicemesh/sdk/pkg/registry/chains/memory" "github.com/networkservicemesh/sdk/pkg/registry/chains/proxydns" "github.com/networkservicemesh/sdk/pkg/registry/common/dnsresolve" - interpose_reg "github.com/networkservicemesh/sdk/pkg/registry/common/interpose" - adapter_registry "github.com/networkservicemesh/sdk/pkg/registry/core/adapters" - "github.com/networkservicemesh/sdk/pkg/registry/core/chain" - "github.com/networkservicemesh/sdk/pkg/tools/addressof" "github.com/networkservicemesh/sdk/pkg/tools/grpcutils" + "github.com/networkservicemesh/sdk/pkg/tools/logger" "github.com/networkservicemesh/sdk/pkg/tools/opentracing" "github.com/networkservicemesh/sdk/pkg/tools/token" ) -const defaultContextTimeout = time.Second * 15 +const ( + defaultContextTimeout = time.Second * 15 + defaultRegistryExpiryDuration = 100 * time.Millisecond +) // Builder implements builder pattern for building NSM Domain type Builder struct { @@ -60,7 +53,6 @@ type Builder struct { nodesCount int DNSDomainName string Resolver dnsresolve.Resolver - supplyForwarder SupplyForwarderFunc supplyNSMgr SupplyNSMgrFunc supplyNSMgrProxy SupplyNSMgrProxyFunc supplyRegistry SupplyRegistryFunc @@ -76,7 +68,6 @@ func NewBuilder(t *testing.T) *Builder { require: require.New(t), Resolver: net.DefaultResolver, supplyNSMgr: nsmgr.NewServer, - supplyForwarder: supplyDummyForwarder, DNSDomainName: "cluster.local", supplyRegistry: memory.NewServer, supplyRegistryProxy: proxydns.NewServer, @@ -107,15 +98,9 @@ func (b *Builder) Build() *Domain { domain.Registry = b.newRegistry(ctx, domain.RegistryProxy.URL) } for i := 0; i < b.nodesCount; i++ { - var node = new(Node) - node.NSMgr = b.newNSMgr(ctx, domain.Registry.URL) - forwarderName := "cross-nse-" + uuid.New().String() - forwarderRegistrationClient := chain.NewNetworkServiceEndpointRegistryClient( - interpose_reg.NewNetworkServiceEndpointRegistryClient(), - adapter_registry.NetworkServiceEndpointServerToClient(node.NSMgr.NetworkServiceEndpointRegistryServer()), - ) - node.Forwarder = b.newCrossConnectNSE(ctx, forwarderName, node.NSMgr.URL, forwarderRegistrationClient) - domain.Nodes = append(domain.Nodes, node) + domain.Nodes = append(domain.Nodes, &Node{ + NSMgr: b.newNSMgr(ctx, domain.Registry.URL), + }) } domain.resources, b.resources = b.resources, nil return domain @@ -163,12 +148,6 @@ func (b *Builder) SetDNSDomainName(name string) *Builder { return b } -// SetForwarderSupplier replaces default dummy forwarder supplier to custom function -func (b *Builder) SetForwarderSupplier(f SupplyForwarderFunc) *Builder { - b.supplyForwarder = f - return b -} - // SetNSMgrProxySupplier replaces default nsmgr-proxy supplier to custom function func (b *Builder) SetNSMgrProxySupplier(f SupplyNSMgrProxyFunc) *Builder { b.supplyNSMgrProxy = f @@ -253,30 +232,6 @@ func serve(ctx context.Context, u *url.URL, register func(server *grpc.Server)) }() } -func (b *Builder) newCrossConnectNSE(ctx context.Context, name string, connectTo *url.URL, forwarderRegistrationClient registryapi.NetworkServiceEndpointRegistryClient) *EndpointEntry { - if b.supplyForwarder == nil { - panic("nodes without forwarder are not supported") - } - listener, err := net.Listen("tcp", "127.0.0.1:0") - b.require.NoError(err) - serveURL := grpcutils.AddressToURL(listener.Addr()) - b.require.NoError(listener.Close()) - - regForwarder, err := forwarderRegistrationClient.Register(context.Background(), ®istryapi.NetworkServiceEndpoint{ - Url: serveURL.String(), - Name: name, - }) - b.require.NoError(err) - - crossNSE := b.supplyForwarder(ctx, regForwarder.Name, b.generateTokenFunc, connectTo, grpc.WithInsecure(), grpc.WithDefaultCallOptions(grpc.WaitForReady(true))) - serve(ctx, serveURL, crossNSE.Register) - logger.Log(ctx).Infof("%v listen on: %v", name, serveURL) - return &EndpointEntry{ - Endpoint: crossNSE, - URL: serveURL, - } -} - func (b *Builder) newRegistryProxy(ctx context.Context, nsmgrProxyURL *url.URL) *RegistryEntry { if b.supplyRegistryProxy == nil { return nil @@ -295,7 +250,7 @@ func (b *Builder) newRegistry(ctx context.Context, proxyRegistryURL *url.URL) *R if b.supplyRegistry == nil { return nil } - result := b.supplyRegistry(ctx, proxyRegistryURL, grpc.WithInsecure(), grpc.WithBlock()) + result := b.supplyRegistry(ctx, defaultRegistryExpiryDuration, proxyRegistryURL, grpc.WithInsecure(), grpc.WithBlock()) serveURL := &url.URL{Scheme: "tcp", Host: "127.0.0.1:0"} serve(ctx, serveURL, result.Register) logger.Log(ctx).Infof("Registry listen on: %v", serveURL) @@ -304,23 +259,3 @@ func (b *Builder) newRegistry(ctx context.Context, proxyRegistryURL *url.URL) *R Registry: result, } } - -func supplyDummyForwarder(ctx context.Context, name string, generateToken token.GeneratorFunc, connectTo *url.URL, dialOptions ...grpc.DialOption) endpoint.Endpoint { - var result endpoint.Endpoint - result = endpoint.NewServer(ctx, - name, - authorize.NewServer(), - generateToken, - // Statically set the url we use to the unix file socket for the NSMgr - clienturl.NewServer(connectTo), - connect.NewServer(ctx, - client.NewCrossConnectClientFactory( - name, - // What to call onHeal - addressof.NetworkServiceClient(adapters.NewServerToClient(result)), - generateToken), - dialOptions..., - ), - ) - return result -} diff --git a/pkg/tools/sandbox/node.go b/pkg/tools/sandbox/node.go new file mode 100644 index 000000000..e26cc8988 --- /dev/null +++ b/pkg/tools/sandbox/node.go @@ -0,0 +1,207 @@ +// Copyright (c) 2020-2021 Doc.ai and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sandbox + +import ( + "context" + "fmt" + "net/url" + + "github.com/google/uuid" + "google.golang.org/grpc" + + "github.com/networkservicemesh/api/pkg/api/networkservice" + "github.com/networkservicemesh/api/pkg/api/registry" + + "github.com/networkservicemesh/sdk/pkg/networkservice/chains/client" + "github.com/networkservicemesh/sdk/pkg/networkservice/chains/endpoint" + "github.com/networkservicemesh/sdk/pkg/networkservice/common/authorize" + "github.com/networkservicemesh/sdk/pkg/networkservice/common/clienturl" + "github.com/networkservicemesh/sdk/pkg/networkservice/common/connect" + "github.com/networkservicemesh/sdk/pkg/networkservice/core/adapters" + "github.com/networkservicemesh/sdk/pkg/registry/common/interpose" + "github.com/networkservicemesh/sdk/pkg/registry/common/null" + "github.com/networkservicemesh/sdk/pkg/registry/common/refresh" + "github.com/networkservicemesh/sdk/pkg/registry/core/chain" + "github.com/networkservicemesh/sdk/pkg/tools/addressof" + "github.com/networkservicemesh/sdk/pkg/tools/grpcutils" + "github.com/networkservicemesh/sdk/pkg/tools/logger" + "github.com/networkservicemesh/sdk/pkg/tools/token" +) + +// Node is a NSMgr with resources +type Node struct { + NSMgr *NSMgrEntry + resources []context.CancelFunc +} + +func (n *Node) cleanup() { + for _, r := range n.resources { + r() + } +} + +// NewForwarder starts a new forwarder and registers it on the node NSMgr +func (n *Node) NewForwarder( + ctx context.Context, + nse *registry.NetworkServiceEndpoint, + generatorFunc token.GeneratorFunc, + additionalFunctionality ...networkservice.NetworkServiceServer, +) (*EndpointEntry, error) { + ep := new(EndpointEntry) + additionalFunctionality = append(additionalFunctionality, + clienturl.NewServer(n.NSMgr.URL), + connect.NewServer(ctx, + client.NewCrossConnectClientFactory( + nse.Name, + // What to call onHeal + addressof.NetworkServiceClient(adapters.NewServerToClient(ep)), + generatorFunc), + grpc.WithInsecure(), grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), + )) + + entry, err := n.newEndpoint(ctx, nse, generatorFunc, true, additionalFunctionality...) + if err != nil { + return nil, err + } + *ep = *entry + + return ep, nil +} + +// NewEndpoint starts a new endpoint and registers it on the node NSMgr +func (n *Node) NewEndpoint( + ctx context.Context, + nse *registry.NetworkServiceEndpoint, + generatorFunc token.GeneratorFunc, + additionalFunctionality ...networkservice.NetworkServiceServer, +) (*EndpointEntry, error) { + return n.newEndpoint(ctx, nse, generatorFunc, false, additionalFunctionality...) +} + +func (n *Node) newEndpoint( + ctx context.Context, + nse *registry.NetworkServiceEndpoint, + generatorFunc token.GeneratorFunc, + isForwarder bool, + additionalFunctionality ...networkservice.NetworkServiceServer, +) (_ *EndpointEntry, err error) { + // 1. Create endpoint server + ep := endpoint.NewServer( + ctx, + nse.Name, + authorize.NewServer(), + generatorFunc, + additionalFunctionality..., + ) + + // 2. Start listening on URL + u := &url.URL{Scheme: "tcp", Host: "127.0.0.1:0"} + if nse.Url != "" { + u, err = url.Parse(nse.Url) + if err != nil { + return nil, err + } + } + + ctx = logger.WithLog(ctx) + serve(ctx, u, ep.Register) + + if nse.Url == "" { + nse.Url = u.String() + } + + // 3. Register to the node NSMgr + cc := n.dialNSMgr(ctx) + + nsClient := registry.NewNetworkServiceRegistryClient(cc) + for _, name := range nse.NetworkServiceNames { + var service *registry.NetworkService + service, err = nsClient.Register(ctx, ®istry.NetworkService{ + Name: name, + Payload: "IP", + }) + if err != nil { + return nil, err + } + n.resources = append(n.resources, func() { + _, _ = nsClient.Unregister(ctx, service) + }) + } + + var interposeClient registry.NetworkServiceEndpointRegistryClient + if isForwarder { + interposeClient = interpose.NewNetworkServiceEndpointRegistryClient() + } else { + interposeClient = null.NewNetworkServiceEndpointRegistryClient() + } + + nseClient := chain.NewNetworkServiceEndpointRegistryClient( + refresh.NewNetworkServiceEndpointRegistryClient( + refresh.WithChainContext(ctx)), + interposeClient, + registry.NewNetworkServiceEndpointRegistryClient(cc), + ) + nse, err = nseClient.Register(ctx, nse) + if err != nil { + return nil, err + } + n.resources = append(n.resources, func() { + _, _ = nseClient.Unregister(ctx, nse) + }) + + if isForwarder { + logger.Log(ctx).Infof("Started listen forwarder %v on %v.", nse.Name, u.String()) + } else { + logger.Log(ctx).Infof("Started listen endpoint %v on %v.", nse.Name, u.String()) + } + + return &EndpointEntry{Endpoint: ep, URL: u}, nil +} + +// NewClient starts a new client and connects it to the node NSMgr +func (n *Node) NewClient( + ctx context.Context, + generatorFunc token.GeneratorFunc, + additionalFunctionality ...networkservice.NetworkServiceClient, +) networkservice.NetworkServiceClient { + return client.NewClient( + ctx, + fmt.Sprintf("nsc-%v", uuid.New().String()), + nil, + generatorFunc, + n.dialNSMgr(ctx), + additionalFunctionality..., + ) +} + +func (n *Node) dialNSMgr(ctx context.Context) grpc.ClientConnInterface { + cc, err := grpc.DialContext(ctx, grpcutils.URLToTarget(n.NSMgr.URL), + grpc.WithInsecure(), + grpc.WithBlock(), + grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), + ) + if err != nil { + panic("failed to dial node NSMgr") + } + + n.resources = append(n.resources, func() { + _ = cc.Close() + }) + + return cc +} diff --git a/pkg/tools/sandbox/types.go b/pkg/tools/sandbox/types.go index 2c3c5e61d..c34ef7504 100644 --- a/pkg/tools/sandbox/types.go +++ b/pkg/tools/sandbox/types.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 Doc.ai and/or its affiliates. +// Copyright (c) 2020-2021 Doc.ai and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 // @@ -19,6 +19,7 @@ package sandbox import ( "context" "net/url" + "time" "github.com/networkservicemesh/api/pkg/api/networkservice" registryapi "github.com/networkservicemesh/api/pkg/api/registry" @@ -41,17 +42,11 @@ type SupplyNSMgrFunc func(context.Context, *registryapi.NetworkServiceEndpoint, type SupplyForwarderFunc func(context.Context, string, token.GeneratorFunc, *url.URL, ...grpc.DialOption) endpoint.Endpoint // SupplyRegistryFunc supplies Registry -type SupplyRegistryFunc func(ctx context.Context, proxyRegistryURL *url.URL, options ...grpc.DialOption) registry.Registry +type SupplyRegistryFunc func(ctx context.Context, expiryDuration time.Duration, proxyRegistryURL *url.URL, options ...grpc.DialOption) registry.Registry // SupplyRegistryProxyFunc supplies registry proxy type SupplyRegistryProxyFunc func(ctx context.Context, dnsResolver dnsresolve.Resolver, handlingDNSDomain string, proxyNSMgrURL *url.URL, options ...grpc.DialOption) registry.Registry -// Node is pair of Forwarder and NSMgr -type Node struct { - Forwarder *EndpointEntry - NSMgr *NSMgrEntry -} - // RegistryEntry is pair of registry.Registry and url.URL type RegistryEntry struct { registry.Registry @@ -83,6 +78,9 @@ type Domain struct { // Cleanup frees all resources related to the domain func (d *Domain) Cleanup() { + for _, node := range d.Nodes { + node.cleanup() + } for _, r := range d.resources { r() } diff --git a/pkg/tools/sandbox/utils.go b/pkg/tools/sandbox/utils.go index 0d7515e20..10857019e 100644 --- a/pkg/tools/sandbox/utils.go +++ b/pkg/tools/sandbox/utils.go @@ -19,27 +19,15 @@ package sandbox import ( "context" "fmt" - "net/url" "time" - "google.golang.org/protobuf/types/known/timestamppb" - "github.com/google/uuid" "google.golang.org/grpc" "google.golang.org/grpc/credentials" "github.com/networkservicemesh/api/pkg/api/networkservice" - "github.com/networkservicemesh/api/pkg/api/networkservice/payload" - "github.com/networkservicemesh/api/pkg/api/registry" "github.com/networkservicemesh/sdk/pkg/networkservice/chains/client" - "github.com/networkservicemesh/sdk/pkg/networkservice/chains/endpoint" - "github.com/networkservicemesh/sdk/pkg/networkservice/chains/nsmgr" - "github.com/networkservicemesh/sdk/pkg/networkservice/common/authorize" - "github.com/networkservicemesh/sdk/pkg/networkservice/common/clienturl" - "github.com/networkservicemesh/sdk/pkg/tools/clienturlctx" - "github.com/networkservicemesh/sdk/pkg/tools/logger" - "github.com/networkservicemesh/sdk/pkg/tools/opentracing" "github.com/networkservicemesh/sdk/pkg/tools/token" ) @@ -48,63 +36,6 @@ func GenerateTestToken(_ credentials.AuthInfo) (tokenValue string, expireTime ti return "TestToken", time.Date(3000, 1, 1, 1, 1, 1, 1, time.UTC), nil } -// NewEndpoint creates endpoint and registers it into passed NSMgr. -func NewEndpoint(ctx context.Context, nse *registry.NetworkServiceEndpoint, generatorFunc token.GeneratorFunc, mgr nsmgr.Nsmgr, additionalFunctionality ...networkservice.NetworkServiceServer) (*EndpointEntry, error) { - ep := endpoint.NewServer(ctx, nse.Name, authorize.NewServer(), generatorFunc, additionalFunctionality...) - - ctx = logger.WithLog(ctx) - - u := &url.URL{Scheme: "tcp", Host: "127.0.0.1:0"} - var err error - if nse.Url != "" { - u, err = url.Parse(nse.Url) - if err != nil { - return nil, err - } - } - serve(ctx, u, ep.Register) - - if nse.Url == "" { - nse.Url = u.String() - } - if nse.ExpirationTime == nil { - nse.ExpirationTime = timestamppb.New(time.Now().Add(time.Hour)) - } - - var reg *registry.NetworkServiceEndpoint - if reg, err = mgr.NetworkServiceEndpointRegistryServer().Register(ctx, nse); err != nil { - return nil, err - } - - nse.Name = reg.Name - nse.ExpirationTime = reg.ExpirationTime - - for _, service := range nse.NetworkServiceNames { - if _, err := mgr.NetworkServiceRegistryServer().Register(ctx, ®istry.NetworkService{ - Name: service, - Payload: payload.IP, - }); err != nil { - return nil, err - } - } - - logger.Log(ctx).Infof("Started listen endpoint %v on %v.", nse.Name, u.String()) - - return &EndpointEntry{Endpoint: ep, URL: u}, nil -} - -// NewClient is a client.NewClient over *url.URL with some fields preset for testing -func NewClient(ctx context.Context, generatorFunc token.GeneratorFunc, connectTo *url.URL, additionalFunctionality ...networkservice.NetworkServiceClient) networkservice.NetworkServiceClient { - return clienturl.NewClient( - clienturlctx.WithClientURL(ctx, connectTo), - client.NewClientFactory( - fmt.Sprintf("nsc-%v", uuid.New().String()), - nil, - generatorFunc, - additionalFunctionality...), - append(opentracing.WithTracingDial(), grpc.WithBlock(), grpc.WithInsecure())...) -} - // NewCrossConnectClientFactory is a client.NewCrossConnectClientFactory with some fields preset for testing func NewCrossConnectClientFactory(generatorFunc token.GeneratorFunc, additionalFunctionality ...networkservice.NetworkServiceClient) func(ctx context.Context, cc grpc.ClientConnInterface) networkservice.NetworkServiceClient { return client.NewCrossConnectClientFactory( From d363212ec709bea5287051e08656b486be3b3798 Mon Sep 17 00:00:00 2001 From: Vladimir Popov Date: Mon, 25 Jan 2021 22:33:43 +0700 Subject: [PATCH 02/10] Rework tests to the new sandbox Signed-off-by: Vladimir Popov --- .../chains/nsmgr/server_test.go | 103 ++++++++---------- .../chains/nsmgrproxy/server_test.go | 39 ++++--- .../core/interdomain/interdomain_ns_test.go | 4 +- .../core/interdomain/interdomain_nse_test.go | 4 +- 4 files changed, 76 insertions(+), 74 deletions(-) diff --git a/pkg/networkservice/chains/nsmgr/server_test.go b/pkg/networkservice/chains/nsmgr/server_test.go index 06a54cc7e..97d3940b8 100644 --- a/pkg/networkservice/chains/nsmgr/server_test.go +++ b/pkg/networkservice/chains/nsmgr/server_test.go @@ -47,16 +47,28 @@ import ( "github.com/networkservicemesh/sdk/pkg/tools/sandbox" ) +func testDomain(ctx context.Context, t *testing.T, nodesCount int) *sandbox.Domain { + domain := sandbox.NewBuilder(t). + SetNodesCount(nodesCount). + SetRegistryProxySupplier(nil). + SetContext(ctx). + Build() + + for _, node := range domain.Nodes { + _, err := node.NewForwarder(ctx, new(registry.NetworkServiceEndpoint), sandbox.GenerateTestToken) + require.NoError(t, err) + } + + return domain +} + func TestNSMGR_RemoteUsecase_Parallel(t *testing.T) { defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - domain := sandbox.NewBuilder(t). - SetNodesCount(2). - SetRegistryProxySupplier(nil). - SetContext(ctx). - Build() + domain := testDomain(ctx, t, 2) defer domain.Cleanup() counter := &counterServer{} @@ -77,10 +89,10 @@ func TestNSMGR_RemoteUsecase_Parallel(t *testing.T) { Name: "final-endpoint", NetworkServiceNames: []string{"my-service-remote"}, } - _, err := sandbox.NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken, domain.Nodes[0].NSMgr, counter) + _, err := domain.Nodes[0].NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken, counter) require.NoError(t, err) }() - nsc := sandbox.NewClient(ctx, sandbox.GenerateTestToken, domain.Nodes[1].NSMgr.URL) + nsc := domain.Nodes[1].NewClient(ctx, sandbox.GenerateTestToken) conn, err := nsc.Request(ctx, request.Clone()) require.NoError(t, err) @@ -113,11 +125,7 @@ func TestNSMGR_SelectsRestartingEndpoint(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - domain := sandbox.NewBuilder(t). - SetNodesCount(1). - SetRegistryProxySupplier(nil). - SetContext(ctx). - Build() + domain := testDomain(ctx, t, 1) defer domain.Cleanup() request := &networkservice.NetworkServiceRequest{ @@ -136,10 +144,10 @@ func TestNSMGR_SelectsRestartingEndpoint(t *testing.T) { NetworkServiceNames: []string{"ns-1"}, } - _, err := sandbox.NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken, domain.Nodes[0].NSMgr, &restartingEndpoint{startTime: time.Now().Add(time.Second * 2)}) + _, err := domain.Nodes[0].NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken, &restartingEndpoint{startTime: time.Now().Add(time.Second * 2)}) require.NoError(t, err) - nsc := sandbox.NewClient(ctx, sandbox.GenerateTestToken, domain.Nodes[0].NSMgr.URL) + nsc := domain.Nodes[0].NewClient(ctx, sandbox.GenerateTestToken) conn, err := nsc.Request(ctx, request.Clone()) require.NoError(t, err) @@ -151,14 +159,11 @@ func TestNSMGR_SelectsRestartingEndpoint(t *testing.T) { func TestNSMGR_RemoteUsecase_BusyEndpoints(t *testing.T) { defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) - ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - domain := sandbox.NewBuilder(t). - SetNodesCount(2). - SetRegistryProxySupplier(nil). - SetContext(ctx). - Build() + domain := testDomain(ctx, t, 2) defer domain.Cleanup() counter := new(counterServer) @@ -181,7 +186,7 @@ func TestNSMGR_RemoteUsecase_BusyEndpoints(t *testing.T) { Name: "final-endpoint-" + strconv.Itoa(id), NetworkServiceNames: []string{"my-service-remote"}, } - _, err := sandbox.NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken, domain.Nodes[1].NSMgr, newBusyEndpoint()) + _, err := domain.Nodes[1].NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken, newBusyEndpoint()) require.NoError(t, err) wg.Done() }(i) @@ -193,10 +198,10 @@ func TestNSMGR_RemoteUsecase_BusyEndpoints(t *testing.T) { Name: "final-endpoint-3", NetworkServiceNames: []string{"my-service-remote"}, } - _, err := sandbox.NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken, domain.Nodes[1].NSMgr, counter) + _, err := domain.Nodes[1].NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken, counter) require.NoError(t, err) }() - nsc := sandbox.NewClient(ctx, sandbox.GenerateTestToken, domain.Nodes[0].NSMgr.URL) + nsc := domain.Nodes[0].NewClient(ctx, sandbox.GenerateTestToken) conn, err := nsc.Request(ctx, request.Clone()) require.NoError(t, err) @@ -218,14 +223,11 @@ func TestNSMGR_RemoteUsecase_BusyEndpoints(t *testing.T) { func TestNSMGR_RemoteUsecase(t *testing.T) { defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - domain := sandbox.NewBuilder(t). - SetNodesCount(2). - SetRegistryProxySupplier(nil). - SetContext(ctx). - Build() + domain := testDomain(ctx, t, 2) defer domain.Cleanup() nseReg := ®istry.NetworkServiceEndpoint{ @@ -234,7 +236,7 @@ func TestNSMGR_RemoteUsecase(t *testing.T) { } counter := &counterServer{} - _, err := sandbox.NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken, domain.Nodes[0].NSMgr, counter) + _, err := domain.Nodes[0].NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken, counter) require.NoError(t, err) request := &networkservice.NetworkServiceRequest{ @@ -248,7 +250,7 @@ func TestNSMGR_RemoteUsecase(t *testing.T) { }, } - nsc := sandbox.NewClient(ctx, sandbox.GenerateTestToken, domain.Nodes[1].NSMgr.URL) + nsc := domain.Nodes[1].NewClient(ctx, sandbox.GenerateTestToken) conn, err := nsc.Request(ctx, request.Clone()) require.NoError(t, err) @@ -295,10 +297,10 @@ func TestNSMGR_ConnectToDeadNSE(t *testing.T) { counter := &counterServer{} nseCtx, killNse := context.WithCancel(ctx) - _, err := sandbox.NewEndpoint(nseCtx, nseReg, sandbox.GenerateTestToken, domain.Nodes[0].NSMgr, counter) + _, err := domain.Nodes[0].NewEndpoint(nseCtx, nseReg, sandbox.GenerateTestToken, counter) require.NoError(t, err) - nsc := sandbox.NewClient(ctx, sandbox.GenerateTestToken, domain.Nodes[0].NSMgr.URL) + nsc := domain.Nodes[0].NewClient(ctx, sandbox.GenerateTestToken) request := &networkservice.NetworkServiceRequest{ MechanismPreferences: []*networkservice.Mechanism{ @@ -329,14 +331,11 @@ func TestNSMGR_ConnectToDeadNSE(t *testing.T) { func TestNSMGR_LocalUsecase(t *testing.T) { defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - domain := sandbox.NewBuilder(t). - SetNodesCount(1). - SetContext(ctx). - SetRegistryProxySupplier(nil). - Build() + domain := testDomain(ctx, t, 1) defer domain.Cleanup() nseReg := ®istry.NetworkServiceEndpoint{ @@ -345,10 +344,10 @@ func TestNSMGR_LocalUsecase(t *testing.T) { } counter := &counterServer{} - _, err := sandbox.NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken, domain.Nodes[0].NSMgr, counter) + _, err := domain.Nodes[0].NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken, counter) require.NoError(t, err) - nsc := sandbox.NewClient(ctx, sandbox.GenerateTestToken, domain.Nodes[0].NSMgr.URL) + nsc := domain.Nodes[0].NewClient(ctx, sandbox.GenerateTestToken) request := &networkservice.NetworkServiceRequest{ MechanismPreferences: []*networkservice.Mechanism{ @@ -387,15 +386,12 @@ func TestNSMGR_LocalUsecase(t *testing.T) { func TestNSMGR_PassThroughRemote(t *testing.T) { defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() const nodesCount = 7 - domain := sandbox.NewBuilder(t). - SetNodesCount(nodesCount). - SetContext(ctx). - SetRegistryProxySupplier(nil). - Build() + domain := testDomain(ctx, t, nodesCount) defer domain.Cleanup() for i := 0; i < nodesCount; i++ { @@ -421,11 +417,11 @@ func TestNSMGR_PassThroughRemote(t *testing.T) { Name: fmt.Sprintf("endpoint-%v", i), NetworkServiceNames: []string{fmt.Sprintf("my-service-remote-%v", i)}, } - _, err := sandbox.NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken, domain.Nodes[i].NSMgr, additionalFunctionality...) + _, err := domain.Nodes[i].NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken, additionalFunctionality...) require.NoError(t, err) } - nsc := sandbox.NewClient(ctx, sandbox.GenerateTestToken, domain.Nodes[nodesCount-1].NSMgr.URL) + nsc := domain.Nodes[nodesCount-1].NewClient(ctx, sandbox.GenerateTestToken) request := &networkservice.NetworkServiceRequest{ MechanismPreferences: []*networkservice.Mechanism{ @@ -449,15 +445,12 @@ func TestNSMGR_PassThroughRemote(t *testing.T) { func TestNSMGR_PassThroughLocal(t *testing.T) { defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() const nsesCount = 7 - domain := sandbox.NewBuilder(t). - SetNodesCount(1). - SetContext(ctx). - SetRegistryProxySupplier(nil). - Build() + domain := testDomain(ctx, t, 1) defer domain.Cleanup() for i := 0; i < nsesCount; i++ { @@ -482,11 +475,11 @@ func TestNSMGR_PassThroughLocal(t *testing.T) { Name: fmt.Sprintf("endpoint-%v", i), NetworkServiceNames: []string{fmt.Sprintf("my-service-remote-%v", i)}, } - _, err := sandbox.NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken, domain.Nodes[0].NSMgr, additionalFunctionality...) + _, err := domain.Nodes[0].NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken, additionalFunctionality...) require.NoError(t, err) } - nsc := sandbox.NewClient(ctx, sandbox.GenerateTestToken, domain.Nodes[0].NSMgr.URL) + nsc := domain.Nodes[0].NewClient(ctx, sandbox.GenerateTestToken) request := &networkservice.NetworkServiceRequest{ MechanismPreferences: []*networkservice.Mechanism{ @@ -552,10 +545,10 @@ func testNSEAndClient( ctx, cancel := context.WithCancel(ctx) defer cancel() - _, err := sandbox.NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken, domain.Nodes[0].NSMgr) + _, err := domain.Nodes[0].NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken) require.NoError(t, err) - nsc := sandbox.NewClient(ctx, sandbox.GenerateTestToken, domain.Nodes[0].NSMgr.URL) + nsc := domain.Nodes[0].NewClient(ctx, sandbox.GenerateTestToken) conn, err := nsc.Request(ctx, &networkservice.NetworkServiceRequest{ MechanismPreferences: []*networkservice.Mechanism{ diff --git a/pkg/networkservice/chains/nsmgrproxy/server_test.go b/pkg/networkservice/chains/nsmgrproxy/server_test.go index f41490444..219e0679e 100644 --- a/pkg/networkservice/chains/nsmgrproxy/server_test.go +++ b/pkg/networkservice/chains/nsmgrproxy/server_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 Doc.ai and/or its affiliates. +// Copyright (c) 2020-2021 Doc.ai and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 // @@ -21,36 +21,45 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + "go.uber.org/goleak" + "github.com/networkservicemesh/api/pkg/api/networkservice" "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/cls" "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/kernel" "github.com/networkservicemesh/api/pkg/api/registry" - "github.com/stretchr/testify/require" - "go.uber.org/goleak" + "github.com/networkservicemesh/sdk/pkg/registry/common/dnsresolve" "github.com/networkservicemesh/sdk/pkg/tools/sandbox" ) +func testDomain(ctx context.Context, t *testing.T, dnsServer dnsresolve.Resolver) *sandbox.Domain { + domain := sandbox.NewBuilder(t). + SetNodesCount(1). + SetContext(ctx). + SetDNSResolver(dnsServer). + Build() + + _, err := domain.Nodes[0].NewForwarder(ctx, new(registry.NetworkServiceEndpoint), sandbox.GenerateTestToken) + require.NoError(t, err) + + return domain +} + func TestNSMGR_InterdomainUseCase(t *testing.T) { defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + const remoteRegistryDomain = "domain2.local.registry" + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() dnsServer := new(sandbox.FakeDNSResolver) - domain1 := sandbox.NewBuilder(t). - SetNodesCount(1). - SetContext(ctx). - SetDNSResolver(dnsServer). - Build() + domain1 := testDomain(ctx, t, dnsServer) defer domain1.Cleanup() - domain2 := sandbox.NewBuilder(t). - SetNodesCount(1). - SetDNSResolver(dnsServer). - SetContext(ctx). - Build() + domain2 := testDomain(ctx, t, dnsServer) defer domain2.Cleanup() require.NoError(t, dnsServer.Register(remoteRegistryDomain, domain2.Registry.URL)) @@ -60,10 +69,10 @@ func TestNSMGR_InterdomainUseCase(t *testing.T) { NetworkServiceNames: []string{"my-service-interdomain"}, } - _, err := sandbox.NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken, domain2.Nodes[0].NSMgr) + _, err := domain2.Nodes[0].NewEndpoint(ctx, nseReg, sandbox.GenerateTestToken) require.NoError(t, err) - nsc := sandbox.NewClient(ctx, sandbox.GenerateTestToken, domain1.Nodes[0].NSMgr.URL) + nsc := domain1.Nodes[0].NewClient(ctx, sandbox.GenerateTestToken) request := &networkservice.NetworkServiceRequest{ MechanismPreferences: []*networkservice.Mechanism{ diff --git a/pkg/registry/core/interdomain/interdomain_ns_test.go b/pkg/registry/core/interdomain/interdomain_ns_test.go index accefb475..6c3678f82 100644 --- a/pkg/registry/core/interdomain/interdomain_ns_test.go +++ b/pkg/registry/core/interdomain/interdomain_ns_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 Doc.ai and/or its affiliates. +// Copyright (c) 2020-2021 Doc.ai and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 // @@ -217,7 +217,7 @@ func TestInterdomainFloatingNetworkServiceRegistry(t *testing.T) { domain3 := sandbox.NewBuilder(t). SetNodesCount(0). - SetRegistrySupplier(func(context.Context, *url.URL, ...grpc.DialOption) registry2.Registry { + SetRegistrySupplier(func(context.Context, time.Duration, *url.URL, ...grpc.DialOption) registry2.Registry { return registry2.NewServer(memory.NewNetworkServiceRegistryServer(), memory.NewNetworkServiceEndpointRegistryServer()) }). SetRegistryProxySupplier(nil). diff --git a/pkg/registry/core/interdomain/interdomain_nse_test.go b/pkg/registry/core/interdomain/interdomain_nse_test.go index 38d845c78..45b753412 100644 --- a/pkg/registry/core/interdomain/interdomain_nse_test.go +++ b/pkg/registry/core/interdomain/interdomain_nse_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020 Doc.ai and/or its affiliates. +// Copyright (c) 2020-2021 Doc.ai and/or its affiliates. // // SPDX-License-Identifier: Apache-2.0 // @@ -223,7 +223,7 @@ func TestInterdomainFloatingNetworkServiceEndpointRegistry(t *testing.T) { domain3 := sandbox.NewBuilder(t). SetNodesCount(0). - SetRegistrySupplier(func(context.Context, *url.URL, ...grpc.DialOption) registry2.Registry { + SetRegistrySupplier(func(context.Context, time.Duration, *url.URL, ...grpc.DialOption) registry2.Registry { return registry2.NewServer(memory.NewNetworkServiceRegistryServer(), memory.NewNetworkServiceEndpointRegistryServer()) }). SetRegistryProxySupplier(nil). From 480f6b5634a82c3ca519f5be92ae9af5dae97401 Mon Sep 17 00:00:00 2001 From: Vladimir Popov Date: Tue, 26 Jan 2021 11:57:41 +0700 Subject: [PATCH 03/10] Add Builder.setupNode Signed-off-by: Vladimir Popov --- .../chains/nsmgr/server_test.go | 57 ++++++++++++------- .../chains/nsmgrproxy/server_test.go | 26 ++++----- pkg/tools/sandbox/README.md | 24 ++++++-- pkg/tools/sandbox/builder.go | 37 ++++++++++-- pkg/tools/sandbox/types.go | 6 +- 5 files changed, 100 insertions(+), 50 deletions(-) diff --git a/pkg/networkservice/chains/nsmgr/server_test.go b/pkg/networkservice/chains/nsmgr/server_test.go index 97d3940b8..6ca8f96ec 100644 --- a/pkg/networkservice/chains/nsmgr/server_test.go +++ b/pkg/networkservice/chains/nsmgr/server_test.go @@ -47,28 +47,17 @@ import ( "github.com/networkservicemesh/sdk/pkg/tools/sandbox" ) -func testDomain(ctx context.Context, t *testing.T, nodesCount int) *sandbox.Domain { - domain := sandbox.NewBuilder(t). - SetNodesCount(nodesCount). - SetRegistryProxySupplier(nil). - SetContext(ctx). - Build() - - for _, node := range domain.Nodes { - _, err := node.NewForwarder(ctx, new(registry.NetworkServiceEndpoint), sandbox.GenerateTestToken) - require.NoError(t, err) - } - - return domain -} - func TestNSMGR_RemoteUsecase_Parallel(t *testing.T) { defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - domain := testDomain(ctx, t, 2) + domain := sandbox.NewBuilder(t). + SetNodesCount(2). + SetRegistryProxySupplier(nil). + SetContext(ctx). + Build() defer domain.Cleanup() counter := &counterServer{} @@ -125,7 +114,11 @@ func TestNSMGR_SelectsRestartingEndpoint(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - domain := testDomain(ctx, t, 1) + domain := sandbox.NewBuilder(t). + SetNodesCount(1). + SetRegistryProxySupplier(nil). + SetContext(ctx). + Build() defer domain.Cleanup() request := &networkservice.NetworkServiceRequest{ @@ -163,7 +156,11 @@ func TestNSMGR_RemoteUsecase_BusyEndpoints(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - domain := testDomain(ctx, t, 2) + domain := sandbox.NewBuilder(t). + SetNodesCount(2). + SetRegistryProxySupplier(nil). + SetContext(ctx). + Build() defer domain.Cleanup() counter := new(counterServer) @@ -227,7 +224,11 @@ func TestNSMGR_RemoteUsecase(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - domain := testDomain(ctx, t, 2) + domain := sandbox.NewBuilder(t). + SetNodesCount(2). + SetRegistryProxySupplier(nil). + SetContext(ctx). + Build() defer domain.Cleanup() nseReg := ®istry.NetworkServiceEndpoint{ @@ -335,7 +336,11 @@ func TestNSMGR_LocalUsecase(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - domain := testDomain(ctx, t, 1) + domain := sandbox.NewBuilder(t). + SetNodesCount(1). + SetContext(ctx). + SetRegistryProxySupplier(nil). + Build() defer domain.Cleanup() nseReg := ®istry.NetworkServiceEndpoint{ @@ -391,7 +396,11 @@ func TestNSMGR_PassThroughRemote(t *testing.T) { defer cancel() const nodesCount = 7 - domain := testDomain(ctx, t, nodesCount) + domain := sandbox.NewBuilder(t). + SetNodesCount(nodesCount). + SetContext(ctx). + SetRegistryProxySupplier(nil). + Build() defer domain.Cleanup() for i := 0; i < nodesCount; i++ { @@ -450,7 +459,11 @@ func TestNSMGR_PassThroughLocal(t *testing.T) { defer cancel() const nsesCount = 7 - domain := testDomain(ctx, t, 1) + domain := sandbox.NewBuilder(t). + SetNodesCount(1). + SetContext(ctx). + SetRegistryProxySupplier(nil). + Build() defer domain.Cleanup() for i := 0; i < nsesCount; i++ { diff --git a/pkg/networkservice/chains/nsmgrproxy/server_test.go b/pkg/networkservice/chains/nsmgrproxy/server_test.go index 219e0679e..e80d0c9d6 100644 --- a/pkg/networkservice/chains/nsmgrproxy/server_test.go +++ b/pkg/networkservice/chains/nsmgrproxy/server_test.go @@ -29,23 +29,9 @@ import ( "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/kernel" "github.com/networkservicemesh/api/pkg/api/registry" - "github.com/networkservicemesh/sdk/pkg/registry/common/dnsresolve" "github.com/networkservicemesh/sdk/pkg/tools/sandbox" ) -func testDomain(ctx context.Context, t *testing.T, dnsServer dnsresolve.Resolver) *sandbox.Domain { - domain := sandbox.NewBuilder(t). - SetNodesCount(1). - SetContext(ctx). - SetDNSResolver(dnsServer). - Build() - - _, err := domain.Nodes[0].NewForwarder(ctx, new(registry.NetworkServiceEndpoint), sandbox.GenerateTestToken) - require.NoError(t, err) - - return domain -} - func TestNSMGR_InterdomainUseCase(t *testing.T) { defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) @@ -56,10 +42,18 @@ func TestNSMGR_InterdomainUseCase(t *testing.T) { dnsServer := new(sandbox.FakeDNSResolver) - domain1 := testDomain(ctx, t, dnsServer) + domain1 := sandbox.NewBuilder(t). + SetNodesCount(1). + SetContext(ctx). + SetDNSResolver(dnsServer). + Build() defer domain1.Cleanup() - domain2 := testDomain(ctx, t, dnsServer) + domain2 := sandbox.NewBuilder(t). + SetNodesCount(1). + SetDNSResolver(dnsServer). + SetContext(ctx). + Build() defer domain2.Cleanup() require.NoError(t, dnsServer.Register(remoteRegistryDomain, domain2.Registry.URL)) diff --git a/pkg/tools/sandbox/README.md b/pkg/tools/sandbox/README.md index d9d2c53c4..728393250 100644 --- a/pkg/tools/sandbox/README.md +++ b/pkg/tools/sandbox/README.md @@ -28,7 +28,6 @@ Solution: SetRegistryProxySupplier(nil). Build() defer localDomain.Cleanup() - localDomain.Nodes[0].NewForwarder(ctx, new(registry.Endpoint), sandbox.GenerateTestToken) localDomain.Nodes[0].NewEndpoint(ctx, ®istry.Endpoint{...}, sandbox.GenerateTestToken, ...myNewEndpointChain) client := localDomain.Nodes[0].NewClient(ctx, sandbox.GenerateTestToken, ...clientChain) ... @@ -56,6 +55,7 @@ Solution: SetNodesCount(1). SetNSMgrProxySupplier(nil). SetRegistryProxySupplier(nil). + SetNodeSetup(nil). Build() defer localDomain.Cleanup() localDomain.Nodes[0].NewForwarder(ctx, new(registry.Endpoint), sandbox.GenerateTestToken, ...myNewForwarderChain) @@ -72,16 +72,30 @@ Solution: SetNodesCount(2). Build() defer localDomain.Cleanup() - for _, node := range localDomain.Nodes { - node.NewForwarder(ctx, new(registry.Endpoint), sandbox.GenerateTestToken) - } localDomain.Nodes[1].NewEndpoint(ctx, ®istry.Endpoint{...}, sandbox.GenerateTestToken, ...remoteEndpointChain) ... ``` +Problem: setup NSMgr, set of Forwarders and set of Endpoints on each node to checking complex scenarios.\ +Solution: +```go + ... + localDomain := sandbox.NewBuilder(t). + SetNodesCount(nodesCount). + SetNSMgrProxySupplier(nil). + SetRegistryProxySupplier(nil). + SetNodeSetup(func(ctx context.Context, node *sandbox.Node) { + node.NewForwarder(ctx, new(registry.Endpoint), sandbox.GenerateTestToken, ...forwarderChain) + node.NewEndpoint(ctx, ®istry.Endpoint{...}, sandbox.GenerateTestToken, ...endpointChain) + }). + Build() + defer localDomain.Cleanup() + ... +``` + ### Setup only local registry -Problem: setup registry and to check API\ +Problem: setup registry and to check API.\ Solution: ```go ... diff --git a/pkg/tools/sandbox/builder.go b/pkg/tools/sandbox/builder.go index 556159897..ee09fce73 100644 --- a/pkg/tools/sandbox/builder.go +++ b/pkg/tools/sandbox/builder.go @@ -57,6 +57,7 @@ type Builder struct { supplyNSMgrProxy SupplyNSMgrProxyFunc supplyRegistry SupplyRegistryFunc supplyRegistryProxy SupplyRegistryProxyFunc + setupNode SetupNodeFunc generateTokenFunc token.GeneratorFunc ctx context.Context } @@ -72,6 +73,7 @@ func NewBuilder(t *testing.T) *Builder { supplyRegistry: memory.NewServer, supplyRegistryProxy: proxydns.NewServer, supplyNSMgrProxy: nsmgrproxy.NewServer, + setupNode: defaultSetupNode(t), generateTokenFunc: GenerateTestToken, } } @@ -85,7 +87,8 @@ func (b *Builder) Build() *Domain { b.resources = append(b.resources, cancel) } ctx = logger.WithLog(ctx) - domain := &Domain{} + + domain := new(Domain) domain.NSMgrProxy = b.newNSMgrProxy(ctx) if domain.NSMgrProxy == nil { domain.RegistryProxy = b.newRegistryProxy(ctx, &url.URL{}) @@ -97,12 +100,13 @@ func (b *Builder) Build() *Domain { } else { domain.Registry = b.newRegistry(ctx, domain.RegistryProxy.URL) } + for i := 0; i < b.nodesCount; i++ { - domain.Nodes = append(domain.Nodes, &Node{ - NSMgr: b.newNSMgr(ctx, domain.Registry.URL), - }) + domain.Nodes = append(domain.Nodes, b.newNode(ctx, domain.Registry.URL)) } + domain.resources, b.resources = b.resources, nil + return domain } @@ -160,6 +164,12 @@ func (b *Builder) SetNSMgrSupplier(f SupplyNSMgrFunc) *Builder { return b } +// SetNodeSetup replaces default node setup to custom function +func (b *Builder) SetNodeSetup(f SetupNodeFunc) *Builder { + b.setupNode = f + return b +} + func (b *Builder) dialContext(ctx context.Context, u *url.URL) *grpc.ClientConn { conn, err := grpc.DialContext(ctx, grpcutils.URLToTarget(u), grpc.WithInsecure(), @@ -259,3 +269,22 @@ func (b *Builder) newRegistry(ctx context.Context, proxyRegistryURL *url.URL) *R Registry: result, } } + +func (b *Builder) newNode(ctx context.Context, registryURL *url.URL) *Node { + node := &Node{ + NSMgr: b.newNSMgr(ctx, registryURL), + } + + if b.setupNode != nil { + b.setupNode(ctx, node) + } + + return node +} + +func defaultSetupNode(t *testing.T) SetupNodeFunc { + return func(ctx context.Context, node *Node) { + _, err := node.NewForwarder(ctx, new(registryapi.NetworkServiceEndpoint), GenerateTestToken) + require.NoError(t, err) + } +} diff --git a/pkg/tools/sandbox/types.go b/pkg/tools/sandbox/types.go index c34ef7504..bc15a3060 100644 --- a/pkg/tools/sandbox/types.go +++ b/pkg/tools/sandbox/types.go @@ -38,15 +38,15 @@ type SupplyNSMgrProxyFunc func(context.Context, string, token.GeneratorFunc, ... // SupplyNSMgrFunc supplies NSMGR type SupplyNSMgrFunc func(context.Context, *registryapi.NetworkServiceEndpoint, networkservice.NetworkServiceServer, token.GeneratorFunc, grpc.ClientConnInterface, ...grpc.DialOption) nsmgr.Nsmgr -// SupplyForwarderFunc supplies Forwarder -type SupplyForwarderFunc func(context.Context, string, token.GeneratorFunc, *url.URL, ...grpc.DialOption) endpoint.Endpoint - // SupplyRegistryFunc supplies Registry type SupplyRegistryFunc func(ctx context.Context, expiryDuration time.Duration, proxyRegistryURL *url.URL, options ...grpc.DialOption) registry.Registry // SupplyRegistryProxyFunc supplies registry proxy type SupplyRegistryProxyFunc func(ctx context.Context, dnsResolver dnsresolve.Resolver, handlingDNSDomain string, proxyNSMgrURL *url.URL, options ...grpc.DialOption) registry.Registry +// SetupNodeFunc setups each node on Builder.Build() stage +type SetupNodeFunc func(ctx context.Context, node *Node) + // RegistryEntry is pair of registry.Registry and url.URL type RegistryEntry struct { registry.Registry From c8bcb78397c4a9ba7a9193909188d30fed831c03 Mon Sep 17 00:00:00 2001 From: Vladimir Popov Date: Tue, 26 Jan 2021 19:54:22 +0700 Subject: [PATCH 04/10] Cleanup NSEs on context close Signed-off-by: Vladimir Popov --- pkg/tools/sandbox/node.go | 27 +++++++++++++++------------ pkg/tools/sandbox/types.go | 3 --- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/pkg/tools/sandbox/node.go b/pkg/tools/sandbox/node.go index e26cc8988..3e1b5b2ec 100644 --- a/pkg/tools/sandbox/node.go +++ b/pkg/tools/sandbox/node.go @@ -45,14 +45,7 @@ import ( // Node is a NSMgr with resources type Node struct { - NSMgr *NSMgrEntry - resources []context.CancelFunc -} - -func (n *Node) cleanup() { - for _, r := range n.resources { - r() - } + NSMgr *NSMgrEntry } // NewForwarder starts a new forwarder and registers it on the node NSMgr @@ -128,6 +121,8 @@ func (n *Node) newEndpoint( // 3. Register to the node NSMgr cc := n.dialNSMgr(ctx) + var resources []context.CancelFunc + nsClient := registry.NewNetworkServiceRegistryClient(cc) for _, name := range nse.NetworkServiceNames { var service *registry.NetworkService @@ -138,7 +133,7 @@ func (n *Node) newEndpoint( if err != nil { return nil, err } - n.resources = append(n.resources, func() { + resources = append(resources, func() { _, _ = nsClient.Unregister(ctx, service) }) } @@ -160,10 +155,17 @@ func (n *Node) newEndpoint( if err != nil { return nil, err } - n.resources = append(n.resources, func() { + resources = append(resources, func() { _, _ = nseClient.Unregister(ctx, nse) }) + go func() { + <-ctx.Done() + for _, resource := range resources { + resource() + } + }() + if isForwarder { logger.Log(ctx).Infof("Started listen forwarder %v on %v.", nse.Name, u.String()) } else { @@ -199,9 +201,10 @@ func (n *Node) dialNSMgr(ctx context.Context) grpc.ClientConnInterface { panic("failed to dial node NSMgr") } - n.resources = append(n.resources, func() { + go func() { + <-ctx.Done() _ = cc.Close() - }) + }() return cc } diff --git a/pkg/tools/sandbox/types.go b/pkg/tools/sandbox/types.go index bc15a3060..6b2cfe99b 100644 --- a/pkg/tools/sandbox/types.go +++ b/pkg/tools/sandbox/types.go @@ -78,9 +78,6 @@ type Domain struct { // Cleanup frees all resources related to the domain func (d *Domain) Cleanup() { - for _, node := range d.Nodes { - node.cleanup() - } for _, r := range d.resources { r() } From beb6b24870bb728ec0165dce2332a09f7505192d Mon Sep 17 00:00:00 2001 From: Vladimir Popov Date: Thu, 28 Jan 2021 18:50:51 +0700 Subject: [PATCH 05/10] Add sandbox registry client, rework endpoint Register/Unregister Signed-off-by: Vladimir Popov --- pkg/tools/sandbox/builder.go | 1 + pkg/tools/sandbox/node.go | 99 ++++++++++++++-------------- pkg/tools/sandbox/registry_client.go | 91 +++++++++++++++++++++++++ 3 files changed, 140 insertions(+), 51 deletions(-) create mode 100644 pkg/tools/sandbox/registry_client.go diff --git a/pkg/tools/sandbox/builder.go b/pkg/tools/sandbox/builder.go index ee09fce73..fc24a24cf 100644 --- a/pkg/tools/sandbox/builder.go +++ b/pkg/tools/sandbox/builder.go @@ -272,6 +272,7 @@ func (b *Builder) newRegistry(ctx context.Context, proxyRegistryURL *url.URL) *R func (b *Builder) newNode(ctx context.Context, registryURL *url.URL) *Node { node := &Node{ + ctx: b.ctx, NSMgr: b.newNSMgr(ctx, registryURL), } diff --git a/pkg/tools/sandbox/node.go b/pkg/tools/sandbox/node.go index 3e1b5b2ec..4420287e6 100644 --- a/pkg/tools/sandbox/node.go +++ b/pkg/tools/sandbox/node.go @@ -21,6 +21,7 @@ import ( "fmt" "net/url" + "github.com/golang/protobuf/ptypes/empty" "github.com/google/uuid" "google.golang.org/grpc" @@ -33,10 +34,6 @@ import ( "github.com/networkservicemesh/sdk/pkg/networkservice/common/clienturl" "github.com/networkservicemesh/sdk/pkg/networkservice/common/connect" "github.com/networkservicemesh/sdk/pkg/networkservice/core/adapters" - "github.com/networkservicemesh/sdk/pkg/registry/common/interpose" - "github.com/networkservicemesh/sdk/pkg/registry/common/null" - "github.com/networkservicemesh/sdk/pkg/registry/common/refresh" - "github.com/networkservicemesh/sdk/pkg/registry/core/chain" "github.com/networkservicemesh/sdk/pkg/tools/addressof" "github.com/networkservicemesh/sdk/pkg/tools/grpcutils" "github.com/networkservicemesh/sdk/pkg/tools/logger" @@ -45,6 +42,7 @@ import ( // Node is a NSMgr with resources type Node struct { + ctx context.Context NSMgr *NSMgrEntry } @@ -120,78 +118,83 @@ func (n *Node) newEndpoint( // 3. Register to the node NSMgr cc := n.dialNSMgr(ctx) + registryClient := NewRegistryClient(ctx, cc, isForwarder) - var resources []context.CancelFunc + go func() { + defer func() { _ = cc.Close() }() + <-ctx.Done() - nsClient := registry.NewNetworkServiceRegistryClient(cc) - for _, name := range nse.NetworkServiceNames { - var service *registry.NetworkService - service, err = nsClient.Register(ctx, ®istry.NetworkService{ - Name: name, - Payload: "IP", - }) - if err != nil { - return nil, err + // Domain is closing, no need to clean up + if n.ctx.Err() != nil { + return } - resources = append(resources, func() { - _, _ = nsClient.Unregister(ctx, service) - }) - } - var interposeClient registry.NetworkServiceEndpointRegistryClient - if isForwarder { - interposeClient = interpose.NewNetworkServiceEndpointRegistryClient() - } else { - interposeClient = null.NewNetworkServiceEndpointRegistryClient() - } + if nse != nil { + if _, unregisterErr := n.unregisterEndpoint(context.Background(), nse, registryClient); unregisterErr != nil { + logger.Log(ctx).Infof("Failed to unregister endpoint %s: %s", nse.Name, unregisterErr.Error()) + } + } + }() - nseClient := chain.NewNetworkServiceEndpointRegistryClient( - refresh.NewNetworkServiceEndpointRegistryClient( - refresh.WithChainContext(ctx)), - interposeClient, - registry.NewNetworkServiceEndpointRegistryClient(cc), - ) - nse, err = nseClient.Register(ctx, nse) + nse, err = n.registerEndpoint(ctx, nse, registryClient) if err != nil { return nil, err } - resources = append(resources, func() { - _, _ = nseClient.Unregister(ctx, nse) - }) - - go func() { - <-ctx.Done() - for _, resource := range resources { - resource() - } - }() if isForwarder { - logger.Log(ctx).Infof("Started listen forwarder %v on %v.", nse.Name, u.String()) + logger.Log(ctx).Infof("Started listen forwarder %s on %s.", nse.Name, u.String()) } else { - logger.Log(ctx).Infof("Started listen endpoint %v on %v.", nse.Name, u.String()) + logger.Log(ctx).Infof("Started listen endpoint %s on %s.", nse.Name, u.String()) } return &EndpointEntry{Endpoint: ep, URL: u}, nil } +func (n *Node) registerEndpoint( + ctx context.Context, + nse *registry.NetworkServiceEndpoint, + registryClient registry.NetworkServiceEndpointRegistryClient, +) (*registry.NetworkServiceEndpoint, error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + return registryClient.Register(ctx, nse) +} + +func (n *Node) unregisterEndpoint( + ctx context.Context, + nse *registry.NetworkServiceEndpoint, + registryClient registry.NetworkServiceEndpointRegistryClient, +) (*empty.Empty, error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + return registryClient.Unregister(ctx, nse) +} + // NewClient starts a new client and connects it to the node NSMgr func (n *Node) NewClient( ctx context.Context, generatorFunc token.GeneratorFunc, additionalFunctionality ...networkservice.NetworkServiceClient, ) networkservice.NetworkServiceClient { + cc := n.dialNSMgr(ctx) + go func() { + defer func() { _ = cc.Close() }() + <-ctx.Done() + }() + return client.NewClient( ctx, fmt.Sprintf("nsc-%v", uuid.New().String()), nil, generatorFunc, - n.dialNSMgr(ctx), + cc, additionalFunctionality..., ) } -func (n *Node) dialNSMgr(ctx context.Context) grpc.ClientConnInterface { +func (n *Node) dialNSMgr(ctx context.Context) *grpc.ClientConn { cc, err := grpc.DialContext(ctx, grpcutils.URLToTarget(n.NSMgr.URL), grpc.WithInsecure(), grpc.WithBlock(), @@ -200,11 +203,5 @@ func (n *Node) dialNSMgr(ctx context.Context) grpc.ClientConnInterface { if err != nil { panic("failed to dial node NSMgr") } - - go func() { - <-ctx.Done() - _ = cc.Close() - }() - return cc } diff --git a/pkg/tools/sandbox/registry_client.go b/pkg/tools/sandbox/registry_client.go new file mode 100644 index 000000000..8ef98bf62 --- /dev/null +++ b/pkg/tools/sandbox/registry_client.go @@ -0,0 +1,91 @@ +// Copyright (c) 2021 Doc.ai and/or its affiliates. +// +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sandbox + +import ( + "context" + + "github.com/golang/protobuf/ptypes/empty" + "github.com/pkg/errors" + "google.golang.org/grpc" + + "github.com/networkservicemesh/api/pkg/api/networkservice/payload" + "github.com/networkservicemesh/api/pkg/api/registry" + + "github.com/networkservicemesh/sdk/pkg/registry/common/interpose" + "github.com/networkservicemesh/sdk/pkg/registry/common/null" + "github.com/networkservicemesh/sdk/pkg/registry/common/refresh" + "github.com/networkservicemesh/sdk/pkg/registry/core/chain" +) + +type registryClient struct { + nsClient registry.NetworkServiceRegistryClient + nseClient registry.NetworkServiceEndpointRegistryClient +} + +// NewRegistryClient returns NSE registry client to register NSE to registry +func NewRegistryClient(ctx context.Context, cc grpc.ClientConnInterface, isForwarder bool) registry.NetworkServiceEndpointRegistryClient { + var interposeClient registry.NetworkServiceEndpointRegistryClient + if isForwarder { + interposeClient = interpose.NewNetworkServiceEndpointRegistryClient() + } else { + interposeClient = null.NewNetworkServiceEndpointRegistryClient() + } + return ®istryClient{ + nsClient: registry.NewNetworkServiceRegistryClient(cc), + nseClient: chain.NewNetworkServiceEndpointRegistryClient( + refresh.NewNetworkServiceEndpointRegistryClient( + refresh.WithChainContext(ctx)), + interposeClient, + registry.NewNetworkServiceEndpointRegistryClient(cc), + ), + } +} + +func (c *registryClient) Register(ctx context.Context, nse *registry.NetworkServiceEndpoint, opts ...grpc.CallOption) (_ *registry.NetworkServiceEndpoint, err error) { + if nse, err = c.nseClient.Register(ctx, nse, opts...); err != nil { + return nil, err + } + for _, name := range nse.NetworkServiceNames { + if _, err = c.nsClient.Register(ctx, ®istry.NetworkService{ + Name: name, + Payload: payload.IP, + }, opts...); err != nil { + return nil, err + } + } + return nse, err +} + +func (c *registryClient) Find(ctx context.Context, query *registry.NetworkServiceEndpointQuery, opts ...grpc.CallOption) (registry.NetworkServiceEndpointRegistry_FindClient, error) { + return c.nseClient.Find(ctx, query, opts...) +} + +func (c *registryClient) Unregister(ctx context.Context, nse *registry.NetworkServiceEndpoint, opts ...grpc.CallOption) (_ *empty.Empty, err error) { + if _, unregisterErr := c.nseClient.Unregister(ctx, nse, opts...); unregisterErr != nil { + err = errors.Wrapf(unregisterErr, "%v\n", err) + } + for _, name := range nse.NetworkServiceNames { + if _, unregisterErr := c.nsClient.Unregister(ctx, ®istry.NetworkService{ + Name: name, + Payload: payload.IP, + }, opts...); unregisterErr != nil { + err = errors.Wrapf(unregisterErr, "%v\n", err) + } + } + return new(empty.Empty), err +} From a5d200e61e50a57c248da0e2c0a464b3fa55f789 Mon Sep 17 00:00:00 2001 From: Vladimir Popov Date: Fri, 29 Jan 2021 17:56:16 +0700 Subject: [PATCH 06/10] Rework endpoint registration in sandbox Signed-off-by: Vladimir Popov --- .../chains/nsmgr/server_test.go | 11 +-- pkg/tools/sandbox/README.md | 4 +- pkg/tools/sandbox/builder.go | 14 ++- pkg/tools/sandbox/node.go | 90 ++++++------------- 4 files changed, 41 insertions(+), 78 deletions(-) diff --git a/pkg/networkservice/chains/nsmgr/server_test.go b/pkg/networkservice/chains/nsmgr/server_test.go index 6ca8f96ec..7c2462117 100644 --- a/pkg/networkservice/chains/nsmgr/server_test.go +++ b/pkg/networkservice/chains/nsmgr/server_test.go @@ -35,7 +35,6 @@ import ( "github.com/networkservicemesh/api/pkg/api/networkservice" "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/cls" kernelmech "github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/kernel" - "github.com/networkservicemesh/api/pkg/api/networkservice/payload" "github.com/networkservicemesh/api/pkg/api/registry" "github.com/networkservicemesh/sdk/pkg/networkservice/common/clienturl" @@ -576,16 +575,8 @@ func testNSEAndClient( _, err = nsc.Close(ctx, conn) require.NoError(t, err) - _, err = domain.Nodes[0].NSMgr.NetworkServiceEndpointRegistryServer().Unregister(ctx, nseReg) + _, err = domain.Nodes[0].EndpointRegistryClient.Unregister(ctx, nseReg) require.NoError(t, err) - - for _, name := range nseReg.NetworkServiceNames { - _, err = domain.Nodes[0].NSMgr.NetworkServiceRegistryServer().Unregister(ctx, ®istry.NetworkService{ - Name: name, - Payload: payload.IP, - }) - require.NoError(t, err) - } } type passThroughClient struct { diff --git a/pkg/tools/sandbox/README.md b/pkg/tools/sandbox/README.md index 728393250..904443256 100644 --- a/pkg/tools/sandbox/README.md +++ b/pkg/tools/sandbox/README.md @@ -58,7 +58,7 @@ Solution: SetNodeSetup(nil). Build() defer localDomain.Cleanup() - localDomain.Nodes[0].NewForwarder(ctx, new(registry.Endpoint), sandbox.GenerateTestToken, ...myNewForwarderChain) + localDomain.Nodes[0].NewForwarder(ctx, ®istry.Endpoint{...}, sandbox.GenerateTestToken, ...myNewForwarderChain) ... ``` @@ -85,7 +85,7 @@ Solution: SetNSMgrProxySupplier(nil). SetRegistryProxySupplier(nil). SetNodeSetup(func(ctx context.Context, node *sandbox.Node) { - node.NewForwarder(ctx, new(registry.Endpoint), sandbox.GenerateTestToken, ...forwarderChain) + node.NewForwarder(ctx, ®istry.Endpoint{...}, sandbox.GenerateTestToken, ...forwarderChain) node.NewEndpoint(ctx, ®istry.Endpoint{...}, sandbox.GenerateTestToken, ...endpointChain) }). Build() diff --git a/pkg/tools/sandbox/builder.go b/pkg/tools/sandbox/builder.go index fc24a24cf..6c0805e33 100644 --- a/pkg/tools/sandbox/builder.go +++ b/pkg/tools/sandbox/builder.go @@ -271,9 +271,14 @@ func (b *Builder) newRegistry(ctx context.Context, proxyRegistryURL *url.URL) *R } func (b *Builder) newNode(ctx context.Context, registryURL *url.URL) *Node { + nsmgrEntry := b.newNSMgr(ctx, registryURL) + nsmgrCC := b.dialContext(ctx, nsmgrEntry.URL) + node := &Node{ - ctx: b.ctx, - NSMgr: b.newNSMgr(ctx, registryURL), + ctx: b.ctx, + NSMgr: nsmgrEntry, + ForwarderRegistryClient: NewRegistryClient(ctx, nsmgrCC, true), + EndpointRegistryClient: NewRegistryClient(ctx, nsmgrCC, false), } if b.setupNode != nil { @@ -285,7 +290,10 @@ func (b *Builder) newNode(ctx context.Context, registryURL *url.URL) *Node { func defaultSetupNode(t *testing.T) SetupNodeFunc { return func(ctx context.Context, node *Node) { - _, err := node.NewForwarder(ctx, new(registryapi.NetworkServiceEndpoint), GenerateTestToken) + nseReg := ®istryapi.NetworkServiceEndpoint{ + Name: uuid.New().String(), + } + _, err := node.NewForwarder(ctx, nseReg, GenerateTestToken) require.NoError(t, err) } } diff --git a/pkg/tools/sandbox/node.go b/pkg/tools/sandbox/node.go index 4420287e6..f05badb72 100644 --- a/pkg/tools/sandbox/node.go +++ b/pkg/tools/sandbox/node.go @@ -21,7 +21,6 @@ import ( "fmt" "net/url" - "github.com/golang/protobuf/ptypes/empty" "github.com/google/uuid" "google.golang.org/grpc" @@ -40,10 +39,12 @@ import ( "github.com/networkservicemesh/sdk/pkg/tools/token" ) -// Node is a NSMgr with resources +// Node is a NSMgr with Forwarder, NSE registry clients type Node struct { - ctx context.Context - NSMgr *NSMgrEntry + ctx context.Context + NSMgr *NSMgrEntry + ForwarderRegistryClient registry.NetworkServiceEndpointRegistryClient + EndpointRegistryClient registry.NetworkServiceEndpointRegistryClient } // NewForwarder starts a new forwarder and registers it on the node NSMgr @@ -112,35 +113,24 @@ func (n *Node) newEndpoint( ctx = logger.WithLog(ctx) serve(ctx, u, ep.Register) - if nse.Url == "" { - nse.Url = u.String() - } - - // 3. Register to the node NSMgr - cc := n.dialNSMgr(ctx) - registryClient := NewRegistryClient(ctx, cc, isForwarder) - - go func() { - defer func() { _ = cc.Close() }() - <-ctx.Done() + nse.Url = u.String() - // Domain is closing, no need to clean up - if n.ctx.Err() != nil { - return - } - - if nse != nil { - if _, unregisterErr := n.unregisterEndpoint(context.Background(), nse, registryClient); unregisterErr != nil { - logger.Log(ctx).Infof("Failed to unregister endpoint %s: %s", nse.Name, unregisterErr.Error()) - } - } - }() + // 3. Register with the node registry client + var registryClient registry.NetworkServiceEndpointRegistryClient + if isForwarder { + registryClient = n.ForwarderRegistryClient + } else { + registryClient = n.EndpointRegistryClient + } - nse, err = n.registerEndpoint(ctx, nse, registryClient) - if err != nil { + var reg *registry.NetworkServiceEndpoint + if reg, err = registryClient.Register(ctx, nse); err != nil { return nil, err } + nse.Name = reg.Name + nse.ExpirationTime = reg.ExpirationTime + if isForwarder { logger.Log(ctx).Infof("Started listen forwarder %s on %s.", nse.Name, u.String()) } else { @@ -150,35 +140,21 @@ func (n *Node) newEndpoint( return &EndpointEntry{Endpoint: ep, URL: u}, nil } -func (n *Node) registerEndpoint( - ctx context.Context, - nse *registry.NetworkServiceEndpoint, - registryClient registry.NetworkServiceEndpointRegistryClient, -) (*registry.NetworkServiceEndpoint, error) { - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - return registryClient.Register(ctx, nse) -} - -func (n *Node) unregisterEndpoint( - ctx context.Context, - nse *registry.NetworkServiceEndpoint, - registryClient registry.NetworkServiceEndpointRegistryClient, -) (*empty.Empty, error) { - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - return registryClient.Unregister(ctx, nse) -} - // NewClient starts a new client and connects it to the node NSMgr func (n *Node) NewClient( ctx context.Context, generatorFunc token.GeneratorFunc, additionalFunctionality ...networkservice.NetworkServiceClient, ) networkservice.NetworkServiceClient { - cc := n.dialNSMgr(ctx) + cc, err := grpc.DialContext(ctx, grpcutils.URLToTarget(n.NSMgr.URL), + grpc.WithInsecure(), + grpc.WithBlock(), + grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), + ) + if err != nil { + logger.Log(ctx).Fatalf("Failed to dial node NSMgr: %s", err.Error()) + } + go func() { defer func() { _ = cc.Close() }() <-ctx.Done() @@ -193,15 +169,3 @@ func (n *Node) NewClient( additionalFunctionality..., ) } - -func (n *Node) dialNSMgr(ctx context.Context) *grpc.ClientConn { - cc, err := grpc.DialContext(ctx, grpcutils.URLToTarget(n.NSMgr.URL), - grpc.WithInsecure(), - grpc.WithBlock(), - grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), - ) - if err != nil { - panic("failed to dial node NSMgr") - } - return cc -} From 69613972b1f3ff1eb5e0773191eb61b6c6c89cce Mon Sep 17 00:00:00 2001 From: Vladimir Popov Date: Mon, 1 Feb 2021 11:55:34 +0700 Subject: [PATCH 07/10] Increase default registry expiration Signed-off-by: Vladimir Popov --- pkg/tools/sandbox/builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tools/sandbox/builder.go b/pkg/tools/sandbox/builder.go index 6c0805e33..5e9cd2e22 100644 --- a/pkg/tools/sandbox/builder.go +++ b/pkg/tools/sandbox/builder.go @@ -43,7 +43,7 @@ import ( const ( defaultContextTimeout = time.Second * 15 - defaultRegistryExpiryDuration = 100 * time.Millisecond + defaultRegistryExpiryDuration = 500 * time.Millisecond ) // Builder implements builder pattern for building NSM Domain From 3ca880b44e1a1042a750f1bbf24bc0f2338715bf Mon Sep 17 00:00:00 2001 From: Vladimir Popov Date: Mon, 1 Feb 2021 19:32:38 +0700 Subject: [PATCH 08/10] Use NSE registry client in sandbox Signed-off-by: Vladimir Popov --- .../chains/nsmgr/server_test.go | 7 ++ pkg/tools/sandbox/builder.go | 6 +- pkg/tools/sandbox/node.go | 42 ++++----- pkg/tools/sandbox/registry_client.go | 91 ------------------- 4 files changed, 32 insertions(+), 114 deletions(-) delete mode 100644 pkg/tools/sandbox/registry_client.go diff --git a/pkg/networkservice/chains/nsmgr/server_test.go b/pkg/networkservice/chains/nsmgr/server_test.go index 7c2462117..9b29d9bde 100644 --- a/pkg/networkservice/chains/nsmgr/server_test.go +++ b/pkg/networkservice/chains/nsmgr/server_test.go @@ -577,6 +577,13 @@ func testNSEAndClient( _, err = domain.Nodes[0].EndpointRegistryClient.Unregister(ctx, nseReg) require.NoError(t, err) + + for _, nsName := range nseReg.NetworkServiceNames { + _, err = domain.Nodes[0].NSRegistryClient.Unregister(ctx, ®istry.NetworkService{ + Name: nsName, + }) + require.NoError(t, err) + } } type passThroughClient struct { diff --git a/pkg/tools/sandbox/builder.go b/pkg/tools/sandbox/builder.go index 5e9cd2e22..25fb70433 100644 --- a/pkg/tools/sandbox/builder.go +++ b/pkg/tools/sandbox/builder.go @@ -32,6 +32,7 @@ import ( "github.com/networkservicemesh/sdk/pkg/networkservice/chains/nsmgr" "github.com/networkservicemesh/sdk/pkg/networkservice/chains/nsmgrproxy" "github.com/networkservicemesh/sdk/pkg/networkservice/common/authorize" + "github.com/networkservicemesh/sdk/pkg/registry/chains/client" "github.com/networkservicemesh/sdk/pkg/registry/chains/memory" "github.com/networkservicemesh/sdk/pkg/registry/chains/proxydns" "github.com/networkservicemesh/sdk/pkg/registry/common/dnsresolve" @@ -277,8 +278,9 @@ func (b *Builder) newNode(ctx context.Context, registryURL *url.URL) *Node { node := &Node{ ctx: b.ctx, NSMgr: nsmgrEntry, - ForwarderRegistryClient: NewRegistryClient(ctx, nsmgrCC, true), - EndpointRegistryClient: NewRegistryClient(ctx, nsmgrCC, false), + ForwarderRegistryClient: client.NewNetworkServiceEndpointRegistryInterposeClient(ctx, nsmgrCC), + EndpointRegistryClient: client.NewNetworkServiceEndpointRegistryClient(ctx, nsmgrCC), + NSRegistryClient: client.NewNetworkServiceRegistryClient(nsmgrCC), } if b.setupNode != nil { diff --git a/pkg/tools/sandbox/node.go b/pkg/tools/sandbox/node.go index f05badb72..5694d7bb8 100644 --- a/pkg/tools/sandbox/node.go +++ b/pkg/tools/sandbox/node.go @@ -25,7 +25,8 @@ import ( "google.golang.org/grpc" "github.com/networkservicemesh/api/pkg/api/networkservice" - "github.com/networkservicemesh/api/pkg/api/registry" + "github.com/networkservicemesh/api/pkg/api/networkservice/payload" + registryapi "github.com/networkservicemesh/api/pkg/api/registry" "github.com/networkservicemesh/sdk/pkg/networkservice/chains/client" "github.com/networkservicemesh/sdk/pkg/networkservice/chains/endpoint" @@ -43,14 +44,15 @@ import ( type Node struct { ctx context.Context NSMgr *NSMgrEntry - ForwarderRegistryClient registry.NetworkServiceEndpointRegistryClient - EndpointRegistryClient registry.NetworkServiceEndpointRegistryClient + ForwarderRegistryClient registryapi.NetworkServiceEndpointRegistryClient + EndpointRegistryClient registryapi.NetworkServiceEndpointRegistryClient + NSRegistryClient registryapi.NetworkServiceRegistryClient } // NewForwarder starts a new forwarder and registers it on the node NSMgr func (n *Node) NewForwarder( ctx context.Context, - nse *registry.NetworkServiceEndpoint, + nse *registryapi.NetworkServiceEndpoint, generatorFunc token.GeneratorFunc, additionalFunctionality ...networkservice.NetworkServiceServer, ) (*EndpointEntry, error) { @@ -66,7 +68,7 @@ func (n *Node) NewForwarder( grpc.WithInsecure(), grpc.WithDefaultCallOptions(grpc.WaitForReady(true)), )) - entry, err := n.newEndpoint(ctx, nse, generatorFunc, true, additionalFunctionality...) + entry, err := n.newEndpoint(ctx, nse, generatorFunc, n.ForwarderRegistryClient, additionalFunctionality...) if err != nil { return nil, err } @@ -78,18 +80,18 @@ func (n *Node) NewForwarder( // NewEndpoint starts a new endpoint and registers it on the node NSMgr func (n *Node) NewEndpoint( ctx context.Context, - nse *registry.NetworkServiceEndpoint, + nse *registryapi.NetworkServiceEndpoint, generatorFunc token.GeneratorFunc, additionalFunctionality ...networkservice.NetworkServiceServer, ) (*EndpointEntry, error) { - return n.newEndpoint(ctx, nse, generatorFunc, false, additionalFunctionality...) + return n.newEndpoint(ctx, nse, generatorFunc, n.EndpointRegistryClient, additionalFunctionality...) } func (n *Node) newEndpoint( ctx context.Context, - nse *registry.NetworkServiceEndpoint, + nse *registryapi.NetworkServiceEndpoint, generatorFunc token.GeneratorFunc, - isForwarder bool, + registryClient registryapi.NetworkServiceEndpointRegistryClient, additionalFunctionality ...networkservice.NetworkServiceServer, ) (_ *EndpointEntry, err error) { // 1. Create endpoint server @@ -116,14 +118,7 @@ func (n *Node) newEndpoint( nse.Url = u.String() // 3. Register with the node registry client - var registryClient registry.NetworkServiceEndpointRegistryClient - if isForwarder { - registryClient = n.ForwarderRegistryClient - } else { - registryClient = n.EndpointRegistryClient - } - - var reg *registry.NetworkServiceEndpoint + var reg *registryapi.NetworkServiceEndpoint if reg, err = registryClient.Register(ctx, nse); err != nil { return nil, err } @@ -131,12 +126,17 @@ func (n *Node) newEndpoint( nse.Name = reg.Name nse.ExpirationTime = reg.ExpirationTime - if isForwarder { - logger.Log(ctx).Infof("Started listen forwarder %s on %s.", nse.Name, u.String()) - } else { - logger.Log(ctx).Infof("Started listen endpoint %s on %s.", nse.Name, u.String()) + for _, nsName := range nse.NetworkServiceNames { + if _, err = n.NSRegistryClient.Register(ctx, ®istryapi.NetworkService{ + Name: nsName, + Payload: payload.IP, + }); err != nil { + return nil, err + } } + logger.Log(ctx).Infof("Started listen endpoint %s on %s.", nse.Name, u.String()) + return &EndpointEntry{Endpoint: ep, URL: u}, nil } diff --git a/pkg/tools/sandbox/registry_client.go b/pkg/tools/sandbox/registry_client.go deleted file mode 100644 index 8ef98bf62..000000000 --- a/pkg/tools/sandbox/registry_client.go +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright (c) 2021 Doc.ai and/or its affiliates. -// -// SPDX-License-Identifier: Apache-2.0 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at: -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sandbox - -import ( - "context" - - "github.com/golang/protobuf/ptypes/empty" - "github.com/pkg/errors" - "google.golang.org/grpc" - - "github.com/networkservicemesh/api/pkg/api/networkservice/payload" - "github.com/networkservicemesh/api/pkg/api/registry" - - "github.com/networkservicemesh/sdk/pkg/registry/common/interpose" - "github.com/networkservicemesh/sdk/pkg/registry/common/null" - "github.com/networkservicemesh/sdk/pkg/registry/common/refresh" - "github.com/networkservicemesh/sdk/pkg/registry/core/chain" -) - -type registryClient struct { - nsClient registry.NetworkServiceRegistryClient - nseClient registry.NetworkServiceEndpointRegistryClient -} - -// NewRegistryClient returns NSE registry client to register NSE to registry -func NewRegistryClient(ctx context.Context, cc grpc.ClientConnInterface, isForwarder bool) registry.NetworkServiceEndpointRegistryClient { - var interposeClient registry.NetworkServiceEndpointRegistryClient - if isForwarder { - interposeClient = interpose.NewNetworkServiceEndpointRegistryClient() - } else { - interposeClient = null.NewNetworkServiceEndpointRegistryClient() - } - return ®istryClient{ - nsClient: registry.NewNetworkServiceRegistryClient(cc), - nseClient: chain.NewNetworkServiceEndpointRegistryClient( - refresh.NewNetworkServiceEndpointRegistryClient( - refresh.WithChainContext(ctx)), - interposeClient, - registry.NewNetworkServiceEndpointRegistryClient(cc), - ), - } -} - -func (c *registryClient) Register(ctx context.Context, nse *registry.NetworkServiceEndpoint, opts ...grpc.CallOption) (_ *registry.NetworkServiceEndpoint, err error) { - if nse, err = c.nseClient.Register(ctx, nse, opts...); err != nil { - return nil, err - } - for _, name := range nse.NetworkServiceNames { - if _, err = c.nsClient.Register(ctx, ®istry.NetworkService{ - Name: name, - Payload: payload.IP, - }, opts...); err != nil { - return nil, err - } - } - return nse, err -} - -func (c *registryClient) Find(ctx context.Context, query *registry.NetworkServiceEndpointQuery, opts ...grpc.CallOption) (registry.NetworkServiceEndpointRegistry_FindClient, error) { - return c.nseClient.Find(ctx, query, opts...) -} - -func (c *registryClient) Unregister(ctx context.Context, nse *registry.NetworkServiceEndpoint, opts ...grpc.CallOption) (_ *empty.Empty, err error) { - if _, unregisterErr := c.nseClient.Unregister(ctx, nse, opts...); unregisterErr != nil { - err = errors.Wrapf(unregisterErr, "%v\n", err) - } - for _, name := range nse.NetworkServiceNames { - if _, unregisterErr := c.nsClient.Unregister(ctx, ®istry.NetworkService{ - Name: name, - Payload: payload.IP, - }, opts...); unregisterErr != nil { - err = errors.Wrapf(unregisterErr, "%v\n", err) - } - } - return new(empty.Empty), err -} From 240776779444db96ef1e4c22896a1ddbe6e1e9ef Mon Sep 17 00:00:00 2001 From: Vladimir Popov Date: Fri, 5 Feb 2021 15:16:00 +0700 Subject: [PATCH 09/10] NSE should be registered after all NS registered Signed-off-by: Vladimir Popov --- pkg/networkservice/chains/nsmgr/server_test.go | 9 +++++---- pkg/tools/sandbox/node.go | 15 ++++++++------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/pkg/networkservice/chains/nsmgr/server_test.go b/pkg/networkservice/chains/nsmgr/server_test.go index 9b29d9bde..cd8aca343 100644 --- a/pkg/networkservice/chains/nsmgr/server_test.go +++ b/pkg/networkservice/chains/nsmgr/server_test.go @@ -578,12 +578,13 @@ func testNSEAndClient( _, err = domain.Nodes[0].EndpointRegistryClient.Unregister(ctx, nseReg) require.NoError(t, err) - for _, nsName := range nseReg.NetworkServiceNames { - _, err = domain.Nodes[0].NSRegistryClient.Unregister(ctx, ®istry.NetworkService{ - Name: nsName, + require.Eventually(t, func() bool { + stream, err := domain.Nodes[0].NSRegistryClient.Find(ctx, ®istry.NetworkServiceQuery{ + NetworkService: new(registry.NetworkService), }) require.NoError(t, err) - } + return len(registry.ReadNetworkServiceList(stream)) == 0 + }, 100*time.Millisecond, 10*time.Millisecond) } type passThroughClient struct { diff --git a/pkg/tools/sandbox/node.go b/pkg/tools/sandbox/node.go index 5694d7bb8..998270ada 100644 --- a/pkg/tools/sandbox/node.go +++ b/pkg/tools/sandbox/node.go @@ -118,13 +118,6 @@ func (n *Node) newEndpoint( nse.Url = u.String() // 3. Register with the node registry client - var reg *registryapi.NetworkServiceEndpoint - if reg, err = registryClient.Register(ctx, nse); err != nil { - return nil, err - } - - nse.Name = reg.Name - nse.ExpirationTime = reg.ExpirationTime for _, nsName := range nse.NetworkServiceNames { if _, err = n.NSRegistryClient.Register(ctx, ®istryapi.NetworkService{ @@ -135,6 +128,14 @@ func (n *Node) newEndpoint( } } + var reg *registryapi.NetworkServiceEndpoint + if reg, err = registryClient.Register(ctx, nse); err != nil { + return nil, err + } + + nse.Name = reg.Name + nse.ExpirationTime = reg.ExpirationTime + logger.Log(ctx).Infof("Started listen endpoint %s on %s.", nse.Name, u.String()) return &EndpointEntry{Endpoint: ep, URL: u}, nil From 0b46ca9d7ebf83f913ee6fa5a3f088edc0553194 Mon Sep 17 00:00:00 2001 From: Vladimir Popov Date: Mon, 8 Feb 2021 15:28:52 +0700 Subject: [PATCH 10/10] Fix imports aliaces Signed-off-by: Vladimir Popov --- .../core/interdomain/interdomain_ns_test.go | 38 +++++++++---------- .../core/interdomain/interdomain_nse_test.go | 38 +++++++++---------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pkg/registry/core/interdomain/interdomain_ns_test.go b/pkg/registry/core/interdomain/interdomain_ns_test.go index 6c3678f82..c7db47ed0 100644 --- a/pkg/registry/core/interdomain/interdomain_ns_test.go +++ b/pkg/registry/core/interdomain/interdomain_ns_test.go @@ -22,12 +22,12 @@ import ( "testing" "time" - "github.com/networkservicemesh/api/pkg/api/registry" + registryapi "github.com/networkservicemesh/api/pkg/api/registry" "github.com/stretchr/testify/require" "go.uber.org/goleak" "google.golang.org/grpc" - registry2 "github.com/networkservicemesh/sdk/pkg/registry" + "github.com/networkservicemesh/sdk/pkg/registry" "github.com/networkservicemesh/sdk/pkg/registry/common/memory" "github.com/networkservicemesh/sdk/pkg/tools/grpcutils" "github.com/networkservicemesh/sdk/pkg/tools/sandbox" @@ -79,7 +79,7 @@ func TestInterdomainNetworkServiceRegistry(t *testing.T) { _, err := domain2.Registry.NetworkServiceRegistryServer().Register( context.Background(), - ®istry.NetworkService{ + ®istryapi.NetworkService{ Name: "ns-1", }, ) @@ -91,17 +91,17 @@ func TestInterdomainNetworkServiceRegistry(t *testing.T) { _ = cc.Close() }() - client := registry.NewNetworkServiceRegistryClient(cc) + client := registryapi.NewNetworkServiceRegistryClient(cc) - stream, err := client.Find(ctx, ®istry.NetworkServiceQuery{ - NetworkService: ®istry.NetworkService{ + stream, err := client.Find(ctx, ®istryapi.NetworkServiceQuery{ + NetworkService: ®istryapi.NetworkService{ Name: "ns-1@" + remoteRegistryDomain, }, }) require.Nil(t, err) - list := registry.ReadNetworkServiceList(stream) + list := registryapi.ReadNetworkServiceList(stream) require.Len(t, list, 1) require.Equal(t, "ns-1@"+remoteRegistryDomain, list[0].Name) @@ -144,7 +144,7 @@ func TestLocalDomain_NetworkServiceRegistry(t *testing.T) { expected, err := domain1.Registry.NetworkServiceRegistryServer().Register( context.Background(), - ®istry.NetworkService{ + ®istryapi.NetworkService{ Name: "ns-1", }, ) @@ -155,17 +155,17 @@ func TestLocalDomain_NetworkServiceRegistry(t *testing.T) { defer func() { _ = cc.Close() }() - client := registry.NewNetworkServiceRegistryClient(cc) + client := registryapi.NewNetworkServiceRegistryClient(cc) - stream, err := client.Find(context.Background(), ®istry.NetworkServiceQuery{ - NetworkService: ®istry.NetworkService{ + stream, err := client.Find(context.Background(), ®istryapi.NetworkServiceQuery{ + NetworkService: ®istryapi.NetworkService{ Name: expected.Name + "@" + localRegistryDomain, }, }) require.Nil(t, err) - list := registry.ReadNetworkServiceList(stream) + list := registryapi.ReadNetworkServiceList(stream) require.Len(t, list, 1) require.Equal(t, expected.Name, list[0].Name) @@ -217,8 +217,8 @@ func TestInterdomainFloatingNetworkServiceRegistry(t *testing.T) { domain3 := sandbox.NewBuilder(t). SetNodesCount(0). - SetRegistrySupplier(func(context.Context, time.Duration, *url.URL, ...grpc.DialOption) registry2.Registry { - return registry2.NewServer(memory.NewNetworkServiceRegistryServer(), memory.NewNetworkServiceEndpointRegistryServer()) + SetRegistrySupplier(func(context.Context, time.Duration, *url.URL, ...grpc.DialOption) registry.Registry { + return registry.NewServer(memory.NewNetworkServiceRegistryServer(), memory.NewNetworkServiceEndpointRegistryServer()) }). SetRegistryProxySupplier(nil). Build() @@ -232,7 +232,7 @@ func TestInterdomainFloatingNetworkServiceRegistry(t *testing.T) { _, err := domain2.Registry.NetworkServiceRegistryServer().Register( context.Background(), - ®istry.NetworkService{ + ®istryapi.NetworkService{ Name: "ns-1@" + floatingRegistryDomain, }, ) @@ -244,17 +244,17 @@ func TestInterdomainFloatingNetworkServiceRegistry(t *testing.T) { _ = cc.Close() }() - client := registry.NewNetworkServiceRegistryClient(cc) + client := registryapi.NewNetworkServiceRegistryClient(cc) - stream, err := client.Find(ctx, ®istry.NetworkServiceQuery{ - NetworkService: ®istry.NetworkService{ + stream, err := client.Find(ctx, ®istryapi.NetworkServiceQuery{ + NetworkService: ®istryapi.NetworkService{ Name: "ns-1@" + floatingRegistryDomain, }, }) require.Nil(t, err) - list := registry.ReadNetworkServiceList(stream) + list := registryapi.ReadNetworkServiceList(stream) require.Len(t, list, 1) require.Equal(t, "ns-1", list[0].Name) diff --git a/pkg/registry/core/interdomain/interdomain_nse_test.go b/pkg/registry/core/interdomain/interdomain_nse_test.go index 45b753412..40f1c6617 100644 --- a/pkg/registry/core/interdomain/interdomain_nse_test.go +++ b/pkg/registry/core/interdomain/interdomain_nse_test.go @@ -23,12 +23,12 @@ import ( "time" "github.com/golang/protobuf/ptypes" - "github.com/networkservicemesh/api/pkg/api/registry" + registryapi "github.com/networkservicemesh/api/pkg/api/registry" "github.com/stretchr/testify/require" "go.uber.org/goleak" "google.golang.org/grpc" - registry2 "github.com/networkservicemesh/sdk/pkg/registry" + "github.com/networkservicemesh/sdk/pkg/registry" "github.com/networkservicemesh/sdk/pkg/registry/common/memory" "github.com/networkservicemesh/sdk/pkg/tools/grpcutils" "github.com/networkservicemesh/sdk/pkg/tools/sandbox" @@ -78,7 +78,7 @@ func TestInterdomainNetworkServiceEndpointRegistry(t *testing.T) { _, err := domain2.Registry.NetworkServiceEndpointRegistryServer().Register( context.Background(), - ®istry.NetworkServiceEndpoint{ + ®istryapi.NetworkServiceEndpoint{ Name: "nse-1", Url: "nsmgr-url", ExpirationTime: expirationTime, @@ -93,17 +93,17 @@ func TestInterdomainNetworkServiceEndpointRegistry(t *testing.T) { _ = cc.Close() }() - client := registry.NewNetworkServiceEndpointRegistryClient(cc) + client := registryapi.NewNetworkServiceEndpointRegistryClient(cc) - stream, err := client.Find(ctx, ®istry.NetworkServiceEndpointQuery{ - NetworkServiceEndpoint: ®istry.NetworkServiceEndpoint{ + stream, err := client.Find(ctx, ®istryapi.NetworkServiceEndpointQuery{ + NetworkServiceEndpoint: ®istryapi.NetworkServiceEndpoint{ Name: "nse-1@" + remoteRegistryDomain, }, }) require.Nil(t, err) - list := registry.ReadNetworkServiceEndpointList(stream) + list := registryapi.ReadNetworkServiceEndpointList(stream) require.Len(t, list, 1) require.Equal(t, "nse-1@nsmgr-url", list[0].Name) @@ -148,7 +148,7 @@ func TestLocalDomain_NetworkServiceEndpointRegistry(t *testing.T) { expected, err := domain1.Registry.NetworkServiceEndpointRegistryServer().Register( context.Background(), - ®istry.NetworkServiceEndpoint{ + ®istryapi.NetworkServiceEndpoint{ Name: "nse-1", Url: "test://publicNSMGRurl", ExpirationTime: expirationTime, @@ -162,17 +162,17 @@ func TestLocalDomain_NetworkServiceEndpointRegistry(t *testing.T) { _ = cc.Close() }() - client := registry.NewNetworkServiceEndpointRegistryClient(cc) + client := registryapi.NewNetworkServiceEndpointRegistryClient(cc) - stream, err := client.Find(context.Background(), ®istry.NetworkServiceEndpointQuery{ - NetworkServiceEndpoint: ®istry.NetworkServiceEndpoint{ + stream, err := client.Find(context.Background(), ®istryapi.NetworkServiceEndpointQuery{ + NetworkServiceEndpoint: ®istryapi.NetworkServiceEndpoint{ Name: expected.Name + "@" + localRegistryDomain, }, }) require.Nil(t, err) - list := registry.ReadNetworkServiceEndpointList(stream) + list := registryapi.ReadNetworkServiceEndpointList(stream) require.Len(t, list, 1) require.Equal(t, expected.Name, list[0].Name) @@ -223,8 +223,8 @@ func TestInterdomainFloatingNetworkServiceEndpointRegistry(t *testing.T) { domain3 := sandbox.NewBuilder(t). SetNodesCount(0). - SetRegistrySupplier(func(context.Context, time.Duration, *url.URL, ...grpc.DialOption) registry2.Registry { - return registry2.NewServer(memory.NewNetworkServiceRegistryServer(), memory.NewNetworkServiceEndpointRegistryServer()) + SetRegistrySupplier(func(context.Context, time.Duration, *url.URL, ...grpc.DialOption) registry.Registry { + return registry.NewServer(memory.NewNetworkServiceRegistryServer(), memory.NewNetworkServiceEndpointRegistryServer()) }). SetRegistryProxySupplier(nil). Build() @@ -238,7 +238,7 @@ func TestInterdomainFloatingNetworkServiceEndpointRegistry(t *testing.T) { _, err := domain2.Registry.NetworkServiceEndpointRegistryServer().Register( context.Background(), - ®istry.NetworkServiceEndpoint{ + ®istryapi.NetworkServiceEndpoint{ Name: "nse-1@" + floatingRegistryDomain, Url: "test://publicNSMGRurl", ExpirationTime: expirationTime, @@ -252,17 +252,17 @@ func TestInterdomainFloatingNetworkServiceEndpointRegistry(t *testing.T) { _ = cc.Close() }() - client := registry.NewNetworkServiceEndpointRegistryClient(cc) + client := registryapi.NewNetworkServiceEndpointRegistryClient(cc) - stream, err := client.Find(ctx, ®istry.NetworkServiceEndpointQuery{ - NetworkServiceEndpoint: ®istry.NetworkServiceEndpoint{ + stream, err := client.Find(ctx, ®istryapi.NetworkServiceEndpointQuery{ + NetworkServiceEndpoint: ®istryapi.NetworkServiceEndpoint{ Name: "nse-1@" + floatingRegistryDomain, }, }) require.Nil(t, err) - list := registry.ReadNetworkServiceEndpointList(stream) + list := registryapi.ReadNetworkServiceEndpointList(stream) require.Len(t, list, 1) require.Equal(t, "nse-1@test://publicNSMGRurl", list[0].Name)