From 859e424cfb2a08aafad38a313b16a7c72a9f02c0 Mon Sep 17 00:00:00 2001
From: Ed Warnicke <hagbard@gmail.com>
Date: Tue, 19 Oct 2021 07:26:25 -0500
Subject: [PATCH] Add retry logic

Signed-off-by: Ed Warnicke <hagbard@gmail.com>
---
 .golangci.yml             |  2 +-
 internal/config/config.go |  2 ++
 main.go                   | 42 +++++++++++++++++++++++++++------------
 3 files changed, 32 insertions(+), 14 deletions(-)

diff --git a/.golangci.yml b/.golangci.yml
index 14f1022..21b86b9 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -29,7 +29,7 @@ linters-settings:
   goimports:
     local-prefixes: github.com/networkservicemesh
   gocyclo:
-    min-complexity: 15
+    min-complexity: 20
   maligned:
     suggest-new: true
   dupl:
diff --git a/internal/config/config.go b/internal/config/config.go
index f810cca..4ff83d8 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -30,6 +30,8 @@ type Config struct {
 	ConnectTo        url.URL       `default:"unix:///var/lib/networkservicemesh/nsm.io.sock" desc:"url to connect to NSM" split_words:"true"`
 	DialTimeout      time.Duration `default:"5s" desc:"timeout to dial NSMgr" split_words:"true"`
 	RequestTimeout   time.Duration `default:"15s" desc:"timeout to request NSE" split_words:"true"`
+	RetryTimeout     time.Duration `default:"0s" desc:"retry timeout" split_words:"true"`
+	RetryInterval    time.Duration `default:"100ms" desc:"retry interval" split_words:"true"`
 	MaxTokenLifetime time.Duration `default:"10m" desc:"maximum lifetime of tokens" split_words:"true"`
 
 	Labels    []string `default:"" desc:"A list of client labels with format key1=val1,key2=val2, will be used a primary list for network services" split_words:"true"`
diff --git a/main.go b/main.go
index 4dd2bf3..7d123c4 100644
--- a/main.go
+++ b/main.go
@@ -25,6 +25,7 @@ import (
 	"os"
 	"os/signal"
 	"syscall"
+	"time"
 
 	nested "github.com/antonfisher/nested-logrus-formatter"
 	"github.com/edwarnicke/grpcfd"
@@ -119,7 +120,6 @@ func main() {
 		grpcfd.WithChainStreamInterceptor(),
 		grpcfd.WithChainUnaryInterceptor(),
 		grpc.WithDefaultCallOptions(
-			grpc.WaitForReady(true),
 			grpc.PerRPCCredentials(token.NewPerRPCCredentials(spiffejwt.TokenGeneratorFunc(source, c.MaxTokenLifetime))),
 		),
 		grpc.WithTransportCredentials(
@@ -226,21 +226,37 @@ func main() {
 			}
 		}
 
-		requestCtx, cancelRequest := context.WithTimeout(ctx, c.RequestTimeout)
-		defer cancelRequest()
-
-		resp, err := nsmClient.Request(requestCtx, request)
-		if err != nil {
-			logger.Fatalf("failed connect to NSMgr: %v", err.Error())
+		retryCtx, retryCancel := context.WithCancel(ctx)
+		if c.RequestTimeout != 0 {
+			retryCtx, retryCancel = context.WithTimeout(ctx, c.RetryTimeout)
+			defer retryCancel()
 		}
 
-		defer func() {
-			closeCtx, cancelClose := context.WithTimeout(ctx, c.RequestTimeout)
-			defer cancelClose()
-			_, _ = nsmClient.Close(closeCtx, resp)
-		}()
+		for {
+			requestCtx, cancelRequest := context.WithTimeout(ctx, c.RequestTimeout)
+			defer cancelRequest()
+
+			resp, err := nsmClient.Request(requestCtx, request)
+			if err != nil {
+				logger.Errorf("failed connect to NSMgr: %v", err.Error())
+				select {
+				case <-retryCtx.Done():
+					logger.Fatalf("failed to connect to %s after %d retries")
+				default:
+					time.Sleep(c.RetryInterval)
+				}
+				continue
+			}
 
-		logger.Infof("successfully connected to %v. Response: %v", u.NetworkService(), resp)
+			defer func() {
+				closeCtx, cancelClose := context.WithTimeout(ctx, c.RequestTimeout)
+				defer cancelClose()
+				_, _ = nsmClient.Close(closeCtx, resp)
+			}()
+
+			logger.Infof("successfully connected to %v. Response: %v", u.NetworkService(), resp)
+			break
+		}
 	}
 
 	// Wait for cancel event to terminate