From d8b842fd79bb4cedbb0e2b0054440228e29c3b1f Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 11 Apr 2024 15:41:49 -0400 Subject: [PATCH 1/8] support for async session creation --- cmd/main.go | 4 ++-- pkg/tunneldriver/driver.go | 43 +++++++++++++++++++++++++++++--------- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 4dc1eb3a..5c908995 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -215,8 +215,8 @@ func runController(ctx context.Context, opts managerOpts) error { } } - td, err := tunneldriver.New( - ctrl.Log.WithName("drivers").WithName("tunnel"), tunneldriver.TunnelDriverOpts{ + td, err := tunneldriver.New(ctx, ctrl.Log.WithName("drivers").WithName("tunnel"), + tunneldriver.TunnelDriverOpts{ ServerAddr: opts.serverAddr, Region: opts.region, }, diff --git a/pkg/tunneldriver/driver.go b/pkg/tunneldriver/driver.go index e093c2c1..b9e94964 100644 --- a/pkg/tunneldriver/driver.go +++ b/pkg/tunneldriver/driver.go @@ -6,10 +6,12 @@ import ( "crypto/x509" "encoding/json" "errors" + "fmt" "io" "net" "os" "path/filepath" + "sync/atomic" "github.com/go-logr/logr" ingressv1alpha1 "github.com/ngrok/kubernetes-ingress-controller/api/ingress/v1alpha1" @@ -43,7 +45,7 @@ const ( // TunnelDriver is a driver for creating and deleting ngrok tunnels type TunnelDriver struct { - session ngrok.Session + session atomic.Pointer[ngrok.Session] tunnels map[string]ngrok.Tunnel } @@ -58,7 +60,7 @@ type TunnelDriverComments struct { } // New creates and initializes a new TunnelDriver -func New(logger logr.Logger, opts TunnelDriverOpts, tunnelComment *TunnelDriverComments) (*TunnelDriver, error) { +func New(ctx context.Context, logger logr.Logger, opts TunnelDriverOpts, tunnelComment *TunnelDriverComments) (*TunnelDriver, error) { comments := []string{} if tunnelComment != nil { @@ -97,14 +99,28 @@ func New(logger logr.Logger, opts TunnelDriverOpts, tunnelComment *TunnelDriverC connOpts = append(connOpts, ngrok.WithCA(caCerts)) } - session, err := ngrok.Connect(context.Background(), connOpts...) - if err != nil { - return nil, err - } - return &TunnelDriver{ - session: session, + td := &TunnelDriver{ tunnels: make(map[string]ngrok.Tunnel), - }, nil + } + + session, err := ngrok.Connect(ctx, connOpts...) + if err == nil { + td.session.Store(&session) + } + + // check auth error, give up early? + go func() { + for ctx.Err() == nil { + session, err := ngrok.Connect(ctx, connOpts...) + if err == nil { + td.session.Store(&session) + break + } + // sleep, check error + } + }() + + return td, nil } // caCerts combines the system ca certs with a directory of custom ca certs @@ -144,6 +160,13 @@ func caCerts() (*x509.CertPool, error) { // CreateTunnel creates and starts a new tunnel in a goroutine. If a tunnel with the same name already exists, // it will be stopped and replaced with a new tunnel unless the labels match. func (td *TunnelDriver) CreateTunnel(ctx context.Context, name string, spec ingressv1alpha1.TunnelSpec) error { + var session ngrok.Session + if sess := td.session.Load(); sess == nil { + return fmt.Errorf("ngrok session not connected") + } else { + session = *sess + } + log := log.FromContext(ctx) if tun, ok := td.tunnels[name]; ok { @@ -156,7 +179,7 @@ func (td *TunnelDriver) CreateTunnel(ctx context.Context, name string, spec ingr defer td.stopTunnel(context.Background(), tun) } - tun, err := td.session.Listen(ctx, td.buildTunnelConfig(spec.Labels, spec.ForwardsTo, spec.AppProtocol)) + tun, err := session.Listen(ctx, td.buildTunnelConfig(spec.Labels, spec.ForwardsTo, spec.AppProtocol)) if err != nil { return err } From 5edbba0ce3dd31b988a6e52636df9f5d913259cb Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 15 Apr 2024 15:20:48 -0400 Subject: [PATCH 2/8] better waiting --- go.mod | 2 +- pkg/tunneldriver/driver.go | 88 +++++++++++++++++++++++++++++--------- 2 files changed, 69 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index b41566e6..127fc646 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/go-logr/logr v1.2.4 github.com/golang/mock v1.4.4 github.com/imdario/mergo v0.3.16 + github.com/jpillora/backoff v1.0.0 github.com/ngrok/ngrok-api-go/v5 v5.3.0 github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.10 @@ -60,7 +61,6 @@ require ( github.com/inconshreveable/log15/v3 v3.0.0-testing.5 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect diff --git a/pkg/tunneldriver/driver.go b/pkg/tunneldriver/driver.go index b9e94964..5ea76f52 100644 --- a/pkg/tunneldriver/driver.go +++ b/pkg/tunneldriver/driver.go @@ -11,9 +11,11 @@ import ( "net" "os" "path/filepath" - "sync/atomic" + "sync" + "time" "github.com/go-logr/logr" + "github.com/jpillora/backoff" ingressv1alpha1 "github.com/ngrok/kubernetes-ingress-controller/api/ingress/v1alpha1" "github.com/ngrok/kubernetes-ingress-controller/internal/version" "golang.org/x/exp/maps" @@ -45,8 +47,10 @@ const ( // TunnelDriver is a driver for creating and deleting ngrok tunnels type TunnelDriver struct { - session atomic.Pointer[ngrok.Session] - tunnels map[string]ngrok.Tunnel + session ngrok.Session + sessionErr error + sessionMu sync.RWMutex + tunnels map[string]ngrok.Tunnel } // TunnelDriverOpts are options for creating a new TunnelDriver @@ -105,24 +109,70 @@ func New(ctx context.Context, logger logr.Logger, opts TunnelDriverOpts, tunnelC session, err := ngrok.Connect(ctx, connOpts...) if err == nil { - td.session.Store(&session) + td.session = session + return td, nil } - // check auth error, give up early? - go func() { - for ctx.Err() == nil { - session, err := ngrok.Connect(ctx, connOpts...) - if err == nil { - td.session.Store(&session) - break - } - // sleep, check error - } - }() + var nerr ngrok.Error + if errors.As(err, &nerr) { + // ngrok specific errors, like auth failed are returned directly + return nil, err + } + td.sessionErr = err + + go td.retryConnectLoop(ctx, connOpts) return td, nil } +func (td *TunnelDriver) retryConnectLoop(ctx context.Context, connOpts []ngrok.ConnectOption) { + boff := &backoff.Backoff{ + Min: 100 * time.Millisecond, + Max: 15 * time.Second, + Jitter: true, + } + tm := time.NewTicker(boff.Duration()) + defer tm.Stop() + + for { + // pause before trying to connect + select { + case <-ctx.Done(): + td.sessionMu.Lock() + defer td.sessionMu.Unlock() + + td.sessionErr = ctx.Err() + return + case <-tm.C: + tm.Reset(boff.Duration()) + } + + if err := td.retryConnect(ctx, connOpts); err == nil { + return + } + } +} + +func (td *TunnelDriver) retryConnect(ctx context.Context, connOpts []ngrok.ConnectOption) error { + session, err := ngrok.Connect(ctx, connOpts...) + + td.sessionMu.Lock() + defer td.sessionMu.Unlock() + + td.sessionErr = err + if err == nil { + td.session = session + } + return td.sessionErr +} + +func (td *TunnelDriver) getSession() (ngrok.Session, error) { + td.sessionMu.RLock() + defer td.sessionMu.RUnlock() + + return td.session, td.sessionErr +} + // caCerts combines the system ca certs with a directory of custom ca certs func caCerts() (*x509.CertPool, error) { systemCertPool, err := x509.SystemCertPool() @@ -160,11 +210,9 @@ func caCerts() (*x509.CertPool, error) { // CreateTunnel creates and starts a new tunnel in a goroutine. If a tunnel with the same name already exists, // it will be stopped and replaced with a new tunnel unless the labels match. func (td *TunnelDriver) CreateTunnel(ctx context.Context, name string, spec ingressv1alpha1.TunnelSpec) error { - var session ngrok.Session - if sess := td.session.Load(); sess == nil { - return fmt.Errorf("ngrok session not connected") - } else { - session = *sess + session, err := td.getSession() + if err != nil { + return fmt.Errorf("ngrok session not yet connected: %w", err) } log := log.FromContext(ctx) From ce126d1b95a86778310ec0024426bce5d9b369e5 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 15 Apr 2024 15:36:40 -0400 Subject: [PATCH 3/8] use timer instead of ticker --- pkg/tunneldriver/driver.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/pkg/tunneldriver/driver.go b/pkg/tunneldriver/driver.go index 5ea76f52..9d666443 100644 --- a/pkg/tunneldriver/driver.go +++ b/pkg/tunneldriver/driver.go @@ -131,24 +131,30 @@ func (td *TunnelDriver) retryConnectLoop(ctx context.Context, connOpts []ngrok.C Max: 15 * time.Second, Jitter: true, } - tm := time.NewTicker(boff.Duration()) + tm := time.NewTimer(boff.Duration()) defer tm.Stop() for { // pause before trying to connect select { case <-ctx.Done(): + // cancel the running timer + if !tm.Stop() { + <-tm.C + } + td.sessionMu.Lock() defer td.sessionMu.Unlock() td.sessionErr = ctx.Err() return case <-tm.C: - tm.Reset(boff.Duration()) - } + if err := td.retryConnect(ctx, connOpts); err == nil { + return + } - if err := td.retryConnect(ctx, connOpts); err == nil { - return + // reset with the next backoff value + tm.Reset(boff.Duration()) } } } From a2a883637f7762b782dcd8fc4f1397c6f00c15a2 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 15 Apr 2024 16:12:25 -0400 Subject: [PATCH 4/8] add logs --- pkg/tunneldriver/driver.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/tunneldriver/driver.go b/pkg/tunneldriver/driver.go index 9d666443..59a482e9 100644 --- a/pkg/tunneldriver/driver.go +++ b/pkg/tunneldriver/driver.go @@ -108,11 +108,13 @@ func New(ctx context.Context, logger logr.Logger, opts TunnelDriverOpts, tunnelC } session, err := ngrok.Connect(ctx, connOpts...) + logger.V(0).Info("session connected", "session", session, "err", err) if err == nil { td.session = session return td, nil } + logger.Error(err, "could not connect to ngrok") var nerr ngrok.Error if errors.As(err, &nerr) { // ngrok specific errors, like auth failed are returned directly From b87bbe4b60c5a2a46c68a760ce53a18428a54f8f Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 16 Apr 2024 10:13:35 -0400 Subject: [PATCH 5/8] support ready state --- cmd/main.go | 14 ++++-- pkg/tunneldriver/driver.go | 91 ++++++++------------------------------ 2 files changed, 30 insertions(+), 75 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 5c908995..fd1b8da1 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -21,6 +21,7 @@ import ( "errors" "flag" "fmt" + "net/http" "net/url" "os" "strings" @@ -312,12 +313,19 @@ func runController(ctx context.Context, opts managerOpts) error { } //+kubebuilder:scaffold:builder + if err := mgr.AddReadyzCheck("readyz", func(req *http.Request) error { + select { + case <-td.Ready(): + return nil + default: + return fmt.Errorf("session not connected") + } + }); err != nil { + return fmt.Errorf("error setting up readyz check: %w", err) + } if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { return fmt.Errorf("error setting up health check: %w", err) } - if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { - return fmt.Errorf("error setting up readyz check: %w", err) - } setupLog.Info("starting manager") if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { diff --git a/pkg/tunneldriver/driver.go b/pkg/tunneldriver/driver.go index 59a482e9..c6143e7a 100644 --- a/pkg/tunneldriver/driver.go +++ b/pkg/tunneldriver/driver.go @@ -11,11 +11,8 @@ import ( "net" "os" "path/filepath" - "sync" - "time" "github.com/go-logr/logr" - "github.com/jpillora/backoff" ingressv1alpha1 "github.com/ngrok/kubernetes-ingress-controller/api/ingress/v1alpha1" "github.com/ngrok/kubernetes-ingress-controller/internal/version" "golang.org/x/exp/maps" @@ -47,10 +44,10 @@ const ( // TunnelDriver is a driver for creating and deleting ngrok tunnels type TunnelDriver struct { - session ngrok.Session - sessionErr error - sessionMu sync.RWMutex - tunnels map[string]ngrok.Tunnel + session ngrok.Session + sessionErr error + sessionReady chan struct{} + tunnels map[string]ngrok.Tunnel } // TunnelDriverOpts are options for creating a new TunnelDriver @@ -104,81 +101,31 @@ func New(ctx context.Context, logger logr.Logger, opts TunnelDriverOpts, tunnelC } td := &TunnelDriver{ - tunnels: make(map[string]ngrok.Tunnel), + sessionReady: make(chan struct{}), + tunnels: make(map[string]ngrok.Tunnel), } - session, err := ngrok.Connect(ctx, connOpts...) - logger.V(0).Info("session connected", "session", session, "err", err) - if err == nil { - td.session = session - return td, nil - } - - logger.Error(err, "could not connect to ngrok") - var nerr ngrok.Error - if errors.As(err, &nerr) { - // ngrok specific errors, like auth failed are returned directly - return nil, err - } - td.sessionErr = err - - go td.retryConnectLoop(ctx, connOpts) + go td.connect(ctx, connOpts) return td, nil } -func (td *TunnelDriver) retryConnectLoop(ctx context.Context, connOpts []ngrok.ConnectOption) { - boff := &backoff.Backoff{ - Min: 100 * time.Millisecond, - Max: 15 * time.Second, - Jitter: true, - } - tm := time.NewTimer(boff.Duration()) - defer tm.Stop() - - for { - // pause before trying to connect - select { - case <-ctx.Done(): - // cancel the running timer - if !tm.Stop() { - <-tm.C - } - - td.sessionMu.Lock() - defer td.sessionMu.Unlock() - - td.sessionErr = ctx.Err() - return - case <-tm.C: - if err := td.retryConnect(ctx, connOpts); err == nil { - return - } - - // reset with the next backoff value - tm.Reset(boff.Duration()) - } - } +func (td *TunnelDriver) Ready() <-chan struct{} { + return td.sessionReady } -func (td *TunnelDriver) retryConnect(ctx context.Context, connOpts []ngrok.ConnectOption) error { - session, err := ngrok.Connect(ctx, connOpts...) - - td.sessionMu.Lock() - defer td.sessionMu.Unlock() - - td.sessionErr = err - if err == nil { - td.session = session - } - return td.sessionErr +func (td *TunnelDriver) connect(ctx context.Context, connOpts []ngrok.ConnectOption) { + td.session, td.sessionErr = ngrok.Connect(ctx, connOpts...) + close(td.sessionReady) } func (td *TunnelDriver) getSession() (ngrok.Session, error) { - td.sessionMu.RLock() - defer td.sessionMu.RUnlock() - - return td.session, td.sessionErr + select { + case <-td.sessionReady: + return td.session, td.sessionErr + default: + return nil, fmt.Errorf("session is trying to connect") + } } // caCerts combines the system ca certs with a directory of custom ca certs @@ -220,7 +167,7 @@ func caCerts() (*x509.CertPool, error) { func (td *TunnelDriver) CreateTunnel(ctx context.Context, name string, spec ingressv1alpha1.TunnelSpec) error { session, err := td.getSession() if err != nil { - return fmt.Errorf("ngrok session not yet connected: %w", err) + return err } log := log.FromContext(ctx) From bc13a7ad87b5a3d2f006e6170f94e7fadb3ddcf3 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 16 Apr 2024 10:38:35 -0400 Subject: [PATCH 6/8] mod tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 127fc646..b41566e6 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/go-logr/logr v1.2.4 github.com/golang/mock v1.4.4 github.com/imdario/mergo v0.3.16 - github.com/jpillora/backoff v1.0.0 github.com/ngrok/ngrok-api-go/v5 v5.3.0 github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.10 @@ -61,6 +60,7 @@ require ( github.com/inconshreveable/log15/v3 v3.0.0-testing.5 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect From f4f257997b06f61c6dbc8c59effd013f3e33877b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 16 Apr 2024 12:00:59 -0400 Subject: [PATCH 7/8] store all session info in atomic --- cmd/main.go | 12 ++--- pkg/tunneldriver/driver.go | 92 +++++++++++++++++++++++++++++++------- 2 files changed, 80 insertions(+), 24 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index fd1b8da1..137e0824 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -38,7 +38,6 @@ import ( clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -314,16 +313,13 @@ func runController(ctx context.Context, opts managerOpts) error { //+kubebuilder:scaffold:builder if err := mgr.AddReadyzCheck("readyz", func(req *http.Request) error { - select { - case <-td.Ready(): - return nil - default: - return fmt.Errorf("session not connected") - } + return td.Ready() }); err != nil { return fmt.Errorf("error setting up readyz check: %w", err) } - if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + if err := mgr.AddHealthzCheck("healthz", func(req *http.Request) error { + return td.Healthy() + }); err != nil { return fmt.Errorf("error setting up health check: %w", err) } diff --git a/pkg/tunneldriver/driver.go b/pkg/tunneldriver/driver.go index c6143e7a..f9aacbef 100644 --- a/pkg/tunneldriver/driver.go +++ b/pkg/tunneldriver/driver.go @@ -11,6 +11,8 @@ import ( "net" "os" "path/filepath" + "strings" + "sync/atomic" "github.com/go-logr/logr" ingressv1alpha1 "github.com/ngrok/kubernetes-ingress-controller/api/ingress/v1alpha1" @@ -44,10 +46,8 @@ const ( // TunnelDriver is a driver for creating and deleting ngrok tunnels type TunnelDriver struct { - session ngrok.Session - sessionErr error - sessionReady chan struct{} - tunnels map[string]ngrok.Tunnel + session atomic.Pointer[sessionState] + tunnels map[string]ngrok.Tunnel } // TunnelDriverOpts are options for creating a new TunnelDriver @@ -60,6 +60,12 @@ type TunnelDriverComments struct { Gateway string `json:"gateway,omitempty"` } +type sessionState struct { + session ngrok.Session + readyErr error + healthErr error +} + // New creates and initializes a new TunnelDriver func New(ctx context.Context, logger logr.Logger, opts TunnelDriverOpts, tunnelComment *TunnelDriverComments) (*TunnelDriver, error) { comments := []string{} @@ -101,30 +107,84 @@ func New(ctx context.Context, logger logr.Logger, opts TunnelDriverOpts, tunnelC } td := &TunnelDriver{ - sessionReady: make(chan struct{}), - tunnels: make(map[string]ngrok.Tunnel), + tunnels: make(map[string]ngrok.Tunnel), } - go td.connect(ctx, connOpts) + td.session.Store(&sessionState{ + readyErr: fmt.Errorf("attempting to connect"), + }) + connOpts = append(connOpts, + ngrok.WithConnectHandler(func(ctx context.Context, sess ngrok.Session) { + td.session.Store(&sessionState{ + session: sess, + }) + }), + ngrok.WithDisconnectHandler(func(ctx context.Context, sess ngrok.Session, err error) { + state := td.session.Load() + + if state.session != nil { + // we already have an established session + // if err is nil this session is going away, otherwise it will reconnect by itself + if err == nil { + td.session.Store(&sessionState{ + healthErr: fmt.Errorf("session closed"), + }) + } + return + } + + if err == nil { + // session is disconnecting, do not override error + if state.healthErr == nil { + td.session.Store(&sessionState{ + healthErr: fmt.Errorf("session closed"), + }) + } + return + } + + // we didn't have a session and we are seeing disconnect error + userErr := strings.HasPrefix(err.Error(), "authentication failed") && !strings.Contains(err.Error(), "internal server error") + if userErr { + // its a user error (e.g. authentication failure), so stop further + td.session.Store(&sessionState{ + healthErr: err, + }) + sess.Close() + } else { + // mark this as connecting error to return from readyz + td.session.Store(&sessionState{ + readyErr: err, + }) + } + }), + ) + go ngrok.Connect(ctx, connOpts...) return td, nil } -func (td *TunnelDriver) Ready() <-chan struct{} { - return td.sessionReady +func (td *TunnelDriver) Ready() error { + state := td.session.Load() + return state.readyErr } -func (td *TunnelDriver) connect(ctx context.Context, connOpts []ngrok.ConnectOption) { - td.session, td.sessionErr = ngrok.Connect(ctx, connOpts...) - close(td.sessionReady) +func (td *TunnelDriver) Healthy() error { + state := td.session.Load() + return state.healthErr } func (td *TunnelDriver) getSession() (ngrok.Session, error) { - select { - case <-td.sessionReady: - return td.session, td.sessionErr + state := td.session.Load() + switch { + case state.session != nil: + return state.session, nil + case state.healthErr != nil: + return nil, state.healthErr + case state.readyErr != nil: + return nil, state.readyErr default: - return nil, fmt.Errorf("session is trying to connect") + return nil, fmt.Errorf("unexpected state") } } From 494ab82d7a92110a986f12d7c5b51a18c1527522 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 16 Apr 2024 14:43:42 -0400 Subject: [PATCH 8/8] preserve the first health error we see --- pkg/tunneldriver/driver.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/tunneldriver/driver.go b/pkg/tunneldriver/driver.go index f9aacbef..28afc723 100644 --- a/pkg/tunneldriver/driver.go +++ b/pkg/tunneldriver/driver.go @@ -123,8 +123,7 @@ func New(ctx context.Context, logger logr.Logger, opts TunnelDriverOpts, tunnelC state := td.session.Load() if state.session != nil { - // we already have an established session - // if err is nil this session is going away, otherwise it will reconnect by itself + // we have established session in the past, so record err only when it is going away if err == nil { td.session.Store(&sessionState{ healthErr: fmt.Errorf("session closed"), @@ -143,6 +142,11 @@ func New(ctx context.Context, logger logr.Logger, opts TunnelDriverOpts, tunnelC return } + if state.healthErr != nil { + // we are already at a terminal error, just keep the first one + return + } + // we didn't have a session and we are seeing disconnect error userErr := strings.HasPrefix(err.Error(), "authentication failed") && !strings.Contains(err.Error(), "internal server error") if userErr {