fix: prevent subsystem failure from stopping otfd

.gitignore

@@ -1,5 +1,5 @@
 # local node shite
-/node_modules
+ui/node_modules
 package.json
 package-lock.json
 # tfenv

docker-compose.yml

@@ -42,7 +42,7 @@ services:
     volumes:
       - ./internal/integration/fixtures:/etc/squid/certs
   pubsub:
-    image: google/cloud-sdk:emulators
+    image: google/cloud-sdk:494.0.0-emulators
     ports:
       - 8085
     stop_signal: SIGINT

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/antchfx/htmlquery v1.3.0
 	github.com/bradleyfalzon/ghinstallation/v2 v2.11.0
 	github.com/buildkite/terminal-to-html v3.2.0+incompatible
-	github.com/cenkalti/backoff/v4 v4.2.1
+	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/coreos/go-oidc/v3 v3.5.0
 	github.com/fatih/color v1.16.0
 	github.com/felixge/httpsnoop v1.0.4
@@ -66,6 +66,7 @@ require (
 	github.com/antchfx/xpath v1.2.3 // indirect
 	github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/deckarep/golang-set/v2 v2.6.0 // indirect

go.sum

@@ -86,8 +86,12 @@ github.com/bradleyfalzon/ghinstallation/v2 v2.11.0 h1:R9d0v+iobRHSaE4wKUnXFiZp53
 github.com/bradleyfalzon/ghinstallation/v2 v2.11.0/go.mod h1:0LWKQwOHewXO/1acI6TtyE0Xc4ObDb2rFN7eHBAG71M=
 github.com/buildkite/terminal-to-html v3.2.0+incompatible h1:WdXzl7ZmYzCAz4pElZosPaUlRTW+qwVx/SkQSCa1jXs=
 github.com/buildkite/terminal-to-html v3.2.0+incompatible/go.mod h1:BFFdFecOxCgjdcarqI+8izs6v85CU/1RA/4Bqh4GR7E=
+github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
+github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
 github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
+github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
+github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=

internal/daemon/subsystem.go

@@ -72,7 +72,8 @@ func (s *Subsystem) Start(ctx context.Context, g *errgroup.Group) error {
 	}
 	// Backoff and retry whenever operation returns an error. If context is
 	// cancelled then it'll stop retrying and return the context error.
-	policy := backoff.WithContext(backoff.NewExponentialBackOff(), ctx)
+	infiniteRetry := backoff.WithMaxElapsedTime(0)
+	policy := backoff.WithContext(backoff.NewExponentialBackOff(infiniteRetry), ctx)
 	g.Go(func() error {
 		return backoff.RetryNotify(op, policy, func(err error, next time.Duration) {
 			s.Error(err, "restarting subsystem", "name", s.Name, "backoff", next)