Added Bridge Status Reporter Service

.changeset/floppy-parts-argue.md

@@ -0,0 +1,5 @@
+---
+"chainlink": minor
+---
+
+Added Bridge Status Reporter Service that polls external adapter status endpoints and emits telemetry events for operational monitoring #nops #added

core/config/app_config.go

@@ -62,6 +62,7 @@ type AppConfig interface {
 	Telemetry() Telemetry
 	CRE() CRE
 	Billing() Billing
+	BridgeStatusReporter() BridgeStatusReporter
 }
 
 type DatabaseBackupMode string

core/config/bridge_status_config.go

@@ -0,0 +1,13 @@
+package config
+
+import "time"
+
+const MinimumPollingInterval = time.Minute
+
+type BridgeStatusReporter interface {
+	Enabled() bool
+	StatusPath() string
+	PollingInterval() time.Duration
+	IgnoreInvalidBridges() bool
+	IgnoreJoblessBridges() bool
+}

core/config/docs/core.toml

@@ -827,3 +827,16 @@ URL = '' # Default
 URL = "localhost:4319" # Default
 # TLSEnabled enables TLS to be used to secure communication with the billing service. This is enabled by default.
 TLSEnabled = true # Default
+
+# BridgeStatusReporter holds settings for the Bridge Status Reporter service.
+[BridgeStatusReporter]
+# Enabled enables the Bridge Status Reporter service that polls bridge status endpoints.
+Enabled = false # Default
+# StatusPath is the path to append to bridge URLs for status polling.
+StatusPath = "/status" # Default
+# PollingInterval is how often to poll bridge status endpoints for status.
+PollingInterval = "5m" # Default
+# IgnoreInvalidBridges skips bridges that return HTTP errors or invalid responses.
+IgnoreInvalidBridges = true # Default
+# IgnoreJoblessBridges skips bridges that have no associated jobs.
+IgnoreJoblessBridges = false # Default

core/config/toml/types.go

@@ -39,30 +39,31 @@ type Core struct {
 	RootDir             *string
 	ShutdownGracePeriod *commonconfig.Duration
 
-	Feature          Feature          `toml:",omitempty"`
-	Database         Database         `toml:",omitempty"`
-	TelemetryIngress TelemetryIngress `toml:",omitempty"`
-	AuditLogger      AuditLogger      `toml:",omitempty"`
-	Log              Log              `toml:",omitempty"`
-	WebServer        WebServer        `toml:",omitempty"`
-	JobDistributor   JobDistributor   `toml:",omitempty"`
-	JobPipeline      JobPipeline      `toml:",omitempty"`
-	FluxMonitor      FluxMonitor      `toml:",omitempty"`
-	OCR2             OCR2             `toml:",omitempty"`
-	OCR              OCR              `toml:",omitempty"`
-	P2P              P2P              `toml:",omitempty"`
-	Keeper           Keeper           `toml:",omitempty"`
-	AutoPprof        AutoPprof        `toml:",omitempty"`
-	Pyroscope        Pyroscope        `toml:",omitempty"`
-	Sentry           Sentry           `toml:",omitempty"`
-	Insecure         Insecure         `toml:",omitempty"`
-	Tracing          Tracing          `toml:",omitempty"`
-	Mercury          Mercury          `toml:",omitempty"`
-	Capabilities     Capabilities     `toml:",omitempty"`
-	Telemetry        Telemetry        `toml:",omitempty"`
-	Workflows        Workflows        `toml:",omitempty"`
-	CRE              CreConfig        `toml:",omitempty"`
-	Billing          Billing          `toml:",omitempty"`
+	Feature              Feature              `toml:",omitempty"`
+	Database             Database             `toml:",omitempty"`
+	TelemetryIngress     TelemetryIngress     `toml:",omitempty"`
+	AuditLogger          AuditLogger          `toml:",omitempty"`
+	Log                  Log                  `toml:",omitempty"`
+	WebServer            WebServer            `toml:",omitempty"`
+	JobDistributor       JobDistributor       `toml:",omitempty"`
+	JobPipeline          JobPipeline          `toml:",omitempty"`
+	FluxMonitor          FluxMonitor          `toml:",omitempty"`
+	OCR2                 OCR2                 `toml:",omitempty"`
+	OCR                  OCR                  `toml:",omitempty"`
+	P2P                  P2P                  `toml:",omitempty"`
+	Keeper               Keeper               `toml:",omitempty"`
+	AutoPprof            AutoPprof            `toml:",omitempty"`
+	Pyroscope            Pyroscope            `toml:",omitempty"`
+	Sentry               Sentry               `toml:",omitempty"`
+	Insecure             Insecure             `toml:",omitempty"`
+	Tracing              Tracing              `toml:",omitempty"`
+	Mercury              Mercury              `toml:",omitempty"`
+	Capabilities         Capabilities         `toml:",omitempty"`
+	Telemetry            Telemetry            `toml:",omitempty"`
+	Workflows            Workflows            `toml:",omitempty"`
+	CRE                  CreConfig            `toml:",omitempty"`
+	Billing              Billing              `toml:",omitempty"`
+	BridgeStatusReporter BridgeStatusReporter `toml:",omitempty"`
 }
 
 // SetFrom updates c with any non-nil values from f. (currently TOML field only!)
@@ -107,6 +108,7 @@ func (c *Core) SetFrom(f *Core) {
 	c.Telemetry.setFrom(&f.Telemetry)
 	c.CRE.setFrom(&f.CRE)
 	c.Billing.setFrom(&f.Billing)
+	c.BridgeStatusReporter.setFrom(&f.BridgeStatusReporter)
 }
 
 func (c *Core) ValidateConfig() (err error) {
@@ -2273,6 +2275,64 @@ func (b *Billing) ValidateConfig() error {
 	return nil
 }
 
+type BridgeStatusReporter struct {
+	Enabled              *bool
+	StatusPath           *string
+	PollingInterval      *commonconfig.Duration
+	IgnoreInvalidBridges *bool
+	IgnoreJoblessBridges *bool
+}
+
+func (e *BridgeStatusReporter) setFrom(f *BridgeStatusReporter) {
+	if f.Enabled != nil {
+		e.Enabled = f.Enabled
+	}
+	if f.StatusPath != nil {
+		e.StatusPath = f.StatusPath
+	}
+	if f.PollingInterval != nil {
+		e.PollingInterval = f.PollingInterval
+	}
+	if f.IgnoreInvalidBridges != nil {
+		e.IgnoreInvalidBridges = f.IgnoreInvalidBridges
+	}
+	if f.IgnoreJoblessBridges != nil {
+		e.IgnoreJoblessBridges = f.IgnoreJoblessBridges
+	}
+}
+
+func (e *BridgeStatusReporter) ValidateConfig() error {
+	if e.Enabled == nil || !*e.Enabled {
+		return nil
+	}
+
+	// Default values when enabled
+	if e.StatusPath == nil || *e.StatusPath == "" {
+		defaultPath := "/status"
+		e.StatusPath = &defaultPath
+	}
+
+	if e.PollingInterval == nil {
+		return configutils.ErrInvalid{Name: "PollingInterval", Value: nil, Msg: "must be set"}
+	}
+
+	if e.PollingInterval.Duration() < config.MinimumPollingInterval {
+		return configutils.ErrInvalid{Name: "PollingInterval", Value: e.PollingInterval.Duration(), Msg: "must be greater than or equal to: " + config.MinimumPollingInterval.String()}
+	}
+
+	if e.IgnoreInvalidBridges == nil {
+		defaultIgnoreInvalid := true
+		e.IgnoreInvalidBridges = &defaultIgnoreInvalid
+	}
+
+	if e.IgnoreJoblessBridges == nil {
+		defaultIgnoreJobless := false
+		e.IgnoreJoblessBridges = &defaultIgnoreJobless
+	}
+
+	return nil
+}
+
 type JobDistributor struct {
 	DisplayName *string
 }

core/config/toml/types_test.go

@@ -6,6 +6,7 @@ import (
 	"net/url"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/pelletier/go-toml/v2"
 	"github.com/stretchr/testify/assert"
@@ -636,3 +637,166 @@ func TestEthKeys_SetFrom(t *testing.T) {
 
 // ptr is a utility function for converting a value to a pointer to the value.
 func ptr[T any](t T) *T { return &t }
+
+func TestBridgeStatusReporter_ValidateConfig(t *testing.T) {
+	testCases := []struct {
+		name        string
+		config      *BridgeStatusReporter
+		expectError bool
+		errorMsg    string
+	}{
+		{
+			name: "disabled with nil fields",
+			config: &BridgeStatusReporter{
+				Enabled:              ptr(false),
+				StatusPath:           nil,
+				PollingInterval:      nil,
+				IgnoreInvalidBridges: nil,
+				IgnoreJoblessBridges: nil,
+			},
+			expectError: false,
+		},
+		{
+			name: "disabled with empty fields",
+			config: &BridgeStatusReporter{
+				Enabled:              ptr(false),
+				StatusPath:           ptr(""),
+				PollingInterval:      durationPtr(0),
+				IgnoreInvalidBridges: ptr(false),
+				IgnoreJoblessBridges: ptr(true),
+			},
+			expectError: false,
+		},
+		{
+			name: "disabled with valid fields",
+			config: &BridgeStatusReporter{
+				Enabled:              ptr(false),
+				StatusPath:           ptr("/status"),
+				PollingInterval:      durationPtr(5 * time.Minute),
+				IgnoreInvalidBridges: ptr(true),
+				IgnoreJoblessBridges: ptr(false),
+			},
+			expectError: false,
+		},
+		{
+			name: "nil enabled (defaults to disabled)",
+			config: &BridgeStatusReporter{
+				Enabled:              nil,
+				StatusPath:           ptr("/status"),
+				PollingInterval:      durationPtr(5 * time.Minute),
+				IgnoreInvalidBridges: ptr(true),
+				IgnoreJoblessBridges: ptr(false),
+			},
+			expectError: false,
+		},
+		// Enabled valid cases with auto-defaulting
+		{
+			name: "enabled with valid config",
+			config: &BridgeStatusReporter{
+				Enabled:              ptr(true),
+				StatusPath:           ptr("/status"),
+				PollingInterval:      durationPtr(5 * time.Minute),
+				IgnoreInvalidBridges: ptr(true),
+				IgnoreJoblessBridges: ptr(false),
+			},
+			expectError: false,
+		},
+		{
+			name: "enabled with nil fields - should fail validation",
+			config: &BridgeStatusReporter{
+				Enabled:              ptr(true),
+				StatusPath:           nil,
+				PollingInterval:      nil,
+				IgnoreInvalidBridges: nil,
+				IgnoreJoblessBridges: nil,
+			},
+			expectError: true,
+			errorMsg:    "must be set",
+		},
+		{
+			name: "enabled with empty status path - should auto-default",
+			config: &BridgeStatusReporter{
+				Enabled:              ptr(true),
+				StatusPath:           ptr(""),
+				PollingInterval:      durationPtr(5 * time.Minute),
+				IgnoreInvalidBridges: ptr(true),
+				IgnoreJoblessBridges: ptr(false),
+			},
+			expectError: false,
+		},
+		{
+			name: "enabled with zero polling interval - should fail validation",
+			config: &BridgeStatusReporter{
+				Enabled:              ptr(true),
+				StatusPath:           ptr("/status"),
+				PollingInterval:      durationPtr(0),
+				IgnoreInvalidBridges: ptr(true),
+				IgnoreJoblessBridges: ptr(false),
+			},
+			expectError: true,
+			errorMsg:    "must be greater than or equal to: 1m",
+		},
+		{
+			name: "enabled with polling interval less than 1 minute - should fail validation",
+			config: &BridgeStatusReporter{
+				Enabled:              ptr(true),
+				StatusPath:           ptr("/status"),
+				PollingInterval:      durationPtr(30 * time.Second),
+				IgnoreInvalidBridges: ptr(true),
+				IgnoreJoblessBridges: ptr(false),
+			},
+			expectError: true,
+			errorMsg:    "must be greater than or equal to: 1m",
+		},
+		{
+			name: "enabled with polling interval exactly 1 minute",
+			config: &BridgeStatusReporter{
+				Enabled:              ptr(true),
+				StatusPath:           ptr("/status"),
+				PollingInterval:      durationPtr(1 * time.Minute),
+				IgnoreInvalidBridges: ptr(true),
+				IgnoreJoblessBridges: ptr(false),
+			},
+			expectError: false,
+		},
+		{
+			name: "enabled with all fields missing - should fail validation",
+			config: &BridgeStatusReporter{
+				Enabled:              ptr(true),
+				StatusPath:           ptr(""),
+				PollingInterval:      durationPtr(0),
+				IgnoreInvalidBridges: nil,
+				IgnoreJoblessBridges: nil,
+			},
+			expectError: true,
+			errorMsg:    "must be greater than or equal to: 1m",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := tc.config.ValidateConfig()
+			if tc.expectError {
+				require.Error(t, err)
+				assert.Contains(t, err.Error(), tc.errorMsg)
+			} else {
+				assert.NoError(t, err)
+
+				// Verify defaults are set when enabled
+				if tc.config.Enabled != nil && *tc.config.Enabled {
+					assert.NotNil(t, tc.config.StatusPath)
+					assert.NotEmpty(t, *tc.config.StatusPath)
+					assert.NotNil(t, tc.config.PollingInterval)
+					assert.GreaterOrEqual(t, tc.config.PollingInterval.Duration(), time.Minute)
+					assert.NotNil(t, tc.config.IgnoreInvalidBridges)
+					assert.NotNil(t, tc.config.IgnoreJoblessBridges)
+				}
+			}
+		})
+	}
+}
+
+func durationPtr(d time.Duration) *commonconfig.Duration {
+	cd := *commonconfig.MustNewDuration(d)
+	return &cd
+}

core/services/chainlink/application.go

@@ -24,6 +24,7 @@ import (
 	"go.uber.org/zap/zapcore"
 	"google.golang.org/grpc/credentials"
 
+	"github.com/smartcontractkit/chainlink-common/pkg/beholder"
 	"github.com/smartcontractkit/chainlink-common/pkg/billing"
 	"github.com/smartcontractkit/chainlink-common/pkg/custmsg"
 	"github.com/smartcontractkit/chainlink-common/pkg/loop"
@@ -64,6 +65,7 @@ import (
 	"github.com/smartcontractkit/chainlink/v2/core/services/keeper"
 	"github.com/smartcontractkit/chainlink/v2/core/services/keystore"
 	"github.com/smartcontractkit/chainlink/v2/core/services/llo/retirement"
+	"github.com/smartcontractkit/chainlink/v2/core/services/nodestatusreporter/bridgestatus"
 	"github.com/smartcontractkit/chainlink/v2/core/services/ocr"
 	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2"
 	"github.com/smartcontractkit/chainlink/v2/core/services/ocrbootstrap"
@@ -649,6 +651,16 @@ func NewApplication(ctx context.Context, opts ApplicationOpts) (Application, err
 		globalLogger.Debug("Off-chain reporting v2 disabled")
 	}
 
+	bridgeStatusReporter := bridgestatus.NewBridgeStatusReporter(
+		cfg.BridgeStatusReporter(),
+		bridgeORM,
+		jobORM,
+		unrestrictedHTTPClient,
+		beholder.GetEmitter(),
+		globalLogger,
+	)
+	srvcs = append(srvcs, bridgeStatusReporter)
+
 	healthChecker := commonservices.NewChecker(static.Version, static.Sha)
 
 	var lbs []utils.DependentAwaiter