INFOPLAT-2449 add telemetry log level config option

.changeset/late-buttons-study.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+#updated: add Telemetry LogLevel config option

core/cmd/shell.go

@@ -115,6 +115,7 @@ func initGlobals(cfgProm config.Prometheus, cfgTracing config.Tracing, cfgTeleme
 				ChipIngressEmitterGRPCEndpoint: cfgTelemetry.ChipIngressEndpoint(),
 				ChipIngressInsecureConnection:  cfgTelemetry.ChipIngressInsecureConnection(),
 				LogStreamingEnabled:            cfgTelemetry.LogStreamingEnabled(),
+				LogLevel:                       cfgTelemetry.LogLevel(),
 			}
 			// note: due to the OTEL specification, all histogram buckets
 			// must be defined when the beholder client is created

core/cmd/shell_local.go

@@ -1122,7 +1122,7 @@ func (s *Shell) beforeNode(c *cli.Context) error {
 			return errors.New("Shell.SetOtelCore is nil")
 		}
 		otelLogger := beholder.GetLogger()
-		logLevel := s.Config.Log().Level()
+		logLevel := s.Config.Telemetry().LogLevel()
 		otelCore := otelzap.NewCore(otelLogger, otelzap.WithLevel(logLevel))
 
 		s.SetOtelCore(&otelCore)

core/config/docs/core.toml

@@ -200,7 +200,7 @@ ListenIP = '0.0.0.0' # Default
 [WebServer.OIDC]
 # ClientID is the ID of the OIDC application registered with the identity provider
 ClientID = 'abcd1234' # Example
-# ProviderURL is the base URL for your OIDC Identity provider. 
+# ProviderURL is the base URL for your OIDC Identity provider.
 ProviderURL = 'https://id[.]example[.]com/oauth2/default' # Example
 # RedirectURL will always be <NODE_BASE_URL>/signin. This needs to match the configuration on the provider side.
 RedirectURL = 'http://localhost:8080/signin' # Example
@@ -222,7 +222,7 @@ UserAPITokenEnabled = false # Default
 UserAPITokenDuration = '240h0m0s' # Default
 
 # Optional LDAP config if WebServer.AuthenticationMethod is set to 'ldap'
-# LDAP queries are all parameterized to support custom LDAP 'dn', 'cn', and attributes 
+# LDAP queries are all parameterized to support custom LDAP 'dn', 'cn', and attributes
 [WebServer.LDAP]
 # ServerTLS defines the option to require the secure ldaps
 ServerTLS = true # Default
@@ -318,7 +318,7 @@ ReaperThreshold = '24h' # Default
 # **ADVANCED**
 # ResultWriteQueueDepth controls how many writes will be buffered before subsequent writes are dropped, for jobs that write results asynchronously for performance reasons, such as OCR.
 ResultWriteQueueDepth = 100 # Default
-# VerboseLogging enables detailed logging of pipeline execution steps. 
+# VerboseLogging enables detailed logging of pipeline execution steps.
 # This can be useful for debugging failed runs without relying on the UI
 # or database.
 #
@@ -464,24 +464,24 @@ TraceLogging = false # Default
 # Enabled enables P2P V2.
 # Note: V1.Enabled is true by default, so it must be set false in order to run V2 only.
 Enabled = true # Default
-# AnnounceAddresses is the addresses the peer will advertise on the network in `host:port` form as accepted by the TCP version of Go’s `net.Dial`. 
-# The addresses should be reachable by other nodes on the network. When attempting to connect to another node, 
+# AnnounceAddresses is the addresses the peer will advertise on the network in `host:port` form as accepted by the TCP version of Go’s `net.Dial`.
+# The addresses should be reachable by other nodes on the network. When attempting to connect to another node,
 # a node will attempt to dial all of the other node’s AnnounceAddresses in round-robin fashion.
 AnnounceAddresses = ['1.2.3.4:9999', '[a52d:0:a88:1274::abcd]:1337'] # Example
 # DefaultBootstrappers is the default bootstrapper peers for libocr's v2 networking stack.
 #
-# Oracle nodes typically only know each other’s PeerIDs, but not their hostnames, IP addresses, or ports. 
-# DefaultBootstrappers are special nodes that help other nodes discover each other’s `AnnounceAddresses` so they can communicate. 
-# Nodes continuously attempt to connect to bootstrappers configured in here. When a node wants to connect to another node 
-# (which it knows only by PeerID, but not by address), it discovers the other node’s AnnounceAddresses from communications 
+# Oracle nodes typically only know each other’s PeerIDs, but not their hostnames, IP addresses, or ports.
+# DefaultBootstrappers are special nodes that help other nodes discover each other’s `AnnounceAddresses` so they can communicate.
+# Nodes continuously attempt to connect to bootstrappers configured in here. When a node wants to connect to another node
+# (which it knows only by PeerID, but not by address), it discovers the other node’s AnnounceAddresses from communications
 # received from its DefaultBootstrappers or other discovered nodes. To facilitate discovery,
 # nodes will regularly broadcast signed announcements containing their PeerID and AnnounceAddresses.
 DefaultBootstrappers = ['12D3KooWMHMRLQkgPbFSYHwD3NBuwtS1AmxhvKVUrcfyaGDASR4U@1.2.3.4:9999', '12D3KooWM55u5Swtpw9r8aFLQHEtw7HR4t44GdNs654ej5gRs2Dh@example.com:1234'] # Example
 # DeltaDial controls how far apart Dial attempts are
 DeltaDial = '15s' # Default
 # DeltaReconcile controls how often a Reconcile message is sent to every peer.
 DeltaReconcile = '1m' # Default
-# ListenAddresses is the addresses the peer will listen to on the network in `host:port` form as accepted by `net.Listen()`, 
+# ListenAddresses is the addresses the peer will listen to on the network in `host:port` form as accepted by `net.Listen()`,
 # but the host and port must be fully specified and cannot be empty. You can specify `0.0.0.0` (IPv4) or `::` (IPv6) to listen on all interfaces, but that is not recommended.
 ListenAddresses = ['1.2.3.4:9999', '[a52d:0:a88:1274::abcd]:1337'] # Example
 
@@ -577,24 +577,24 @@ TraceLogging = false # Default
 [Capabilities.Peering.V2]
 # Enabled enables P2P V2.
 Enabled = false # Default
-# AnnounceAddresses is the addresses the peer will advertise on the network in `host:port` form as accepted by the TCP version of Go’s `net.Dial`. 
-# The addresses should be reachable by other nodes on the network. When attempting to connect to another node, 
+# AnnounceAddresses is the addresses the peer will advertise on the network in `host:port` form as accepted by the TCP version of Go’s `net.Dial`.
+# The addresses should be reachable by other nodes on the network. When attempting to connect to another node,
 # a node will attempt to dial all of the other node’s AnnounceAddresses in round-robin fashion.
 AnnounceAddresses = ['1.2.3.4:9999', '[a52d:0:a88:1274::abcd]:1337'] # Example
 # DefaultBootstrappers is the default bootstrapper peers for libocr's v2 networking stack.
 #
-# Oracle nodes typically only know each other’s PeerIDs, but not their hostnames, IP addresses, or ports. 
-# DefaultBootstrappers are special nodes that help other nodes discover each other’s `AnnounceAddresses` so they can communicate. 
-# Nodes continuously attempt to connect to bootstrappers configured in here. When a node wants to connect to another node 
-# (which it knows only by PeerID, but not by address), it discovers the other node’s AnnounceAddresses from communications 
+# Oracle nodes typically only know each other’s PeerIDs, but not their hostnames, IP addresses, or ports.
+# DefaultBootstrappers are special nodes that help other nodes discover each other’s `AnnounceAddresses` so they can communicate.
+# Nodes continuously attempt to connect to bootstrappers configured in here. When a node wants to connect to another node
+# (which it knows only by PeerID, but not by address), it discovers the other node’s AnnounceAddresses from communications
 # received from its DefaultBootstrappers or other discovered nodes. To facilitate discovery,
 # nodes will regularly broadcast signed announcements containing their PeerID and AnnounceAddresses.
 DefaultBootstrappers = ['12D3KooWMHMRLQkgPbFSYHwD3NBuwtS1AmxhvKVUrcfyaGDASR4U@1.2.3.4:9999', '12D3KooWM55u5Swtpw9r8aFLQHEtw7HR4t44GdNs654ej5gRs2Dh@example.com:1234'] # Example
 # DeltaDial controls how far apart Dial attempts are
 DeltaDial = '15s' # Default
 # DeltaReconcile controls how often a Reconcile message is sent to every peer.
 DeltaReconcile = '1m' # Default
-# ListenAddresses is the addresses the peer will listen to on the network in `host:port` form as accepted by `net.Listen()`, 
+# ListenAddresses is the addresses the peer will listen to on the network in `host:port` form as accepted by `net.Listen()`,
 # but the host and port must be fully specified and cannot be empty. You can specify `0.0.0.0` (IPv4) or `::` (IPv6) to listen on all interfaces, but that is not recommended.
 ListenAddresses = ['1.2.3.4:9999', '[a52d:0:a88:1274::abcd]:1337'] # Example
 
@@ -766,15 +766,15 @@ VerboseLogging = false # Default
 # LatestReportTTL controls how "stale" we will allow a price to be e.g. if
 # set to 1s, a new price will always be fetched if the last result was
 # from 1 second ago or older.
-# 
+#
 # Another way of looking at it is such: the cache will _never_ return a
 # price that was queried from now-LatestReportTTL or before.
-# 
+#
 # Setting to zero disables caching entirely.
 LatestReportTTL = "1s" # Default
 # MaxStaleAge is that maximum amount of time that a value can be stale
 # before it is deleted from the cache (a form of garbage collection).
-# 
+#
 # This should generally be set to something much larger than
 # LatestReportTTL. Setting to zero disables garbage collection.
 MaxStaleAge = "1h" # Default
@@ -844,6 +844,8 @@ ChipIngressInsecureConnection = false # Default
 
 # HeartbeatInterval is the interval at which a the application heartbeat is sent to telemetry backends.
 HeartbeatInterval = '1s' # Default
+# LogLevel sets the log level for telemetry streaming (debug, info, warn, error, crit, panic, fatal)
+LogLevel = "info" # Default
 # LogStreamingEnabled enables log streaming to the OTel log exporter
 LogStreamingEnabled = false # Default
 
@@ -859,7 +861,7 @@ WsURL = "streams.url" # Example
 RestURL = "streams.url" # Example
 
 [CRE.WorkflowFetcher]
-# URL is override URL for the workflow fetcher service. 
+# URL is override URL for the workflow fetcher service.
 URL = '' # Default
 
 [CRE.Linking]

core/config/telemetry_config.go

@@ -1,6 +1,10 @@
 package config
 
-import "time"
+import (
+	"time"
+
+	"go.uber.org/zap/zapcore"
+)
 
 type Telemetry interface {
 	Enabled() bool
@@ -15,4 +19,5 @@ type Telemetry interface {
 	ChipIngressInsecureConnection() bool
 	HeartbeatInterval() time.Duration
 	LogStreamingEnabled() bool
+	LogLevel() zapcore.Level
 }

core/config/toml/types.go

@@ -438,6 +438,7 @@ func (p *P2PKey) SetFrom(f *P2PKey) (err error) {
 	}
 	return nil
 }
+
 func (p *P2PKey) validateMerge(f *P2PKey) (err error) {
 	if p.JSON != nil && f.JSON != nil {
 		err = errors.Join(err, configutils.ErrOverride{Name: "JSON"})
@@ -683,8 +684,10 @@ func (l *DatabaseLock) Mode() string {
 
 func (l *DatabaseLock) ValidateConfig() (err error) {
 	if l.LeaseRefreshInterval.Duration() > l.LeaseDuration.Duration()/2 {
-		err = errors.Join(err, configutils.ErrInvalid{Name: "LeaseRefreshInterval", Value: l.LeaseRefreshInterval.String(),
-			Msg: fmt.Sprintf("must be less than or equal to half of LeaseDuration (%s)", l.LeaseDuration.String())})
+		err = errors.Join(err, configutils.ErrInvalid{
+			Name: "LeaseRefreshInterval", Value: l.LeaseRefreshInterval,
+			Msg: fmt.Sprintf("must be less than or equal to half of LeaseDuration (%s)", l.LeaseDuration),
+		})
 	}
 	return
 }
@@ -2431,6 +2434,7 @@ type Telemetry struct {
 	ChipIngressEndpoint           *string
 	ChipIngressInsecureConnection *bool
 	HeartbeatInterval             *commonconfig.Duration
+	LogLevel                      *string
 	LogStreamingEnabled           *bool
 }
 
@@ -2471,6 +2475,9 @@ func (b *Telemetry) setFrom(f *Telemetry) {
 	if v := f.LogStreamingEnabled; v != nil {
 		b.LogStreamingEnabled = v
 	}
+	if v := f.LogLevel; v != nil {
+		b.LogLevel = v
+	}
 }
 
 func (b *Telemetry) ValidateConfig() (err error) {

core/services/chainlink/config_telemetry.go

@@ -5,6 +5,8 @@ import (
 	"maps"
 	"time"
 
+	"go.uber.org/zap/zapcore"
+
 	"github.com/smartcontractkit/chainlink/v2/core/config/toml"
 	"github.com/smartcontractkit/chainlink/v2/core/static"
 )
@@ -108,3 +110,15 @@ func (b *telemetryConfig) LogStreamingEnabled() bool {
 	}
 	return *b.s.LogStreamingEnabled
 }
+
+func (b *telemetryConfig) LogLevel() zapcore.Level {
+	if b.s.LogLevel == nil {
+		return zapcore.InfoLevel // Default log level
+	}
+
+	var level zapcore.Level
+	if err := level.Set(*b.s.LogLevel); err != nil {
+		return zapcore.InfoLevel // Fallback to info level on invalid input
+	}
+	return level
+}

core/services/chainlink/config_telemetry_test.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"go.uber.org/zap/zapcore"
 
 	"github.com/smartcontractkit/chainlink-common/pkg/config"
 	"github.com/smartcontractkit/chainlink/v2/core/config/toml"
@@ -261,3 +262,25 @@ func TestTelemetryConfig_LogStreamingEnabled(t *testing.T) {
 		})
 	}
 }
+
+func TestTelemetryConfig_LogLevel(t *testing.T) {
+	tests := []struct {
+		name      string
+		telemetry toml.Telemetry
+		expected  zapcore.Level
+	}{
+		{"LogLevelSet", toml.Telemetry{LogLevel: ptr("debug")}, zapcore.DebugLevel},
+		{"LogLevelInfo", toml.Telemetry{LogLevel: ptr("info")}, zapcore.InfoLevel},
+		{"LogLevelWarn", toml.Telemetry{LogLevel: ptr("warn")}, zapcore.WarnLevel},
+		{"LogLevelError", toml.Telemetry{LogLevel: ptr("error")}, zapcore.ErrorLevel},
+		{"LogLevelNil", toml.Telemetry{LogLevel: nil}, zapcore.InfoLevel},
+		{"LogLevelInvalid", toml.Telemetry{LogLevel: ptr("invalid")}, zapcore.InfoLevel},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tc := telemetryConfig{s: tt.telemetry}
+			assert.Equal(t, tt.expected, tc.LogLevel())
+		})
+	}
+}

core/services/chainlink/config_test.go

@@ -612,6 +612,7 @@ func TestConfig_Marshal(t *testing.T) {
 		ChipIngressInsecureConnection: ptr(false),
 		HeartbeatInterval:             commoncfg.MustNewDuration(1 * time.Second),
 		LogStreamingEnabled:           ptr(false),
+		LogLevel:                      ptr("info"),
 	}
 	full.CRE = toml.CreConfig{
 		UseLocalTimeProvider: ptr(true),

core/services/chainlink/testdata/config-empty-effective.toml

@@ -348,6 +348,7 @@ EmitterExportTimeout = '1s'
 ChipIngressEndpoint = ''
 ChipIngressInsecureConnection = false
 HeartbeatInterval = '1s'
+LogLevel = 'info'
 LogStreamingEnabled = false
 
 [Workflows]

core/services/chainlink/testdata/config-full.toml

@@ -361,6 +361,7 @@ EmitterExportTimeout = '1s'
 ChipIngressEndpoint = 'example.com/chip-ingress'
 ChipIngressInsecureConnection = false
 HeartbeatInterval = '1s'
+LogLevel = 'info'
 LogStreamingEnabled = false
 
 [Telemetry.ResourceAttributes]

core/services/chainlink/testdata/config-multi-chain-effective.toml

@@ -348,6 +348,7 @@ EmitterExportTimeout = '1s'
 ChipIngressEndpoint = ''
 ChipIngressInsecureConnection = false
 HeartbeatInterval = '1s'
+LogLevel = 'info'
 LogStreamingEnabled = false
 
 [Workflows]

core/web/resolver/testdata/config-empty-effective.toml

@@ -348,6 +348,7 @@ EmitterExportTimeout = '1s'
 ChipIngressEndpoint = ''
 ChipIngressInsecureConnection = false
 HeartbeatInterval = '1s'
+LogLevel = 'info'
 LogStreamingEnabled = false
 
 [Workflows]

core/web/resolver/testdata/config-full.toml

@@ -361,6 +361,7 @@ EmitterExportTimeout = '1s'
 ChipIngressEndpoint = 'example.com/chip-ingress'
 ChipIngressInsecureConnection = false
 HeartbeatInterval = '1s'
+LogLevel = 'info'
 LogStreamingEnabled = false
 
 [Telemetry.ResourceAttributes]

core/web/resolver/testdata/config-multi-chain-effective.toml

@@ -348,6 +348,7 @@ EmitterExportTimeout = '1s'
 ChipIngressEndpoint = ''
 ChipIngressInsecureConnection = false
 HeartbeatInterval = '1s'
+LogLevel = 'info'
 LogStreamingEnabled = false
 
 [Workflows]

docs/CONFIG.md

@@ -2312,6 +2312,7 @@ EmitterExportTimeout = '1s' # Default
 ChipIngressEndpoint = '' # Default
 ChipIngressInsecureConnection = false # Default
 HeartbeatInterval = '1s' # Default
+LogLevel = "info" # Default
 LogStreamingEnabled = false # Default
 ```
 Telemetry holds OTEL settings.
@@ -2380,6 +2381,12 @@ HeartbeatInterval = '1s' # Default
 ```
 HeartbeatInterval is the interval at which a the application heartbeat is sent to telemetry backends.
 
+### LogLevel
+```toml
+LogLevel = "info" # Default
+```
+LogLevel sets the log level for telemetry streaming (debug, info, warn, error, crit, panic, fatal)
+
 ### LogStreamingEnabled
 ```toml
 LogStreamingEnabled = false # Default

plugins/loop_registry.go

@@ -129,6 +129,7 @@ func (m *LoopRegistry) Register(id string) (*RegisteredLoop, error) {
 		envCfg.ChipIngressEndpoint = m.cfgTelemetry.ChipIngressEndpoint()
 		envCfg.ChipIngressInsecureConnection = m.cfgTelemetry.ChipIngressInsecureConnection()
 		envCfg.TelemetryLogStreamingEnabled = m.cfgTelemetry.LogStreamingEnabled()
+		envCfg.TelemetryLogLevel = m.cfgTelemetry.LogLevel()
 	}
 	m.lggr.Debugf("Registered loopp %q with port %d", id, envCfg.PrometheusPort)
 

plugins/loop_registry_test.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/require"
+	"go.uber.org/zap/zapcore"
 
 	"github.com/smartcontractkit/chainlink-common/pkg/beholder"
 	commonconfig "github.com/smartcontractkit/chainlink-common/pkg/config"
@@ -73,6 +74,7 @@ func (m mockCfgTelemetry) HeartbeatInterval() time.Duration {
 }
 
 func (m mockCfgTelemetry) LogStreamingEnabled() bool { return false }
+func (m mockCfgTelemetry) LogLevel() zapcore.Level   { return zapcore.InfoLevel }
 
 type mockCfgDatabase struct{}
 

testdata/scripts/config/merge_raw_configs.txtar

@@ -495,6 +495,7 @@ EmitterExportTimeout = '1s'
 ChipIngressEndpoint = ''
 ChipIngressInsecureConnection = false
 HeartbeatInterval = '1s'
+LogLevel = 'info'
 LogStreamingEnabled = false
 
 [Workflows]

testdata/scripts/node/validate/default.txtar

@@ -360,6 +360,7 @@ EmitterExportTimeout = '1s'
 ChipIngressEndpoint = ''
 ChipIngressInsecureConnection = false
 HeartbeatInterval = '1s'
+LogLevel = 'info'
 LogStreamingEnabled = false
 
 [Workflows]

testdata/scripts/node/validate/defaults-override.txtar

@@ -421,6 +421,7 @@ EmitterExportTimeout = '1s'
 ChipIngressEndpoint = ''
 ChipIngressInsecureConnection = false
 HeartbeatInterval = '1s'
+LogLevel = 'info'
 LogStreamingEnabled = false
 
 [Workflows]

testdata/scripts/node/validate/disk-based-logging-disabled.txtar

@@ -404,6 +404,7 @@ EmitterExportTimeout = '1s'
 ChipIngressEndpoint = ''
 ChipIngressInsecureConnection = false
 HeartbeatInterval = '1s'
+LogLevel = 'info'
 LogStreamingEnabled = false
 
 [Workflows]