diff --git a/cmd/otelcol/main.go b/cmd/otelcol/main.go index c5b82b255f..0b872822eb 100644 --- a/cmd/otelcol/main.go +++ b/cmd/otelcol/main.go @@ -80,6 +80,9 @@ func main() { ResolverSettings: confmap.ResolverSettings{ URIs: collectorSettings.ResolverURIs(), Providers: map[string]confmap.Provider{ + discovery.PropertyScheme(): configprovider.NewConfigSourceConfigMapProvider( + discovery.PropertyProvider(), zap.NewNop(), info, hooks, configsources.Get()..., + ), discovery.ConfigDScheme(): configprovider.NewConfigSourceConfigMapProvider( discovery.ConfigDProvider(), zap.NewNop(), // The service logger is not available yet, setting it to Nop. diff --git a/go.mod b/go.mod index 379c5f8e7d..a1bc636718 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/signalfx/splunk-otel-collector go 1.19 require ( + github.com/alecthomas/participle/v2 v2.0.0-beta.5 github.com/antonmedv/expr v1.9.0 github.com/apache/pulsar-client-go v0.9.0 github.com/cenkalti/backoff/v4 v4.2.0 @@ -152,7 +153,6 @@ require ( github.com/Shopify/sarama v1.37.2 // indirect github.com/Showmax/go-fqdn v1.0.0 // indirect github.com/StackExchange/wmi v1.2.1 // indirect - github.com/alecthomas/participle/v2 v2.0.0-beta.5 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 // indirect github.com/apache/thrift v0.17.0 // indirect diff --git a/internal/confmapprovider/discovery/discoverer.go b/internal/confmapprovider/discovery/discoverer.go index 41d3201768..8484b17962 100644 --- a/internal/confmapprovider/discovery/discoverer.go +++ b/internal/confmapprovider/discovery/discoverer.go @@ -19,6 +19,7 @@ import ( "encoding/base64" "fmt" "os" + "strings" "sync" "time" @@ -42,11 +43,15 @@ import ( "github.com/signalfx/splunk-otel-collector/internal/common/discovery" "github.com/signalfx/splunk-otel-collector/internal/components" + "github.com/signalfx/splunk-otel-collector/internal/confmapprovider/discovery/properties" "github.com/signalfx/splunk-otel-collector/internal/receiver/discoveryreceiver" "github.com/signalfx/splunk-otel-collector/internal/version" ) -const durationEnvVar = "SPLUNK_DISCOVERY_DURATION" +const ( + durationEnvVar = "SPLUNK_DISCOVERY_DURATION" + logLevelEnvVar = "SPLUNK_DISCOVERY_LOG_LEVEL" +) // discoverer provides the mechanism for a "preflight" collector service // that will stand up the observers and discovery receivers based on the .discovery.yaml @@ -64,9 +69,12 @@ type discoverer struct { unexpandedReceiverEntries map[component.ID]map[component.ID]map[string]any discoveredConfig map[component.ID]map[string]any discoveredObservers map[component.ID]discovery.StatusType - info component.BuildInfo - duration time.Duration - mu sync.Mutex + // propertiesConf is a store of all properties from cmdline args and env vars + // that's merged with receiver/observer configs before creation + propertiesConf *confmap.Conf + info component.BuildInfo + duration time.Duration + mu sync.Mutex } func newDiscoverer(logger *zap.Logger) (*discoverer, error) { @@ -87,7 +95,6 @@ func newDiscoverer(logger *zap.Logger) (*discoverer, error) { if err != nil { return (*discoverer)(nil), err } - m := &discoverer{ logger: logger, info: info, @@ -102,9 +109,31 @@ func newDiscoverer(logger *zap.Logger) (*discoverer, error) { discoveredConfig: map[component.ID]map[string]any{}, discoveredObservers: map[component.ID]discovery.StatusType{}, } + m.propertiesConf = m.propertiesConfFromEnv() return m, nil } +func (d *discoverer) propertiesConfFromEnv() *confmap.Conf { + propertiesConf := confmap.New() + for _, env := range os.Environ() { + equalsIdx := strings.Index(env, "=") + if equalsIdx != -1 && len(env) > equalsIdx+1 { + envVar := env[:equalsIdx] + if envVar == logLevelEnvVar || envVar == durationEnvVar { + continue + } + if p, ok, e := properties.NewPropertyFromEnvVar(envVar, env[equalsIdx+1:]); ok { + if e != nil { + d.logger.Info(fmt.Sprintf("invalid discovery property environment variable %q", env), zap.Error(e)) + continue + } + propertiesConf.Merge(confmap.NewFromStringMap(p.ToStringMap())) + } + } + } + return propertiesConf +} + // discover will create all .discovery.yaml components, start them, wait the configured // duration, and tear them down before returning the discovery config. func (d *discoverer) discover(cfg *Config) (map[string]any, error) { @@ -197,19 +226,38 @@ func (d *discoverer) createDiscoveryReceiversAndObservers(cfg *Config) (map[comp discoveryReceiverRaw := map[string]any{} receivers := map[string]any{} + receiversPropertiesConf := confmap.New() + if d.propertiesConf.IsSet("receivers") { + receiversPropertiesConf, err = d.propertiesConf.Sub("receivers") + if err != nil { + return nil, nil, fmt.Errorf("failed obtaining receivers properties config: %w", err) + } + } for receiverID, receiver := range cfg.ReceiversToDiscover { if ok, err = d.updateReceiverForObserver(receiverID, receiver, observerID); err != nil { return nil, nil, err } else if !ok { continue } - d.addUnexpandedReceiverConfig(receiverID, observerID, receiver.Entry.ToStringMap()) - receivers[receiverID.String()] = receiver.Entry.ToStringMap() + receiverEntry := receiver.Entry.ToStringMap() + if receiversPropertiesConf.IsSet(receiverID.String()) { + receiverPropertiesConf, e := receiversPropertiesConf.Sub(receiverID.String()) + if e != nil { + return nil, nil, fmt.Errorf("failed obtaining receiver properties config: %w", e) + } + entryConf := confmap.NewFromStringMap(receiverEntry) + if err = entryConf.Merge(receiverPropertiesConf); err != nil { + return nil, nil, fmt.Errorf("failed merging receiver properties config: %w", err) + } + receiverEntry = entryConf.ToStringMap() + } + + d.addUnexpandedReceiverConfig(receiverID, observerID, receiverEntry) + receivers[receiverID.String()] = receiverEntry } discoveryReceiverRaw["receivers"] = receivers discoveryReceiverConfMap := confmap.NewFromStringMap(discoveryReceiverRaw) - if err = d.expandConverter.Convert(context.Background(), discoveryReceiverConfMap); err != nil { return nil, nil, fmt.Errorf("error converting environment variables in receiver config: %w", err) } @@ -241,6 +289,24 @@ func (d *discoverer) createObserver(observerID component.ID, cfg *Config) (otelc observerConfig := observerFactory.CreateDefaultConfig() observerCfgMap := confmap.NewFromStringMap(cfg.DiscoveryObservers[observerID].ToStringMap()) + + if d.propertiesConf.IsSet("extensions") { + propertiesConf, e := d.propertiesConf.Sub("extensions") + if e != nil { + return nil, fmt.Errorf("failed obtaining extensions properties config: %w", e) + } + if propertiesConf.IsSet(observerID.String()) { + propertiesConf, e = propertiesConf.Sub(observerID.String()) + if e != nil { + return nil, fmt.Errorf("failed obtaining observer properties config: %w", e) + } + if err = observerCfgMap.Merge(propertiesConf); err != nil { + return nil, fmt.Errorf("failed merging observer properties config: %w", err) + } + cfg.DiscoveryObservers[observerID] = ExtensionEntry{observerCfgMap.ToStringMap()} + } + } + if err = d.expandConverter.Convert(context.Background(), observerCfgMap); err != nil { return nil, fmt.Errorf("error converting environment variables in %q config: %w", observerID.String(), err) } @@ -323,7 +389,7 @@ func (d *discoverer) discoveryConfig(cfg *Config) (map[string]any, error) { } } if receiverAdded { - dCfg.Merge(confmap.NewFromStringMap(map[string]any{ + if err := dCfg.Merge(confmap.NewFromStringMap(map[string]any{ "service": map[string]any{ "pipelines": map[string]any{ "metrics": map[string]any{ @@ -331,7 +397,9 @@ func (d *discoverer) discoveryConfig(cfg *Config) (map[string]any, error) { }, }, }, - })) + })); err != nil { + return nil, fmt.Errorf("failed adding receiver_creator/discovery to metrics pipeline: %w", err) + } } extensions := confmap.NewFromStringMap(map[string]any{"extensions": map[string]any{}}) @@ -573,10 +641,12 @@ func (d *discoverer) ConsumeLogs(_ context.Context, ld plog.Logs) error { func determineCurrentStatus(current, observed discovery.StatusType) discovery.StatusType { switch { + case current == discovery.Successful: + // once successful never revert case observed == discovery.Successful: current = discovery.Successful - case current == discovery.Failed && observed == discovery.Partial: - current = discovery.Partial + case current == discovery.Partial: + // only update if observed successful (above) default: current = observed } diff --git a/internal/confmapprovider/discovery/discoverer_test.go b/internal/confmapprovider/discovery/discoverer_test.go index ff559f3cc3..b5abdb973e 100644 --- a/internal/confmapprovider/discovery/discoverer_test.go +++ b/internal/confmapprovider/discovery/discoverer_test.go @@ -15,6 +15,7 @@ package discovery import ( + "fmt" "os" "strings" "testing" @@ -25,6 +26,8 @@ import ( "go.opentelemetry.io/collector/confmap" "go.uber.org/zap" "go.uber.org/zap/zaptest/observer" + + "github.com/signalfx/splunk-otel-collector/internal/common/discovery" ) func TestDiscovererDurationFromEnv(t *testing.T) { @@ -140,3 +143,23 @@ func TestMergeEntries(t *testing.T) { "five.key": "five.val", }, first) } + +func TestDetermineCurrentStatus(t *testing.T) { + for _, test := range []struct { + current, observed, expected discovery.StatusType + }{ + {"failed", "failed", "failed"}, + {"failed", "partial", "partial"}, + {"failed", "successful", "successful"}, + {"partial", "failed", "partial"}, + {"partial", "partial", "partial"}, + {"partial", "successful", "successful"}, + {"successful", "failed", "successful"}, + {"successful", "partial", "successful"}, + {"successful", "successful", "successful"}, + } { + t.Run(fmt.Sprintf("%s:%s->%s", test.current, test.observed, test.expected), func(t *testing.T) { + require.Equal(t, test.expected, determineCurrentStatus(test.current, test.observed)) + }) + } +} diff --git a/internal/confmapprovider/discovery/properties/env_var.go b/internal/confmapprovider/discovery/properties/env_var.go new file mode 100644 index 0000000000..7e5173445a --- /dev/null +++ b/internal/confmapprovider/discovery/properties/env_var.go @@ -0,0 +1,57 @@ +// Copyright Splunk, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package properties + +import ( + "fmt" + + "github.com/alecthomas/participle/v2" + "github.com/alecthomas/participle/v2/lexer" +) + +// SPLUNK_DISCOVERY_RECEIVERS_receiver_x2d_type_x2f_receiver_x2d_name_CONFIG_field_x3a_x3a_subfield=val +// SPLUNK_DISCOVERY_EXTENSIONS_observer_x2d_type_x2f_observer_x2d_name_CONFIG_field_x3a_x3a_subfield=val + +var envVarLex = lexer.MustSimple([]lexer.SimpleRule{ + {Name: "Underscore", Pattern: `_`}, + {Name: "String", Pattern: `[^_]+`}, +}) + +var envVarParser = participle.MustBuild[EnvVarProperty]( + participle.Lexer(envVarLex), + participle.UseLookahead(participle.MaxLookahead), +) + +type EnvVarProperty struct { + ComponentType string `parser:"'SPLUNK' Underscore 'DISCOVERY' Underscore @('RECEIVERS' | 'EXTENSIONS') Underscore"` + Component EnvVarComponentID `parser:"@@"` + Key string `parser:"Underscore 'CONFIG' Underscore @(String|Underscore)+"` + Val string +} + +type EnvVarComponentID struct { + Type string `parser:"@~(Underscore (?= 'CONFIG'))+"` + // _x2f_ -> '/' + Name string `parser:"(Underscore 'x2f' Underscore @(~(?= Underscore (?= 'CONFIG'))+|''))?"` +} + +func NewEnvVarProperty(property, val string) (*EnvVarProperty, error) { + p, err := envVarParser.ParseString("SPLUNK_DISCOVERY", property) + if err != nil { + return nil, fmt.Errorf("invalid property env var (parsing error): %w", err) + } + p.Val = val + return p, nil +} diff --git a/internal/confmapprovider/discovery/properties/env_var_test.go b/internal/confmapprovider/discovery/properties/env_var_test.go new file mode 100644 index 0000000000..1a69a61d77 --- /dev/null +++ b/internal/confmapprovider/discovery/properties/env_var_test.go @@ -0,0 +1,93 @@ +// Copyright Splunk, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package properties + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestEnvVarPropertyEBNF(t *testing.T) { + require.Equal(t, `EnvVarProperty = "SPLUNK" "DISCOVERY" ("RECEIVERS" | "EXTENSIONS") EnvVarComponentID "CONFIG" ( | )+ . +EnvVarComponentID = ~( (?= "CONFIG"))+ ( "x2f" (~(?= (?= "CONFIG"))+ | ""))? .`, envVarParser.String()) +} + +func TestValidEnvVarProperties(t *testing.T) { + for _, tt := range []struct { + expected *Property + envVar string + }{ + {envVar: "SPLUNK_DISCOVERY_RECEIVERS_receiver_x2d_type_x2f__CONFIG_one", + expected: &Property{ + stringMap: map[string]any{ + "receivers": map[string]any{ + "receiver-type": map[string]any{ + "config": map[string]any{ + "one": "val"}, + }, + }, + }, + ComponentType: "receivers", + Component: ComponentID{Type: "receiver-type"}, + Key: "one", + Val: "val", + }, + }, + {envVar: "SPLUNK_DISCOVERY_EXTENSIONS_extension_x2e_type_x2f_extension____name_CONFIG_one_x3a__x3a_two", + expected: &Property{ + stringMap: map[string]any{ + "extensions": map[string]any{ + "extension.type/extension____name": map[string]any{ + "one": map[string]any{ + "two": "val", + }, + }, + }, + }, + ComponentType: "extensions", + Component: ComponentID{Type: "extension.type", Name: "extension____name"}, + Key: "one::two", + Val: "val", + }, + }, + } { + t.Run(tt.envVar, func(t *testing.T) { + p, ok, err := NewPropertyFromEnvVar(tt.envVar, "val") + require.True(t, ok) + require.NoError(t, err) + require.NotNil(t, p) + require.Equal(t, tt.expected, p) + }) + } +} + +func TestInvalidEnvVarProperties(t *testing.T) { + for _, tt := range []struct { + envVar, expectedError string + }{ + {envVar: "SPLUNK_DISCOVERY_NOTVALIDCOMPONENT_TYPE_CONFIG_ONE", expectedError: "invalid env var property (parsing error): invalid property env var (parsing error): SPLUNK_DISCOVERY:1:18: unexpected token \"NOTVALIDCOMPONENT\" (expected (\"RECEIVERS\" | \"EXTENSIONS\") EnvVarComponentID \"CONFIG\" ( | )+)"}, + {envVar: "SPLUNK_DISCOVERY_RECEIVERS_TYPE_NOTCONFIG_ONE", expectedError: "invalid env var property (parsing error): invalid property env var (parsing error): SPLUNK_DISCOVERY:1:46: unexpected token \"\" (expected \"CONFIG\" ( | )+)"}, + {envVar: "SPLUNK_DISCOVERY_EXTENSIONS_TYPE_x2f_NAME_CONFIG_", expectedError: "invalid env var property (parsing error): invalid property env var (parsing error): SPLUNK_DISCOVERY:1:50: sub-expression ( | )+ must match at least once"}, + } { + t.Run(tt.envVar, func(t *testing.T) { + p, ok, err := NewPropertyFromEnvVar(tt.envVar, "val") + require.True(t, ok) + require.Error(t, err) + require.EqualError(t, err, tt.expectedError) + require.Nil(t, p) + }) + } +} diff --git a/internal/confmapprovider/discovery/properties/property.go b/internal/confmapprovider/discovery/properties/property.go new file mode 100644 index 0000000000..b2e4afbc0d --- /dev/null +++ b/internal/confmapprovider/discovery/properties/property.go @@ -0,0 +1,208 @@ +// Copyright Splunk, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package properties + +import ( + "encoding/binary" + "encoding/hex" + "fmt" + "regexp" + "strings" + + "github.com/alecthomas/participle/v2" + "github.com/alecthomas/participle/v2/lexer" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/confmap" + "go.uber.org/multierr" + "gopkg.in/yaml.v2" +) + +// Discovery properties are the method of configuring individual components for discovery mode. +// They are available as commandline --set options and provide equivalent environment variables. +// They are always of the format: +// splunk.discovery.receivers..config.(<::subfield>)*=value +// splunk.discovery.extensions..config.(<::subfield>)*=value +// with corresponding env var: +// SPLUNK_DISCOVERY_RECEIVERS_receiver_x2d_type_x2f_receiver_x2d_name_CONFIG_field_x3a__x3a_subfield=value +// SPLUNK_DISCOVERY_EXTENSIONS_observer_x2d_type_x2f_observer_x2d_name_CONFIG_field_x3a__x3a_subfield=value + +// Parsing properties requires lookaheads (backtracking), which isn't possible in re2. Using participle we +// can define a simple lexer and grammar to establish the Property type as an ast. +var lex = lexer.MustSimple([]lexer.SimpleRule{ + {Name: "Dot", Pattern: `\.`}, + {Name: "ForwardSlash", Pattern: `/`}, + {Name: "Whitespace", Pattern: `\s+`}, + {Name: "String", Pattern: `[^./]*`}, +}) + +var parser = participle.MustBuild[Property]( + participle.Lexer(lex), + participle.Elide("Whitespace"), + participle.UseLookahead(participle.MaxLookahead), +) + +// Property is the ast for a parsed property. +// TODO: support rules and resource_attributes instead of just embedded config +type Property struct { + stringMap map[string]any + ComponentType string `parser:"'splunk' Dot 'discovery' Dot @('receivers' | 'extensions') Dot"` + Component ComponentID `parser:"@@"` + Key string `parser:"Dot 'config' Dot @(String|Dot|ForwardSlash)+"` + Val string +} + +type ComponentID struct { + Type string `parser:"@~(ForwardSlash | (Dot (?= 'config')))+"` + Name string `parser:"(ForwardSlash @(~(Dot (?= 'config'))+)*)?"` +} + +func NewProperty(property, val string) (*Property, error) { + p, err := parser.ParseString("splunk.discovery", property) + if err != nil { + return nil, fmt.Errorf("invalid property (parsing error): %w", err) + } + p.Val = val + var dst map[string]any + cfgItem := []byte(fmt.Sprintf("%s: %s", p.Key, val)) + if err = yaml.Unmarshal(cfgItem, &dst); err != nil { + return nil, fmt.Errorf("failed unmarshaling property %q: %w", p.Key, err) + } + config := confmap.NewFromStringMap(dst).ToStringMap() + if p.ComponentType == "receivers" { + config = map[string]any{"config": config} + } + p.stringMap = map[string]any{ + p.ComponentType: map[string]any{ + component.NewIDWithName(component.Type(p.Component.Type), p.Component.Name).String(): config, + }, + } + return p, nil +} + +// ToEnvVar will output the equivalent env var property for informational purposes. +func (p *Property) ToEnvVar() string { + envVar := envVarPrefixS + envVar = fmt.Sprintf("%s%s_", envVar, strings.ToUpper(p.ComponentType)) + envVar = fmt.Sprintf("%s%s", envVar, wordify(p.Component.Type)) + if p.Component.Name != "" { + envVar = fmt.Sprintf("%s%s", envVar, wordify(fmt.Sprintf("/%s", p.Component.Name))) + } + envVar = fmt.Sprintf("%s_CONFIG_", envVar) + return fmt.Sprintf("%s%s", envVar, wordify(p.Key)) +} + +// ToStringMap() will return a map[string]any equivalent to the property's root-level confmap.ToStringMap() +func (p *Property) ToStringMap() map[string]any { + if p != nil { + return p.stringMap + } + return nil +} + +const ( + envVarPrefixS = "SPLUNK_DISCOVERY_" +) + +var ( + envVarPrefixRE = regexp.MustCompile(fmt.Sprintf("^%s", envVarPrefixS)) + envVarHexRE = regexp.MustCompile("_x[0-9a-fA-F]+_") +) + +func NewPropertyFromEnvVar(envVar, val string) (*Property, bool, error) { + if !envVarPrefixRE.MatchString(envVar) { + return nil, false, nil + } + evp, err := NewEnvVarProperty(envVar, val) + if err != nil { + return nil, true, fmt.Errorf("invalid env var property (parsing error): %w", err) + } + + cid, err := unwordify(evp.Component.Type) + if err != nil { + return nil, true, fmt.Errorf("failed parsing env var property component id type: %w", err) + } + + if evp.Component.Name != "" { + cidName, e := unwordify(evp.Component.Name) + if e != nil { + return nil, true, fmt.Errorf("failed parsing env var property component id name: %w", err) + } + cid = fmt.Sprintf("%s/%s", cid, cidName) + } + + key, err := unwordify(evp.Key) + if err != nil { + return nil, true, fmt.Errorf("failed parsing env var property key: %w", err) + } + + property := fmt.Sprintf("splunk.discovery.%s.%s.config.%s", strings.ToLower(evp.ComponentType), cid, key) + + prop, err := NewProperty(property, val) + return prop, true, err +} + +// wordify takes an arbitrary string (utf8) and will hex encode any rune not in \w, escaping with `_x_`. +func wordify(text string) string { + var wordified []rune + for _, c := range text { + // encoded all non-word characters to hex + if c != '_' && c < '0' || (c > '9') && (c < 'A') || (c > 'Z') && (c < 'a') || (c > 'z') { + b := make([]byte, 4) + binary.BigEndian.PutUint32(b, uint32(c)) + hexEncoded := make([]byte, len(b)*2) // hex.EncodedLen + hex.Encode(hexEncoded, b) + + // strip all leading '0' unless evenness at stake + for len(hexEncoded) > 0 && hexEncoded[0] == '0' { + if len(hexEncoded) > 1 { + if hexEncoded[1] != '0' && len(hexEncoded)%2 == 0 { + break + } + } + hexEncoded = hexEncoded[1:] + } + for _, r := range fmt.Sprintf("_x%s_", hexEncoded) { + wordified = append(wordified, r) + } + } else { + wordified = append(wordified, c) + } + } + return string(wordified) +} + +// unwordify takes any string, expanding `_x<.>_` content as hex-decoded utf8 strings +func unwordify(text string) (string, error) { + var err error + unwordified := envVarHexRE.ReplaceAllStringFunc(text, func(s string) string { + s = s[2 : len(s)-1] + decoded, e := hex.DecodeString(s) + if e != nil { + err = multierr.Combine(err, fmt.Errorf("%q: %w", s, e)) + return "" + } + // left pad if too short for uint32 conversion + for len(decoded) < 4 { + decoded = append([]byte{0}, decoded...) + } + r := int32(binary.BigEndian.Uint32(decoded)) + return fmt.Sprintf("%c", r) + }) + if err != nil { + return "", fmt.Errorf("failed parsing env var hex-encoded content: %w", err) + } + return unwordified, nil + +} diff --git a/internal/confmapprovider/discovery/properties/property_test.go b/internal/confmapprovider/discovery/properties/property_test.go new file mode 100644 index 0000000000..59bb5d7997 --- /dev/null +++ b/internal/confmapprovider/discovery/properties/property_test.go @@ -0,0 +1,200 @@ +// Copyright Splunk, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package properties + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPropertyEBNF(t *testing.T) { + require.Equal(t, `Property = "splunk" "discovery" ("receivers" | "extensions") ComponentID "config" ( | | )+ . +ComponentID = ~( | ( (?= "config")))+ ( ~( (?= "config"))+*)? .`, parser.String()) +} + +func TestWordifyHappyPath(t *testing.T) { + cBytes, err := os.ReadFile(filepath.Join(".", "testdata", "utf8-corpus.txt")) + require.NoError(t, err) + corpus := string(cBytes) + nonWordRE := regexp.MustCompile(`[^\w]+`) + require.True(t, nonWordRE.MatchString(corpus)) + + wordifiedCorpus := wordify(corpus) + require.False(t, nonWordRE.MatchString(wordifiedCorpus)) + + unwordifiedCorpus, err := unwordify(wordifiedCorpus) + require.NoError(t, err) + require.Equal(t, corpus, unwordifiedCorpus) +} + +func TestUnwordifyInvalidHexInEscapedForm(t *testing.T) { + notHex := "_xnothex__xgggg " + out, err := unwordify(notHex) + require.NoError(t, err) + require.Equal(t, notHex, out) + + notHex = "_xfffff__xa_" + out, err = unwordify(notHex) + assert.Empty(t, out) + require.EqualError( + t, err, + `failed parsing env var hex-encoded content: "fffff": encoding/hex: odd length hex string; "a": encoding/hex: odd length hex string`, + ) +} + +func TestValidProperties(t *testing.T) { + for _, tt := range []struct { + expected *Property + key string + val string + }{ + {key: "splunk.discovery.receivers.receivertype.config.key", val: "val", + expected: &Property{ + ComponentType: "receivers", + Component: ComponentID{Type: "receivertype"}, + Key: "key", + Val: "val", + stringMap: map[string]any{ + "receivers": map[string]any{ + "receivertype": map[string]any{ + "config": map[string]any{ + "key": "val", + }, + }, + }, + }, + }, + }, + {key: "splunk.discovery.extensions.extension-type/extensionname.config.key", val: "val", + expected: &Property{ + ComponentType: "extensions", + Component: ComponentID{Type: "extension-type", Name: "extensionname"}, + Key: "key", + Val: "val", + stringMap: map[string]any{ + "extensions": map[string]any{ + "extension-type/extensionname": map[string]any{ + "key": "val", + }, + }, + }, + }, + }, + {key: "splunk.discovery.receivers.receivertype/.config.key", val: "val", + expected: &Property{ + ComponentType: "receivers", + Component: ComponentID{Type: "receivertype"}, + Key: "key", + Val: "val", + stringMap: map[string]any{ + "receivers": map[string]any{ + "receivertype": map[string]any{ + "config": map[string]any{ + "key": "val", + }, + }, + }, + }, + }, + }, + {key: "splunk.discovery.receivers.receiver_type/config.config.one::two::three", val: "val", + expected: &Property{ + ComponentType: "receivers", + Component: ComponentID{Type: "receiver_type", Name: "config"}, + Key: "one::two::three", + Val: "val", + stringMap: map[string]any{ + "receivers": map[string]any{ + "receiver_type/config": map[string]any{ + "config": map[string]any{ + "one": map[string]any{"two": map[string]any{"three": "val"}}, + }, + }, + }, + }, + }, + }, + {key: "splunk.discovery.receivers.receiver.type////.config.one::config", val: "val", + expected: &Property{ + ComponentType: "receivers", + Component: ComponentID{Type: "receiver.type", Name: "///"}, + Key: "one::config", + Val: "val", + stringMap: map[string]any{ + "receivers": map[string]any{ + "receiver.type////": map[string]any{ + "config": map[string]any{ + "one": map[string]any{"config": "val"}}, + }, + }, + }, + }, + }, + {key: "splunk.discovery.extensions.extension--0-1-with-config-in-type-_x64__x86_🙈🙉🙊4:000x0;;0;;0;;-___-----type/e/x/t/e%nso<=n=>nam/e-with-config.config.o::n::e.config", val: "val", + expected: &Property{ + ComponentType: "extensions", + Component: ComponentID{Type: "extension--0-1-with-config-in-type-_x64__x86_🙈🙉🙊4:000x0;;0;;0;;-___-----type", Name: "e/x/t/e%nso<=n=>nam/e-with-config"}, + Key: "o::n::e.config", + Val: "val", + stringMap: map[string]any{ + "extensions": map[string]any{ + "extension--0-1-with-config-in-type-_x64__x86_🙈🙉🙊4:000x0;;0;;0;;-___-----type/e/x/t/e%nso<=n=>nam/e-with-config": map[string]any{ + "o": map[string]any{"n": map[string]any{"e.config": "val"}}}, + }, + }, + }, + }, + } { + t.Run(fmt.Sprintf("%s=%s", tt.key, tt.val), func(t *testing.T) { + p, err := NewProperty(tt.key, tt.val) + require.NoError(t, err) + if tt.expected == nil { + require.Nil(t, p) + return + } + require.NotNil(t, p) + require.Equal(t, *tt.expected, *p) + + // confirm env var rendering and equivalence + envVarP, ok, err := NewPropertyFromEnvVar(p.ToEnvVar(), tt.val) + require.True(t, ok) + require.NoError(t, err) + require.Equal(t, p, envVarP) + }) + } +} + +func TestInvalidProperties(t *testing.T) { + for _, tt := range []struct { + property, expectedError string + }{ + {property: "splunk.discovery.invalid", expectedError: "invalid property (parsing error): splunk.discovery:1:18: unexpected token \"invalid\" (expected (\"receivers\" | \"extensions\") ComponentID \"config\" ( | | )+)"}, + {property: "splunk.discovery.extensions.config.one.two", expectedError: "invalid property (parsing error): splunk.discovery:1:43: unexpected token \"\" (expected \"config\" ( | | )+)"}, + {property: "splunk.discovery.receivers.type/name.config", expectedError: "invalid property (parsing error): splunk.discovery:1:44: unexpected token \"\" (expected ( | | )+)"}, + } { + t.Run(tt.property, func(t *testing.T) { + p, err := NewProperty(tt.property, "val") + require.Error(t, err) + require.EqualError(t, err, tt.expectedError) + require.Nil(t, p) + }) + } +} diff --git a/internal/confmapprovider/discovery/properties/testdata/utf8-corpus.txt b/internal/confmapprovider/discovery/properties/testdata/utf8-corpus.txt new file mode 100644 index 0000000000..c47ea50f33 --- /dev/null +++ b/internal/confmapprovider/discovery/properties/testdata/utf8-corpus.txt @@ -0,0 +1,204 @@ +(taken from https://www.w3.org/2001/06/utf-8-test/UTF-8-demo.html) + +UTF-8 encoded sample plain-text file +‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾ + +Markus Kuhn [ˈmaʳkʊs kuːn] <mkuhn@acm.org> — 1999-08-20 + + +The ASCII compatible UTF-8 encoding of ISO 10646 and Unicode +plain-text files is defined in RFC 2279 and in ISO 10646-1 Annex R. + + +Using Unicode/UTF-8, you can write in emails and source code things such as + +Mathematics and Sciences: + + ∮ E⋅da = Q, n → ∞, ∑ f(i) = ∏ g(i), ∀x∈ℝ: ⌈x⌉ = −⌊−x⌋, α ∧ ¬β = ¬(¬α ∨ β), + + ℕ ⊆ ℕ₀ ⊂ ℤ ⊂ ℚ ⊂ ℝ ⊂ ℂ, ⊥ < a ≠ b ≡ c ≤ d ≪ ⊤ ⇒ (A ⇔ B), + + 2H₂ + O₂ ⇌ 2H₂O, R = 4.7 kΩ, ⌀ 200 mm + +Linguistics and dictionaries: + + ði ıntəˈnæʃənəl fəˈnɛtık əsoʊsiˈeıʃn + Y [ˈʏpsilɔn], Yen [jɛn], Yoga [ˈjoːgɑ] + +APL: + + ((V⍳V)=⍳⍴V)/V←,V ⌷←⍳→⍴∆∇⊃‾⍎⍕⌈ + +Nicer typography in plain text files: + + ╔══════════════════════════════════════════╗ + ║ ║ + ║ • ‘single’ and “double” quotes ║ + ║ ║ + ║ • Curly apostrophes: “We’ve been here” ║ + ║ ║ + ║ • Latin-1 apostrophe and accents: '´` ║ + ║ ║ + ║ • ‚deutsche‘ „Anführungszeichen“ ║ + ║ ║ + ║ • †, ‡, ‰, •, 3–4, —, −5/+5, ™, … ║ + ║ ║ + ║ • ASCII safety test: 1lI|, 0OD, 8B ║ + ║ ╭─────────╮ ║ + ║ • the euro symbol: │ 14.95 € │ ║ + ║ ╰─────────╯ ║ + ╚══════════════════════════════════════════╝ + +Greek (in Polytonic): + + The Greek anthem: + + Σὲ γνωρίζω ἀπὸ τὴν κόψη + τοῦ σπαθιοῦ τὴν τρομερή, + σὲ γνωρίζω ἀπὸ τὴν ὄψη + ποὺ μὲ βία μετράει τὴ γῆ. + + ᾿Απ᾿ τὰ κόκκαλα βγαλμένη + τῶν ῾Ελλήνων τὰ ἱερά + καὶ σὰν πρῶτα ἀνδρειωμένη + χαῖρε, ὦ χαῖρε, ᾿Ελευθεριά! + + From a speech of Demosthenes in the 4th century BC: + + Οὐχὶ ταὐτὰ παρίσταταί μοι γιγνώσκειν, ὦ ἄνδρες ᾿Αθηναῖοι, + ὅταν τ᾿ εἰς τὰ πράγματα ἀποβλέψω καὶ ὅταν πρὸς τοὺς + λόγους οὓς ἀκούω· τοὺς μὲν γὰρ λόγους περὶ τοῦ + τιμωρήσασθαι Φίλιππον ὁρῶ γιγνομένους, τὰ δὲ πράγματ᾿ + εἰς τοῦτο προήκοντα, ὥσθ᾿ ὅπως μὴ πεισόμεθ᾿ αὐτοὶ + πρότερον κακῶς σκέψασθαι δέον. οὐδέν οὖν ἄλλο μοι δοκοῦσιν + οἱ τὰ τοιαῦτα λέγοντες ἢ τὴν ὑπόθεσιν, περὶ ἧς βουλεύεσθαι, + οὐχὶ τὴν οὖσαν παριστάντες ὑμῖν ἁμαρτάνειν. ἐγὼ δέ, ὅτι μέν + ποτ᾿ ἐξῆν τῇ πόλει καὶ τὰ αὑτῆς ἔχειν ἀσφαλῶς καὶ Φίλιππον + τιμωρήσασθαι, καὶ μάλ᾿ ἀκριβῶς οἶδα· ἐπ᾿ ἐμοῦ γάρ, οὐ πάλαι + γέγονεν ταῦτ᾿ ἀμφότερα· νῦν μέντοι πέπεισμαι τοῦθ᾿ ἱκανὸν + προλαβεῖν ἡμῖν εἶναι τὴν πρώτην, ὅπως τοὺς συμμάχους + σώσομεν. ἐὰν γὰρ τοῦτο βεβαίως ὑπάρξῃ, τότε καὶ περὶ τοῦ + τίνα τιμωρήσεταί τις καὶ ὃν τρόπον ἐξέσται σκοπεῖν· πρὶν δὲ + τὴν ἀρχὴν ὀρθῶς ὑποθέσθαι, μάταιον ἡγοῦμαι περὶ τῆς + τελευτῆς ὁντινοῦν ποιεῖσθαι λόγον. + + Δημοσθένους, Γ´ ᾿Ολυνθιακὸς + +Georgian: + + From a Unicode conference invitation: + + გთხოვთ ახლავე გაიაროთ რეგისტრაცია Unicode-ის მეათე საერთაშორისო + კონფერენციაზე დასასწრებად, რომელიც გაიმართება 10-12 მარტს, + ქ. მაინცში, გერმანიაში. კონფერენცია შეჰკრებს ერთად მსოფლიოს + ექსპერტებს ისეთ დარგებში როგორიცაა ინტერნეტი და Unicode-ი, + ინტერნაციონალიზაცია და ლოკალიზაცია, Unicode-ის გამოყენება + ოპერაციულ სისტემებსა, და გამოყენებით პროგრამებში, შრიფტებში, + ტექსტების დამუშავებასა და მრავალენოვან კომპიუტერულ სისტემებში. + +Russian: + + From a Unicode conference invitation: + + Зарегистрируйтесь сейчас на Десятую Международную Конференцию по + Unicode, которая состоится 10-12 марта 1997 года в Майнце в Германии. + Конференция соберет широкий круг экспертов по вопросам глобального + Интернета и Unicode, локализации и интернационализации, воплощению и + применению Unicode в различных операционных системах и программных + приложениях, шрифтах, верстке и многоязычных компьютерных системах. + +Thai (UCS Level 2): + + Excerpt from a poetry on The Romance of The Three Kingdoms (a Chinese + classic 'San Gua'): + + [----------------------------|------------------------] + ๏ แผ่นดินฮั่นเสื่อมโทรมแสนสังเวช พระปกเกศกองบู๊กู้ขึ้นใหม่ + สิบสองกษัตริย์ก่อนหน้าแลถัดไป สององค์ไซร้โง่เขลาเบาปัญญา + ทรงนับถือขันทีเป็นที่พึ่ง บ้านเมืองจึงวิปริตเป็นนักหนา + โฮจิ๋นเรียกทัพทั่วหัวเมืองมา หมายจะฆ่ามดชั่วตัวสำคัญ + เหมือนขับไสไล่เสือจากเคหา รับหมาป่าเข้ามาเลยอาสัญ + ฝ่ายอ้องอุ้นยุแยกให้แตกกัน ใช้สาวนั้นเป็นชนวนชื่นชวนใจ + พลันลิฉุยกุยกีกลับก่อเหตุ ช่างอาเพศจริงหนาฟ้าร้องไห้ + ต้องรบราฆ่าฟันจนบรรลัย ฤๅหาใครค้ำชูกู้บรรลังก์ ฯ + + (The above is a two-column text. If combining characters are handled + correctly, the lines of the second column should be aligned with the + | character above.) + +Ethiopian: + + Proverbs in the Amharic language: + + ሰማይ አይታረስ ንጉሥ አይከሰስ። + ብላ ካለኝ እንደአባቴ በቆመጠኝ። + ጌጥ ያለቤቱ ቁምጥና ነው። + ደሀ በሕልሙ ቅቤ ባይጠጣ ንጣት በገደለው። + የአፍ ወለምታ በቅቤ አይታሽም። + አይጥ በበላ ዳዋ ተመታ። + ሲተረጉሙ ይደረግሙ። + ቀስ በቀስ፥ ዕንቁላል በእግሩ ይሄዳል። + ድር ቢያብር አንበሳ ያስር። + ሰው እንደቤቱ እንጅ እንደ ጉረቤቱ አይተዳደርም። + እግዜር የከፈተውን ጉሮሮ ሳይዘጋው አይድርም። + የጎረቤት ሌባ፥ ቢያዩት ይስቅ ባያዩት ያጠልቅ። + ሥራ ከመፍታት ልጄን ላፋታት። + ዓባይ ማደሪያ የለው፥ ግንድ ይዞ ይዞራል። + የእስላም አገሩ መካ የአሞራ አገሩ ዋርካ። + ተንጋሎ ቢተፉ ተመልሶ ባፉ። + ወዳጅህ ማር ቢሆን ጨርስህ አትላሰው። + እግርህን በፍራሽህ ልክ ዘርጋ። + +Runes: + + ᚻᛖ ᚳᚹᚫᚦ ᚦᚫᛏ ᚻᛖ ᛒᚢᛞᛖ ᚩᚾ ᚦᚫᛗ ᛚᚪᚾᛞᛖ ᚾᚩᚱᚦᚹᛖᚪᚱᛞᚢᛗ ᚹᛁᚦ ᚦᚪ ᚹᛖᛥᚫ + + (Old English, which transcribed into Latin reads 'He cwaeth that he + bude thaem lande northweardum with tha Westsae.' and means 'He said + that he lived in the northern land near the Western Sea.') + +Braille: + + ⡌⠁⠧⠑ ⠼⠁⠒ ⡍⠜⠇⠑⠹⠰⠎ ⡣⠕⠌ + + ⡍⠜⠇⠑⠹ ⠺⠁⠎ ⠙⠑⠁⠙⠒ ⠞⠕ ⠃⠑⠛⠔ ⠺⠊⠹⠲ ⡹⠻⠑ ⠊⠎ ⠝⠕ ⠙⠳⠃⠞ + ⠱⠁⠞⠑⠧⠻ ⠁⠃⠳⠞ ⠹⠁⠞⠲ ⡹⠑ ⠗⠑⠛⠊⠌⠻ ⠕⠋ ⠙⠊⠎ ⠃⠥⠗⠊⠁⠇ ⠺⠁⠎ + ⠎⠊⠛⠝⠫ ⠃⠹ ⠹⠑ ⠊⠇⠻⠛⠹⠍⠁⠝⠂ ⠹⠑ ⠊⠇⠻⠅⠂ ⠹⠑ ⠥⠝⠙⠻⠞⠁⠅⠻⠂ + ⠁⠝⠙ ⠹⠑ ⠡⠊⠑⠋ ⠍⠳⠗⠝⠻⠲ ⡎⠊⠗⠕⠕⠛⠑ ⠎⠊⠛⠝⠫ ⠊⠞⠲ ⡁⠝⠙ + ⡎⠊⠗⠕⠕⠛⠑⠰⠎ ⠝⠁⠍⠑ ⠺⠁⠎ ⠛⠕⠕⠙ ⠥⠏⠕⠝ ⠰⡡⠁⠝⠛⠑⠂ ⠋⠕⠗ ⠁⠝⠹⠹⠔⠛ ⠙⠑ + ⠡⠕⠎⠑ ⠞⠕ ⠏⠥⠞ ⠙⠊⠎ ⠙⠁⠝⠙ ⠞⠕⠲ + + ⡕⠇⠙ ⡍⠜⠇⠑⠹ ⠺⠁⠎ ⠁⠎ ⠙⠑⠁⠙ ⠁⠎ ⠁ ⠙⠕⠕⠗⠤⠝⠁⠊⠇⠲ + + ⡍⠔⠙⠖ ⡊ ⠙⠕⠝⠰⠞ ⠍⠑⠁⠝ ⠞⠕ ⠎⠁⠹ ⠹⠁⠞ ⡊ ⠅⠝⠪⠂ ⠕⠋ ⠍⠹ + ⠪⠝ ⠅⠝⠪⠇⠫⠛⠑⠂ ⠱⠁⠞ ⠹⠻⠑ ⠊⠎ ⠏⠜⠞⠊⠊⠥⠇⠜⠇⠹ ⠙⠑⠁⠙ ⠁⠃⠳⠞ + ⠁ ⠙⠕⠕⠗⠤⠝⠁⠊⠇⠲ ⡊ ⠍⠊⠣⠞ ⠙⠁⠧⠑ ⠃⠑⠲ ⠔⠊⠇⠔⠫⠂ ⠍⠹⠎⠑⠇⠋⠂ ⠞⠕ + ⠗⠑⠛⠜⠙ ⠁ ⠊⠕⠋⠋⠔⠤⠝⠁⠊⠇ ⠁⠎ ⠹⠑ ⠙⠑⠁⠙⠑⠌ ⠏⠊⠑⠊⠑ ⠕⠋ ⠊⠗⠕⠝⠍⠕⠝⠛⠻⠹ + ⠔ ⠹⠑ ⠞⠗⠁⠙⠑⠲ ⡃⠥⠞ ⠹⠑ ⠺⠊⠎⠙⠕⠍ ⠕⠋ ⠳⠗ ⠁⠝⠊⠑⠌⠕⠗⠎ + ⠊⠎ ⠔ ⠹⠑ ⠎⠊⠍⠊⠇⠑⠆ ⠁⠝⠙ ⠍⠹ ⠥⠝⠙⠁⠇⠇⠪⠫ ⠙⠁⠝⠙⠎ + ⠩⠁⠇⠇ ⠝⠕⠞ ⠙⠊⠌⠥⠗⠃ ⠊⠞⠂ ⠕⠗ ⠹⠑ ⡊⠳⠝⠞⠗⠹⠰⠎ ⠙⠕⠝⠑ ⠋⠕⠗⠲ ⡹⠳ + ⠺⠊⠇⠇ ⠹⠻⠑⠋⠕⠗⠑ ⠏⠻⠍⠊⠞ ⠍⠑ ⠞⠕ ⠗⠑⠏⠑⠁⠞⠂ ⠑⠍⠏⠙⠁⠞⠊⠊⠁⠇⠇⠹⠂ ⠹⠁⠞ + ⡍⠜⠇⠑⠹ ⠺⠁⠎ ⠁⠎ ⠙⠑⠁⠙ ⠁⠎ ⠁ ⠙⠕⠕⠗⠤⠝⠁⠊⠇⠲ + + (The first couple of paragraphs of "A Christmas Carol" by Dickens) + +Compact font selection example text: + + ABCDEFGHIJKLMNOPQRSTUVWXYZ /0123456789 + abcdefghijklmnopqrstuvwxyz £©µÀÆÖÞßéöÿ + –—‘“”„†•…‰™œŠŸž€ ΑΒΓΔΩαβγδω АБВГДабвгд + ∀∂∈ℝ∧∪≡∞ ↑↗↨↻⇣ ┐┼╔╘░►☺♀ fi�⑀₂ἠḂӥẄɐː⍎אԱა + +Greetings in various languages: + + Hello world, Καλημέρα κόσμε, コンニチハ + +Box drawing alignment tests: █ + ▉ + ╔══╦══╗ ┌──┬──┐ ╭──┬──╮ ╭──┬──╮ ┏━━┳━━┓ ┎┒┏┑ ╷ ╻ ┏┯┓ ┌┰┐ ▊ ╱╲╱╲╳╳╳ + ║┌─╨─┐║ │╔═╧═╗│ │╒═╪═╕│ │╓─╁─╖│ ┃┌─╂─┐┃ ┗╃╄┙ ╶┼╴╺╋╸┠┼┨ ┝╋┥ ▋ ╲╱╲╱╳╳╳ + ║│╲ ╱│║ │║ ║│ ││ │ ││ │║ ┃ ║│ ┃│ ╿ │┃ ┍╅╆┓ ╵ ╹ ┗┷┛ └┸┘ ▌ ╱╲╱╲╳╳╳ + ╠╡ ╳ ╞╣ ├╢ ╟┤ ├┼─┼─┼┤ ├╫─╂─╫┤ ┣┿╾┼╼┿┫ ┕┛┖┚ ┌┄┄┐ ╎ ┏┅┅┓ ┋ ▍ ╲╱╲╱╳╳╳ + ║│╱ ╲│║ │║ ║│ ││ │ ││ │║ ┃ ║│ ┃│ ╽ │┃ ░░▒▒▓▓██ ┊ ┆ ╎ ╏ ┇ ┋ ▎ + ║└─╥─┘║ │╚═╤═╝│ │╘═╪═╛│ │╙─╀─╜│ ┃└─╂─┘┃ ░░▒▒▓▓██ ┊ ┆ ╎ ╏ ┇ ┋ ▏ + ╚══╩══╝ └──┴──┘ ╰──┴──╯ ╰──┴──╯ ┗━━┻━━┛ └╌╌┘ ╎ ┗╍╍┛ ┋ ▁▂▃▄▅▆▇█ diff --git a/internal/confmapprovider/discovery/provider.go b/internal/confmapprovider/discovery/provider.go index 613a7f5199..b9e173fc4d 100644 --- a/internal/confmapprovider/discovery/provider.go +++ b/internal/confmapprovider/discovery/provider.go @@ -24,6 +24,7 @@ import ( "go.uber.org/zap" "go.uber.org/zap/zapcore" + "github.com/signalfx/splunk-otel-collector/internal/confmapprovider/discovery/properties" "github.com/signalfx/splunk-otel-collector/internal/settings" ) @@ -34,6 +35,8 @@ type Provider interface { ConfigDProvider() confmap.Provider DiscoveryModeScheme() string DiscoveryModeProvider() confmap.Provider + PropertyScheme() string + PropertyProvider() confmap.Provider } type providerShim struct { @@ -63,7 +66,7 @@ func New() (Provider, error) { m := &mapProvider{configs: map[string]*Config{}} zapConfig := zap.NewProductionConfig() logLevel := zap.WarnLevel - if ll, ok := os.LookupEnv("SPLUNK_DISCOVERY_LOG_LEVEL"); ok { + if ll, ok := os.LookupEnv(logLevelEnvVar); ok { if l, err := zapcore.ParseLevel(ll); err == nil { logLevel = l } @@ -94,16 +97,28 @@ func (m *mapProvider) DiscoveryModeProvider() confmap.Provider { } } +func (m *mapProvider) PropertyProvider() confmap.Provider { + return providerShim{ + scheme: m.PropertyScheme(), + retrieve: m.retrieve(m.PropertyScheme()), + } +} + func (m *mapProvider) retrieve(scheme string) func(context.Context, string, confmap.WatcherFunc) (*confmap.Retrieved, error) { return func(ctx context.Context, uri string, _ confmap.WatcherFunc) (*confmap.Retrieved, error) { schemePrefix := fmt.Sprintf("%s:", scheme) if !strings.HasPrefix(uri, schemePrefix) { return nil, fmt.Errorf("uri %q is not supported by %s provider", uri, scheme) } - configDir := uri[len(schemePrefix):] + + uriVal := uri[len(schemePrefix):] + if schemePrefix == fmt.Sprintf("%s:", settings.PropertyScheme) { + return m.parsedProperty(uriVal) + } var cfg *Config var ok bool + configDir := uriVal if cfg, ok = m.configs[configDir]; !ok { cfg = NewConfig(m.logger) if err := cfg.Load(configDir); err != nil { @@ -135,3 +150,22 @@ func (m *mapProvider) ConfigDScheme() string { func (m *mapProvider) DiscoveryModeScheme() string { return settings.DiscoveryModeScheme } + +func (m *mapProvider) PropertyScheme() string { + return settings.PropertyScheme +} + +func (m *mapProvider) parsedProperty(rawProperty string) (*confmap.Retrieved, error) { + // split property from value + equalsIdx := strings.Index(rawProperty, "=") + if equalsIdx == -1 || len(rawProperty) <= equalsIdx+1 { + return nil, fmt.Errorf("invalid discovery property %q not of form =", rawProperty) + } + prop, err := properties.NewProperty(rawProperty[:equalsIdx], rawProperty[equalsIdx+1:]) + if err != nil { + return nil, fmt.Errorf("invalid discovery property: %w", err) + } + m.discoverer.propertiesConf.Merge(confmap.NewFromStringMap(prop.ToStringMap())) + // return nil confmap to satisfy signature + return confmap.NewRetrieved(nil) +} diff --git a/internal/settings/settings.go b/internal/settings/settings.go index ed7a2de71b..27c05470ee 100644 --- a/internal/settings/settings.go +++ b/internal/settings/settings.go @@ -53,19 +53,22 @@ const ( DefaultMemoryTotalMiB = 512 DiscoveryModeScheme = "splunk.discovery" + PropertyScheme = "splunk.property" ConfigDScheme = "splunk.configd" ) type Settings struct { - configPaths *stringArrayFlagValue - setProperties *stringArrayFlagValue - configDir *stringPointerFlagValue - colCoreArgs []string - versionFlag bool - noConvertConfig bool - configD bool - discoveryMode bool - dryRun bool + configPaths *stringArrayFlagValue + setOptionArguments *stringArrayFlagValue + configDir *stringPointerFlagValue + colCoreArgs []string + setProperties []string + discoveryProperties []string + versionFlag bool + noConvertConfig bool + configD bool + discoveryMode bool + dryRun bool } func New(args []string) (*Settings, error) { @@ -101,6 +104,10 @@ func (s *Settings) ResolverURIs() []string { configDir := getConfigDir(s) + for _, property := range s.discoveryProperties { + configPaths = append(configPaths, fmt.Sprintf("%s:%s", PropertyScheme, property)) + } + if s.configD { configPaths = append(configPaths, fmt.Sprintf("%s:%s", ConfigDScheme, configDir)) } @@ -138,7 +145,7 @@ func getConfigDir(f *Settings) string { // ConfMapConverters returns confmap.Converters for the collector core service. func (s *Settings) ConfMapConverters() []confmap.Converter { confMapConverters := []confmap.Converter{ - configconverter.NewOverwritePropertiesConverter(s.setProperties.value), + configconverter.NewOverwritePropertiesConverter(s.setProperties), } if !s.noConvertConfig { confMapConverters = append( @@ -168,15 +175,15 @@ func parseArgs(args []string) (*Settings, error) { flagSet := flag.NewFlagSet("otelcol", flag.ContinueOnError) settings := &Settings{ - configPaths: new(stringArrayFlagValue), - setProperties: new(stringArrayFlagValue), - configDir: new(stringPointerFlagValue), + configPaths: new(stringArrayFlagValue), + setOptionArguments: new(stringArrayFlagValue), + configDir: new(stringPointerFlagValue), } flagSet.Var(settings.configPaths, "config", "Locations to the config file(s), "+ "note that only a single location can be set per flag entry e.g. --config=/path/to/first "+ "--config=path/to/second.") - flagSet.Var(settings.setProperties, "set", "Set arbitrary component config property. "+ + flagSet.Var(settings.setOptionArguments, "set", "Set arbitrary component config property. "+ "The component has to be defined in the config file and the flag has a higher precedence. "+ "Array config properties are overridden and maps are joined. Example --set=processors.batch.timeout=2s") flagSet.BoolVar(&settings.dryRun, "dry-run", false, "Don't run the service, just show the configuration") @@ -208,12 +215,25 @@ func parseArgs(args []string) (*Settings, error) { return nil, err } + settings.setProperties, settings.discoveryProperties = parseSetOptionArguments(settings.setOptionArguments.value) + // Pass flags that are handled by the collector core service as raw command line arguments. settings.colCoreArgs = flagSetToArgs(colCoreFlags, flagSet) return settings, nil } +func parseSetOptionArguments(arguments []string) (setProperties, discoveryProperties []string) { + for _, arg := range arguments { + if strings.HasPrefix(arg, "splunk.discovery") { + discoveryProperties = append(discoveryProperties, arg) + } else { + setProperties = append(setProperties, arg) + } + } + return +} + // flagSetToArgs takes a list of flag names and returns a list of corresponding command line arguments // using values from the provided flagSet. // The flagSet must be populated (flagSet.Parse is called), otherwise the returned list of arguments will be empty. diff --git a/internal/settings/settings_test.go b/internal/settings/settings_test.go index df2bb3933b..ac45e2aa1e 100644 --- a/internal/settings/settings_test.go +++ b/internal/settings/settings_test.go @@ -88,8 +88,10 @@ func TestNewSettingsNoConvertConfig(t *testing.T) { "--config", configPath, "--config", anotherConfigPath, "--set", "foo", + "--set", "splunk.discovery.receiver.receiver-type/name.config.field.one=val.one", "--set", "bar", "--set=baz", + "--set", "splunk.discovery.receiver.receiver-type/name.config.field.two=val.two", "--feature-gates", "foo", "--feature-gates", "-bar", }) @@ -98,11 +100,19 @@ func TestNewSettingsNoConvertConfig(t *testing.T) { require.True(t, settings.noConvertConfig) require.Equal(t, []string{configPath, anotherConfigPath}, settings.configPaths.value) - require.Equal(t, []string{"foo", "bar", "baz"}, settings.setProperties.value) - - require.Equal(t, []string{configPath, anotherConfigPath}, settings.ResolverURIs()) + require.Equal(t, []string{"foo", "bar", "baz"}, settings.setProperties) + require.Equal(t, []string{ + "splunk.discovery.receiver.receiver-type/name.config.field.one=val.one", + "splunk.discovery.receiver.receiver-type/name.config.field.two=val.two", + }, settings.discoveryProperties) + + require.Equal(t, []string{ + configPath, anotherConfigPath, + "splunk.property:splunk.discovery.receiver.receiver-type/name.config.field.one=val.one", + "splunk.property:splunk.discovery.receiver.receiver-type/name.config.field.two=val.two", + }, settings.ResolverURIs()) require.Equal(t, []confmap.Converter{ - configconverter.NewOverwritePropertiesConverter(settings.setProperties.value), + configconverter.NewOverwritePropertiesConverter(settings.setProperties), }, settings.ConfMapConverters()) require.Equal(t, []string{"--feature-gates", "foo", "--feature-gates", "-bar"}, settings.ColCoreArgs()) } @@ -124,11 +134,12 @@ func TestNewSettingsConvertConfig(t *testing.T) { require.False(t, settings.noConvertConfig) require.Equal(t, []string{configPath, anotherConfigPath}, settings.configPaths.value) - require.Equal(t, []string{"foo", "bar", "baz"}, settings.setProperties.value) + require.Equal(t, []string{"foo", "bar", "baz"}, settings.setProperties) + require.Equal(t, []string(nil), settings.discoveryProperties) require.Equal(t, []string{configPath, anotherConfigPath}, settings.ResolverURIs()) require.Equal(t, []confmap.Converter{ - configconverter.NewOverwritePropertiesConverter(settings.setProperties.value), + configconverter.NewOverwritePropertiesConverter(settings.setProperties), configconverter.RemoveBallastKey{}, configconverter.MoveOTLPInsecureKey{}, configconverter.MoveHecTLS{}, diff --git a/tests/general/discoverymode/docker_observer_discovery_test.go b/tests/general/discoverymode/docker_observer_discovery_test.go index 12f65e78d3..e88d7b215a 100644 --- a/tests/general/discoverymode/docker_observer_discovery_test.go +++ b/tests/general/discoverymode/docker_observer_discovery_test.go @@ -68,7 +68,10 @@ func TestDockerObserver(t *testing.T) { "DOCKER_DOMAIN_SOCKET": "unix:///var/run/dock.e.r.sock", "LABEL_ONE_VALUE": "actual.label.one.value", "LABEL_TWO_VALUE": "actual.label.two.value", - }).WithArgs("--discovery", "--config-dir", "/opt/config.d") + }).WithArgs( + "--discovery", "--config-dir", "/opt/config.d", + "--set", `splunk.discovery.extensions.docker_observer.config.endpoint=${DOCKER_DOMAIN_SOCKET}`, + ) }, ) defer shutdown() @@ -138,6 +141,7 @@ func TestDockerObserver(t *testing.T) { }, }, }, + "splunk.property": map[string]any{}, } require.Equal(t, expectedInitial, cc.InitialConfig(t, 55554)) @@ -192,8 +196,9 @@ func TestDockerObserver(t *testing.T) { require.Equal(t, expectedEffective, cc.EffectiveConfig(t, 55554)) sc, stdout, stderr := cc.Container.AssertExec(t, 25*time.Second, - "bash", "-c", "SPLUNK_DISCOVERY_LOG_LEVEL=error SPLUNK_DEBUG_CONFIG_SERVER=false /otelcol --config-dir /opt/config.d --discovery --dry-run", - ) + "bash", "-c", `SPLUNK_DISCOVERY_LOG_LEVEL=error SPLUNK_DEBUG_CONFIG_SERVER=false \ +SPLUNK_DISCOVERY_EXTENSIONS_docker_observer_CONFIG_endpoint=\${DOCKER_DOMAIN_SOCKET} \ +/otelcol --config-dir /opt/config.d --discovery --dry-run`) require.Equal(t, `exporters: otlp: endpoint: ${OTLP_ENDPOINT} diff --git a/tests/general/discoverymode/host_observer_discovery_test.go b/tests/general/discoverymode/host_observer_discovery_test.go index bed15b20d9..56b06c93d7 100644 --- a/tests/general/discoverymode/host_observer_discovery_test.go +++ b/tests/general/discoverymode/host_observer_discovery_test.go @@ -72,9 +72,14 @@ func TestHostObserver(t *testing.T) { "INTERNAL_PROMETHEUS_PORT": fmt.Sprintf("%d", promPort), // confirm that debug logging doesn't affect runtime "SPLUNK_DISCOVERY_LOG_LEVEL": "debug", - "LABEL_ONE_VALUE": "actual.label.one.value", - "LABEL_TWO_VALUE": "actual.label.two.value", - }).WithArgs("--discovery", "--config-dir", "/opt/config.d") + "LABEL_ONE_VALUE": "actual.label.one.value.from.env.var", + "LABEL_TWO_VALUE": "actual.label.two.value.from.env.var", + }).WithArgs( + "--discovery", + "--config-dir", "/opt/config.d", + "--set", "splunk.discovery.receivers.prometheus_simple.config.labels::label.three=actual.label.three.value.from.cmdline.property", + "--set", "splunk.discovery.extensions.host_observer.config.refresh_interval=1s", + ) }, ) defer shutdown() @@ -137,8 +142,9 @@ func TestHostObserver(t *testing.T) { "config": map[string]any{ "collection_interval": "1s", "labels": map[string]any{ - "label.one": "${LABEL_ONE_VALUE}", - "label.two": "${LABEL_TWO_VALUE}", + "label.one": "${LABEL_ONE_VALUE}", + "label.two": "${LABEL_TWO_VALUE}", + "label.three": "actual.label.three.value.from.cmdline.property", }, }, "resource_attributes": map[string]any{}, @@ -157,6 +163,7 @@ func TestHostObserver(t *testing.T) { }, }, }, + "splunk.property": map[string]any{}, } require.Equal(t, expectedInitial, cc.InitialConfig(t, 55554)) @@ -196,8 +203,9 @@ func TestHostObserver(t *testing.T) { "config": map[string]any{ "collection_interval": "1s", "labels": map[string]any{ - "label.one": "actual.label.one.value", - "label.two": "actual.label.two.value", + "label.one": "actual.label.one.value.from.env.var", + "label.two": "actual.label.two.value.from.env.var", + "label.three": "actual.label.three.value.from.cmdline.property", }, }, "resource_attributes": map[string]any{}, @@ -211,8 +219,12 @@ func TestHostObserver(t *testing.T) { require.Equal(t, expectedEffective, cc.EffectiveConfig(t, 55554)) sc, stdout, stderr := cc.Container.AssertExec(t, 15*time.Second, - "bash", "-c", "SPLUNK_DISCOVERY_LOG_LEVEL=error SPLUNK_DEBUG_CONFIG_SERVER=false /otelcol --config-dir /opt/config.d --discovery --dry-run", - ) + "bash", "-c", `SPLUNK_DISCOVERY_LOG_LEVEL=error SPLUNK_DEBUG_CONFIG_SERVER=false \ +REFRESH_INTERVAL=1s \ +SPLUNK_DISCOVERY_DURATION=9s \ +SPLUNK_DISCOVERY_RECEIVERS_prometheus_simple_CONFIG_labels_x3a__x3a_label_x2e_three=actual.label.three.value.from.env.var.property \ +SPLUNK_DISCOVERY_EXTENSIONS_host_observer_CONFIG_refresh_interval=\$REFRESH_INTERVAL \ +/otelcol --config-dir /opt/config.d --discovery --dry-run`) require.Equal(t, `exporters: otlp: endpoint: ${OTLP_ENDPOINT} @@ -220,7 +232,7 @@ func TestHostObserver(t *testing.T) { insecure: true extensions: host_observer: - refresh_interval: 1s + refresh_interval: $REFRESH_INTERVAL receivers: receiver_creator/discovery: receivers: @@ -229,6 +241,7 @@ receivers: collection_interval: 1s labels: label.one: ${LABEL_ONE_VALUE} + label.three: actual.label.three.value.from.env.var.property label.two: ${LABEL_TWO_VALUE} resource_attributes: {} rule: type == "hostport" and command contains "otelcol" and port == ${INTERNAL_PROMETHEUS_PORT} @@ -247,7 +260,7 @@ service: metrics: address: "" level: none -`, stdout) - require.Contains(t, stderr, "Discovering for next 10s...\nDiscovery complete.\n") +`, stdout, fmt.Sprintf("unexpected --dry-run: %s", stderr)) + require.Contains(t, stderr, "Discovering for next 9s...\nDiscovery complete.\n") require.Zero(t, sc) } diff --git a/tests/general/discoverymode/k8s_observer_discovery_test.go b/tests/general/discoverymode/k8s_observer_discovery_test.go index ab1b1b1c1d..e782ccbbe6 100644 --- a/tests/general/discoverymode/k8s_observer_discovery_test.go +++ b/tests/general/discoverymode/k8s_observer_discovery_test.go @@ -102,7 +102,11 @@ func TestK8sObserver(t *testing.T) { expectedMetrics := tc.ResourceMetrics("k8s-observer-smart-agent-redis.yaml") require.NoError(t, tc.OTLPReceiverSink.AssertAllMetricsReceived(t, *expectedMetrics, 30*time.Second)) - stdout, stderr, err := cluster.Kubectl("exec", "-n", namespace, collectorPodName, "--", "bash", "-c", "SPLUNK_DEBUG_CONFIG_SERVER=false /otelcol --config=/config/config.yaml --config-dir=/config.d --discovery --dry-run") + stdout, stderr, err := cluster.Kubectl( + "exec", "-n", namespace, collectorPodName, "--", "bash", "-c", + `SPLUNK_DEBUG_CONFIG_SERVER=false \ +SPLUNK_DISCOVERY_RECEIVERS_smartagent_CONFIG_extraDimensions_x3a__x3a_three_x2e_key='three.value.from.env.var.property' \ +/otelcol --config=/config/config.yaml --config-dir=/config.d --discovery --dry-run`) require.NoError(t, err) require.Equal(t, `exporters: otlp: @@ -117,6 +121,8 @@ receivers: receivers: smartagent: config: + extraDimensions: + three.key: three.value.from.env.var.property type: collectd/redis resource_attributes: one.key: one.value @@ -315,8 +321,12 @@ func daemonSetManifest(cluster *kubeutils.KindCluster, namespace, serviceAccount Labels: map[string]string{"label.key": "label.value"}, Containers: []corev1.Container{ { - Image: testutils.GetCollectorImageOrSkipTest(cluster.Testcase), - Command: []string{"/otelcol", "--config=/config/config.yaml", "--config-dir=/config.d", "--discovery"}, + Image: testutils.GetCollectorImageOrSkipTest(cluster.Testcase), + Command: []string{ + "/otelcol", "--config=/config/config.yaml", "--config-dir=/config.d", "--discovery", + // TODO update w/ resource_attributes when supported + "--set", "splunk.discovery.receivers.smartagent.config.extraDimensions::three.key='three.value.from.cmdline.property'", + }, Env: []corev1.EnvVar{ {Name: "OTLP_ENDPOINT", Value: otlpEndpoint}, {Name: "TARGET_POD_NAME", Value: "target.redis"}, diff --git a/tests/general/discoverymode/testdata/docker-observer-config.d/extensions/docker-observer.discovery.yaml b/tests/general/discoverymode/testdata/docker-observer-config.d/extensions/docker-observer.discovery.yaml index 8f408301a2..ffb0049fc3 100644 --- a/tests/general/discoverymode/testdata/docker-observer-config.d/extensions/docker-observer.discovery.yaml +++ b/tests/general/discoverymode/testdata/docker-observer-config.d/extensions/docker-observer.discovery.yaml @@ -1,2 +1,2 @@ docker_observer: - endpoint: ${DOCKER_DOMAIN_SOCKET} \ No newline at end of file + endpoint: overwritten.by.cmdline.property.that.references.env.var \ No newline at end of file diff --git a/tests/general/discoverymode/testdata/host-observer-config.d/extensions/host-observer.discovery.yaml b/tests/general/discoverymode/testdata/host-observer-config.d/extensions/host-observer.discovery.yaml index 27540a048c..5ca281f8dd 100644 --- a/tests/general/discoverymode/testdata/host-observer-config.d/extensions/host-observer.discovery.yaml +++ b/tests/general/discoverymode/testdata/host-observer-config.d/extensions/host-observer.discovery.yaml @@ -1,2 +1,2 @@ host_observer: - refresh_interval: 1s \ No newline at end of file + refresh_interval: overrwritten by property \ No newline at end of file diff --git a/tests/general/discoverymode/testdata/host-observer-config.d/receivers/internal-prometheus.discovery.yaml b/tests/general/discoverymode/testdata/host-observer-config.d/receivers/internal-prometheus.discovery.yaml index 52d9936dfe..3798fe88cb 100644 --- a/tests/general/discoverymode/testdata/host-observer-config.d/receivers/internal-prometheus.discovery.yaml +++ b/tests/general/discoverymode/testdata/host-observer-config.d/receivers/internal-prometheus.discovery.yaml @@ -6,6 +6,7 @@ prometheus_simple: collection_interval: invalid labels: label.one: ${LABEL_ONE_VALUE} + label.three: overwritten by discovery property host_observer: collection_interval: 1s labels: diff --git a/tests/general/discoverymode/testdata/resource_metrics/host-observer-internal-prometheus.yaml b/tests/general/discoverymode/testdata/resource_metrics/host-observer-internal-prometheus.yaml index d57ce0304f..66caaf49ee 100644 --- a/tests/general/discoverymode/testdata/resource_metrics/host-observer-internal-prometheus.yaml +++ b/tests/general/discoverymode/testdata/resource_metrics/host-observer-internal-prometheus.yaml @@ -10,8 +10,9 @@ resource_metrics: type: DoubleMonotonicCumulativeSum attributes: exporter: logging - label.one: actual.label.one.value - label.two: actual.label.two.value + label.one: actual.label.one.value.from.env.var + label.two: actual.label.two.value.from.env.var + label.three: actual.label.three.value.from.cmdline.property service_instance_id: service_name: otelcol service_version: \ No newline at end of file diff --git a/tests/general/discoverymode/testdata/resource_metrics/k8s-observer-smart-agent-redis.yaml b/tests/general/discoverymode/testdata/resource_metrics/k8s-observer-smart-agent-redis.yaml index 772a0f1e35..de19286be7 100644 --- a/tests/general/discoverymode/testdata/resource_metrics/k8s-observer-smart-agent-redis.yaml +++ b/tests/general/discoverymode/testdata/resource_metrics/k8s-observer-smart-agent-redis.yaml @@ -14,3 +14,4 @@ resource_metrics: plugin: redis_info plugin_instance: system.type: redis + three.key: three.value.from.cmdline.property