Skip to content

Commit

Permalink
Support set flag for component config
Browse files Browse the repository at this point in the history 
Signed-off-by: Pavol Loffay <ploffay@redhat.com>
  • Loading branch information
@pavolloffay
pavolloffay committed Aug 27, 2020
1 parent eb0896e commit 4d8f6c5
Show file tree
Hide file tree
Showing 9 changed files with 477 additions and 63 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ otelcol:

.PHONY: run
run:
GO111MODULE=on go run --race ./cmd/otelcol/... --config ${RUN_CONFIG}
GO111MODULE=on go run --race ./cmd/otelcol/... --config ${RUN_CONFIG} --set processors.batch.timeout=70s --set=processors.batch.send_batch_max_size=600 --set=receivers.otlp.protocols.grpc.endpoint=localhost:12365 --set=processor.queued_retry.queue_size=15

.PHONY: docker-component # Not intended to be used directly
docker-component: check-component
Expand Down
156 changes: 101 additions & 55 deletions config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,19 +266,10 @@ func loadExtensions(v *viper.Viper, factories map[configmodels.Type]component.Ex
extensionCfg.SetType(typeStr)
extensionCfg.SetName(fullName)

// Unmarshal only the subconfig for this exporter.
componentConfig, err := ViperSubExact(extensionsConfig, key)
if err != nil {
if err := unmarshallExtension(extensionsConfig, extensionCfg); err != nil {
return nil, err
}

// Now that the default config struct is created we can Unmarshal into it
// and it will apply user-defined config on top of the default.
unm := unmarshaler(factory)
if err := unm(componentConfig, extensionCfg); err != nil {
return nil, errorUnmarshalError(extensionsKeyName, fullName, err)
}

if extensions[fullName] != nil {
return nil, errorDuplicateName(extensionsKeyName, fullName)
}
Expand Down Expand Up @@ -319,23 +310,6 @@ func loadService(v *viper.Viper) (configmodels.Service, error) {
return service, nil
}

// LoadReceiver loads a receiver config from componentConfig using the provided factories.
func LoadReceiver(componentConfig *viper.Viper, typeStr configmodels.Type, fullName string, factory component.ReceiverFactory) (configmodels.Receiver, error) {
// Create the default config for this receiver.
receiverCfg := factory.CreateDefaultConfig()
receiverCfg.SetType(typeStr)
receiverCfg.SetName(fullName)

// Now that the default config struct is created we can Unmarshal into it
// and it will apply user-defined config on top of the default.
unm := unmarshaler(factory)
if err := unm(componentConfig, receiverCfg); err != nil {
return nil, errorUnmarshalError(receiversKeyName, fullName, err)
}

return receiverCfg, nil
}

func loadReceivers(v *viper.Viper, factories map[configmodels.Type]component.ReceiverFactory) (configmodels.Receivers, error) {
// Get the list of all "receivers" sub vipers from config source.
receiversConfig, err := ViperSubExact(v, receiversKeyName)
Expand Down Expand Up @@ -364,14 +338,11 @@ func loadReceivers(v *viper.Viper, factories map[configmodels.Type]component.Rec
return nil, errorUnknownType(receiversKeyName, typeStr, fullName)
}

// Unmarshal only the subconfig for this exporter.
componentConfig, err := ViperSubExact(receiversConfig, key)
if err != nil {
return nil, err
}

receiverCfg, err := LoadReceiver(componentConfig, typeStr, fullName, factory)
receiverCfg := factory.CreateDefaultConfig()
receiverCfg.SetType(typeStr)
receiverCfg.SetName(fullName)

err = unmarshallReceiver(receiversConfig, receiverCfg)
if err != nil {
// LoadReceiver already wraps the error.
return nil, err
Expand Down Expand Up @@ -419,19 +390,10 @@ func loadExporters(v *viper.Viper, factories map[configmodels.Type]component.Exp
exporterCfg.SetType(typeStr)
exporterCfg.SetName(fullName)

// Unmarshal only the subconfig for this exporter.
componentConfig, err := ViperSubExact(exportersConfig, key)
if err != nil {
if err := unmarshallExporter(exportersConfig, exporterCfg); err != nil {
return nil, err
}

// Now that the default config struct is created we can Unmarshal into it
// and it will apply user-defined config on top of the default.
unm := unmarshaler(factory)
if err := unm(componentConfig, exporterCfg); err != nil {
return nil, errorUnmarshalError(exportersKeyName, fullName, err)
}

if exporters[fullName] != nil {
return nil, errorDuplicateName(exportersKeyName, fullName)
}
Expand All @@ -442,6 +404,54 @@ func loadExporters(v *viper.Viper, factories map[configmodels.Type]component.Exp
return exporters, nil
}

func unmarshallReceiver(receiversConfig *viper.Viper, receiver configmodels.Receiver) error {
componentConfig, err := ViperSubExact(receiversConfig, receiver.Name())
if err != nil {
return err
}
unm := unmarshaler(receiver)
if err := unm(componentConfig, receiver); err != nil {
return errorUnmarshalError(receiversKeyName, receiver.Name(), err)
}
return nil
}

func unmarshallProcessor(processorsConfig *viper.Viper, processor configmodels.Processor) error {
componentConfig, err := ViperSubExact(processorsConfig, processor.Name())
if err != nil {
return err
}
unm := unmarshaler(processor)
if err := unm(componentConfig, processor); err != nil {
return errorUnmarshalError(processorsKeyName, processor.Name(), err)
}
return nil
}

func unmarshallExporter(exportersConfig *viper.Viper, exporter configmodels.Exporter) error {
componentConfig, err := ViperSubExact(exportersConfig, exporter.Name())
if err != nil {
return nil
}
unm := unmarshaler(exporter)
if err := unm(componentConfig, exporter); err != nil {
return errorUnmarshalError(exportersKeyName, exporter.Name(), err)
}
return nil
}

func unmarshallExtension(extensionsConfig *viper.Viper, extension configmodels.Extension) error {
componentConfig, err := ViperSubExact(extensionsConfig, extension.Name())
if err != nil {
return err
}
unm := unmarshaler(extension)
if err := unm(componentConfig, extension); err != nil {
return errorUnmarshalError(extensionsKeyName, extension.Name(), err)
}
return nil
}

func loadProcessors(v *viper.Viper, factories map[configmodels.Type]component.ProcessorFactory) (configmodels.Processors, error) {
// Get the list of all "processors" sub vipers from config source.
processorsConfig, err := ViperSubExact(v, processorsKeyName)
Expand Down Expand Up @@ -475,17 +485,8 @@ func loadProcessors(v *viper.Viper, factories map[configmodels.Type]component.Pr
processorCfg.SetType(typeStr)
processorCfg.SetName(fullName)

// Unmarshal only the subconfig for this processor.
componentConfig, vsErr := ViperSubExact(processorsConfig, key)
if vsErr != nil {
return nil, vsErr
}

// Now that the default config struct is created we can Unmarshal into it
// and it will apply user-defined config on top of the default.
unm := unmarshaler(factory)
if err := unm(componentConfig, processorCfg); err != nil {
return nil, errorUnmarshalError(processorsKeyName, fullName, err)
if err := unmarshallProcessor(processorsConfig, processorCfg); err != nil {
return nil, err
}

if processors[fullName] != nil {
Expand Down Expand Up @@ -801,3 +802,48 @@ func ViperSubExact(v *viper.Viper, key string) (*viper.Viper, error) {
msg: fmt.Sprintf("unexpected sub-config value kind for key:%s value:%v kind:%v)", key, data, reflect.TypeOf(data).Kind()),
}
}

// ApplyChange applies changes from viper to config.
func ApplyChange(v *viper.Viper, config *configmodels.Config) error {
receiversConfig, err := ViperSubExact(v, receiversKeyName)
if err != nil {
return err
}
expandEnvConfig(receiversConfig)
for _, receiver := range config.Receivers {
if err := unmarshallReceiver(receiversConfig, receiver); err != nil {
return err
}
}
processorsConfig, err := ViperSubExact(v, processorsKeyName)
if err != nil {
return nil
}
expandEnvConfig(processorsConfig)
for _, processor := range config.Processors {
if err := unmarshallProcessor(processorsConfig, processor); err != nil {
return err
}
}
exportersConfig, err := ViperSubExact(v, exportersKeyName)
if err != nil {
return nil
}
expandEnvConfig(exportersConfig)
for _, exp := range config.Exporters {
if err := unmarshallExporter(exportersConfig, exp); err != nil {
return err
}
}
extensionsConfig, err := ViperSubExact(v, extensionsKeyName)
if err != nil {
return nil
}
expandEnvConfig(extensionsConfig)
for _, ext := range config.Extensions {
if err := unmarshallExtension(extensionsConfig, ext); err != nil {
return err
}
}
return nil
}
147 changes: 147 additions & 0 deletions config/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"path/filepath"
"testing"

"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
Expand Down Expand Up @@ -581,3 +582,149 @@ func loadConfigFile(t *testing.T, fileName string, factories component.Factories
}
return cfg, ValidateConfig(cfg, zap.NewNop())
}

func TestApplyChanges_errs(t *testing.T) {
v := viper.New()
t.Run("empty_config", func(t *testing.T) {
require.NoError(t, ApplyChange(v, &configmodels.Config{}))
})
t.Run("err_receiver_should_be_pointer", func(t *testing.T) {
cfg := &configmodels.Config{
Receivers: configmodels.Receivers{
"foo": invalidEntity{},
},
}
require.Error(t, ApplyChange(v, cfg))
})
t.Run("err_processor_should_be_pointer", func(t *testing.T) {
cfg := &configmodels.Config{
Processors: configmodels.Processors{
"foo": invalidEntity{},
},
}
require.Error(t, ApplyChange(v, cfg))
})
t.Run("err_exporter_should_be_pointer", func(t *testing.T) {
cfg := &configmodels.Config{
Exporters: configmodels.Exporters{
"foo": invalidEntity{},
},
}
require.Error(t, ApplyChange(v, cfg))
})
t.Run("err_extension_should_be_pointer", func(t *testing.T) {
cfg := &configmodels.Config{
Extensions: configmodels.Extensions{
"foo": invalidEntity{},
},
}
require.Error(t, ApplyChange(v, cfg))
})
}

func TestApplyConfig(t *testing.T) {
v := viper.New()
v.Set("receivers.test.property", "receiver_val")
// arrays are overridden
v.Set("receivers.test.arr", "val1,val2")
// maps are joined
v.Set("receivers.test.map.key1", "val1")
v.Set("receivers.test.map.key2", "val2")
v.Set("receivers.test/foo.property", "receiver_foo_val")
v.Set("processors.test.property", "processor_val")
v.Set("exporters.test.property", "exporter_val")
v.Set("extensions.test.property", "extension_val")

cfg := &configmodels.Config{
Receivers: configmodels.Receivers{
"test": &testEntity{
OtherProperty: "remains_here",
Arr: []string{"does_not_remain_here"},
Map: map[string]string{"remains_here": "val"},
},
"test/foo": &testEntity{
name: "foo",
},
},
Processors: configmodels.Processors{
"test": &testEntity{},
},
Exporters: configmodels.Exporters{
"test": &testEntity{},
},
Extensions: configmodels.Extensions{
"test": &testEntity{},
},
}
require.NoError(t, ApplyChange(v, cfg))
assert.Equal(t, &configmodels.Config{
Receivers: configmodels.Receivers{
"test": &testEntity{
Property: "receiver_val",
OtherProperty: "remains_here",
Arr: []string{"val1", "val2"},
Map: map[string]string{"remains_here": "val", "key1": "val1", "key2": "val2"},
},
"test/foo": &testEntity{
name: "foo",
Property: "receiver_foo_val",
},
},
Processors: configmodels.Processors{
"test": &testEntity{
Property: "processor_val",
},
},
Exporters: configmodels.Exporters{
"test": &testEntity{
Property: "exporter_val",
},
},
Extensions: configmodels.Extensions{
"test": &testEntity{
Property: "extension_val",
},
},
}, cfg)
}

type testEntity struct {
name string
Property string `mapstructure:"property"`
OtherProperty string `mapstructure:"other_property"`
Arr []string `mapstructure:"arr"`
Map map[string]string `mapstructure:"map"`
}

var _ configmodels.NamedEntity = (*testEntity)(nil)

func (i *testEntity) Type() configmodels.Type {
return "test"
}
func (i *testEntity) SetType(typeStr configmodels.Type) {
}
func (i *testEntity) Name() string {
if i.name == "" {
return "test"
}
return "test/" + i.name
}
func (i *testEntity) SetName(name string) {
i.name = name
}

type invalidEntity struct {
}

var _ configmodels.NamedEntity = (*invalidEntity)(nil)

func (i invalidEntity) Type() configmodels.Type {
return ""
}
func (i invalidEntity) SetType(typeStr configmodels.Type) {
}
func (i invalidEntity) Name() string {
return ""
}
func (i invalidEntity) SetName(name string) {
}
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ require (
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/cast v1.3.1
github.com/spf13/cobra v1.0.0
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.7.1
github.com/stretchr/testify v1.6.1
github.com/tcnksm/ghr v0.13.0
Expand Down
Loading

0 comments on commit 4d8f6c5

Please sign in to comment.