Add a file config source (#397)

* Add a file config source

* Fix Linux tests

* PR Feedback

go.mod

@@ -4,6 +4,7 @@ go 1.16
 
 require (
 	github.com/cenkalti/backoff/v4 v4.1.0
+	github.com/fsnotify/fsnotify v1.4.9
 	github.com/go-playground/universal-translator v0.17.0 // indirect
 	github.com/go-zookeeper/zk v1.0.2
 	github.com/gogo/googleapis v1.4.0 // indirect

internal/configsource/fileconfigsource/README.md

@@ -0,0 +1,46 @@
+# File Config Source (Alpha)
+
+Use the file config source to inject YAML fragments or scalars into the
+configuration.
+
+## Configuration
+
+Under the top-level `config_sources:` component mapping use `file:` or
+`file/<name>:` to create a file config source capable of reading from
+subsequently specified files and detecting their changes:
+
+
+```yaml
+config_sources:
+  file:
+```
+
+By default, the config source will monitor for updates on the used files
+and will trigger a configuration reload when they are updated.
+Configuration reload causes temporary interruption of the data flow during
+the time taken to shut down the current pipeline configuration and start the
+new one. 
+Optionally, the file config source can be configured to delete the injected file
+(typically to remove secrets from the file system) as soon as its value is read
+or to not watch for changes to the file.
+
+```yaml
+config_sources:
+  file:
+
+components:
+  component_0:
+    # Default usage: configuration will be reloaded if the file
+    # '/etc/configs/component_field' is changed.
+    component_field: ${file:/etc/configs/component_field} 
+
+  component_1:
+    # Use the 'delete' parameter to force the removal of files with
+    # secrets that shouldn't stay in the OS.
+    component_field: ${file:/etc/configs/secret?delete=true} 
+
+  component_2:
+    # Use the 'disable_watch' parameter to avoid reloading the configuration
+    # if the file is changed.
+    component_field: ${file:/etc/configs/secret?disable_watch=true} 
+```

internal/configsource/fileconfigsource/config.go

@@ -0,0 +1,25 @@
+// Copyright Splunk, Inc.
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fileconfigsource
+
+import (
+	"github.com/signalfx/splunk-otel-collector/internal/configprovider"
+)
+
+// Config holds the configuration for the creation of file config source objects.
+type Config struct {
+	*configprovider.Settings
+}

internal/configsource/fileconfigsource/config_test.go

@@ -0,0 +1,68 @@
+// Copyright Splunk, Inc.
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fileconfigsource
+
+import (
+	"context"
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/config"
+	"go.uber.org/zap"
+
+	"github.com/signalfx/splunk-otel-collector/internal/configprovider"
+)
+
+func TestFileConfigSourceLoadConfig(t *testing.T) {
+	fileName := path.Join("testdata", "config.yaml")
+	v, err := config.NewParserFromFile(fileName)
+	require.NoError(t, err)
+
+	factories := map[config.Type]configprovider.Factory{
+		typeStr: NewFactory(),
+	}
+
+	actualSettings, err := configprovider.Load(context.Background(), v, factories)
+	require.NoError(t, err)
+
+	expectedSettings := map[string]configprovider.ConfigSettings{
+		"file": &Config{
+			Settings: &configprovider.Settings{
+				TypeVal: "file",
+				NameVal: "file",
+			},
+		},
+		"file/named": &Config{
+			Settings: &configprovider.Settings{
+				TypeVal: "file",
+				NameVal: "file/named",
+			},
+		},
+	}
+
+	require.Equal(t, expectedSettings, actualSettings)
+
+	params := configprovider.CreateParams{
+		Logger: zap.NewNop(),
+	}
+	cfgSrcs, err := configprovider.Build(context.Background(), actualSettings, params, factories)
+	require.NoError(t, err)
+	for k := range expectedSettings {
+		assert.Contains(t, cfgSrcs, k)
+	}
+}

internal/configsource/fileconfigsource/configsource.go

@@ -0,0 +1,35 @@
+// Copyright Splunk, Inc.
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fileconfigsource
+
+import (
+	"context"
+
+	"go.opentelemetry.io/collector/config/experimental/configsource"
+	"go.uber.org/zap"
+)
+
+type fileConfigSource struct{}
+
+var _ configsource.ConfigSource = (*fileConfigSource)(nil)
+
+func (f *fileConfigSource) NewSession(context.Context) (configsource.Session, error) {
+	return newSession()
+}
+
+func newConfigSource(_ *zap.Logger, _ *Config) (*fileConfigSource, error) {
+	return &fileConfigSource{}, nil
+}

internal/configsource/fileconfigsource/configsource_test.go

@@ -0,0 +1,100 @@
+// Copyright Splunk, Inc.
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fileconfigsource
+
+import (
+	"context"
+	"io/ioutil"
+	"path"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/config"
+	"go.opentelemetry.io/collector/config/experimental/configsource"
+	"go.uber.org/zap"
+
+	"github.com/signalfx/splunk-otel-collector/internal/configprovider"
+)
+
+func TestFileConfigSourceNew(t *testing.T) {
+	cfgSrc, err := newConfigSource(zap.NewNop(), &Config{})
+	require.NoError(t, err)
+	require.NotNil(t, cfgSrc)
+}
+
+func TestFileConfigSource_End2End(t *testing.T) {
+	file := path.Join("testdata", "file_config_source_end_2_end.yaml")
+	p, err := config.NewParserFromFile(file)
+	require.NoError(t, err)
+	require.NotNil(t, p)
+
+	factories := configprovider.Factories{
+		"file": NewFactory(),
+	}
+	m, err := configprovider.NewManager(p, zap.NewNop(), component.DefaultBuildInfo(), factories)
+	require.NoError(t, err)
+	require.NotNil(t, m)
+
+	ctx := context.Background()
+	r, err := m.Resolve(ctx, p)
+	require.NoError(t, err)
+	require.NotNil(t, r)
+	t.Cleanup(func() {
+		assert.NoError(t, m.Close(ctx))
+	})
+
+	var watchErr error
+	watchDone := make(chan struct{})
+	go func() {
+		defer close(watchDone)
+		watchErr = m.WatchForUpdate()
+	}()
+	m.WaitForWatcher()
+
+	fileWithExpectedData := path.Join("testdata", "file_config_source_end_2_end_expected.yaml")
+	expected, err := config.NewParserFromFile(fileWithExpectedData)
+	require.NoError(t, err)
+	require.NotNil(t, expected)
+
+	assert.Equal(t, expected.ToStringMap(), r.ToStringMap())
+
+	// Touch one of the files to trigger an update.
+	yamlDataFile := path.Join("testdata", "yaml_data_file")
+	touchFile(t, yamlDataFile)
+
+	select {
+	case <-watchDone:
+	case <-time.After(3 * time.Second):
+		require.Fail(t, "expected file change notification didn't happen")
+	}
+
+	assert.ErrorIs(t, watchErr, configsource.ErrValueUpdated)
+
+	// Value should not have changed, resolve it again and confirm.
+	r, err = m.Resolve(ctx, p)
+	require.NoError(t, err)
+	require.NotNil(t, r)
+	assert.Equal(t, expected.ToStringMap(), r.ToStringMap())
+}
+
+func touchFile(t *testing.T, file string) {
+	contents, err := ioutil.ReadFile(file)
+	require.NoError(t, err)
+	require.NoError(t, ioutil.WriteFile(file, contents, 0))
+}

internal/configsource/fileconfigsource/factory.go

@@ -0,0 +1,51 @@
+// Copyright Splunk, Inc.
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//       http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fileconfigsource
+
+import (
+	"context"
+
+	"go.opentelemetry.io/collector/config"
+	"go.opentelemetry.io/collector/config/experimental/configsource"
+
+	"github.com/signalfx/splunk-otel-collector/internal/configprovider"
+)
+
+const (
+	// The "type" of file config sources in configuration.
+	typeStr = "file"
+)
+
+type fileFactory struct{}
+
+func (f *fileFactory) Type() config.Type {
+	return typeStr
+}
+
+func (f *fileFactory) CreateDefaultConfig() configprovider.ConfigSettings {
+	return &Config{
+		Settings: configprovider.NewSettings(typeStr),
+	}
+}
+
+func (f *fileFactory) CreateConfigSource(_ context.Context, params configprovider.CreateParams, cfg configprovider.ConfigSettings) (configsource.ConfigSource, error) {
+	return newConfigSource(params.Logger, cfg.(*Config))
+}
+
+// NewFactory creates a factory for Vault ConfigSource objects.
+func NewFactory() configprovider.Factory {
+	return &fileFactory{}
+}

internal/configsource/fileconfigsource/factory_test.go

@@ -0,0 +1,39 @@
+// Copyright Splunk, Inc.
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package fileconfigsource
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"go.opentelemetry.io/collector/config"
+	"go.uber.org/zap"
+
+	"github.com/signalfx/splunk-otel-collector/internal/configprovider"
+)
+
+func TestFileConfigSourceFactory_CreateConfigSource(t *testing.T) {
+	factory := NewFactory()
+	assert.Equal(t, config.Type("file"), factory.Type())
+	createParams := configprovider.CreateParams{
+		Logger: zap.NewNop(),
+	}
+
+	actual, err := factory.CreateConfigSource(context.Background(), createParams, &Config{})
+	assert.NoError(t, err)
+	assert.NotNil(t, actual)
+}