diff --git a/docs/sources/administration/provisioning.md b/docs/sources/administration/provisioning.md index c09c3caec8249..aa84863bf7ae1 100644 --- a/docs/sources/administration/provisioning.md +++ b/docs/sources/administration/provisioning.md @@ -30,33 +30,19 @@ Checkout the [configuration](/installation/configuration) page for more informat ### Using Environment Variables -All options in the configuration file (listed below) can be overridden -using environment variables using the syntax: +It is possible to use environment variable interpolation in all 3 provisioning config types. Allowed syntax +is either `$ENV_VAR_NAME` or `${ENV_VAR_NAME}` and can be used only for values not for keys or bigger parts +of the configs. It is not available in the dashboards definition files just the dashboard provisioning +configuration. +Example: -```bash -GF__ -``` - -Where the section name is the text within the brackets. Everything -should be upper case and `.` should be replaced by `_`. For example, given these configuration settings: - -```bash -# default section -instance_name = ${HOSTNAME} - -[security] -admin_user = admin - -[auth.google] -client_secret = 0ldS3cretKey -``` - -Overriding will be done like so: - -```bash -export GF_DEFAULT_INSTANCE_NAME=my-instance -export GF_SECURITY_ADMIN_USER=true -export GF_AUTH_GOOGLE_CLIENT_SECRET=newS3cretKey +```yaml +datasources: +- name: Graphite + url: http://localhost:$PORT + user: $USER + secureJsonData: + password: $PASSWORD ```
diff --git a/pkg/services/provisioning/dashboards/config_reader_test.go b/pkg/services/provisioning/dashboards/config_reader_test.go index 8ce322a76079f..18d8022d62d04 100644 --- a/pkg/services/provisioning/dashboards/config_reader_test.go +++ b/pkg/services/provisioning/dashboards/config_reader_test.go @@ -1,6 +1,7 @@ package dashboards import ( + "os" "testing" "github.com/grafana/grafana/pkg/log" @@ -18,8 +19,10 @@ func TestDashboardsAsConfig(t *testing.T) { logger := log.New("test-logger") Convey("Can read config file version 1 format", func() { + _ = os.Setenv("TEST_VAR", "general") cfgProvider := configReader{path: simpleDashboardConfig, log: logger} cfg, err := cfgProvider.readConfig() + _ = os.Unsetenv("TEST_VAR") So(err, ShouldBeNil) validateDashboardAsConfig(t, cfg) diff --git a/pkg/services/provisioning/dashboards/testdata/test-configs/dashboards-from-disk/dev-dashboards.yaml b/pkg/services/provisioning/dashboards/testdata/test-configs/dashboards-from-disk/dev-dashboards.yaml index ee3d8ea73b0fa..42b6ecd75a5e2 100644 --- a/pkg/services/provisioning/dashboards/testdata/test-configs/dashboards-from-disk/dev-dashboards.yaml +++ b/pkg/services/provisioning/dashboards/testdata/test-configs/dashboards-from-disk/dev-dashboards.yaml @@ -1,7 +1,7 @@ apiVersion: 1 providers: -- name: 'general dashboards' +- name: '$TEST_VAR dashboards' orgId: 2 folder: 'developers' folderUid: 'xyz' diff --git a/pkg/services/provisioning/dashboards/types.go b/pkg/services/provisioning/dashboards/types.go index d5364c33509d4..547e5a4427c26 100644 --- a/pkg/services/provisioning/dashboards/types.go +++ b/pkg/services/provisioning/dashboards/types.go @@ -1,6 +1,7 @@ package dashboards import ( + "github.com/grafana/grafana/pkg/services/provisioning/values" "time" "github.com/grafana/grafana/pkg/components/simplejson" @@ -42,15 +43,15 @@ type DashboardAsConfigV1 struct { } type DashboardProviderConfigs struct { - Name string `json:"name" yaml:"name"` - Type string `json:"type" yaml:"type"` - OrgId int64 `json:"orgId" yaml:"orgId"` - Folder string `json:"folder" yaml:"folder"` - FolderUid string `json:"folderUid" yaml:"folderUid"` - Editable bool `json:"editable" yaml:"editable"` - Options map[string]interface{} `json:"options" yaml:"options"` - DisableDeletion bool `json:"disableDeletion" yaml:"disableDeletion"` - UpdateIntervalSeconds int64 `json:"updateIntervalSeconds" yaml:"updateIntervalSeconds"` + Name values.StringValue `json:"name" yaml:"name"` + Type values.StringValue `json:"type" yaml:"type"` + OrgId values.Int64Value `json:"orgId" yaml:"orgId"` + Folder values.StringValue `json:"folder" yaml:"folder"` + FolderUid values.StringValue `json:"folderUid" yaml:"folderUid"` + Editable values.BoolValue `json:"editable" yaml:"editable"` + Options values.JSONValue `json:"options" yaml:"options"` + DisableDeletion values.BoolValue `json:"disableDeletion" yaml:"disableDeletion"` + UpdateIntervalSeconds values.Int64Value `json:"updateIntervalSeconds" yaml:"updateIntervalSeconds"` } func createDashboardJson(data *simplejson.Json, lastModified time.Time, cfg *DashboardsAsConfig, folderId int64) (*dashboards.SaveDashboardDTO, error) { @@ -94,15 +95,15 @@ func (dc *DashboardAsConfigV1) mapToDashboardAsConfig() []*DashboardsAsConfig { for _, v := range dc.Providers { r = append(r, &DashboardsAsConfig{ - Name: v.Name, - Type: v.Type, - OrgId: v.OrgId, - Folder: v.Folder, - FolderUid: v.FolderUid, - Editable: v.Editable, - Options: v.Options, - DisableDeletion: v.DisableDeletion, - UpdateIntervalSeconds: v.UpdateIntervalSeconds, + Name: v.Name.Value(), + Type: v.Type.Value(), + OrgId: v.OrgId.Value(), + Folder: v.Folder.Value(), + FolderUid: v.FolderUid.Value(), + Editable: v.Editable.Value(), + Options: v.Options.Value(), + DisableDeletion: v.DisableDeletion.Value(), + UpdateIntervalSeconds: v.UpdateIntervalSeconds.Value(), }) } diff --git a/pkg/services/provisioning/datasources/config_reader_test.go b/pkg/services/provisioning/datasources/config_reader_test.go index 07c8d68e75c14..6aba826221415 100644 --- a/pkg/services/provisioning/datasources/config_reader_test.go +++ b/pkg/services/provisioning/datasources/config_reader_test.go @@ -1,6 +1,7 @@ package datasources import ( + "os" "testing" "github.com/grafana/grafana/pkg/bus" @@ -146,8 +147,10 @@ func TestDatasourceAsConfig(t *testing.T) { }) Convey("can read all properties from version 1", func() { + _ = os.Setenv("TEST_VAR", "name") cfgProvifer := &configReader{log: log.New("test logger")} cfg, err := cfgProvifer.readConfig(allProperties) + _ = os.Unsetenv("TEST_VAR") if err != nil { t.Fatalf("readConfig return an error %v", err) } diff --git a/pkg/services/provisioning/datasources/testdata/all-properties/all-properties.yaml b/pkg/services/provisioning/datasources/testdata/all-properties/all-properties.yaml index b92b81f70792d..abd1253f8399f 100644 --- a/pkg/services/provisioning/datasources/testdata/all-properties/all-properties.yaml +++ b/pkg/services/provisioning/datasources/testdata/all-properties/all-properties.yaml @@ -1,7 +1,7 @@ apiVersion: 1 datasources: - - name: name + - name: $TEST_VAR type: type access: proxy orgId: 2 diff --git a/pkg/services/provisioning/datasources/types.go b/pkg/services/provisioning/datasources/types.go index f619fd5f0054b..bbd072052dd70 100644 --- a/pkg/services/provisioning/datasources/types.go +++ b/pkg/services/provisioning/datasources/types.go @@ -4,6 +4,7 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/provisioning/values" ) type ConfigVersion struct { @@ -64,8 +65,8 @@ type DeleteDatasourceConfigV0 struct { } type DeleteDatasourceConfigV1 struct { - OrgId int64 `json:"orgId" yaml:"orgId"` - Name string `json:"name" yaml:"name"` + OrgId values.Int64Value `json:"orgId" yaml:"orgId"` + Name values.StringValue `json:"name" yaml:"name"` } type DataSourceFromConfigV0 struct { @@ -89,23 +90,23 @@ type DataSourceFromConfigV0 struct { } type DataSourceFromConfigV1 struct { - OrgId int64 `json:"orgId" yaml:"orgId"` - Version int `json:"version" yaml:"version"` - Name string `json:"name" yaml:"name"` - Type string `json:"type" yaml:"type"` - Access string `json:"access" yaml:"access"` - Url string `json:"url" yaml:"url"` - Password string `json:"password" yaml:"password"` - User string `json:"user" yaml:"user"` - Database string `json:"database" yaml:"database"` - BasicAuth bool `json:"basicAuth" yaml:"basicAuth"` - BasicAuthUser string `json:"basicAuthUser" yaml:"basicAuthUser"` - BasicAuthPassword string `json:"basicAuthPassword" yaml:"basicAuthPassword"` - WithCredentials bool `json:"withCredentials" yaml:"withCredentials"` - IsDefault bool `json:"isDefault" yaml:"isDefault"` - JsonData map[string]interface{} `json:"jsonData" yaml:"jsonData"` - SecureJsonData map[string]string `json:"secureJsonData" yaml:"secureJsonData"` - Editable bool `json:"editable" yaml:"editable"` + OrgId values.Int64Value `json:"orgId" yaml:"orgId"` + Version values.IntValue `json:"version" yaml:"version"` + Name values.StringValue `json:"name" yaml:"name"` + Type values.StringValue `json:"type" yaml:"type"` + Access values.StringValue `json:"access" yaml:"access"` + Url values.StringValue `json:"url" yaml:"url"` + Password values.StringValue `json:"password" yaml:"password"` + User values.StringValue `json:"user" yaml:"user"` + Database values.StringValue `json:"database" yaml:"database"` + BasicAuth values.BoolValue `json:"basicAuth" yaml:"basicAuth"` + BasicAuthUser values.StringValue `json:"basicAuthUser" yaml:"basicAuthUser"` + BasicAuthPassword values.StringValue `json:"basicAuthPassword" yaml:"basicAuthPassword"` + WithCredentials values.BoolValue `json:"withCredentials" yaml:"withCredentials"` + IsDefault values.BoolValue `json:"isDefault" yaml:"isDefault"` + JsonData values.JSONValue `json:"jsonData" yaml:"jsonData"` + SecureJsonData values.StringMapValue `json:"secureJsonData" yaml:"secureJsonData"` + Editable values.BoolValue `json:"editable" yaml:"editable"` } func (cfg *DatasourcesAsConfigV1) mapToDatasourceFromConfig(apiVersion int64) *DatasourcesAsConfig { @@ -119,36 +120,47 @@ func (cfg *DatasourcesAsConfigV1) mapToDatasourceFromConfig(apiVersion int64) *D for _, ds := range cfg.Datasources { r.Datasources = append(r.Datasources, &DataSourceFromConfig{ - OrgId: ds.OrgId, - Name: ds.Name, - Type: ds.Type, - Access: ds.Access, - Url: ds.Url, - Password: ds.Password, - User: ds.User, - Database: ds.Database, - BasicAuth: ds.BasicAuth, - BasicAuthUser: ds.BasicAuthUser, - BasicAuthPassword: ds.BasicAuthPassword, - WithCredentials: ds.WithCredentials, - IsDefault: ds.IsDefault, - JsonData: ds.JsonData, - SecureJsonData: ds.SecureJsonData, - Editable: ds.Editable, - Version: ds.Version, + OrgId: ds.OrgId.Value(), + Name: ds.Name.Value(), + Type: ds.Type.Value(), + Access: ds.Access.Value(), + Url: ds.Url.Value(), + Password: ds.Password.Value(), + User: ds.User.Value(), + Database: ds.Database.Value(), + BasicAuth: ds.BasicAuth.Value(), + BasicAuthUser: ds.BasicAuthUser.Value(), + BasicAuthPassword: ds.BasicAuthPassword.Value(), + WithCredentials: ds.WithCredentials.Value(), + IsDefault: ds.IsDefault.Value(), + JsonData: ds.JsonData.Value(), + SecureJsonData: ds.SecureJsonData.Value(), + Editable: ds.Editable.Value(), + Version: ds.Version.Value(), }) - if ds.Password != "" { - cfg.log.Warn("[Deprecated] the use of password field is deprecated. Please use secureJsonData.password", "datasource name", ds.Name) + + // Using Raw value for the warnings here so that even if it uses env interpolation and the env var is empty + // it will still warn + if len(ds.Password.Raw) > 0 { + cfg.log.Warn( + "[Deprecated] the use of password field is deprecated. Please use secureJsonData.password", + "datasource name", + ds.Name.Value(), + ) } - if ds.BasicAuthPassword != "" { - cfg.log.Warn("[Deprecated] the use of basicAuthPassword field is deprecated. Please use secureJsonData.basicAuthPassword", "datasource name", ds.Name) + if len(ds.BasicAuthPassword.Raw) > 0 { + cfg.log.Warn( + "[Deprecated] the use of basicAuthPassword field is deprecated. Please use secureJsonData.basicAuthPassword", + "datasource name", + ds.Name.Value(), + ) } } for _, ds := range cfg.DeleteDatasources { r.DeleteDatasources = append(r.DeleteDatasources, &DeleteDatasourceConfig{ - OrgId: ds.OrgId, - Name: ds.Name, + OrgId: ds.OrgId.Value(), + Name: ds.Name.Value(), }) } diff --git a/pkg/services/provisioning/notifiers/alert_notifications.go b/pkg/services/provisioning/notifiers/alert_notifications.go index 7b595a3d32f24..3659a73832ee5 100644 --- a/pkg/services/provisioning/notifiers/alert_notifications.go +++ b/pkg/services/provisioning/notifiers/alert_notifications.go @@ -131,39 +131,6 @@ func (dc *NotificationProvisioner) mergeNotifications(notificationToMerge []*not return nil } -func (cfg *notificationsAsConfig) mapToNotificationFromConfig() *notificationsAsConfig { - r := ¬ificationsAsConfig{} - if cfg == nil { - return r - } - - for _, notification := range cfg.Notifications { - r.Notifications = append(r.Notifications, ¬ificationFromConfig{ - Uid: notification.Uid, - OrgId: notification.OrgId, - OrgName: notification.OrgName, - Name: notification.Name, - Type: notification.Type, - IsDefault: notification.IsDefault, - Settings: notification.Settings, - DisableResolveMessage: notification.DisableResolveMessage, - Frequency: notification.Frequency, - SendReminder: notification.SendReminder, - }) - } - - for _, notification := range cfg.DeleteNotifications { - r.DeleteNotifications = append(r.DeleteNotifications, &deleteNotificationConfig{ - Uid: notification.Uid, - OrgId: notification.OrgId, - OrgName: notification.OrgName, - Name: notification.Name, - }) - } - - return r -} - func (dc *NotificationProvisioner) applyChanges(configPath string) error { configs, err := dc.cfgProvider.readConfig(configPath) if err != nil { diff --git a/pkg/services/provisioning/notifiers/config_reader.go b/pkg/services/provisioning/notifiers/config_reader.go index c1b4cbf9f29b5..896d5105cedba 100644 --- a/pkg/services/provisioning/notifiers/config_reader.go +++ b/pkg/services/provisioning/notifiers/config_reader.go @@ -63,7 +63,7 @@ func (cr *configReader) parseNotificationConfig(path string, file os.FileInfo) ( return nil, err } - var cfg *notificationsAsConfig + var cfg *notificationsAsConfigV0 err = yaml.Unmarshal(yamlFile, &cfg) if err != nil { return nil, err diff --git a/pkg/services/provisioning/notifiers/config_reader_test.go b/pkg/services/provisioning/notifiers/config_reader_test.go index 87645ee7d31b4..d705bd0d93d9d 100644 --- a/pkg/services/provisioning/notifiers/config_reader_test.go +++ b/pkg/services/provisioning/notifiers/config_reader_test.go @@ -1,6 +1,7 @@ package notifiers import ( + "os" "testing" "github.com/grafana/grafana/pkg/log" @@ -43,8 +44,10 @@ func TestNotificationAsConfig(t *testing.T) { }) Convey("Can read correct properties", func() { + _ = os.Setenv("TEST_VAR", "default") cfgProvifer := &configReader{log: log.New("test logger")} cfg, err := cfgProvifer.readConfig(correct_properties) + _ = os.Unsetenv("TEST_VAR") if err != nil { t.Fatalf("readConfig return an error %v", err) } diff --git a/pkg/services/provisioning/notifiers/testdata/test-configs/correct-properties/correct-properties.yaml b/pkg/services/provisioning/notifiers/testdata/test-configs/correct-properties/correct-properties.yaml index af0736f35a4a6..1d846f64473ba 100644 --- a/pkg/services/provisioning/notifiers/testdata/test-configs/correct-properties/correct-properties.yaml +++ b/pkg/services/provisioning/notifiers/testdata/test-configs/correct-properties/correct-properties.yaml @@ -1,5 +1,5 @@ notifiers: - - name: default-slack-notification + - name: $TEST_VAR-slack-notification type: slack uid: notifier1 org_id: 2 @@ -39,4 +39,4 @@ delete_notifiers: org_id: 0 uid: "notifier3" - name: Deleted notification with whitespaces in name - uid: "notifier4" \ No newline at end of file + uid: "notifier4" diff --git a/pkg/services/provisioning/notifiers/types.go b/pkg/services/provisioning/notifiers/types.go index f788da79c79af..48ff9a6ce1d95 100644 --- a/pkg/services/provisioning/notifiers/types.go +++ b/pkg/services/provisioning/notifiers/types.go @@ -1,30 +1,61 @@ package notifiers -import "github.com/grafana/grafana/pkg/components/simplejson" +import ( + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/services/provisioning/values" +) +// notificationsAsConfig is normalized data object for notifications config data. Any config version should be mappable +// to this type. type notificationsAsConfig struct { - Notifications []*notificationFromConfig `json:"notifiers" yaml:"notifiers"` - DeleteNotifications []*deleteNotificationConfig `json:"delete_notifiers" yaml:"delete_notifiers"` + Notifications []*notificationFromConfig + DeleteNotifications []*deleteNotificationConfig } type deleteNotificationConfig struct { - Uid string `json:"uid" yaml:"uid"` - Name string `json:"name" yaml:"name"` - OrgId int64 `json:"org_id" yaml:"org_id"` - OrgName string `json:"org_name" yaml:"org_name"` + Uid string + Name string + OrgId int64 + OrgName string } type notificationFromConfig struct { - Uid string `json:"uid" yaml:"uid"` - OrgId int64 `json:"org_id" yaml:"org_id"` - OrgName string `json:"org_name" yaml:"org_name"` - Name string `json:"name" yaml:"name"` - Type string `json:"type" yaml:"type"` - SendReminder bool `json:"send_reminder" yaml:"send_reminder"` - DisableResolveMessage bool `json:"disable_resolve_message" yaml:"disable_resolve_message"` - Frequency string `json:"frequency" yaml:"frequency"` - IsDefault bool `json:"is_default" yaml:"is_default"` - Settings map[string]interface{} `json:"settings" yaml:"settings"` + Uid string + OrgId int64 + OrgName string + Name string + Type string + SendReminder bool + DisableResolveMessage bool + Frequency string + IsDefault bool + Settings map[string]interface{} +} + +// notificationsAsConfigV0 is mapping for zero version configs. This is mapped to its normalised version. +type notificationsAsConfigV0 struct { + Notifications []*notificationFromConfigV0 `json:"notifiers" yaml:"notifiers"` + DeleteNotifications []*deleteNotificationConfigV0 `json:"delete_notifiers" yaml:"delete_notifiers"` +} + +type deleteNotificationConfigV0 struct { + Uid values.StringValue `json:"uid" yaml:"uid"` + Name values.StringValue `json:"name" yaml:"name"` + OrgId values.Int64Value `json:"org_id" yaml:"org_id"` + OrgName values.StringValue `json:"org_name" yaml:"org_name"` +} + +type notificationFromConfigV0 struct { + Uid values.StringValue `json:"uid" yaml:"uid"` + OrgId values.Int64Value `json:"org_id" yaml:"org_id"` + OrgName values.StringValue `json:"org_name" yaml:"org_name"` + Name values.StringValue `json:"name" yaml:"name"` + Type values.StringValue `json:"type" yaml:"type"` + SendReminder values.BoolValue `json:"send_reminder" yaml:"send_reminder"` + DisableResolveMessage values.BoolValue `json:"disable_resolve_message" yaml:"disable_resolve_message"` + Frequency values.StringValue `json:"frequency" yaml:"frequency"` + IsDefault values.BoolValue `json:"is_default" yaml:"is_default"` + Settings values.JSONValue `json:"settings" yaml:"settings"` } func (notification notificationFromConfig) SettingsToJson() *simplejson.Json { @@ -36,3 +67,38 @@ func (notification notificationFromConfig) SettingsToJson() *simplejson.Json { } return settings } + +// mapToNotificationFromConfig maps config syntax to normalized notificationsAsConfig object. Every version +// of the config syntax should have this function. +func (cfg *notificationsAsConfigV0) mapToNotificationFromConfig() *notificationsAsConfig { + r := ¬ificationsAsConfig{} + if cfg == nil { + return r + } + + for _, notification := range cfg.Notifications { + r.Notifications = append(r.Notifications, ¬ificationFromConfig{ + Uid: notification.Uid.Value(), + OrgId: notification.OrgId.Value(), + OrgName: notification.OrgName.Value(), + Name: notification.Name.Value(), + Type: notification.Type.Value(), + IsDefault: notification.IsDefault.Value(), + Settings: notification.Settings.Value(), + DisableResolveMessage: notification.DisableResolveMessage.Value(), + Frequency: notification.Frequency.Value(), + SendReminder: notification.SendReminder.Value(), + }) + } + + for _, notification := range cfg.DeleteNotifications { + r.DeleteNotifications = append(r.DeleteNotifications, &deleteNotificationConfig{ + Uid: notification.Uid.Value(), + OrgId: notification.OrgId.Value(), + OrgName: notification.OrgName.Value(), + Name: notification.Name.Value(), + }) + } + + return r +} diff --git a/pkg/services/provisioning/values/values.go b/pkg/services/provisioning/values/values.go new file mode 100644 index 0000000000000..48c7cd6444e63 --- /dev/null +++ b/pkg/services/provisioning/values/values.go @@ -0,0 +1,206 @@ +// A set of value types to use in provisioning. They add custom unmarshaling logic that puts the string values +// through os.ExpandEnv. +// Usage: +// type Data struct { +// Field StringValue `yaml:"field"` // Instead of string +// } +// d := &Data{} +// // unmarshal into d +// d.Field.Value() // returns the final interpolated value from the yaml file +// +package values + +import ( + "github.com/pkg/errors" + "os" + "reflect" + "strconv" +) + +type IntValue struct { + value int + Raw string +} + +func (val *IntValue) UnmarshalYAML(unmarshal func(interface{}) error) error { + interpolated, err := getInterpolated(unmarshal) + if err != nil { + return err + } + if len(interpolated.value) == 0 { + // To keep the same behaviour as the yaml lib which just does not set the value if it is empty. + return nil + } + val.Raw = interpolated.raw + val.value, err = strconv.Atoi(interpolated.value) + return errors.Wrap(err, "cannot convert value int") +} + +func (val *IntValue) Value() int { + return val.value +} + +type Int64Value struct { + value int64 + Raw string +} + +func (val *Int64Value) UnmarshalYAML(unmarshal func(interface{}) error) error { + interpolated, err := getInterpolated(unmarshal) + if err != nil { + return err + } + if len(interpolated.value) == 0 { + // To keep the same behaviour as the yaml lib which just does not set the value if it is empty. + return nil + } + val.Raw = interpolated.raw + val.value, err = strconv.ParseInt(interpolated.value, 10, 64) + return err +} + +func (val *Int64Value) Value() int64 { + return val.value +} + +type StringValue struct { + value string + Raw string +} + +func (val *StringValue) UnmarshalYAML(unmarshal func(interface{}) error) error { + interpolated, err := getInterpolated(unmarshal) + if err != nil { + return err + } + val.Raw = interpolated.raw + val.value = interpolated.value + return err +} + +func (val *StringValue) Value() string { + return val.value +} + +type BoolValue struct { + value bool + Raw string +} + +func (val *BoolValue) UnmarshalYAML(unmarshal func(interface{}) error) error { + interpolated, err := getInterpolated(unmarshal) + if err != nil { + return err + } + val.Raw = interpolated.raw + val.value, err = strconv.ParseBool(interpolated.value) + return err +} + +func (val *BoolValue) Value() bool { + return val.value +} + +type JSONValue struct { + value map[string]interface{} + Raw map[string]interface{} +} + +func (val *JSONValue) UnmarshalYAML(unmarshal func(interface{}) error) error { + unmarshaled := make(map[string]interface{}) + err := unmarshal(unmarshaled) + if err != nil { + return err + } + val.Raw = unmarshaled + interpolated := make(map[string]interface{}) + for key, val := range unmarshaled { + interpolated[key] = tranformInterface(val) + } + val.value = interpolated + return err +} + +func (val *JSONValue) Value() map[string]interface{} { + return val.value +} + +type StringMapValue struct { + value map[string]string + Raw map[string]string +} + +func (val *StringMapValue) UnmarshalYAML(unmarshal func(interface{}) error) error { + unmarshaled := make(map[string]string) + err := unmarshal(unmarshaled) + if err != nil { + return err + } + val.Raw = unmarshaled + interpolated := make(map[string]string) + for key, val := range unmarshaled { + interpolated[key] = interpolateValue(val) + } + val.value = interpolated + return err +} + +func (val *StringMapValue) Value() map[string]string { + return val.value +} + +// tranformInterface tries to transform any interface type into proper value with env expansion. It travers maps and +// slices and the actual interpolation is done on all simple string values in the structure. It returns a copy of any +// map or slice value instead of modifying them in place. +func tranformInterface(i interface{}) interface{} { + switch reflect.TypeOf(i).Kind() { + case reflect.Slice: + return transformSlice(i.([]interface{})) + case reflect.Map: + return transformMap(i.(map[interface{}]interface{})) + case reflect.String: + return interpolateValue(i.(string)) + default: + // Was int, float or some other value that we do not need to do any transform on. + return i + } +} + +func transformSlice(i []interface{}) interface{} { + var transformed []interface{} + for _, val := range i { + transformed = append(transformed, tranformInterface(val)) + } + return transformed +} + +func transformMap(i map[interface{}]interface{}) interface{} { + transformed := make(map[interface{}]interface{}) + for key, val := range i { + transformed[key] = tranformInterface(val) + } + return transformed +} + +// interpolateValue returns final value after interpolation. At the moment only env var interpolation is done +// here but in the future something like interpolation from file could be also done here. +func interpolateValue(val string) string { + return os.ExpandEnv(val) +} + +type interpolated struct { + value string + raw string +} + +// getInterpolated unmarshals the value as string and runs interpolation on it. It is the responsibility of each +// value type to convert this string value to appropriate type. +func getInterpolated(unmarshal func(interface{}) error) (*interpolated, error) { + var raw string + err := unmarshal(&raw) + if err != nil { + return &interpolated{}, err + } + value := interpolateValue(raw) + return &interpolated{raw: raw, value: value}, nil +} diff --git a/pkg/services/provisioning/values/values_test.go b/pkg/services/provisioning/values/values_test.go new file mode 100644 index 0000000000000..284069be39584 --- /dev/null +++ b/pkg/services/provisioning/values/values_test.go @@ -0,0 +1,210 @@ +package values + +import ( + . "github.com/smartystreets/goconvey/convey" + "gopkg.in/yaml.v2" + "os" + "testing" +) + +func TestValues(t *testing.T) { + Convey("Values", t, func() { + os.Setenv("INT", "1") + os.Setenv("STRING", "test") + os.Setenv("BOOL", "true") + + Convey("IntValue", func() { + type Data struct { + Val IntValue `yaml:"val"` + } + d := &Data{} + + Convey("Should unmarshal simple number", func() { + unmarshalingTest(`val: 1`, d) + So(d.Val.Value(), ShouldEqual, 1) + So(d.Val.Raw, ShouldEqual, "1") + }) + + Convey("Should unmarshal env var", func() { + unmarshalingTest(`val: $INT`, d) + So(d.Val.Value(), ShouldEqual, 1) + So(d.Val.Raw, ShouldEqual, "$INT") + }) + + Convey("Should ignore empty value", func() { + unmarshalingTest(`val: `, d) + So(d.Val.Value(), ShouldEqual, 0) + So(d.Val.Raw, ShouldEqual, "") + }) + }) + + Convey("StringValue", func() { + type Data struct { + Val StringValue `yaml:"val"` + } + d := &Data{} + + Convey("Should unmarshal simple string", func() { + unmarshalingTest(`val: test`, d) + So(d.Val.Value(), ShouldEqual, "test") + So(d.Val.Raw, ShouldEqual, "test") + }) + + Convey("Should unmarshal env var", func() { + unmarshalingTest(`val: $STRING`, d) + So(d.Val.Value(), ShouldEqual, "test") + So(d.Val.Raw, ShouldEqual, "$STRING") + }) + + Convey("Should ignore empty value", func() { + unmarshalingTest(`val: `, d) + So(d.Val.Value(), ShouldEqual, "") + So(d.Val.Raw, ShouldEqual, "") + }) + }) + + Convey("BoolValue", func() { + type Data struct { + Val BoolValue `yaml:"val"` + } + d := &Data{} + + Convey("Should unmarshal bool value", func() { + unmarshalingTest(`val: true`, d) + So(d.Val.Value(), ShouldBeTrue) + So(d.Val.Raw, ShouldEqual, "true") + }) + + Convey("Should unmarshal explicit string", func() { + unmarshalingTest(`val: "true"`, d) + So(d.Val.Value(), ShouldBeTrue) + So(d.Val.Raw, ShouldEqual, "true") + }) + + Convey("Should unmarshal env var", func() { + unmarshalingTest(`val: $BOOL`, d) + So(d.Val.Value(), ShouldBeTrue) + So(d.Val.Raw, ShouldEqual, "$BOOL") + }) + + Convey("Should ignore empty value", func() { + unmarshalingTest(`val: `, d) + So(d.Val.Value(), ShouldBeFalse) + So(d.Val.Raw, ShouldEqual, "") + }) + }) + + Convey("JSONValue", func() { + + type Data struct { + Val JSONValue `yaml:"val"` + } + d := &Data{} + + Convey("Should unmarshal variable nesting", func() { + doc := ` + val: + one: 1 + two: $STRING + three: + - 1 + - two + - three: + inside: $STRING + four: + nested: + onemore: $INT + multiline: > + Some text with $STRING + anchor: &label $INT + anchored: *label + ` + unmarshalingTest(doc, d) + + type anyMap = map[interface{}]interface{} + So(d.Val.Value(), ShouldResemble, map[string]interface{}{ + "one": 1, + "two": "test", + "three": []interface{}{ + 1, "two", anyMap{ + "three": anyMap{ + "inside": "test", + }, + }, + }, + "four": anyMap{ + "nested": anyMap{ + "onemore": "1", + }, + }, + "multiline": "Some text with test\n", + "anchor": "1", + "anchored": "1", + }) + + So(d.Val.Raw, ShouldResemble, map[string]interface{}{ + "one": 1, + "two": "$STRING", + "three": []interface{}{ + 1, "two", anyMap{ + "three": anyMap{ + "inside": "$STRING", + }, + }, + }, + "four": anyMap{ + "nested": anyMap{ + "onemore": "$INT", + }, + }, + "multiline": "Some text with $STRING\n", + "anchor": "$INT", + "anchored": "$INT", + }) + }) + }) + + Convey("StringMapValue", func() { + type Data struct { + Val StringMapValue `yaml:"val"` + } + d := &Data{} + + Convey("Should unmarshal mapping", func() { + doc := ` + val: + one: 1 + two: "test string" + three: $STRING + four: true + ` + unmarshalingTest(doc, d) + So(d.Val.Value(), ShouldResemble, map[string]string{ + "one": "1", + "two": "test string", + "three": "test", + "four": "true", + }) + + So(d.Val.Raw, ShouldResemble, map[string]string{ + "one": "1", + "two": "test string", + "three": "$STRING", + "four": "true", + }) + + }) + }) + + Reset(func() { + os.Unsetenv("INT") + os.Unsetenv("STRING") + os.Unsetenv("BOOL") + }) + }) +} + +func unmarshalingTest(document string, out interface{}) { + err := yaml.Unmarshal([]byte(document), out) + So(err, ShouldBeNil) +}