adding support for a collector config file.

README.md

@@ -128,7 +128,7 @@ docker exec scrutiny /scrutiny/bin/scrutiny-collector-metrics run
 ```
 
 # Configuration
-We support a global YAML configuration file that must be located at /scrutiny/config/scrutiny.yaml
+We support a global YAML configuration file that must be located at `/scrutiny/config/scrutiny.yaml`
 
 Check the [example.scrutiny.yml](example.scrutiny.yaml) file for a fully commented version.
 

collector/cmd/collector-metrics/collector-metrics.go

@@ -3,6 +3,8 @@ package main
 import (
 	"fmt"
 	"github.com/analogj/scrutiny/collector/pkg/collector"
+	"github.com/analogj/scrutiny/collector/pkg/config"
+	"github.com/analogj/scrutiny/collector/pkg/errors"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/version"
 	"github.com/sirupsen/logrus"
 	"io"
@@ -20,6 +22,20 @@ var goarch string
 
 func main() {
 
+	config, err := config.Create()
+	if err != nil {
+		fmt.Printf("FATAL: %+v\n", err)
+		os.Exit(1)
+	}
+
+	//we're going to load the config file manually, since we need to validate it.
+	err = config.ReadConfig("/scrutiny/config/collector.yaml") // Find and read the config file
+	if _, ok := err.(errors.ConfigFileMissingError); ok {      // Handle errors reading the config file
+		//ignore "could not find config file"
+	} else if err != nil {
+		os.Exit(1)
+	}
+
 	cli.CommandHelpTemplate = `NAME:
    {{.HelpName}} - {{.Usage}}
 USAGE:
@@ -75,6 +91,17 @@ OPTIONS:
 				Name:  "run",
 				Usage: "Run the scrutiny smartctl metrics collector",
 				Action: func(c *cli.Context) error {
+					if c.IsSet("config") {
+						err = config.ReadConfig(c.String("config")) // Find and read the config file
+						if err != nil {                             // Handle errors reading the config file
+							//ignore "could not find config file"
+							fmt.Printf("Could not find config file at specified path: %s", c.String("config"))
+							return err
+						}
+					}
+					if c.IsSet("host-id") {
+						config.Set("host.id", c.String("host-id")) // set/override the host-id using CLI.
+					}
 
 					collectorLogger := logrus.WithFields(logrus.Fields{
 						"type": "metrics",
@@ -97,6 +124,7 @@ OPTIONS:
 					}
 
 					metricCollector, err := collector.CreateMetricsCollector(
+						config,
 						collectorLogger,
 						c.String("api-endpoint"),
 					)
@@ -109,6 +137,10 @@ OPTIONS:
 				},
 
 				Flags: []cli.Flag{
+					&cli.StringFlag{
+						Name:  "config",
+						Usage: "Specify the path to the devices file",
+					},
 					&cli.StringFlag{
 						Name:    "api-endpoint",
 						Usage:   "The api server endpoint",
@@ -128,12 +160,18 @@ OPTIONS:
 						Usage:   "Enable debug logging",
 						EnvVars: []string{"COLLECTOR_DEBUG", "DEBUG"},
 					},
+
+					&cli.BoolFlag{
+						Name:    "host-id",
+						Usage:   "Host identifier/label, used for grouping devices",
+						EnvVars: []string{"COLLECTOR_HOST_ID"},
+					},
 				},
 			},
 		},
 	}
 
-	err := app.Run(os.Args)
+	err = app.Run(os.Args)
 	if err != nil {
 		log.Fatal(color.HiRedString("ERROR: %v", err))
 	}

collector/pkg/collector/metrics.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"github.com/analogj/scrutiny/collector/pkg/common"
+	"github.com/analogj/scrutiny/collector/pkg/config"
 	"github.com/analogj/scrutiny/collector/pkg/detect"
 	"github.com/analogj/scrutiny/collector/pkg/errors"
 	"github.com/analogj/scrutiny/collector/pkg/models"
@@ -16,17 +17,19 @@ import (
 )
 
 type MetricsCollector struct {
+	config config.Interface
 	BaseCollector
 	apiEndpoint *url.URL
 }
 
-func CreateMetricsCollector(logger *logrus.Entry, apiEndpoint string) (MetricsCollector, error) {
+func CreateMetricsCollector(appConfig config.Interface, logger *logrus.Entry, apiEndpoint string) (MetricsCollector, error) {
 	apiEndpointUrl, err := url.Parse(apiEndpoint)
 	if err != nil {
 		return MetricsCollector{}, err
 	}
 
 	sc := MetricsCollector{
+		config:      appConfig,
 		apiEndpoint: apiEndpointUrl,
 		BaseCollector: BaseCollector{
 			logger: logger,
@@ -49,6 +52,7 @@ func (mc *MetricsCollector) Run() error {
 
 	deviceDetector := detect.Detect{
 		Logger: mc.logger,
+		Config: mc.config,
 	}
 	detectedStorageDevices, err := deviceDetector.Start()
 	if err != nil {

collector/pkg/config/config.go

@@ -0,0 +1,100 @@
+package config
+
+import (
+	"github.com/analogj/go-util/utils"
+	"github.com/analogj/scrutiny/collector/pkg/errors"
+	"github.com/analogj/scrutiny/collector/pkg/models"
+	"github.com/mitchellh/mapstructure"
+	"github.com/spf13/viper"
+	"log"
+	"os"
+)
+
+// When initializing this class the following methods must be called:
+// Config.New
+// Config.Init
+// This is done automatically when created via the Factory.
+type configuration struct {
+	*viper.Viper
+}
+
+//Viper uses the following precedence order. Each item takes precedence over the item below it:
+// explicit call to Set
+// flag
+// env
+// config
+// key/value store
+// default
+
+func (c *configuration) Init() error {
+	c.Viper = viper.New()
+	//set defaults
+	c.SetDefault("host.id", "")
+
+	c.SetDefault("devices", []string{})
+
+	//c.SetDefault("collect.short.command", "-a -o on -S on")
+
+	//if you want to load a non-standard location system config file (~/drawbridge.yml), use ReadConfig
+	c.SetConfigType("yaml")
+	//c.SetConfigName("drawbridge")
+	//c.AddConfigPath("$HOME/")
+
+	//CLI options will be added via the `Set()` function
+	return nil
+}
+
+func (c *configuration) ReadConfig(configFilePath string) error {
+	configFilePath, err := utils.ExpandPath(configFilePath)
+	if err != nil {
+		return err
+	}
+
+	if !utils.FileExists(configFilePath) {
+		log.Printf("No configuration file found at %v. Using Defaults.", configFilePath)
+		return errors.ConfigFileMissingError("The configuration file could not be found.")
+	}
+
+	//validate config file contents
+	//err = c.ValidateConfigFile(configFilePath)
+	//if err != nil {
+	//	log.Printf("Config file at `%v` is invalid: %s", configFilePath, err)
+	//	return err
+	//}
+
+	log.Printf("Loading configuration file: %s", configFilePath)
+
+	config_data, err := os.Open(configFilePath)
+	if err != nil {
+		log.Printf("Error reading configuration file: %s", err)
+		return err
+	}
+
+	err = c.MergeConfig(config_data)
+	if err != nil {
+		return err
+	}
+
+	return c.ValidateConfig()
+}
+
+// This function ensures that the merged config works correctly.
+func (c *configuration) ValidateConfig() error {
+
+	//TODO:
+	// check that device prefix matches OS
+	// check that schema of config file is valid
+
+	return nil
+}
+
+func (c *configuration) GetScanOverrides() []models.ScanOverride {
+	// we have to support 2 types of device types.
+	// - simple device type (device_type: 'sat')
+	// and list of device types (type: \n- 3ware,0 \n- 3ware,1 \n- 3ware,2)
+	// GetString will return "" if this is a list of device types.
+
+	overrides := []models.ScanOverride{}
+	c.UnmarshalKey("devices", &overrides, func(c *mapstructure.DecoderConfig) { c.WeaklyTypedInput = true })
+	return overrides
+}

collector/pkg/config/config_test.go

@@ -0,0 +1,64 @@
+package config_test
+
+import (
+	"github.com/analogj/scrutiny/collector/pkg/config"
+	"github.com/analogj/scrutiny/collector/pkg/models"
+	"github.com/stretchr/testify/require"
+	"path"
+	"testing"
+)
+
+func TestConfiguration_GetScanOverrides_Simple(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	testConfig, _ := config.Create()
+
+	//test
+	err := testConfig.ReadConfig(path.Join("testdata", "simple_device.yaml"))
+	require.NoError(t, err, "should correctly load simple device config")
+	scanOverrides := testConfig.GetScanOverrides()
+
+	//assert
+	require.Equal(t, []models.ScanOverride{{Device: "/dev/sda", DeviceType: []string{"sat"}, Ignore: false}}, scanOverrides)
+}
+
+func TestConfiguration_GetScanOverrides_Ignore(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	testConfig, _ := config.Create()
+
+	//test
+	err := testConfig.ReadConfig(path.Join("testdata", "ignore_device.yaml"))
+	require.NoError(t, err, "should correctly load ignore device config")
+	scanOverrides := testConfig.GetScanOverrides()
+
+	//assert
+	require.Equal(t, []models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Ignore: true}}, scanOverrides)
+}
+
+func TestConfiguration_GetScanOverrides_Raid(t *testing.T) {
+	t.Parallel()
+
+	//setup
+	testConfig, _ := config.Create()
+
+	//test
+	err := testConfig.ReadConfig(path.Join("testdata", "raid_device.yaml"))
+	require.NoError(t, err, "should correctly load ignore device config")
+	scanOverrides := testConfig.GetScanOverrides()
+
+	//assert
+	require.Equal(t, []models.ScanOverride{
+		{
+			Device:     "/dev/bus/0",
+			DeviceType: []string{"megaraid,14", "megaraid,15", "megaraid,18", "megaraid,19", "megaraid,20", "megaraid,21"},
+			Ignore:     false,
+		},
+		{
+			Device:     "/dev/twa0",
+			DeviceType: []string{"3ware,0", "3ware,1", "3ware,2", "3ware,3", "3ware,4", "3ware,5"},
+			Ignore:     false,
+		}}, scanOverrides)
+}

collector/pkg/config/factory.go

@@ -0,0 +1,9 @@
+package config
+
+func Create() (Interface, error) {
+	config := new(configuration)
+	if err := config.Init(); err != nil {
+		return nil, err
+	}
+	return config, nil
+}

collector/pkg/config/interface.go

@@ -0,0 +1,26 @@
+package config
+
+import (
+	"github.com/analogj/scrutiny/collector/pkg/models"
+	"github.com/spf13/viper"
+)
+
+// Create mock using:
+// mockgen -source=collector/pkg/config/interface.go -destination=collector/pkg/config/mock/mock_config.go
+type Interface interface {
+	Init() error
+	ReadConfig(configFilePath string) error
+	Set(key string, value interface{})
+	SetDefault(key string, value interface{})
+
+	AllSettings() map[string]interface{}
+	IsSet(key string) bool
+	Get(key string) interface{}
+	GetBool(key string) bool
+	GetInt(key string) int
+	GetString(key string) string
+	GetStringSlice(key string) []string
+	UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error
+
+	GetScanOverrides() []models.ScanOverride
+}