Merge pull request #352 from AnalogJ/beta

README.md

@@ -239,9 +239,9 @@ scrutiny-collector-metrics run --debug --log-file /tmp/collector.log
 | linux-arm-6 | :white_check_mark: |  |
 | linux-arm-7 | :white_check_mark: | web/collector only. see [#236](https://github.com/AnalogJ/scrutiny/issues/236)  |
 | linux-arm64 | :white_check_mark: | :white_check_mark: |
-| freebsd-amd64 | collector only. see [#238](https://github.com/AnalogJ/scrutiny/issues/238) |  | 
-| macos-amd64 |  | :white_check_mark: |
-| macos-arm64 |  | :white_check_mark: |
+| freebsd-amd64 | :white_check_mark: |  | 
+| macos-amd64 | :white_check_mark: | :white_check_mark: |
+| macos-arm64 | :white_check_mark: | :white_check_mark: |
 | windows-amd64 | :white_check_mark: | WIP, see [#15](https://github.com/AnalogJ/scrutiny/issues/15) |
 | windows-arm64 | :white_check_mark: |  |
 

collector/cmd/collector-metrics/collector-metrics.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
 	"github.com/analogj/scrutiny/collector/pkg/collector"
 	"github.com/analogj/scrutiny/collector/pkg/config"
@@ -120,26 +121,16 @@ OPTIONS:
 						config.Set("api.endpoint", apiEndpoint)
 					}
 
-					collectorLogger := logrus.WithFields(logrus.Fields{
-						"type": "metrics",
-					})
-
-					if level, err := logrus.ParseLevel(config.GetString("log.level")); err == nil {
-						logrus.SetLevel(level)
-					} else {
-						logrus.SetLevel(logrus.InfoLevel)
-					}
-
-					if config.IsSet("log.file") && len(config.GetString("log.file")) > 0 {
-						logFile, err := os.OpenFile(config.GetString("log.file"), os.O_CREATE|os.O_WRONLY, 0644)
-						if err != nil {
-							logrus.Errorf("Failed to open log file %s for output: %s", config.GetString("log.file"), err)
-							return err
-						}
+					collectorLogger, logFile, err := CreateLogger(config)
+					if logFile != nil {
 						defer logFile.Close()
-						logrus.SetOutput(io.MultiWriter(os.Stderr, logFile))
+					}
+					if err != nil {
+						return err
 					}
 
+					settingsData, err := json.MarshalIndent(config.AllSettings(), "", "\t")
+					collectorLogger.Debug(string(settingsData), err)
 					metricCollector, err := collector.CreateMetricsCollector(
 						config,
 						collectorLogger,
@@ -192,5 +183,28 @@ OPTIONS:
 	if err != nil {
 		log.Fatal(color.HiRedString("ERROR: %v", err))
 	}
+}
+
+func CreateLogger(appConfig config.Interface) (*logrus.Entry, *os.File, error) {
+	logger := logrus.WithFields(logrus.Fields{
+		"type": "metrics",
+	})
+
+	if level, err := logrus.ParseLevel(appConfig.GetString("log.level")); err == nil {
+		logger.Logger.SetLevel(level)
+	} else {
+		logger.Logger.SetLevel(logrus.InfoLevel)
+	}
 
+	var logFile *os.File
+	var err error
+	if appConfig.IsSet("log.file") && len(appConfig.GetString("log.file")) > 0 {
+		logFile, err = os.OpenFile(appConfig.GetString("log.file"), os.O_CREATE|os.O_WRONLY, 0644)
+		if err != nil {
+			logger.Logger.Errorf("Failed to open log file %s for output: %s", appConfig.GetString("log.file"), err)
+			return nil, logFile, err
+		}
+		logger.Logger.SetOutput(io.MultiWriter(os.Stderr, logFile))
+	}
+	return logger, logFile, nil
 }

docker/Dockerfile.collector

@@ -14,7 +14,7 @@ RUN make binary-clean binary-collector
 
 ########
 FROM debian:bullseye-slim as runtime
-WORKDIR /scrutiny
+WORKDIR /opt/scrutiny
 ENV PATH="/opt/scrutiny/bin:${PATH}"
 
 RUN apt-get update && apt-get install -y cron smartmontools ca-certificates tzdata && update-ca-certificates

docs/INSTALL_SYNOLOGY_COLLECTOR.md

@@ -91,9 +91,13 @@ wget https://raw.githubusercontent.com/smartmontools/smartmontools/master/smartm
 ```
 #!/bin/bash
 
-/volume1/\@Entware/scrutiny/bin/scrutiny-collector-metrics-linux-arm64 run --config /volume1/\@Entware/scrutiny/config/collector.yaml
+/volume1/\@Entware/scrutiny/bin/scrutiny-collector-metrics-linux-arm64 run --config /volume1/\@Entware/scrutiny/conf/collector.yaml
 ```
 
+**Make `run_collect.sh` executable**
+
+`chmod +x /volume1/\@Entware/scrutiny/bin/run_collect.sh`
+
 ## Set up Synology to run a scheduled task. 
 
 Log in to DSM and do the following:
@@ -131,4 +135,4 @@ Frequency: <Your desired frequency>
 
 ## Troubleshooting
 
-If you have any issues with your devices being detected, or incorrect data, please take a look at [TROUBLESHOOTING_DEVICE_COLLECTOR.md](./TROUBLESHOOTING_DEVICE_COLLECTOR.md)
+If you have any issues with your devices being detected, or incorrect data, please take a look at [TROUBLESHOOTING_DEVICE_COLLECTOR.md](./TROUBLESHOOTING_DEVICE_COLLECTOR.md)

docs/TROUBLESHOOTING_INFLUXDB.md

@@ -1,7 +1,19 @@
 # InfluxDB Troubleshooting
 
-## Installation 
-InfluxDB is a required dependency for Scrutiny v0.4.0+. 
+## Why??
+
+Scrutiny has many features, but the relevant one to this conversation is the "S.M.A.R.T metric tracking for historical
+trends". Basically Scrutiny not only shows you the current SMART values, but how they've changed over weeks, months (or
+even years).
+
+To efficiently handle that data at scale (and to make my life easier as a developer) I decided to add InfluxDB as a
+dependency. It's a dedicated timeseries database, as opposed to the general purpose sqlite DB I used before. I also did
+a bunch of testing and analysis before I made the change. With InfluxDB the memory footprint for Scrutiny (at idle) is ~
+100mb, which is still fairly reasonable.
+
+## Installation
+
+InfluxDB is a required dependency for Scrutiny v0.4.0+.
 
 https://docs.influxdata.com/influxdb/v2.2/install/
 

docs/TROUBLESHOOTING_NOTIFICATIONS.md

@@ -21,5 +21,6 @@ SCRUTINY_DEVICE_NAME - eg. /dev/sda
 SCRUTINY_DEVICE_TYPE - ATA/SCSI/NVMe
 SCRUTINY_DEVICE_SERIAL - eg. WDDJ324KSO
 SCRUTINY_MESSAGE - eg. "Scrutiny SMART error notification for device: %s\nFailure Type: %s\nDevice Name: %s\nDevice Serial: %s\nDevice Type: %s\nDate: %s"
+SCRUTINY_HOST_ID - (optional) eg. "my-custom-host-id"
 ```
 

docs/TROUBLESHOOTING_UDEV.md

@@ -0,0 +1,18 @@
+# Operating systems without udev
+
+Some operating systems do not come with `udev` out of the box, for example Alpine Linux. In these instances you will not be able to bind `/run/udev` to the container for sharing device metadata. Some operating systems offer `udev` as a package that can be installed separately, or an alternative (such as `eudev` in the case of Alpine Linux) that provides the same functionality.
+
+To install `eudev` in Alpine Linux (run as root):
+
+```
+apk add eudev
+setup-udev
+```
+
+Once your `udev` implementation is installed, create `/run/udev` with the following command:
+
+```
+udevadm trigger
+```
+
+On Alpine Linux, this also has the benefit of creating symlinks to device serial numbers in `/dev/disk/by-id`.

docs/dbdiagram.io.txt

@@ -1,62 +1,88 @@
 
 // SQLite Table(s)
-Table device {
-  created_at timestamp
-
-  wwn varchar [pk]
-
-  //user provided
-  label varchar
-  host_id varchar
-
-  // smartctl provided
-  device_name varchar
-  manufacturer varchar
-  model_name varchar
-  interface_type varchar
-  interface_speed varchar
-  serial_number varchar
-  firmware varchar
-  rotational_speed varchar
-  capacity varchar
-  form_factor varchar
-  smart_support varchar
-  device_protocol varchar
-  device_type varchar
 
+Table Device  {
+	//GORM attributes, see: http://gorm.io/docs/conventions.html
+	CreatedAt time
+	UpdatedAt time
+	DeletedAt time
+
+	WWN string
+
+	DeviceName     string
+	DeviceUUID     string
+	DeviceSerialID string
+	DeviceLabel    string
+
+	Manufacturer   string
+	ModelName      string
+	InterfaceType  string
+	InterfaceSpeed string
+	SerialNumber   string
+	Firmware       string
+	RotationSpeed  int
+	Capacity       int64
+	FormFactor     string
+	SmartSupport   bool
+	DeviceProtocol string//protocol determines which smart attribute types are available (ATA, NVMe, SCSI)
+	DeviceType     string//device type is used for querying with -d/t flag, should only be used by collector.
+
+	// User provided metadata
+	Label  string
+	HostId string
+
+	// Data set by Scrutiny
+	DeviceStatus enum
 }
 
+Table Setting {
+	//GORM attributes, see: http://gorm.io/docs/conventions.html
 
-// InfluxDB Tables
-Table device_temperature {
-  //timestamp
-  created_at timestamp
-
-  //tags (indexed & queryable)
-  device_wwn varchar [pk]
-
-  //fields
-  temp bigint
- }
-
+	SettingKeyName  string
+    SettingKeyDescription  string
+	SettingDataType string
 
-Table smart_ata_results {
-  //timestamp
-  created_at timestamp
-
-  //tags (indexed & queryable)
-  device_wwn varchar [pk]
-  smart_status varchar
-  scrutiny_status varchar
+	SettingValueNumeric int64
+	SettingValueString  string
+}
 
 
+// InfluxDB Tables
+Table SmartTemperature {
+    Date time
+	DeviceWWN      string //(tag)
+	Temp int64
+}
 
-  //fields
-  temp bigint
-  power_on_hours bigint
-  power_cycle_count bigint
 
+Table Smart {
+    Date           time
+	DeviceWWN      string //(tag)
+	DeviceProtocol string
+
+	//Metrics (fields)
+	Temp            int64
+	PowerOnHours    int64
+	PowerCycleCount int64
+
+	//Smart Status
+	Status enum
+
+	//SMART Attributes (fields)
+	Attr_ID_AttributeId int
+	Attr_ID_Value       int64
+	Attr_ID_Threshold   int64
+	Attr_ID_Worst       int64
+    Attr_ID_RawValue    int64
+	Attr_ID_RawString   string
+	Attr_ID_WhenFailed  string
+	//Generated data
+	Attr_ID_TransformedValue int64
+	Attr_ID_Status           enum
+	Attr_ID_StatusReason     string
+	Attr_ID_FailureRate      float64
 
  }
 
-Ref: device.wwn < smart_ata_results.device_wwn
+Ref: Device.WWN < Smart.DeviceWWN
+Ref: Device.WWN < SmartTemperature.DeviceWWN

example.scrutiny.yaml

@@ -73,8 +73,6 @@ log:
 #    - "join://shoutrrr:api-key@join/?devices=device1[,device2, ...][&icon=icon][&title=title]"
 #    - "script:///file/path/on/disk"
 #    - "https://www.example.com/path"
-#  filter_attributes: 'all' # options: 'all' or  'critical'
-#  level: 'fail' # options: 'fail', 'fail_scrutiny', 'fail_smart'
 
 ########################################################################################################################
 # FEATURES COMING SOON

webapp/backend/cmd/scrutiny/scrutiny.go

@@ -1,12 +1,15 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/config"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/errors"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/version"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/web"
-	log "github.com/sirupsen/logrus"
+	"github.com/sirupsen/logrus"
+	"io"
+	"log"
 	"os"
 	"time"
 
@@ -107,7 +110,18 @@ OPTIONS:
 						config.Set("log.file", c.String("log-file"))
 					}
 
-					webServer := web.AppEngine{Config: config}
+					webLogger, logFile, err := CreateLogger(config)
+					if logFile != nil {
+						defer logFile.Close()
+					}
+					if err != nil {
+						return err
+					}
+
+					settingsData, err := json.Marshal(config.AllSettings())
+					webLogger.Debug(string(settingsData), err)
+
+					webServer := web.AppEngine{Config: config, Logger: webLogger}
 
 					return webServer.Start()
 				},
@@ -140,3 +154,27 @@ OPTIONS:
 	}
 
 }
+
+func CreateLogger(appConfig config.Interface) (*logrus.Entry, *os.File, error) {
+	logger := logrus.WithFields(logrus.Fields{
+		"type": "web",
+	})
+	//set default log level
+	if level, err := logrus.ParseLevel(appConfig.GetString("log.level")); err == nil {
+		logger.Logger.SetLevel(level)
+	} else {
+		logger.Logger.SetLevel(logrus.InfoLevel)
+	}
+
+	var logFile *os.File
+	var err error
+	if appConfig.IsSet("log.file") && len(appConfig.GetString("log.file")) > 0 {
+		logFile, err = os.OpenFile(appConfig.GetString("log.file"), os.O_CREATE|os.O_WRONLY, 0644)
+		if err != nil {
+			logger.Logger.Errorf("Failed to open log file %s for output: %s", appConfig.GetString("log.file"), err)
+			return nil, logFile, err
+		}
+		logger.Logger.SetOutput(io.MultiWriter(os.Stderr, logFile))
+	}
+	return logger, logFile, nil
+}