started working on migration code.

go.mod

@@ -7,11 +7,13 @@ require (
 	github.com/containrrr/shoutrrr v0.4.4
 	github.com/fatih/color v1.10.0
 	github.com/gin-gonic/gin v1.6.3
+	github.com/go-gormigrate/gormigrate/v2 v2.0.0
 	github.com/golang/mock v1.4.3
 	github.com/google/uuid v1.2.0 // indirect
 	github.com/hashicorp/serf v0.8.2 // indirect
 	github.com/influxdata/influxdb-client-go/v2 v2.2.3
 	github.com/jaypipes/ghw v0.6.1
+	github.com/jinzhu/gorm v1.9.16
 	github.com/klauspost/compress v1.12.1 // indirect
 	github.com/kvz/logstreamer v0.0.0-20150507115422-a635b98146f0 // indirect
 	github.com/mattn/go-sqlite3 v1.14.4 // indirect

webapp/backend/pkg/database/migrations/m20201107210306/device.go

@@ -0,0 +1,47 @@
+package m20201107210306
+
+import (
+	"time"
+)
+
+// Deprecated: m20201107210306.Device is deprecated, only used by db migrations
+type Device struct {
+	//GORM attributes, see: http://gorm.io/docs/conventions.html
+	CreatedAt time.Time
+	UpdatedAt time.Time
+	DeletedAt *time.Time
+
+	WWN    string `json:"wwn" gorm:"primary_key"`
+	HostId string `json:"host_id"`
+
+	DeviceName     string  `json:"device_name"`
+	Manufacturer   string  `json:"manufacturer"`
+	ModelName      string  `json:"model_name"`
+	InterfaceType  string  `json:"interface_type"`
+	InterfaceSpeed string  `json:"interface_speed"`
+	SerialNumber   string  `json:"serial_number"`
+	Firmware       string  `json:"firmware"`
+	RotationSpeed  int     `json:"rotational_speed"`
+	Capacity       int64   `json:"capacity"`
+	FormFactor     string  `json:"form_factor"`
+	SmartSupport   bool    `json:"smart_support"`
+	DeviceProtocol string  `json:"device_protocol"` //protocol determines which smart attribute types are available (ATA, NVMe, SCSI)
+	DeviceType     string  `json:"device_type"`     //device type is used for querying with -d/t flag, should only be used by collector.
+	SmartResults   []Smart `gorm:"foreignkey:DeviceWWN" json:"smart_results"`
+}
+
+const DeviceProtocolAta = "ATA"
+const DeviceProtocolScsi = "SCSI"
+const DeviceProtocolNvme = "NVMe"
+
+func (dv *Device) IsAta() bool {
+	return dv.DeviceProtocol == DeviceProtocolAta
+}
+
+func (dv *Device) IsScsi() bool {
+	return dv.DeviceProtocol == DeviceProtocolScsi
+}
+
+func (dv *Device) IsNvme() bool {
+	return dv.DeviceProtocol == DeviceProtocolNvme
+}

webapp/backend/pkg/database/migrations/m20201107210306/smart.go

@@ -0,0 +1,26 @@
+package m20201107210306
+
+import (
+	"gorm.io/gorm"
+	"time"
+)
+
+// Deprecated: m20201107210306.Smart is deprecated, only used by db migrations
+type Smart struct {
+	gorm.Model
+
+	DeviceWWN string `json:"device_wwn"`
+	Device    Device `json:"-" gorm:"foreignkey:DeviceWWN"` // use DeviceWWN as foreign key
+
+	TestDate    time.Time `json:"date"`
+	SmartStatus string    `json:"smart_status"` // SmartStatusPassed or SmartStatusFailed
+
+	//Metrics
+	Temp            int64 `json:"temp"`
+	PowerOnHours    int64 `json:"power_on_hours"`
+	PowerCycleCount int64 `json:"power_cycle_count"`
+
+	AtaAttributes  []SmartAtaAttribute  `json:"ata_attributes" gorm:"foreignkey:SmartId"`
+	NvmeAttributes []SmartNvmeAttribute `json:"nvme_attributes" gorm:"foreignkey:SmartId"`
+	ScsiAttributes []SmartScsiAttribute `json:"scsi_attributes" gorm:"foreignkey:SmartId"`
+}

webapp/backend/pkg/database/migrations/m20201107210306/smart_ata_attribute.go

@@ -0,0 +1,26 @@
+package m20201107210306
+
+import "gorm.io/gorm"
+
+// Deprecated: m20201107210306.SmartAtaAttribute is deprecated, only used by db migrations
+type SmartAtaAttribute struct {
+	gorm.Model
+
+	SmartId int    `json:"smart_id"`
+	Smart   Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key
+
+	AttributeId int    `json:"attribute_id"`
+	Name        string `json:"name"`
+	Value       int    `json:"value"`
+	Worst       int    `json:"worst"`
+	Threshold   int    `json:"thresh"`
+	RawValue    int64  `json:"raw_value"`
+	RawString   string `json:"raw_string"`
+	WhenFailed  string `json:"when_failed"`
+
+	TransformedValue int64               `json:"transformed_value"`
+	Status           string              `gorm:"-" json:"status,omitempty"`
+	StatusReason     string              `gorm:"-" json:"status_reason,omitempty"`
+	FailureRate      float64             `gorm:"-" json:"failure_rate,omitempty"`
+	History          []SmartAtaAttribute `gorm:"-" json:"history,omitempty"`
+}

webapp/backend/pkg/database/migrations/m20201107210306/smart_nvme_attribute.go

@@ -0,0 +1,22 @@
+package m20201107210306
+
+import "gorm.io/gorm"
+
+// Deprecated: m20201107210306.SmartNvmeAttribute is deprecated, only used by db migrations
+type SmartNvmeAttribute struct {
+	gorm.Model
+
+	SmartId int    `json:"smart_id"`
+	Smart   Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key
+
+	AttributeId string `json:"attribute_id"` //json string from smartctl
+	Name        string `json:"name"`
+	Value       int    `json:"value"`
+	Threshold   int    `json:"thresh"`
+
+	TransformedValue int64                `json:"transformed_value"`
+	Status           string               `gorm:"-" json:"status,omitempty"`
+	StatusReason     string               `gorm:"-" json:"status_reason,omitempty"`
+	FailureRate      float64              `gorm:"-" json:"failure_rate,omitempty"`
+	History          []SmartNvmeAttribute `gorm:"-" json:"history,omitempty"`
+}

webapp/backend/pkg/database/migrations/m20201107210306/smart_scsci_attribute.go

@@ -0,0 +1,22 @@
+package m20201107210306
+
+import "gorm.io/gorm"
+
+// Deprecated: m20201107210306.SmartScsiAttribute is deprecated, only used by db migrations
+type SmartScsiAttribute struct {
+	gorm.Model
+
+	SmartId int    `json:"smart_id"`
+	Smart   Device `json:"-" gorm:"foreignkey:SmartId"` // use SmartId as foreign key
+
+	AttributeId string `json:"attribute_id"` //json string from smartctl
+	Name        string `json:"name"`
+	Value       int    `json:"value"`
+	Threshold   int    `json:"thresh"`
+
+	TransformedValue int64                `json:"transformed_value"`
+	Status           string               `gorm:"-" json:"status,omitempty"`
+	StatusReason     string               `gorm:"-" json:"status_reason,omitempty"`
+	FailureRate      float64              `gorm:"-" json:"failure_rate,omitempty"`
+	History          []SmartScsiAttribute `gorm:"-" json:"history,omitempty"`
+}

webapp/backend/pkg/database/scrutiny_repository.go

@@ -57,17 +57,18 @@ func NewScrutinyRepository(appConfig config.Interface, globalLogger logrus.Field
 	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 	// Gorm/SQLite setup
 	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-	fmt.Printf("Trying to connect to database stored: %s\n", appConfig.GetString("web.database.location"))
+	globalLogger.Infof("Trying to connect to scrutiny sqlite db: %s\n", appConfig.GetString("web.database.location"))
 	database, err := gorm.Open(sqlite.Open(appConfig.GetString("web.database.location")), &gorm.Config{
 		//TODO: figure out how to log database queries again.
 		//Logger: logger
+		DisableForeignKeyConstraintWhenMigrating: true,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("Failed to connect to database! - %v", err)
 	}
+	globalLogger.Infof("Successfully connected to scrutiny sqlite db: %s\n", appConfig.GetString("web.database.location"))
 
 	//database.SetLogger()
-	database.AutoMigrate(&models.Device{})
 
 	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 	// InfluxDB setup
@@ -143,6 +144,16 @@ func NewScrutinyRepository(appConfig config.Interface, globalLogger logrus.Field
 	if err != nil {
 		return nil, err
 	}
+
+	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+	// InfluxDB & SQLite migrations
+	////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+	//database.AutoMigrate(&models.Device{})
+	err = deviceRepo.Migrate(backgroundContext)
+	if err != nil {
+		return nil, err
+	}
+
 	return &deviceRepo, nil
 }
 
@@ -241,15 +252,46 @@ func (sr *scrutinyRepository) GetSummary(ctx context.Context) (map[string]*model
 	// Get parser flux query result
 	//appConfig.GetString("web.influxdb.bucket")
 	queryStr := fmt.Sprintf(`
-  import "influxdata/influxdb/schema"
-  from(bucket: "%s")
-  |> range(start: -1y, stop: now())
-  |> filter(fn: (r) => r["_measurement"] == "smart" )
-  |> filter(fn: (r) => r["_field"] == "temp" or r["_field"] == "power_on_hours" or r["_field"] == "date")
-  |> last()
-  |> schema.fieldsAsCols()
-  |> group(columns: ["device_wwn"])
-  |> yield(name: "last")
+  	import "influxdata/influxdb/schema"
+  	bucketBaseName = "%s"
+
+	dailyData = from(bucket: bucketBaseName)
+	|> range(start: -10y, stop: now())
+	|> filter(fn: (r) => r["_measurement"] == "smart" )
+	|> filter(fn: (r) => r["_field"] == "temp" or r["_field"] == "power_on_hours" or r["_field"] == "date")
+	|> last()
+	|> schema.fieldsAsCols()
+	|> group(columns: ["device_wwn"])
+
+	weeklyData = from(bucket: bucketBaseName + "_weekly")
+	|> range(start: -10y, stop: now())
+	|> filter(fn: (r) => r["_measurement"] == "smart" )
+	|> filter(fn: (r) => r["_field"] == "temp" or r["_field"] == "power_on_hours" or r["_field"] == "date")
+	|> last()
+	|> schema.fieldsAsCols()
+	|> group(columns: ["device_wwn"])
+
+	monthlyData = from(bucket: bucketBaseName + "_monthly")
+	|> range(start: -10y, stop: now())
+	|> filter(fn: (r) => r["_measurement"] == "smart" )
+	|> filter(fn: (r) => r["_field"] == "temp" or r["_field"] == "power_on_hours" or r["_field"] == "date")
+	|> last()
+	|> schema.fieldsAsCols()
+	|> group(columns: ["device_wwn"])
+
+	yearlyData = from(bucket: bucketBaseName + "_yearly")
+	|> range(start: -10y, stop: now())
+	|> filter(fn: (r) => r["_measurement"] == "smart" )
+	|> filter(fn: (r) => r["_field"] == "temp" or r["_field"] == "power_on_hours" or r["_field"] == "date")
+	|> last()
+	|> schema.fieldsAsCols()
+	|> group(columns: ["device_wwn"])
+
+	union(tables: [dailyData, weeklyData, monthlyData, yearlyData])
+	|> sort(columns: ["_time"], desc: false)
+	|> group(columns: ["device_wwn"])
+	|> last(column: "device_wwn")
+	|> yield(name: "last")
 		`,
 		sr.appConfig.GetString("web.influxdb.bucket"),
 	)
@@ -357,7 +399,7 @@ func (sr *scrutinyRepository) lookupNestedDurationKeys(durationKey string) []str
 		return []string{DURATION_KEY_WEEK, DURATION_KEY_MONTH, DURATION_KEY_YEAR}
 	case DURATION_KEY_FOREVER:
 		//data stored before the last year
-		return []string{DURATION_KEY_WEEK, DURATION_KEY_MONTH, DURATION_KEY_YEAR}
+		return []string{DURATION_KEY_WEEK, DURATION_KEY_MONTH, DURATION_KEY_YEAR, DURATION_KEY_FOREVER}
 	}
 	return []string{DURATION_KEY_WEEK}
 }

webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go

@@ -6,8 +6,10 @@ import (
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/collector"
 	"github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements"
 	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
+	"github.com/influxdata/influxdb-client-go/v2/api"
 	log "github.com/sirupsen/logrus"
 	"strings"
+	"time"
 )
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -22,20 +24,14 @@ func (sr *scrutinyRepository) SaveSmartAttributes(ctx context.Context, wwn strin
 	}
 
 	tags, fields := deviceSmartData.Flatten()
-	p := influxdb2.NewPoint("smart",
-		tags,
-		fields,
-		deviceSmartData.Date)
 
 	// write point immediately
-	return deviceSmartData, sr.influxWriteApi.WritePoint(ctx, p)
+	return deviceSmartData, sr.saveDatapoint(sr.influxWriteApi, "smart", tags, fields, deviceSmartData.Date, ctx)
 }
 
 func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, attributes []string) ([]measurements.Smart, error) {
 	// Get SMartResults from InfluxDB
 
-	fmt.Println("GetDeviceDetails from INFLUXDB")
-
 	//TODO: change the filter startrange to a real number.
 
 	// Get parser flux query result
@@ -47,18 +43,13 @@ func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn
 
 	result, err := sr.influxQueryApi.Query(ctx, queryStr)
 	if err == nil {
-		fmt.Println("GetDeviceDetails NO EROR")
-
 		// Use Next() to iterate over query result lines
 		for result.Next() {
-			fmt.Println("GetDeviceDetails NEXT")
-
 			// Observe when there is new grouping key producing new table
 			if result.TableChanged() {
 				//fmt.Printf("table: %s\n", result.TableMetadata().String())
 			}
 
-			fmt.Printf("DECODINIG TABLE VALUES: %v", result.Record().Values())
 			smartData, err := measurements.NewSmartFromInfluxDB(result.Record().Values())
 			if err != nil {
 				return nil, err
@@ -93,31 +84,49 @@ func (sr *scrutinyRepository) GetSmartAttributeHistory(ctx context.Context, wwn
 // Helper Methods
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
+func (sr *scrutinyRepository) saveDatapoint(influxWriteApi api.WriteAPIBlocking, measurement string, tags map[string]string, fields map[string]interface{}, date time.Time, ctx context.Context) error {
+	//sr.logger.Debugf("Storing datapoint in measurement '%s'. tags: %d fields: %d", measurement, len(tags), len(fields))
+	p := influxdb2.NewPoint(measurement,
+		tags,
+		fields,
+		date)
+
+	// write point immediately
+	return influxWriteApi.WritePoint(ctx, p)
+}
+
 func (sr *scrutinyRepository) aggregateSmartAttributesQuery(wwn string, durationKey string) string {
 
 	/*
 
 		import "influxdata/influxdb/schema"
 		weekData = from(bucket: "metrics")
-		  |> range(start: -1w, stop: now())
-		  |> filter(fn: (r) => r["_measurement"] == "smart" )
-		  |> filter(fn: (r) => r["device_wwn"] == "%s" )
-		  |> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
-		  |> group(columns: ["device_wwn"])
-		  |> toInt()
+		|> range(start: -1w, stop: now())
+		|> filter(fn: (r) => r["_measurement"] == "smart" )
+		|> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" )
+		|> schema.fieldsAsCols()
 
 		monthData = from(bucket: "metrics_weekly")
-		  |> range(start: -1mo, stop: now())
-		  |> filter(fn: (r) => r["_measurement"] == "smart" )
-		  |> filter(fn: (r) => r["device_wwn"] == "%s" )
-		  |> aggregateWindow(every: 1h, fn: mean, createEmpty: false)
-		  |> group(columns: ["device_wwn"])
-		  |> toInt()
-
-		union(tables: [weekData, monthData])
-		  |> group(columns: ["device_wwn"])
-		  |> sort(columns: ["_time"], desc: false)
-		  |> schema.fieldsAsCols()
+		|> range(start: -1mo, stop: -1w)
+		|> filter(fn: (r) => r["_measurement"] == "smart" )
+		|> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" )
+		|> schema.fieldsAsCols()
+
+		yearData = from(bucket: "metrics_monthly")
+		|> range(start: -1y, stop: -1mo)
+		|> filter(fn: (r) => r["_measurement"] == "smart" )
+		|> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" )
+		|> schema.fieldsAsCols()
+
+		foreverData = from(bucket: "metrics_yearly")
+		|> range(start: -10y, stop: -1y)
+		|> filter(fn: (r) => r["_measurement"] == "smart" )
+		|> filter(fn: (r) => r["device_wwn"] == "0x5000c5002df89099" )
+		|> schema.fieldsAsCols()
+
+		union(tables: [weekData, monthData, yearData, foreverData])
+		|> sort(columns: ["_time"], desc: false)
+		|> yield(name: "last")
 
 	*/