Skip to content

Commit

Permalink
feat: add tags to the Volume and Cluster dashboards (#3273)
Browse files Browse the repository at this point in the history
* feat: add tags field in volume table

* feat: add tags in cluster dashboard

* feat: minor change

* feat: minor change

* feat: update volume dashboard with tags

* feat: adding volume_tags and cluster_tags metrics

* feat: skip zapi counters

* feat: sorted slice

* feat: join with volume_labels

* feat: update cluster dashboard

* feat: update test
  • Loading branch information
Hardikl authored Nov 18, 2024
1 parent 3e51f38 commit 1135ee5
Show file tree
Hide file tree
Showing 15 changed files with 345 additions and 101 deletions.
93 changes: 93 additions & 0 deletions cmd/collectors/rest/plugins/cluster/cluster.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
package cluster

import (
"github.com/netapp/harvest/v2/cmd/poller/plugin"
"github.com/netapp/harvest/v2/pkg/conf"
"github.com/netapp/harvest/v2/pkg/matrix"
"github.com/netapp/harvest/v2/pkg/slogx"
"github.com/netapp/harvest/v2/pkg/tree/node"
"github.com/netapp/harvest/v2/pkg/util"
"log/slog"
"strings"
)

type Cluster struct {
*plugin.AbstractPlugin
tags *matrix.Matrix
}

func New(p *plugin.AbstractPlugin) plugin.Plugin {
return &Cluster{AbstractPlugin: p}
}

func (c *Cluster) Init(_ conf.Remote) error {
var err error

if err := c.InitAbc(); err != nil {
return err
}

c.tags = matrix.New(c.Parent+".Cluster", "cluster", "cluster")
exportOptions := node.NewS("export_options")
instanceKeys := exportOptions.NewChildS("instance_keys", "")
instanceKeys.NewChildS("", "tag")
c.tags.SetExportOptions(exportOptions)
_, err = c.tags.NewMetricFloat64("tags", "tags")
if err != nil {
c.SLogger.Error("add metric", slogx.Err(err))
return err
}

return nil
}

func (c *Cluster) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) {
data := dataMap[c.Object]

// Based on the tags array, cluster_tags instances/metrics would be created
c.handleTags(data)

return []*matrix.Matrix{c.tags}, nil, nil
}

func (c *Cluster) handleTags(data *matrix.Matrix) {
var (
tagInstance *matrix.Instance
err error
)

// Purge and reset data
c.tags.PurgeInstances()
c.tags.Reset()

// Set all global labels
c.tags.SetGlobalLabels(data.GetGlobalLabels())

// Based on the tags array, cluster_tags instances/metrics would be created.
for _, cluster := range data.GetInstances() {
if tags := cluster.GetLabel("tags"); tags != "" {
for _, tag := range strings.Split(tags, ",") {
tagInstanceKey := data.GetGlobalLabels()["cluster"] + tag
if tagInstance, err = c.tags.NewInstance(tagInstanceKey); err != nil {
c.SLogger.Error(
"Failed to create tag instance",
slogx.Err(err),
slog.String("tagInstanceKey", tagInstanceKey),
)
return
}

tagInstance.SetLabel("tag", tag)

m := c.tags.GetMetric("tags")
// populate numeric data
value := 1.0
if err = m.SetValueFloat64(tagInstance, value); err != nil {
c.SLogger.Error("Failed to parse value", slogx.Err(err), slog.Float64("value", value))
} else {
c.SLogger.Debug("added value", slog.Float64("value", value))
}
}
}
}
}
89 changes: 85 additions & 4 deletions cmd/collectors/rest/plugins/volume/volume.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/tidwall/gjson"
"log/slog"
"strconv"
"strings"
"time"
)

Expand All @@ -28,8 +29,10 @@ type Volume struct {
*plugin.AbstractPlugin
currentVal int
client *rest.Client
aggrsMap map[string]bool // aggregate-name -> exist map
aggrsMap map[string]bool // aggregate-name -> exist map
volTagMap map[string]volumeTag // volume-key -> volumeTag map
arw *matrix.Matrix
tags *matrix.Matrix
includeConstituents bool
isArwSupportedVersion bool
}
Expand All @@ -45,6 +48,12 @@ type volumeInfo struct {
isDestinationCloud string
}

type volumeTag struct {
vol string
svm string
tags string
}

func New(p *plugin.AbstractPlugin) plugin.Plugin {
return &Volume{AbstractPlugin: p}
}
Expand All @@ -58,6 +67,7 @@ func (v *Volume) Init(remote conf.Remote) error {
}

v.aggrsMap = make(map[string]bool)
v.volTagMap = make(map[string]volumeTag)

// Assigned the value to currentVal so that plugin would be invoked first time to populate cache.
v.currentVal = v.SetPluginInterval()
Expand Down Expand Up @@ -87,6 +97,19 @@ func (v *Volume) Init(remote conf.Remote) error {
return err
}

v.tags = matrix.New(v.Parent+".Volume", "volume", "volume")
exportOptions = node.NewS("export_options")
instanceKeys = exportOptions.NewChildS("instance_keys", "")
instanceKeys.NewChildS("", "tag")
instanceKeys.NewChildS("", "svm")
instanceKeys.NewChildS("", "volume")
v.tags.SetExportOptions(exportOptions)
_, err = v.tags.NewMetricFloat64("tags", "tags")
if err != nil {
v.SLogger.Error("add metric", slogx.Err(err))
return err
}

// Read template to decide inclusion of flexgroup constituents
v.includeConstituents = collectors.ReadPluginKey(v.Params, "include_constituents")
// ARW feature is supported from 9.10 onwards, If we ask this field in Rest call in plugin, then it will be failed.
Expand Down Expand Up @@ -121,28 +144,40 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util
if err != nil {
v.SLogger.Error("Failed to collect volume info data", slogx.Err(err))
} else {
// update volume instance labels
// update volume instance labels and populate volTagMap
v.updateVolumeLabels(data, volumeMap)
}

// parse anti_ransomware_start_time, antiRansomwareState for all volumes and export at cluster level
v.handleARWProtection(data)

// Based on the tags array in volTagMap, volume_tags instances/metrics would be created
v.handleTags(data.GetGlobalLabels())

v.currentVal++
return []*matrix.Matrix{v.arw}, v.client.Metadata, nil
return []*matrix.Matrix{v.arw, v.tags}, v.client.Metadata, nil
}

func (v *Volume) updateVolumeLabels(data *matrix.Matrix, volumeMap map[string]volumeInfo) {
var err error

// clear volTagMap
clear(v.volTagMap)

cloneSplitEstimateMetric := data.GetMetric("clone_split_estimate")
if cloneSplitEstimateMetric == nil {
if cloneSplitEstimateMetric, err = data.NewMetricFloat64("clone_split_estimate"); err != nil {
v.SLogger.Error("error while creating clone split estimate metric", slogx.Err(err))
return
}
}
for _, volume := range data.GetInstances() {
for vKey, volume := range data.GetInstances() {
// update volumeTagMap used to update tag details
svm := volume.GetLabel("svm")
vol := volume.GetLabel("volume")
tags := volume.GetLabel("tags")
v.volTagMap[vKey] = volumeTag{vol: vol, svm: svm, tags: tags}

if !volume.IsExportable() {
continue
}
Expand Down Expand Up @@ -344,3 +379,49 @@ func (v *Volume) updateAggrMap(disks []gjson.Result) {
}
}
}

func (v *Volume) handleTags(globalLabels map[string]string) {
var (
tagInstance *matrix.Instance
err error
)

// Purge and reset data
v.tags.PurgeInstances()
v.tags.Reset()

// Set all global labels
v.tags.SetGlobalLabels(globalLabels)

// Based on the tags array, volume_tags instances/metrics would be created.
for _, volume := range v.volTagMap {
svm := volume.svm
vol := volume.vol
if tags := volume.tags; tags != "" {
for _, tag := range strings.Split(tags, ",") {
tagInstanceKey := globalLabels["cluster"] + svm + vol + tag
if tagInstance, err = v.tags.NewInstance(tagInstanceKey); err != nil {
v.SLogger.Error(
"Failed to create tag instance",
slogx.Err(err),
slog.String("tagInstanceKey", tagInstanceKey),
)
return
}

tagInstance.SetLabel("tag", tag)
tagInstance.SetLabel("volume", vol)
tagInstance.SetLabel("svm", svm)

m := v.tags.GetMetric("tags")
// populate numeric data
value := 1.0
if err = m.SetValueFloat64(tagInstance, value); err != nil {
v.SLogger.Error("Failed to parse value", slogx.Err(err), slog.Float64("value", value))
} else {
v.SLogger.Debug("added value", slog.Float64("value", value))
}
}
}
}
}
3 changes: 3 additions & 0 deletions cmd/collectors/rest/rest.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"github.com/netapp/harvest/v2/cmd/collectors"
"github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/aggregate"
"github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/certificate"
"github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/cluster"
"github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/disk"
"github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/health"
"github.com/netapp/harvest/v2/cmd/collectors/rest/plugins/metroclustercheck"
Expand Down Expand Up @@ -484,6 +485,8 @@ func (r *Rest) LoadPlugin(kind string, abc *plugin.AbstractPlugin) plugin.Plugin
switch kind {
case "Aggregate":
return aggregate.New(abc)
case "Cluster":
return cluster.New(abc)
case "Disk":
return disk.New(abc)
case "Health":
Expand Down
5 changes: 3 additions & 2 deletions cmd/collectors/restperf/plugins/volume/volume.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ func (v *Volume) fetchVolumes() map[string]string {

href := rest.NewHrefBuilder().
APIPath(query).
Fields([]string{"volume", "volume_style_extended"}).
Fields([]string{"volume", "vserver", "volume_style_extended"}).
Filter([]string{"is_constituent=*"}).
MaxRecords(collectors.DefaultBatchSize).
Build()
Expand All @@ -101,7 +101,8 @@ func (v *Volume) fetchVolumes() map[string]string {
}
styleExtended := volume.Get("volume_style_extended").String()
name := volume.Get("volume").String()
volumesMap[name] = styleExtended
svm := volume.Get("vserver").String()
volumesMap[svm+name] = styleExtended
}

return volumesMap
Expand Down
8 changes: 4 additions & 4 deletions cmd/collectors/restperf/plugins/volume/volume_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,26 +37,26 @@ func runVolumeTest(t *testing.T, createVolume func(params *node.Node) plugin.Plu
instance1.SetLabel("volume", "RahulTest__0001")
instance1.SetLabel("svm", "svm1")
instance1.SetLabel("aggr", "aggr1")
volumesMap["RahulTest__0001"] = "flexgroup_constituent"
volumesMap["svm1"+"RahulTest__0001"] = "flexgroup_constituent"

instance2, _ := data.NewInstance("RahulTest__0002")
instance2.SetLabel("volume", "RahulTest__0002")
instance2.SetLabel("svm", "svm1")
instance2.SetLabel("aggr", "aggr2")
volumesMap["RahulTest__0002"] = "flexgroup_constituent"
volumesMap["svm1"+"RahulTest__0002"] = "flexgroup_constituent"

instance3, _ := data.NewInstance("RahulTest__0003")
instance3.SetLabel("volume", "RahulTest__0003")
instance3.SetLabel("svm", "svm1")
instance3.SetLabel("aggr", "aggr3")
volumesMap["RahulTest__0003"] = "flexgroup_constituent"
volumesMap["svm1"+"RahulTest__0003"] = "flexgroup_constituent"

// Create a simple volume instance
simpleInstance, _ := data.NewInstance("SimpleVolume")
simpleInstance.SetLabel("volume", "SimpleVolume")
simpleInstance.SetLabel("svm", "svm1")
simpleInstance.SetLabel("aggr", "aggr4")
volumesMap["SimpleVolume"] = "flexvol"
volumesMap["svm1"+"SimpleVolume"] = "flexvol"

// Create latency and ops metrics
latencyMetric, _ := data.NewMetricFloat64("read_latency")
Expand Down
12 changes: 7 additions & 5 deletions cmd/collectors/volume.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,11 @@ func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string

for _, i := range data.GetInstances() {
volName := i.GetLabel("volume")
switch volumesMap[volName] {
svmName := i.GetLabel("svm")
switch volumesMap[svmName+volName] {
case "flexgroup_constituent":
match := flexgroupRegex.FindStringSubmatch(volName)
key := i.GetLabel("svm") + "." + match[1]
key := svmName + "." + match[1]
if cache.GetInstance(key) == nil {
fg, _ := cache.NewInstance(key)
fg.SetLabels(maps.Clone(i.GetLabels()))
Expand Down Expand Up @@ -72,7 +73,7 @@ func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string
i.SetExportable(includeConstituents)
case "flexvol":
i.SetLabel(style, "flexvol")
key := i.GetLabel("svm") + "." + volName
key := svmName + "." + volName
flexvolInstance, err := volumeAggrmetric.NewInstance(key)
if err != nil {
logger.Error("Failed to create new instance", slogx.Err(err), slog.String("key", key))
Expand All @@ -91,11 +92,12 @@ func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string
recordFGFalse := make(map[string]*set.Set)
for _, i := range data.GetInstances() {
volName := i.GetLabel("volume")
if volumesMap[volName] != "flexgroup_constituent" {
svmName := i.GetLabel("svm")
if volumesMap[svmName+volName] != "flexgroup_constituent" {
continue
}
match := flexgroupRegex.FindStringSubmatch(volName)
key := i.GetLabel("svm") + "." + match[1]
key := svmName + "." + match[1]

flexgroupInstance := volumeAggrmetric.GetInstance(key)
if flexgroupInstance != nil {
Expand Down
4 changes: 3 additions & 1 deletion cmd/collectors/zapiperf/plugins/volume/volume.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ func (v *Volume) fetchVolumes() map[string]string {
volumeAttributes := node.NewXMLS("desired-attributes")
volumeIDAttributes := node.NewXMLS("volume-id-attributes")
volumeIDAttributes.NewChildS("name", "")
volumeIDAttributes.NewChildS("owning-vserver-name", "")
volumeIDAttributes.NewChildS("style-extended", "")
volumeAttributes.AddChild(volumeIDAttributes)
desired.AddChild(volumeAttributes)
Expand Down Expand Up @@ -119,7 +120,8 @@ func (v *Volume) fetchVolumes() map[string]string {
for _, volume := range volumes {
styleExtended := volume.GetChildS("volume-id-attributes").GetChildContentS("style-extended")
name := volume.GetChildS("volume-id-attributes").GetChildContentS("name")
volumesMap[name] = styleExtended
svm := volume.GetChildS("volume-id-attributes").GetChildContentS("owning-vserver-name")
volumesMap[svm+name] = styleExtended
}
}

Expand Down
1 change: 1 addition & 0 deletions cmd/tools/generate/counter.go
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@ var (
"_labels",
"volume_arw_status",
"ALERTS",
"_tags",
}

// Exclude extra metrics for ZAPI
Expand Down
1 change: 1 addition & 0 deletions cmd/tools/grafana/dashboard_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -699,6 +699,7 @@ func checkVariablesHaveAll(t *testing.T, path string, data []byte) {
}
exceptionForAllValues := map[string]bool{
"cmode/security.json": true,
"cmode/cluster.json": true,
}

if exceptionToAll[ShortPath(path)] {
Expand Down
Loading

0 comments on commit 1135ee5

Please sign in to comment.