Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add tags to the Volume and Cluster dashboards #3273

Merged
merged 14 commits into from
Nov 18, 2024
Prev Previous commit
Next Next commit
feat: join with volume_labels
  • Loading branch information
Hardikl committed Nov 14, 2024
commit 8b4edbf941b0246de903c6f4d9755f5c80aa5c80
30 changes: 7 additions & 23 deletions cmd/collectors/restperf/plugins/volume/volume.go
Original file line number Diff line number Diff line change
@@ -8,8 +8,6 @@ import (
"github.com/netapp/harvest/v2/pkg/matrix"
"github.com/netapp/harvest/v2/pkg/util"
"log/slog"
"sort"
"strings"
"time"
)

@@ -19,7 +17,7 @@ type Volume struct {
styleType string
includeConstituents bool
client *rest.Client
volumesMap map[string]collectors.VolumeData // volume-name -> {volume-extended-style, tags} map
volumesMap map[string]string // volume-name -> volume-extended-style map
}

func New(p *plugin.AbstractPlugin) plugin.Plugin {
@@ -38,7 +36,7 @@ func (v *Volume) Init(remote conf.Remote) error {
v.styleType = "type"
}

v.volumesMap = make(map[string]collectors.VolumeData)
v.volumesMap = make(map[string]string)

// Assigned the value to currentVal so that plugin would be invoked first time to populate cache.
v.currentVal = v.SetPluginInterval()
@@ -63,7 +61,6 @@ func (v *Volume) Init(remote conf.Remote) error {
func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) {
data := dataMap[v.Object]
style := v.styleType
tags := "tags"
opsKeyPrefix := "temp_"
if v.currentVal >= v.PluginInvocationRate {
v.currentVal = 0
@@ -73,16 +70,16 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util
}

v.currentVal++
return collectors.ProcessFlexGroupData(v.SLogger, data, style, tags, v.includeConstituents, opsKeyPrefix, v.volumesMap)
return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix, v.volumesMap)
}

func (v *Volume) fetchVolumes() map[string]collectors.VolumeData {
volumesMap := make(map[string]collectors.VolumeData)
func (v *Volume) fetchVolumes() map[string]string {
volumesMap := make(map[string]string)
query := "api/private/cli/volume"

href := rest.NewHrefBuilder().
APIPath(query).
Fields([]string{"volume", "vserver", "volume_style_extended", "tags"}).
Fields([]string{"volume", "vserver", "volume_style_extended"}).
Filter([]string{"is_constituent=*"}).
MaxRecords(collectors.DefaultBatchSize).
Build()
@@ -98,27 +95,14 @@ func (v *Volume) fetchVolumes() map[string]collectors.VolumeData {
}

for _, volume := range records {
var tags string
if !volume.IsObject() {
v.SLogger.Warn("volume is not object, skipping", slog.String("type", volume.Type.String()))
continue
}
styleExtended := volume.Get("volume_style_extended").String()
name := volume.Get("volume").String()
svm := volume.Get("vserver").String()
tagData := volume.Get("tags")
if tagData.IsArray() {
var labelArray []string
for _, r := range tagData.Array() {
labelString := r.String()
labelArray = append(labelArray, labelString)
}
sort.Strings(labelArray)
tags = strings.Join(labelArray, ",")
} else {
tags = tagData.String()
}
volumesMap[svm+name] = collectors.VolumeData{Style: styleExtended, Tags: tags}
volumesMap[svm+name] = styleExtended
}

return volumesMap
13 changes: 6 additions & 7 deletions cmd/collectors/restperf/plugins/volume/volume_test.go
Original file line number Diff line number Diff line change
@@ -17,15 +17,14 @@ import (

const OpsKeyPrefix = "temp_"
const StyleType = "style"
const Tags = "tags"
const PollerName = "test"

// Common test logic for RestPerf/ZapiPerf Volume plugin
func runVolumeTest(t *testing.T, createVolume func(params *node.Node) plugin.Plugin, includeConstituents string, expectedCount int, setMetricNaN bool) {
params := node.NewS("Volume")
params.NewChildS("include_constituents", includeConstituents)
v := createVolume(params)
volumesMap := make(map[string]collectors.VolumeData)
volumesMap := make(map[string]string)

// Initialize the plugin
if err := v.Init(conf.Remote{}); err != nil {
@@ -38,26 +37,26 @@ func runVolumeTest(t *testing.T, createVolume func(params *node.Node) plugin.Plu
instance1.SetLabel("volume", "RahulTest__0001")
instance1.SetLabel("svm", "svm1")
instance1.SetLabel("aggr", "aggr1")
volumesMap["svm1"+"RahulTest__0001"] = collectors.VolumeData{Style: "flexgroup_constituent"}
volumesMap["svm1"+"RahulTest__0001"] = "flexgroup_constituent"

instance2, _ := data.NewInstance("RahulTest__0002")
instance2.SetLabel("volume", "RahulTest__0002")
instance2.SetLabel("svm", "svm1")
instance2.SetLabel("aggr", "aggr2")
volumesMap["svm1"+"RahulTest__0002"] = collectors.VolumeData{Style: "flexgroup_constituent"}
volumesMap["svm1"+"RahulTest__0002"] = "flexgroup_constituent"

instance3, _ := data.NewInstance("RahulTest__0003")
instance3.SetLabel("volume", "RahulTest__0003")
instance3.SetLabel("svm", "svm1")
instance3.SetLabel("aggr", "aggr3")
volumesMap["svm1"+"RahulTest__0003"] = collectors.VolumeData{Style: "flexgroup_constituent"}
volumesMap["svm1"+"RahulTest__0003"] = "flexgroup_constituent"

// Create a simple volume instance
simpleInstance, _ := data.NewInstance("SimpleVolume")
simpleInstance.SetLabel("volume", "SimpleVolume")
simpleInstance.SetLabel("svm", "svm1")
simpleInstance.SetLabel("aggr", "aggr4")
volumesMap["svm1"+"SimpleVolume"] = collectors.VolumeData{Style: "flexvol"}
volumesMap["svm1"+"SimpleVolume"] = "flexvol"

// Create latency and ops metrics
latencyMetric, _ := data.NewMetricFloat64("read_latency")
@@ -89,7 +88,7 @@ func runVolumeTest(t *testing.T, createVolume func(params *node.Node) plugin.Plu

// Run the plugin
boolValue, _ := strconv.ParseBool(includeConstituents)
output, _, err := collectors.ProcessFlexGroupData(slog.Default(), data, StyleType, Tags, boolValue, OpsKeyPrefix, volumesMap)
output, _, err := collectors.ProcessFlexGroupData(slog.Default(), data, StyleType, boolValue, OpsKeyPrefix, volumesMap)
if err != nil {
t.Fatalf("Run method failed: %v", err)
}
13 changes: 3 additions & 10 deletions cmd/collectors/volume.go
Original file line number Diff line number Diff line change
@@ -14,12 +14,7 @@ import (

var flexgroupRegex = regexp.MustCompile(`^(.*)__(\d{4})$`)

type VolumeData struct {
Style string
Tags string
}

func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string, tags string, includeConstituents bool, opsKeyPrefix string, volumesMap map[string]VolumeData) ([]*matrix.Matrix, *util.Metadata, error) {
func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string, includeConstituents bool, opsKeyPrefix string, volumesMap map[string]string) ([]*matrix.Matrix, *util.Metadata, error) {
var err error

if volumesMap == nil {
@@ -46,9 +41,7 @@ func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string
for _, i := range data.GetInstances() {
volName := i.GetLabel("volume")
svmName := i.GetLabel("svm")
volData := volumesMap[svmName+volName]
i.SetLabel(tags, volData.Tags)
switch volData.Style {
switch volumesMap[svmName+volName] {
case "flexgroup_constituent":
match := flexgroupRegex.FindStringSubmatch(volName)
key := svmName + "." + match[1]
@@ -100,7 +93,7 @@ func ProcessFlexGroupData(logger *slog.Logger, data *matrix.Matrix, style string
for _, i := range data.GetInstances() {
volName := i.GetLabel("volume")
svmName := i.GetLabel("svm")
if volumesMap[svmName+volName].Style != "flexgroup_constituent" {
if volumesMap[svmName+volName] != "flexgroup_constituent" {
continue
}
match := flexgroupRegex.FindStringSubmatch(volName)
17 changes: 9 additions & 8 deletions cmd/collectors/zapiperf/plugins/volume/volume.go
Original file line number Diff line number Diff line change
@@ -23,7 +23,7 @@ type Volume struct {
styleType string
includeConstituents bool
client *zapi.Client
volumesMap map[string]collectors.VolumeData // volume-name -> {volume-extended-style, tags} map
volumesMap map[string]string // volume-name -> volume-extended-style map
}

func New(p *plugin.AbstractPlugin) plugin.Plugin {
@@ -46,7 +46,7 @@ func (v *Volume) Init(remote conf.Remote) error {
return nil
}

v.volumesMap = make(map[string]collectors.VolumeData)
v.volumesMap = make(map[string]string)

// Assigned the value to currentVal so that plugin would be invoked first time to populate cache.
v.currentVal = v.SetPluginInterval()
@@ -64,7 +64,6 @@ func (v *Volume) Init(remote conf.Remote) error {
func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util.Metadata, error) {
data := dataMap[v.Object]
style := v.styleType
tags := "tags"
opsKeyPrefix := "temp_"
if v.currentVal >= v.PluginInvocationRate {
v.currentVal = 0
@@ -74,17 +73,17 @@ func (v *Volume) Run(dataMap map[string]*matrix.Matrix) ([]*matrix.Matrix, *util
}

v.currentVal++
return collectors.ProcessFlexGroupData(v.SLogger, data, style, tags, v.includeConstituents, opsKeyPrefix, v.volumesMap)
return collectors.ProcessFlexGroupData(v.SLogger, data, style, v.includeConstituents, opsKeyPrefix, v.volumesMap)
}

func (v *Volume) fetchVolumes() map[string]collectors.VolumeData {
func (v *Volume) fetchVolumes() map[string]string {
var (
result *node.Node
volumes []*node.Node
volumesMap map[string]collectors.VolumeData
volumesMap map[string]string
)

volumesMap = make(map[string]collectors.VolumeData)
volumesMap = make(map[string]string)
query := "volume-get-iter"
tag := "initial"
request := node.NewXMLS(query)
@@ -93,6 +92,7 @@ func (v *Volume) fetchVolumes() map[string]collectors.VolumeData {
volumeAttributes := node.NewXMLS("desired-attributes")
volumeIDAttributes := node.NewXMLS("volume-id-attributes")
volumeIDAttributes.NewChildS("name", "")
volumeIDAttributes.NewChildS("owning-vserver-name", "")
volumeIDAttributes.NewChildS("style-extended", "")
volumeAttributes.AddChild(volumeIDAttributes)
desired.AddChild(volumeAttributes)
@@ -120,7 +120,8 @@ func (v *Volume) fetchVolumes() map[string]collectors.VolumeData {
for _, volume := range volumes {
styleExtended := volume.GetChildS("volume-id-attributes").GetChildContentS("style-extended")
name := volume.GetChildS("volume-id-attributes").GetChildContentS("name")
volumesMap[name] = collectors.VolumeData{Style: styleExtended, Tags: ""}
svm := volume.GetChildS("volume-id-attributes").GetChildContentS("owning-vserver-name")
volumesMap[svm+name] = styleExtended
}
}

1 change: 1 addition & 0 deletions cmd/tools/generate/counter.go
Original file line number Diff line number Diff line change
@@ -172,6 +172,7 @@ var (
"_labels",
"volume_arw_status",
"ALERTS",
"_tags",
}

// Exclude extra metrics for ZAPI
1 change: 1 addition & 0 deletions cmd/tools/grafana/dashboard_test.go
Original file line number Diff line number Diff line change
@@ -700,6 +700,7 @@ func checkVariablesHaveAll(t *testing.T, path string, data []byte) {
exceptionForAllValues := map[string]bool{
"cmode/security.json": true,
"cmode/cluster.json": true,
"cmode/volume.json": true,
}

if exceptionToAll[ShortPath(path)] {
1 change: 0 additions & 1 deletion conf/restperf/9.12.0/volume.yaml
Original file line number Diff line number Diff line change
@@ -78,5 +78,4 @@ export_options:
- node
- style
- svm
- tags
- volume
Loading