Beats dashboard should use custom index patterns when `setup.dashboar…

…ds.index` is set (elastic#27901)

## What does this PR do?

This PR adds a new check to `make check` to make sure all dashboards can handle custom index names. It is the responsibility of the dashboard author to keep the dashboard useable if `setup.dashboards.index` is configured.

The PR contains additional fixes to make sure existing dashboards can be used with custom index names. Also, the test coverage was increased as it is a quite big PR.

The PR also decodes a few fields to make dashboards easier to review.

## Why is it important?

The bug prevents users from setting custom index names and use our dashboards at the same time.

Closes elastic#21232

CHANGELOG.next.asciidoc

@@ -210,6 +210,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix bug with cgroups hierarchy override path in cgroups {pull}27620[27620]
 - Beat `setup kibana` command may use the elasticsearch API key defined in `output.elasticsearch.api_key`. {issue}24015[24015] {pull}27540[27540]
 - Seperate namespaces for V1 and V2 controller paths {pull}27676[27676]
+- Beats dashboards use custom index when `setup.dashboards.index` is set. {issue}21232[21232] {pull}27901[27901]
 
 *Auditbeat*
 

dev-tools/mage/check.go

@@ -36,6 +36,7 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/dev-tools/mage/gotool"
+	"github.com/elastic/beats/v7/libbeat/dashboards"
 	"github.com/elastic/beats/v7/libbeat/processors/dissect"
 )
 
@@ -260,6 +261,14 @@ func checkDashboardForErrors(file string, d []byte) bool {
 		fmt.Println("  ", err)
 	}
 
+	replaced := dashboards.ReplaceIndexInDashboardObject("my-test-index-*", d)
+	if bytes.Contains(replaced, []byte(BeatName+"-*")) {
+		hasErrors = true
+		fmt.Printf(">> Cannot modify all index pattern references in dashboard - %s\n", file)
+		fmt.Println("Please edit the dashboard override function named ReplaceIndexInDashboardObject in libbeat.")
+		fmt.Println(string(replaced))
+	}
+
 	return hasErrors
 }
 

libbeat/dashboards/decode.go

@@ -30,11 +30,13 @@ import (
 
 var (
 	responseToDecode = []string{
-		"attributes.uiStateJSON",
-		"attributes.visState",
+		"attributes.kibanaSavedObjectMeta.searchSourceJSON",
+		"attributes.layerListJSON",
+		"attributes.mapStateJSON",
 		"attributes.optionsJSON",
 		"attributes.panelsJSON",
-		"attributes.kibanaSavedObjectMeta.searchSourceJSON",
+		"attributes.uiStateJSON",
+		"attributes.visState",
 	}
 )
 
@@ -76,15 +78,51 @@ func decodeLine(line []byte) []byte {
 	if err != nil {
 		return line
 	}
+	o = decodeObject(o)
+	o = decodeEmbeddableConfig(o)
+
+	return []byte(o.String())
+}
+
+func decodeObject(o common.MapStr) common.MapStr {
 	for _, key := range responseToDecode {
 		// All fields are optional, so errors are not caught
 		err := decodeValue(o, key)
 		if err != nil {
 			logger := logp.NewLogger("dashboards")
 			logger.Debugf("Error while decoding dashboard objects: %+v", err)
+			continue
 		}
 	}
-	return []byte(o.String())
+
+	return o
+}
+
+func decodeEmbeddableConfig(o common.MapStr) common.MapStr {
+	p, err := o.GetValue("attributes.panelsJSON")
+	if err != nil {
+		return o
+	}
+
+	if panels, ok := p.([]interface{}); ok {
+		for i, pan := range panels {
+			if panel, ok := pan.(map[string]interface{}); ok {
+				panelObj := common.MapStr(panel)
+				embedded, err := panelObj.GetValue("embeddableConfig")
+				if err != nil {
+					continue
+				}
+				if embeddedConfig, ok := embedded.(map[string]interface{}); ok {
+					embeddedConfigObj := common.MapStr(embeddedConfig)
+					panelObj.Put("embeddableConfig", decodeObject(embeddedConfigObj))
+					panels[i] = panelObj
+				}
+			}
+		}
+		o.Put("attributes.panelsJSON", panels)
+	}
+
+	return o
 }
 
 func decodeValue(data common.MapStr, key string) error {

libbeat/dashboards/modify_json.go

@@ -21,6 +21,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"regexp"
 
 	"github.com/pkg/errors"
 
@@ -46,11 +47,6 @@ type JSONObject struct {
 	Attributes JSONObjectAttribute `json:"attributes"`
 }
 
-// JSONFormat contains a list of JSON object
-type JSONFormat struct {
-	Objects []JSONObject `json:"objects"`
-}
-
 // ReplaceIndexInIndexPattern replaces an index in a dashboard content body
 func ReplaceIndexInIndexPattern(index string, content common.MapStr) (err error) {
 	if index == "" {
@@ -128,43 +124,62 @@ func ReplaceIndexInSavedObject(logger *logp.Logger, index string, kibanaSavedObj
 		}
 		kibanaSavedObject["searchSourceJSON"] = searchSourceJSON
 	}
-	if visStateJSON, ok := kibanaSavedObject["visState"].(string); ok {
-		visStateJSON = ReplaceIndexInVisState(logger, index, visStateJSON)
-		kibanaSavedObject["visState"] = visStateJSON
+	if visState, ok := kibanaSavedObject["visState"].(map[string]interface{}); ok {
+		kibanaSavedObject["visState"] = ReplaceIndexInVisState(logger, index, visState)
 	}
 
 	return kibanaSavedObject
 }
 
-// ReplaceIndexInVisState replaces index appearing in visState params objects
-func ReplaceIndexInVisState(logger *logp.Logger, index string, visStateJSON string) string {
-
-	var visState map[string]interface{}
-	err := json.Unmarshal([]byte(visStateJSON), &visState)
-	if err != nil {
-		logger.Errorf("Fail to unmarshal visState: %v", err)
-		return visStateJSON
-	}
+var timeLionIdxRegexp = regexp.MustCompile(`index=\".*beat-\*\"`)
 
+// ReplaceIndexInVisState replaces index appearing in visState params objects
+func ReplaceIndexInVisState(logger *logp.Logger, index string, visState map[string]interface{}) map[string]interface{} {
 	params, ok := visState["params"].(map[string]interface{})
 	if !ok {
-		return visStateJSON
+		return visState
 	}
 
 	// Don't set it if it was not set before
-	if pattern, ok := params["index_pattern"].(string); !ok || len(pattern) == 0 {
-		return visStateJSON
+	if pattern, ok := params["index_pattern"].(string); ok && len(pattern) != 0 {
+		params["index_pattern"] = index
+	}
+
+	if s, ok := params["series"].([]interface{}); ok {
+		for i, ser := range s {
+			if series, ok := ser.(map[string]interface{}); ok {
+				if _, ok := series["series_index_pattern"]; !ok {
+					continue
+				}
+				series["series_index_pattern"] = index
+				s[i] = series
+			}
+		}
+		params["series"] = s
 	}
 
-	params["index_pattern"] = index
+	if annotations, ok := params["annotations"].([]interface{}); ok {
+		for i, ann := range annotations {
+			annotation, ok := ann.(map[string]interface{})
+			if !ok {
+				continue
+			}
+			if _, ok = annotation["index_pattern"]; !ok {
+				continue
+			}
+			annotation["index_pattern"] = index
+			annotations[i] = annotation
+		}
+		params["annotations"] = annotations
+	}
 
-	d, err := json.Marshal(visState)
-	if err != nil {
-		logger.Errorf("Fail to marshal visState: %v", err)
-		return visStateJSON
+	if expr, ok := params["expression"].(string); ok {
+		params["expression"] = timeLionIdxRegexp.ReplaceAllString(expr, `index="`+index+`"`)
 	}
 
-	return string(d)
+	visState["params"] = replaceIndexInParamControls(logger, index, params)
+
+	return visState
 }
 
 // ReplaceIndexInDashboardObject replaces references to the index pattern in dashboard objects
@@ -195,10 +210,28 @@ func ReplaceIndexInDashboardObject(index string, content []byte) []byte {
 		attributes["kibanaSavedObjectMeta"] = ReplaceIndexInSavedObject(logger, index, kibanaSavedObject)
 	}
 
-	if visState, ok := attributes["visState"].(string); ok {
+	if visState, ok := attributes["visState"].(map[string]interface{}); ok {
 		attributes["visState"] = ReplaceIndexInVisState(logger, index, visState)
 	}
 
+	if layerListJSON, ok := attributes["layerListJSON"].([]interface{}); ok {
+		attributes["layerListJSON"] = replaceIndexInLayerListJSON(logger, index, layerListJSON)
+	}
+
+	if mapStateJSON, ok := attributes["mapStateJSON"].(map[string]interface{}); ok {
+		attributes["mapStateJSON"] = replaceIndexInMapStateJSON(logger, index, mapStateJSON)
+	}
+
+	if panelsJSON, ok := attributes["panelsJSON"].([]interface{}); ok {
+		attributes["panelsJSON"] = replaceIndexInPanelsJSON(logger, index, panelsJSON)
+	}
+
+	objectMap["attributes"] = attributes
+
+	if references, ok := objectMap["references"].([]interface{}); ok {
+		objectMap["references"] = replaceIndexInReferences(index, references)
+	}
+
 	b, err := json.Marshal(objectMap)
 	if err != nil {
 		logger.Error("Error marshaling modified dashboard: %+v", err)
@@ -208,6 +241,121 @@ func ReplaceIndexInDashboardObject(index string, content []byte) []byte {
 	return b
 }
 
+func replaceIndexInLayerListJSON(logger *logp.Logger, index string, layerListJSON []interface{}) []interface{} {
+	for i, layerListElem := range layerListJSON {
+		elem, ok := layerListElem.(map[string]interface{})
+		if !ok {
+			continue
+		}
+
+		if joins, ok := elem["joins"].([]interface{}); ok {
+			for j, join := range joins {
+				if pos, ok := join.(map[string]interface{}); ok {
+					for key, val := range pos {
+						if joinElems, ok := val.(map[string]interface{}); ok {
+							if _, ok := joinElems["indexPatternTitle"]; ok {
+								joinElems["indexPatternTitle"] = index
+								pos[key] = joinElems
+							}
+						}
+					}
+					joins[j] = pos
+				}
+			}
+			elem["joins"] = joins
+		}
+		if descriptor, ok := elem["sourceDescriptor"].(map[string]interface{}); ok {
+			if _, ok := descriptor["indexPatternId"]; ok {
+				descriptor["indexPatternId"] = index
+			}
+			elem["sourceDescriptor"] = descriptor
+		}
+
+		layerListJSON[i] = elem
+	}
+	return layerListJSON
+}
+
+func replaceIndexInMapStateJSON(logger *logp.Logger, index string, mapState map[string]interface{}) map[string]interface{} {
+	if filters, ok := mapState["filters"].([]interface{}); ok {
+		for i, f := range filters {
+			if filter, ok := f.(map[string]interface{}); ok {
+				if meta, ok := filter["meta"].(map[string]interface{}); ok {
+					if _, ok := meta["index"]; !ok {
+						continue
+					}
+					meta["index"] = index
+					filter["meta"] = meta
+				}
+				filters[i] = filter
+			}
+		}
+		mapState["filters"] = filters
+	}
+
+	return mapState
+}
+
+func replaceIndexInPanelsJSON(logger *logp.Logger, index string, panelsJSON []interface{}) []interface{} {
+	for i, p := range panelsJSON {
+		if panel, ok := p.(map[string]interface{}); ok {
+			config, ok := panel["embeddableConfig"].(map[string]interface{})
+			if !ok {
+				continue
+			}
+			if configAttr, ok := config["attributes"].(map[string]interface{}); ok {
+				if references, ok := configAttr["references"].([]interface{}); ok {
+					configAttr["references"] = replaceIndexInReferences(index, references)
+				}
+				if layerListJSON, ok := configAttr["layerListJSON"].([]interface{}); ok {
+					configAttr["layerListJSON"] = replaceIndexInLayerListJSON(logger, index, layerListJSON)
+				}
+				config["attributes"] = configAttr
+			}
+
+			if savedVis, ok := config["savedVis"].(map[string]interface{}); ok {
+				if params, ok := savedVis["params"].(map[string]interface{}); ok {
+					savedVis["params"] = replaceIndexInParamControls(logger, index, params)
+				}
+				config["savedVis"] = savedVis
+			}
+
+			panel["embeddableConfig"] = config
+			panelsJSON[i] = panel
+		}
+	}
+	return panelsJSON
+}
+
+func replaceIndexInParamControls(logger *logp.Logger, index string, params map[string]interface{}) map[string]interface{} {
+	if controlsList, ok := params["controls"].([]interface{}); ok {
+		for i, ctrl := range controlsList {
+			if control, ok := ctrl.(map[string]interface{}); ok {
+				if _, ok := control["indexPattern"]; ok {
+					control["indexPattern"] = index
+					controlsList[i] = control
+				}
+			}
+		}
+		params["controls"] = controlsList
+	}
+	return params
+}
+
+func replaceIndexInReferences(index string, references []interface{}) []interface{} {
+	for i, ref := range references {
+		if reference, ok := ref.(map[string]interface{}); ok {
+			if refType, ok := reference["type"].(string); ok {
+				if refType == "index-pattern" {
+					reference["id"] = index
+				}
+			}
+			references[i] = reference
+		}
+	}
+	return references
+}
+
 func EncodeJSONObjects(content []byte) []byte {
 	logger := logp.NewLogger("dashboards")
 

libbeat/dashboards/modify_json_test.go

@@ -77,9 +77,34 @@ func TestReplaceIndexInDashboardObject(t *testing.T) {
 			[]byte(`{"attributes":{"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"index\":\"otherindex-*\"}"}}}`),
 		},
 		{
-			[]byte(`{"attributes":{"kibanaSavedObjectMeta":{"visState":"{\"params\":{\"index_pattern\":\"metricbeat-*\"}}"}}}`),
+			[]byte(`{"attributes":{"layerListJSON":[{"joins":[{"leftField":"iso2","right":{"indexPatternTitle":"filebeat-*"}}]}]}}`),
 			"otherindex-*",
-			[]byte(`{"attributes":{"kibanaSavedObjectMeta":{"visState":"{\"params\":{\"index_pattern\":\"otherindex-*\"}}"}}}`),
+			[]byte(`{"attributes":{"layerListJSON":[{"joins":[{"leftField":"iso2","right":{"indexPatternTitle":"otherindex-*"}}]}]}}`),
+		},
+		{
+			[]byte(`{"attributes":{"panelsJSON":[{"embeddableConfig":{"attributes":{"references":[{"id":"filebeat-*","type":"index-pattern"}]}}}]}}`),
+			"otherindex-*",
+			[]byte(`{"attributes":{"panelsJSON":[{"embeddableConfig":{"attributes":{"references":[{"id":"otherindex-*","type":"index-pattern"}]}}}]}}`),
+		},
+		{
+			[]byte(`{"attributes":{},"references":[{"id":"auditbeat-*","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}]}`),
+			"otherindex-*",
+			[]byte(`{"attributes":{},"references":[{"id":"otherindex-*","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}]}`),
+		},
+		{
+			[]byte(`{"attributes":{"visState":{"params":{"index_pattern":"winlogbeat-*"}}}}`),
+			"otherindex-*",
+			[]byte(`{"attributes":{"visState":{"params":{"index_pattern":"otherindex-*"}}}}`),
+		},
+		{
+			[]byte(`{"attributes":{"visState":{"params":{"series":[{"series_index_pattern":"filebeat-*"}]}}}}`),
+			"otherindex-*",
+			[]byte(`{"attributes":{"visState":{"params":{"series":[{"series_index_pattern":"otherindex-*"}]}}}}`),
+		},
+		{
+			[]byte(`{"attributes":{"mapStateJSON":{"filters":[{"meta":{"index":"filebeat-*"}}]}}}`),
+			"otherindex-*",
+			[]byte(`{"attributes":{"mapStateJSON":{"filters":[{"meta":{"index":"otherindex-*"}}]}}}`),
 		},
 	}