Move generated Kibana files to _meta/kibana.generated (#7261)

Currently all dashboards end up under `_meta/kibana` in each Beat. The problem with this directory is that for some Beats it contains the automatically collected and generated files and for others the original. This prevents us from having decoded JSON files for all dashboards across Beats.

Another benefit of having the `kibana.generated` directory is that now `_meta/kibana` can become the place for dashboards across modules in one Beat without having to worry about overwriting / removing these dashboards.

Follow up PR's will be needed to implement this change in each Beat and remove `_meta/kibana` from the ignored directory. Some Beats will not have the directory.

.gitignore

@@ -8,6 +8,7 @@
 /*/fields.yml
 /*/*.template*.json
 **/html_docs
+/*/_meta/kibana.generated
 
 # Files
 .DS_Store

libbeat/kibana/index_pattern_generator.go

@@ -152,7 +152,7 @@ func dumpToFile(f string, pattern common.MapStr) error {
 }
 
 func createTargetDir(baseDir string, version common.Version) string {
-	targetDir := filepath.Join(baseDir, "_meta", "kibana", getVersionPath(version), "index-pattern")
+	targetDir := filepath.Join(baseDir, "_meta", "kibana.generated", getVersionPath(version), "index-pattern")
 	if _, err := os.Stat(targetDir); os.IsNotExist(err) {
 		os.MkdirAll(targetDir, 0777)
 	}

libbeat/kibana/index_pattern_generator_test.go

@@ -30,7 +30,7 @@ func TestNewGenerator(t *testing.T) {
 	assert.Equal(t, filepath.Join(beatDir, "fields.yml"), generator.fieldsYaml)
 
 	// creates file dir and sets name
-	expectedDir := filepath.Join(beatDir, "_meta/kibana/6/index-pattern")
+	expectedDir := filepath.Join(beatDir, "_meta/kibana.generated/6/index-pattern")
 	assert.Equal(t, expectedDir, generator.targetDir)
 	_, err = os.Stat(generator.targetDir)
 	assert.NoError(t, err)
@@ -40,7 +40,7 @@ func TestNewGenerator(t *testing.T) {
 	generator, err = NewGenerator("beat-index", "mybeat.", beatDir, "7.0", *v)
 	assert.NoError(t, err)
 
-	expectedDir = filepath.Join(beatDir, "_meta/kibana/5/index-pattern")
+	expectedDir = filepath.Join(beatDir, "_meta/kibana.generated/5/index-pattern")
 	assert.Equal(t, expectedDir, generator.targetDir)
 	_, err = os.Stat(generator.targetDir)
 
@@ -127,8 +127,8 @@ func TestGenerate(t *testing.T) {
 	}
 
 	tests := []map[string]string{
-		{"existing": "beat-5.json", "created": "_meta/kibana/5/index-pattern/beat.json"},
-		{"existing": "beat-6.json", "created": "_meta/kibana/6/index-pattern/beat.json"},
+		{"existing": "beat-5.json", "created": "_meta/kibana.generated/5/index-pattern/beat.json"},
+		{"existing": "beat-6.json", "created": "_meta/kibana.generated/6/index-pattern/beat.json"},
 	}
 	testGenerate(t, beatDir, tests, true)
 }
@@ -152,8 +152,8 @@ func TestGenerateExtensive(t *testing.T) {
 	}
 
 	tests := []map[string]string{
-		{"existing": "metricbeat-5.json", "created": "_meta/kibana/5/index-pattern/metricbeat.json"},
-		{"existing": "metricbeat-6.json", "created": "_meta/kibana/6/index-pattern/metricbeat.json"},
+		{"existing": "metricbeat-5.json", "created": "_meta/kibana.generated/5/index-pattern/metricbeat.json"},
+		{"existing": "metricbeat-6.json", "created": "_meta/kibana.generated/6/index-pattern/metricbeat.json"},
 	}
 	testGenerate(t, beatDir, tests, false)
 }

libbeat/scripts/Makefile

@@ -149,8 +149,7 @@ clean:: ## @build  Cleans up all files generated by the build steps
 	@rm -f docker-compose.yml.lock
 	@rm -f ${BEAT_NAME} ${BEAT_NAME}.test ${BEAT_NAME}.exe ${BEAT_NAME}.test.exe
 	@rm -f _meta/fields.generated.yml fields.yml
-	@rm -f $(PWD)/_meta/kibana/5/index-pattern/${BEAT_NAME}.json
-	@rm -f $(PWD)/_meta/kibana/6/index-pattern/${BEAT_NAME}.json
+	@rm -f $(PWD)/_meta/kibana.generated
 	@rm -f ${BEAT_NAME}.template*.json
 
 .PHONY: ci
@@ -343,9 +342,12 @@ endif
 	@mkdir -p docs
 	@${PYTHON_ENV}/bin/python ${ES_BEATS}/libbeat/scripts/generate_fields_docs.py $(PWD) ${BEAT_TITLE} ${ES_BEATS}
 
-	@# Generate Kibana index pattern
-	@mkdir -p $(PWD)/_meta/kibana/5/index-pattern
-	@mkdir -p $(PWD)/_meta/kibana/6/index-pattern
+	@# Generate Kibana index pattern and copy dashboard files
+	if [ -d $(PWD)/_meta/kibana ]; then \
+		cp -r $(PWD)/_meta/kibana/ $(PWD)/_meta/kibana.generated; \
+	fi
+	@mkdir -p $(PWD)/_meta/kibana.generated/5/index-pattern
+	@mkdir -p $(PWD)/_meta/kibana.generated/6/index-pattern
 	@go run ${ES_BEATS}/dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go -index '${BEAT_INDEX_PREFIX}-*' -beat-name ${BEAT_NAME} -beat-dir $(PWD) -version ${BEAT_VERSION}
 
 .PHONY: docs
@@ -366,7 +368,7 @@ ${ES_BEATS}/dev-tools/cmd/dashboards/export_dashboards:
 
 .PHONY: import-dashboards
 import-dashboards: update ${BEAT_NAME}
-	${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME} setup -E setup.dashboards.directory=${PWD}/_meta/kibana -E setup.kibana.host=${KIBANA_URL} --dashboards
+	${BEAT_GOPATH}/src/${BEAT_PATH}/${BEAT_NAME} setup -E setup.dashboards.directory=${PWD}/_meta/kibana.generated -E setup.kibana.host=${KIBANA_URL} --dashboards
 
 ### CONTAINER ENVIRONMENT ####
 
@@ -417,9 +419,9 @@ install-home:
 		rsync -av _meta/module.generated/ ${HOME_PREFIX}/module/; \
 		chmod -R go-w ${HOME_PREFIX}/module/; \
 	fi
-	if [ -d _meta/kibana ]; then \
+	if [ -d _meta/kibana.generated ]; then \
 		install -d -m 755 ${HOME_PREFIX}/kibana; \
-		rsync -av _meta/kibana/ ${HOME_PREFIX}/kibana/; \
+		rsync -av _meta/kibana.generated/ ${HOME_PREFIX}/kibana/; \
 	fi
 
 # Prepares for packaging. Builds binaries and creates homedir data
@@ -541,7 +543,7 @@ package-all: package-elastic package-oss
 
 package-dashboards: package-setup
 	mkdir -p ${BUILD_DIR}
-	cp -r _meta/kibana ${BUILD_DIR}/dashboards
+	cp -r _meta/kibana.generated ${BUILD_DIR}/dashboards
 	# build the dashboards package
 	BEAT_NAME=${BEAT_NAME} BUILD_DIR=${BUILD_DIR} SNAPSHOT=$(SNAPSHOT) $(MAKE) -C ${ES_BEATS}/dev-tools/packer package-dashboards ${shell pwd}/build/upload/build_id.txt