Add the option to generate the template into a file (#4323)

Part of #3654.

This adds two settings: `setup.template.output_to_file.path` and `setup.template.output_to_file.version`. The version refers to the ES version and is optional, we'll use the Beats version if not specified. I put it under `output_to_file` to make it clear that it only applies when outputting to a file:

To generate a config, one can do:

```
./metricbeat -e -E setup.template.output_to_file.path=template.json
```

In the current version, the Beat automatically stops after generating the template, but the output might be slightly confusing:

```
2017/05/16 09:55:02.043671 load.go:116: INFO Template for Elasticsearch 6.0.0-alpha2 written to: template.json
2017/05/16 09:55:02.043717 beat.go:538: CRIT Exiting: Stopping after successfully writing the template to the file.
Exiting: Stopping after successfully writing the template to the file.
```

IMO this is better than the alternative of leaving it running.

To generate the template for the 2.x version, one can do:

```
./metricbeat -e -E setup.template.output_to_file.path=template.json -E setup.template.output_to_file.version=2.4.0
```

Remaining TODOs:

* [x] System test
* [x] Docs
* [x] Changelog

CHANGELOG.asciidoc

@@ -54,6 +54,7 @@ https://github.com/elastic/beats/compare/v6.0.0-alpha1...master[Check the HEAD d
 *Affecting all Beats*
 
 - Added the possibility to set Elasticsearch mapping template settings from the Beat configuration file. {pull}4284[4284] {pull}4317[4317]
+- Add the option to write the generated Elasticsearch mapping template into a file. {pull}4323[4323]
 
 *Filebeat*
 

libbeat/beat/beat.go

@@ -469,8 +469,30 @@ func (b *Beat) registerTemplateLoading() error {
 		}
 	}
 
-	esConfig := b.Config.Output["elasticsearch"]
+	// Check if outputting to file is enabled, and output to file if it is
+	if b.Config.Template != nil && b.Config.Template.Enabled() {
+		var cfg template.TemplateConfig
+		err := b.Config.Template.Unpack(&cfg)
+		if err != nil {
+			return fmt.Errorf("unpacking template config fails: %v", err)
+		}
+		if len(cfg.OutputToFile.Path) > 0 {
+			// output to file is enabled
+			loader, err := template.NewLoader(b.Config.Template, nil, b.Info)
+			if err != nil {
+				return fmt.Errorf("Error creating Elasticsearch template loader: %v", err)
+			}
+			err = loader.Generate()
+			if err != nil {
+				return fmt.Errorf("Error generating template: %v", err)
+			}
 
+			// XXX: Should we kill the Beat here or just continue?
+			return fmt.Errorf("Stopping after successfully writing the template to the file.")
+		}
+	}
+
+	esConfig := b.Config.Output["elasticsearch"]
 	// Loads template by default if esOutput is enabled
 	if (b.Config.Template == nil && esConfig.Enabled()) || (b.Config.Template != nil && b.Config.Template.Enabled()) {
 		if esConfig == nil || !esConfig.Enabled() {
@@ -487,7 +509,7 @@ func (b *Beat) registerTemplateLoading() error {
 
 			loader, err := template.NewLoader(b.Config.Template, esClient, b.Info)
 			if err != nil {
-				return fmt.Errorf("Error creating Elasticsearch template: %v", err)
+				return fmt.Errorf("Error creating Elasticsearch template loader: %v", err)
 			}
 
 			err = loader.Load()

libbeat/docs/template-config.asciidoc

@@ -49,3 +49,25 @@ setup.template.overwrite: false
 setup.template.settings:
   _source.enabled: false
 ----------------------------------------------------------------------
+
+*`output_to_file.path`*:: If this option is set, {beatname_uc} generates the Elasticsearch template
+  JSON object and writes into a file at the specified path. Immediately after writing the file,
+  {beatname_uc} exists with the exit code 1.
+
+For example, you can generate a template file ready to be uploaded to Elasticsearch like this:
+
+["source","yaml",subs="attributes,callouts"]
+----------------------------------------------------------------------
+./{beatname_lc} -e -E "setup.template.output_to_file.path={beatname_lc}.template.json"
+----------------------------------------------------------------------
+
+*`output_to_file.version`*:: The Elasticsearch version for which to generate the template file. By
+default, the {beatname_uc} version is used. This setting is only used if `output_to_file.path` is
+also set.
+
+For example, the following generates a template file for Elasticsearch 5.4:
+
+["source","yaml",subs="attributes,callouts"]
+----------------------------------------------------------------------
+./{beatname_lc} -e -E "setup.template.output_to_file.path={beatname_lc}.template.json" -E "setup.template.output_to_file.version=5.4.0"
+----------------------------------------------------------------------

libbeat/template/config.go

@@ -5,8 +5,15 @@ type TemplateConfig struct {
 	Name         string           `config:"name"`
 	Fields       string           `config:"fields"`
 	Overwrite    bool             `config:"overwrite"`
-	OutputToFile string           `config:"output_to_file"`
 	Settings     templateSettings `config:"settings"`
+	OutputToFile OutputToFile     `config:"output_to_file"`
+}
+
+// OutputToFile contains the configuration options for generating
+// and writing the template into a file.
+type OutputToFile struct {
+	Path    string `config:"path"`
+	Version string `config:"version"`
 }
 
 type templateSettings struct {

libbeat/template/load.go

@@ -1,7 +1,9 @@
 package template
 
 import (
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/logp"
@@ -38,7 +40,7 @@ func NewLoader(cfg *common.Config, client ESClient, beatInfo common.BeatInfo) (*
 	}, nil
 }
 
-// loadTemplate checks if the index mapping template should be loaded
+// Load checks if the index mapping template should be loaded
 // In case the template is not already loaded or overwriting is enabled, the
 // template is written to index
 func (l *Loader) Load() error {
@@ -80,6 +82,43 @@ func (l *Loader) Load() error {
 	return nil
 }
 
+// Generate generates the template and writes it to a file based on the configuration
+// from `output_to_file`.
+func (l *Loader) Generate() error {
+	if l.config.OutputToFile.Version == "" {
+		l.config.OutputToFile.Version = l.beatInfo.Version
+	}
+
+	if l.config.Name == "" {
+		l.config.Name = l.beatInfo.Beat
+	}
+
+	tmpl, err := New(l.beatInfo.Version, l.config.OutputToFile.Version, l.config.Name, l.config.Settings)
+	if err != nil {
+		return fmt.Errorf("error creating template instance: %v", err)
+	}
+
+	fieldsPath := paths.Resolve(paths.Config, l.config.Fields)
+
+	output, err := tmpl.Load(fieldsPath)
+	if err != nil {
+		return fmt.Errorf("error creating template from file %s: %v", fieldsPath, err)
+	}
+
+	jsonBytes, err := json.MarshalIndent(output, "", "  ")
+	if err != nil {
+		return fmt.Errorf("error marshaling template: %v", err)
+	}
+
+	err = ioutil.WriteFile(l.config.OutputToFile.Path, jsonBytes, 0644)
+	if err != nil {
+		return fmt.Errorf("error writing to file %s: %v", l.config.OutputToFile.Path, err)
+	}
+
+	logp.Info("Template for Elasticsearch %s written to: %s", l.config.OutputToFile.Version, l.config.OutputToFile.Path)
+	return nil
+}
+
 // LoadTemplate loads a template into Elasticsearch overwriting the existing
 // template if it exists. If you wish to not overwrite an existing template
 // then use CheckTemplate prior to calling this method.

libbeat/template/load_test.go

@@ -0,0 +1,112 @@
+// +build !integration
+
+package template
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/version"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGenerateTemplate(t *testing.T) {
+
+	// Load template
+	absPath, err := filepath.Abs("../")
+	assert.NotNil(t, absPath)
+	assert.Nil(t, err)
+
+	beatInfo := common.BeatInfo{
+		Beat:    "testbeat",
+		Version: version.GetDefaultVersion(),
+	}
+
+	dir, err := ioutil.TempDir("", "test-template")
+	assert.NoError(t, err)
+	defer os.RemoveAll(dir)
+	path := filepath.Join(dir, "template.json")
+
+	config := newConfigFrom(t, TemplateConfig{
+		Enabled: true,
+		Fields:  absPath + "/fields.yml",
+		OutputToFile: OutputToFile{
+			Path: path,
+		},
+	})
+
+	loader, err := NewLoader(config, nil, beatInfo)
+	assert.NoError(t, err)
+
+	err = loader.Generate()
+	assert.NoError(t, err)
+
+	// Read it back to check it
+	fp, err := os.Open(path)
+	assert.NoError(t, err)
+	jsonParser := json.NewDecoder(fp)
+	var parsed common.MapStr
+	err = jsonParser.Decode(&parsed)
+	assert.NoError(t, err)
+
+	val, err := parsed.GetValue("mappings._default_._meta.version")
+	assert.NoError(t, err)
+	assert.Equal(t, val.(string), version.GetDefaultVersion())
+
+}
+
+func TestGenerateTemplateWithVersion(t *testing.T) {
+
+	// Load template
+	absPath, err := filepath.Abs("../")
+	assert.NotNil(t, absPath)
+	assert.Nil(t, err)
+
+	beatInfo := common.BeatInfo{
+		Beat:    "testbeat",
+		Version: version.GetDefaultVersion(),
+	}
+
+	dir, err := ioutil.TempDir("", "test-template")
+	assert.NoError(t, err)
+	defer os.RemoveAll(dir)
+	path := filepath.Join(dir, "template.json")
+
+	config := newConfigFrom(t, TemplateConfig{
+		Enabled: true,
+		Fields:  absPath + "/fields.yml",
+		OutputToFile: OutputToFile{
+			Path:    path,
+			Version: "2.4.0",
+		},
+	})
+
+	loader, err := NewLoader(config, nil, beatInfo)
+	assert.NoError(t, err)
+
+	err = loader.Generate()
+	assert.NoError(t, err)
+
+	// Read it back to check it
+	fp, err := os.Open(path)
+	assert.NoError(t, err)
+	jsonParser := json.NewDecoder(fp)
+	var parsed common.MapStr
+	err = jsonParser.Decode(&parsed)
+	assert.NoError(t, err)
+
+	// check a setting specific to that version
+	val, err := parsed.GetValue("mappings._default_._all.norms.enabled")
+	assert.NoError(t, err)
+	assert.Equal(t, val.(bool), false)
+}
+
+func newConfigFrom(t *testing.T, from interface{}) *common.Config {
+	cfg, err := common.NewConfigFrom(from)
+	assert.NoError(t, err)
+	return cfg
+}

libbeat/tests/system/base.py

@@ -9,4 +9,11 @@ def setUpClass(self):
         self.beat_name = "mockbeat"
         self.beat_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))
         self.test_binary = self.beat_path + "/libbeat.test"
+        self.beats = [
+            "filebeat",
+            "heartbeat",
+            "metricbeat",
+            "packetbeat",
+            "winlogbeat"
+        ]
         super(BaseTest, self).setUpClass()

libbeat/tests/system/test_dashboard.py

@@ -5,7 +5,7 @@
 import subprocess
 import unittest
 import re
-from nose.plugins.skip import Skip, SkipTest
+from nose.plugins.skip import SkipTest
 
 
 INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False)
@@ -20,9 +20,7 @@ def test_load_dashboard(self):
         """
         Test loading dashboards for all beats
         """
-        beats = ["metricbeat", "packetbeat", "filebeat", "winlogbeat"]
-
-        for beat in beats:
+        for beat in self.beats:
             command = "go run ../../../dev-tools/cmd/import_dashboards/import_dashboards.go -es http://" + \
                 self.get_elasticsearch_host() + " -dir ../../../" + beat + "/_meta/kibana"
 
@@ -51,9 +49,7 @@ def test_export_dashboard(self):
         # In addition, this test should not write to the beats directory but to a
         # temporary directory and check the files there.
 
-        beats = ["metricbeat", "packetbeat", "filebeat", "winlogbeat"]
-
-        for beat in beats:
+        for beat in self.beats:
             if os.name == "nt":
                 path = "..\..\..\\" + beat + "\etc\kibana"
             else:

libbeat/tests/system/test_template.py

@@ -0,0 +1,47 @@
+from base import BaseTest
+
+import os
+import json
+
+
+class Test(BaseTest):
+
+    def test_generate_templates(self):
+        """
+        Generates templates from other Beats.
+        """
+        self.render_config_template()
+
+        output_json = os.path.join(self.working_dir, "template.json")
+        fields_yml = "../../../../fields.yml"
+
+        exit_code = self.run_beat(extra_args=[
+            "-E", "setup.template.output_to_file.path={}".format(output_json),
+            "-E", "setup.template.fields={}".format(fields_yml)])
+        assert exit_code == 1
+
+        # check json file
+        with open(output_json) as f:
+            tmpl = json.load(f)
+        assert "mappings" in tmpl
+
+    def test_generate_templates_v5(self):
+        """
+        Generates templates from other Beats.
+        """
+        self.render_config_template()
+
+        output_json = os.path.join(self.working_dir, "template-5x.json")
+        fields_yml = "../../../../fields.yml"
+
+        exit_code = self.run_beat(extra_args=[
+            "-E", "setup.template.output_to_file.path={}".format(output_json),
+            "-E", "setup.template.output_to_file.version=5.0.0".format(output_json),
+            "-E", "setup.template.fields={}".format(fields_yml)])
+        assert exit_code == 1
+
+        # check json file
+        with open(output_json) as f:
+            tmpl = json.load(f)
+        assert "mappings" in tmpl
+        assert tmpl["mappings"]["_default_"]["_all"]["norms"]["enabled"] is False