Update beats framework to 675f2b0 + ECS

NOTICE.txt

@@ -264,7 +264,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 --------------------------------------------------------------------
 Dependency: github.com/elastic/beats
 Version: master
-Revision: 122fb9364c9fc9d1d6508a864e132396180f9c38
+Revision: 675f2b000b067ad9fb2ee2fbe2bb3d98ecfab30c
 License type (autodetected): Apache-2.0
 ./vendor/github.com/elastic/beats/LICENSE.txt:
 --------------------------------------------------------------------
@@ -2378,7 +2378,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------
 Dependency: golang.org/x/tools
-Version: release-branch.go1.10
+Version: release-branch.go1.11
 Revision: release-branch.go1.10
 License type (autodetected): BSD-3-Clause
 ./vendor/golang.org/x/tools/LICENSE:

_beats/.go-version

@@ -1 +1 @@
-1.10.3
+1.11.2

_beats/dev-tools/cmd/dashboards/export_dashboards.go

@@ -89,6 +89,7 @@ func Export(client *http.Client, conn string, spaceID string, dashboard string,
 	for _, obj := range objects {
 		o := obj.(common.MapStr)
 
+		// All fields are optional, so errors are not catched
 		decodeValue(o, "attributes.uiStateJSON")
 		decodeValue(o, "attributes.visState")
 		decodeValue(o, "attributes.optionsJSON")
@@ -104,23 +105,27 @@ func Export(client *http.Client, conn string, spaceID string, dashboard string,
 		return err
 	}
 
-	err = ioutil.WriteFile(out, []byte(data.StringToPrint()), 0666)
+	err = ioutil.WriteFile(out, []byte(data.StringToPrint()), 0644)
 	if !quiet {
 		log.Printf("The dashboard %s was exported under the %s file\n", dashboard, out)
 	}
 	return err
 }
 
-func decodeValue(data common.MapStr, key string) {
+func decodeValue(data common.MapStr, key string) error {
 	v, err := data.GetValue(key)
 	if err != nil {
-		return
+		return err
 	}
 	s := v.(string)
 	var d interface{}
-	json.Unmarshal([]byte(s), &d)
+	err = json.Unmarshal([]byte(s), &d)
+	if err != nil {
+		return fmt.Errorf("error decoding %s: %v", key, err)
+	}
 
 	data.Put(key, d)
+	return nil
 }
 
 func ReadManifest(file string) ([]map[string]string, error) {

_beats/dev-tools/cmd/kibana_index_pattern/kibana_index_pattern.go

@@ -31,7 +31,7 @@ var usageText = `
 Usage: kibana_index_pattern [flags]
   kibana_index_pattern generates Kibana index patterns from the Beat's
   fields.yml file. It will create a index pattern file that is usable with both
-  Kibana 5.x and 6.x.
+  Kibana 6.x and 7.x.
 Options:
 `[1:]
 
@@ -67,11 +67,12 @@ func main() {
 		log.Fatal("Index pattern must be set (-index).")
 	}
 
-	version5, _ := common.NewVersion("5.0.0")
-	version6, _ := common.NewVersion("6.0.0")
-	versions := []common.Version{*version5, *version6}
+	versions := []string{
+		"6.0.0",
+	}
 	for _, version := range versions {
-		indexPattern, err := kibana.NewGenerator(indexPattern, beatName, fieldsYAMLFile, outputDir, beatVersion, version)
+		version, _ := common.NewVersion(version)
+		indexPattern, err := kibana.NewGenerator(indexPattern, beatName, fieldsYAMLFile, outputDir, beatVersion, *version)
 		if err != nil {
 			log.Fatal(err)
 		}

_beats/dev-tools/ecs-migration.yml

@@ -85,8 +85,170 @@
   alias: true
   copy_to: false
 
+- from: system.syslog.hostname
+  to: host.hostname
+  alias: true
+  copy_to: false
+
+- from: system.syslog.program
+  to: process.name
+  alias: true
+  copy_to: false
+
+- from: system.syslog.pid
+  to: process.pid
+  alias: true
+  copy_to: false
+
+- from: system.syslog.message
+  to: message
+  alias: true
+  copy_to: false
+
 # From Auditbeat's auditd module.
 - from: source.hostname
   to: source.domain
   alias: true
   copy_to: false
+
+- from: iis.access.server_ip
+  to: destination.ip
+  alias: true
+  copy_to: false
+
+- from: iis.access.remote_ip
+  to: source.ip
+  alias: true
+  copy_to: false
+
+- from: iis.access.url
+  to: url.path
+  alias: true
+  copy_to: false
+
+- from: iis.access.query_string
+  to: url.query
+  alias: true
+  copy_to: false
+
+- from: iis.access.port
+  to: destination.port
+  alias: true
+  copy_to: false
+
+- from: iis.access.user_name
+  to: user.name
+  alias: true
+  copy_to: false
+
+- from: iis.access.hostname
+  to: destination.domain
+  alias: true
+  copy_to: false
+
+- from: iis.access.user_agent.original
+  to: user_agent.original
+  alias: true
+  copy_to: false
+
+- from: iis.access.geoip.continent_name
+  to: source.geo.continent_name
+  alias: true
+  copy_to: false
+
+- from: iis.access.geoip.country_iso_code
+  to: source.geo.country_iso_code
+  alias: true
+  copy_to: false
+
+- from: iis.access.geoip.location
+  to: source.geo.location
+  alias: true
+  copy_to: false
+
+- from: iis.access.geoip.region_name
+  to: source.geo.region_name
+  alias: true
+  copy_to: false
+
+- from: iis.access.geoip.city_name
+  to: source.geo.city_name
+  alias: true
+  copy_to: false
+
+- from: iis.access.geoip.region_iso_code
+  to: source.geo.region_iso_code
+  alias: true
+  copy_to: false
+
+# Note: `http` is not officially in ECS yet
+
+- from: iis.access.method
+  to: http.request.method
+  alias: true
+  copy_to: false
+
+- from: iis.access.response_code
+  to: http.response.status_code
+  alias: true
+  copy_to: false
+
+- from: iis.access.referrer
+  to: http.request.referrer
+  alias: true
+  copy_to: false
+
+- from: haproxy.client.port
+  to: source.port
+  alias: true
+  copy_to: false
+
+- from: haproxy.process_name
+  to: process.name
+  alias: true
+  copy_to: false
+
+- from: haproxy.pid
+  to: process.pid
+  alias: true
+  copy_to: false
+
+- from: haproxy.destination.ip
+  to: destination.ip
+  alias: true
+  copy_to: false
+
+- from: haproxy.destination.port
+  to: destination.port
+  alias: true
+  copy_to: false
+
+- from: haproxy.geoip.continent_name
+  to: source.geo.continent_name
+  alias: true
+  copy_to: false
+
+- from: haproxy.geoip.country_iso_code
+  to: source.geo.country_iso_code
+  alias: true
+  copy_to: false
+
+- from: haproxy.geoip.location
+  to: source.geo.location
+  alias: true
+  copy_to: false
+
+- from: haproxy.geoip.region_name
+  to: source.geo.region_name
+  alias: true
+  copy_to: false
+
+- from: haproxy.geoip.city_name
+  to: source.geo.city_name
+  alias: true
+  copy_to: false
+
+- from: haproxy.geoip.region_iso_code
+  to: source.geo.region_iso_code
+  alias: true
+  copy_to: false

_beats/dev-tools/jenkins_release.sh

@@ -18,7 +18,6 @@ trap cleanup EXIT
 
 # This controls the defaults used the Jenkins package job. They can be
 # overridden by setting them in the environement prior to running this script.
-export BEAT_VERSION_QUALIFIER="${VERSION_QUALIFIER:-}"
 export SNAPSHOT="${SNAPSHOT:-true}"
 export PLATFORMS="${PLATFORMS:-+linux/armv7 +linux/ppc64le +linux/s390x +linux/mips64}"
 

_beats/dev-tools/make/mage.mk

@@ -0,0 +1,11 @@
+MAGE_PRESENT := $(shell command -v mage 2> /dev/null)
+MAGE_IMPORT_PATH?=github.com/elastic/beats/vendor/github.com/magefile/mage
+export MAGE_IMPORT_PATH
+
+.PHONY: mage
+mage:
+ifndef MAGE_PRESENT
+	go install ${MAGE_IMPORT_PATH}
+	@-mage -clean 2> /dev/null
+endif
+	@true

_beats/dev-tools/make/xpack.mk

@@ -0,0 +1,38 @@
+# This Makefile is for x-pack Beats. Its only responsibility is to provide
+# compatibility with existing Jenkins and Travis setups.
+
+#
+# Variables
+#
+PWD           := $(CURDIR)
+.DEFAULT_GOAL := help
+
+#
+# Includes
+#
+include $(ES_BEATS)/dev-tools/make/mage.mk
+
+#
+# Targets
+#
+.PHONY: clean
+clean: mage
+	mage clean
+
+.PHONY: fmt
+fmt: mage
+	mage fmt
+
+.PHONY: check
+check: mage
+	mage check
+
+.PHONY: testsuite
+testsuite: mage
+	mage update build unitTest integTest
+
+# Default target.
+.PHONY: help
+help:
+	@echo Use mage rather than make. Here are the available mage targets:
+	@mage -l

_beats/dev-tools/packaging/package_test.go

@@ -80,7 +80,8 @@ func TestDeb(t *testing.T) {
 }
 
 func TestTar(t *testing.T) {
-	tars := getFiles(t, regexp.MustCompile(`\.tar\.gz$`))
+	// Regexp matches *-arch.tar.gz, but not *-arch.docker.tar.gz
+	tars := getFiles(t, regexp.MustCompile(`-\w+\.tar\.gz$`))
 	for _, tar := range tars {
 		checkTar(t, tar)
 	}
@@ -94,7 +95,7 @@ func TestZip(t *testing.T) {
 }
 
 func TestDocker(t *testing.T) {
-	dockers := getFiles(t, regexp.MustCompile(`\.docker.tar$`))
+	dockers := getFiles(t, regexp.MustCompile(`\.docker\.tar\.gz$`))
 	for _, docker := range dockers {
 		checkDocker(t, docker)
 	}
@@ -514,7 +515,13 @@ func readDocker(dockerFile string) (*packageFile, *dockerInfo, error) {
 	var info *dockerInfo
 	layers := make(map[string]*packageFile)
 
-	tarReader := tar.NewReader(file)
+	gzipReader, err := gzip.NewReader(file)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer gzipReader.Close()
+
+	tarReader := tar.NewReader(gzipReader)
 	for {
 		header, err := tarReader.Next()
 		if err != nil {