Merge remote-tracking branch 'upstream/master' into feature/ci-pipeli…

…ne-2.0

* upstream/master:
  [Metricbeat][test] Disable ec2 flaky test (elastic#20959)
  Check if tracer is active before starting a transaction (elastic#20852)
  [Elastic Agent] Add support for variable replacement from providers (elastic#20839)
  Only request wildcard expansion for hidden indices if supported (elastic#20938)
  [Ingest Manager] New agent structure (symlinks) (elastic#20400)
  [Ingest Manager] Print a message confirming shutdown (elastic#20948)
  Skip flaky test on unix input (elastic#20942)
  [Ingest Manager] Align introspect-inspect naming in code (elastic#20952)
  [Filebeat][zeek] Map new x509 fields for ssl module (elastic#20927)
  [CI] fix regression with variable name (elastic#20930)
  [Autodiscover] Handle input-not-finished errors in config reload (elastic#20915)
  [Ingest Manager] Remove Success from fleet contract (elastic#20449)

.ci/packaging.groovy

@@ -252,7 +252,7 @@ def publishPackages(baseDir){
 * baseDir=name1/name2/name3-> return name2
 */
 def getBeatsName(baseDir) {
-  return basedir.replace('x-pack/', '')
+  return baseDir.replace('x-pack/', '')
 }
 
 def withBeatsEnv(Closure body) {

CHANGELOG.next.asciidoc

@@ -168,6 +168,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - [Metricbeat][Kubernetes] Change cluster_ip field from ip to keyword. {pull}20571[20571]
 - Rename cloud.provider `az` value to `azure` inside the add_cloud_metadata processor. {pull}20689[20689]
 - Add missing country_name geo field in `add_host_metadata` and `add_observer_metadata` processors. {issue}20796[20796] {pull}20811[20811]
+- [Autodiscover] Handle input-not-finished errors in config reload. {pull}20915[20915]
 - Explicitly detect missing variables in autodiscover configuration, log them at the debug level. {issue}20568[20568] {pull}20898[20898]
 
 *Auditbeat*
@@ -337,6 +338,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix resource tags in aws cloudwatch metricset {issue}20326[20326]  {pull}20385[20385]
 - Fix ec2 disk and network metrics to use Sum statistic method. {pull}20680[20680]
 - Fill cloud.account.name with accountID if account alias doesn't exist. {pull}20736[20736]
+- The `elasticsearch/index` metricset only requests wildcard expansion for hidden indices if the monitored Elasticsearch cluster supports it. {pull}20938[20938]
 
 *Packetbeat*
 
@@ -551,6 +553,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Avoid goroutine leaks in Filebeat readers. {issue}19193[19193] {pull}20455[20455]
 - Convert httpjson to v2 input {pull}20226[20226]
 - Improve Zeek x509 module with `x509` ECS mappings {pull}20867[20867]
+- Improve Zeek SSL module with `x509` ECS mappings {pull}20927[20927]
 
 *Heartbeat*
 

NOTICE.txt

@@ -13971,11 +13971,11 @@ Contents of probable licence file $GOMODCACHE/github.com/xdg/scram@v0.0.0-201808
 
 --------------------------------------------------------------------------------
 Dependency : go.elastic.co/apm
-Version: v1.7.2
+Version: v1.8.1-0.20200902013556-b34fe04da73f
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/go.elastic.co/apm@v1.7.2/LICENSE:
+Contents of probable licence file $GOMODCACHE/go.elastic.co/apm@v1.8.1-0.20200902013556-b34fe04da73f/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -37344,11 +37344,11 @@ SOFTWARE.
 
 --------------------------------------------------------------------------------
 Dependency : go.elastic.co/fastjson
-Version: v1.0.0
+Version: v1.1.0
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/go.elastic.co/fastjson@v1.0.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/go.elastic.co/fastjson@v1.1.0/LICENSE:
 
 Copyright 2018 Elasticsearch BV
 

dev-tools/mage/dmgbuilder.go

@@ -109,6 +109,11 @@ func (b *dmgBuilder) buildBeatPkg() error {
 
 	// Copy files into the packaging root and set their mode.
 	for _, f := range b.Files {
+		if f.Symlink {
+			// not supported, handling symlink in post/pre install scripts
+			continue
+		}
+
 		target := filepath.Join(beatPkgRoot, f.Target)
 		if err := Copy(f.Source, target); err != nil {
 			if f.SkipOnMissing && errors.Is(err, os.ErrNotExist) {

dev-tools/mage/pkgtypes.go

@@ -108,6 +108,7 @@ type PackageFile struct {
 	Dep           func(PackageSpec) error `yaml:"-" hash:"-" json:"-"`       // Dependency to invoke during Evaluate.
 	Owner         string                  `yaml:"owner,omitempty"`           // File Owner, for user and group name (rpm only).
 	SkipOnMissing bool                    `yaml:"skip_on_missing,omitempty"` // Prevents build failure if the file is missing.
+	Symlink       bool                    `yaml:"symlink"`                   // Symlink marks file as a symlink pointing from target to source.
 }
 
 // OSArchNames defines the names of architectures for use in packages.
@@ -476,6 +477,10 @@ func copyInstallScript(spec PackageSpec, script string, local *string) error {
 		*local = strings.TrimSuffix(*local, ".tmpl")
 	}
 
+	if strings.HasSuffix(*local, "."+spec.Name) {
+		*local = strings.TrimSuffix(*local, "."+spec.Name)
+	}
+
 	if err := spec.ExpandFile(script, createDir(*local)); err != nil {
 		return errors.Wrap(err, "failed to copy install script to package dir")
 	}
@@ -539,6 +544,11 @@ func PackageZip(spec PackageSpec) error {
 
 	// Add files to zip.
 	for _, pkgFile := range spec.Files {
+		if pkgFile.Symlink {
+			// not supported on zip archives
+			continue
+		}
+
 		if err := addFileToZip(w, baseDir, pkgFile); err != nil {
 			p, _ := filepath.Abs(pkgFile.Source)
 			return errors.Wrapf(err, "failed adding file=%+v to zip", p)
@@ -584,11 +594,32 @@ func PackageTarGz(spec PackageSpec) error {
 
 	// Add files to tar.
 	for _, pkgFile := range spec.Files {
+		if pkgFile.Symlink {
+			continue
+		}
+
 		if err := addFileToTar(w, baseDir, pkgFile); err != nil {
 			return errors.Wrapf(err, "failed adding file=%+v to tar", pkgFile)
 		}
 	}
 
+	// same for symlinks so they can point to files in tar
+	for _, pkgFile := range spec.Files {
+		if !pkgFile.Symlink {
+			continue
+		}
+
+		tmpdir, err := ioutil.TempDir("", "TmpSymlinkDropPath")
+		if err != nil {
+			return err
+		}
+		defer os.RemoveAll(tmpdir)
+
+		if err := addSymlinkToTar(tmpdir, w, baseDir, pkgFile); err != nil {
+			return errors.Wrapf(err, "failed adding file=%+v to tar", pkgFile)
+		}
+	}
+
 	if err := w.Close(); err != nil {
 		return err
 	}
@@ -882,6 +913,56 @@ func addFileToTar(ar *tar.Writer, baseDir string, pkgFile PackageFile) error {
 	})
 }
 
+// addSymlinkToTar adds a symlink file  to a tar archive.
+func addSymlinkToTar(tmpdir string, ar *tar.Writer, baseDir string, pkgFile PackageFile) error {
+	// create symlink we can work with later, header will be updated later
+	link := filepath.Join(tmpdir, "link")
+	target := tmpdir
+	if err := os.Symlink(target, link); err != nil {
+		return err
+	}
+
+	return filepath.Walk(link, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			if pkgFile.SkipOnMissing && os.IsNotExist(err) {
+				return nil
+			}
+
+			return err
+		}
+
+		header, err := tar.FileInfoHeader(info, info.Name())
+		if err != nil {
+			return err
+		}
+		header.Uname, header.Gname = "root", "root"
+		header.Uid, header.Gid = 0, 0
+
+		if info.Mode().IsRegular() && pkgFile.Mode > 0 {
+			header.Mode = int64(pkgFile.Mode & os.ModePerm)
+		} else if info.IsDir() {
+			header.Mode = int64(0755)
+		}
+
+		header.Name = filepath.Join(baseDir, pkgFile.Target)
+		if filepath.IsAbs(pkgFile.Target) {
+			header.Name = pkgFile.Target
+		}
+
+		header.Linkname = pkgFile.Source
+		header.Typeflag = tar.TypeSymlink
+
+		if mg.Verbose() {
+			log.Println("Adding", os.FileMode(header.Mode), header.Name)
+		}
+		if err := ar.WriteHeader(header); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
 // PackageDMG packages the Beat into a .dmg file containing an installer pkg
 // and uninstaller app.
 func PackageDMG(spec PackageSpec) error {

dev-tools/mage/settings.go

@@ -85,6 +85,7 @@ var (
 		"beat_doc_branch":   BeatDocBranch,
 		"beat_version":      BeatQualifiedVersion,
 		"commit":            CommitHash,
+		"commit_short":      CommitHashShort,
 		"date":              BuildDate,
 		"elastic_beats_dir": ElasticBeatsDir,
 		"go_version":        GoVersion,
@@ -239,6 +240,15 @@ func CommitHash() (string, error) {
 	return commitHash, err
 }
 
+// CommitHashShort returns the short length git commit hash.
+func CommitHashShort() (string, error) {
+	shortHash, err := CommitHash()
+	if len(shortHash) > 6 {
+		shortHash = shortHash[:6]
+	}
+	return shortHash, err
+}
+
 var (
 	elasticBeatsDirValue string
 	elasticBeatsDirErr   error