diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy
index 57a99a74f63..2be78aac68f 100644
--- a/.ci/packaging.groovy
+++ b/.ci/packaging.groovy
@@ -252,7 +252,7 @@ def publishPackages(baseDir){
 * baseDir=name1/name2/name3-> return name2
 */
 def getBeatsName(baseDir) {
-  return basedir.replace('x-pack/', '')
+  return baseDir.replace('x-pack/', '')
 }
 
 def withBeatsEnv(Closure body) {
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
index 37f98a01533..ec86340f3d1 100644
--- a/CHANGELOG.next.asciidoc
+++ b/CHANGELOG.next.asciidoc
@@ -168,6 +168,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - [Metricbeat][Kubernetes] Change cluster_ip field from ip to keyword. {pull}20571[20571]
 - Rename cloud.provider `az` value to `azure` inside the add_cloud_metadata processor. {pull}20689[20689]
 - Add missing country_name geo field in `add_host_metadata` and `add_observer_metadata` processors. {issue}20796[20796] {pull}20811[20811]
+- [Autodiscover] Handle input-not-finished errors in config reload. {pull}20915[20915]
 - Explicitly detect missing variables in autodiscover configuration, log them at the debug level. {issue}20568[20568] {pull}20898[20898]
 
 *Auditbeat*
@@ -337,6 +338,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix resource tags in aws cloudwatch metricset {issue}20326[20326]  {pull}20385[20385]
 - Fix ec2 disk and network metrics to use Sum statistic method. {pull}20680[20680]
 - Fill cloud.account.name with accountID if account alias doesn't exist. {pull}20736[20736]
+- The `elasticsearch/index` metricset only requests wildcard expansion for hidden indices if the monitored Elasticsearch cluster supports it. {pull}20938[20938]
 
 *Packetbeat*
 
@@ -551,6 +553,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Avoid goroutine leaks in Filebeat readers. {issue}19193[19193] {pull}20455[20455]
 - Convert httpjson to v2 input {pull}20226[20226]
 - Improve Zeek x509 module with `x509` ECS mappings {pull}20867[20867]
+- Improve Zeek SSL module with `x509` ECS mappings {pull}20927[20927]
 
 *Heartbeat*
 
diff --git a/NOTICE.txt b/NOTICE.txt
index d7f5b0d8b14..33eaad14ad7 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -13971,11 +13971,11 @@ Contents of probable licence file $GOMODCACHE/github.com/xdg/scram@v0.0.0-201808
 
 --------------------------------------------------------------------------------
 Dependency : go.elastic.co/apm
-Version: v1.7.2
+Version: v1.8.1-0.20200902013556-b34fe04da73f
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/go.elastic.co/apm@v1.7.2/LICENSE:
+Contents of probable licence file $GOMODCACHE/go.elastic.co/apm@v1.8.1-0.20200902013556-b34fe04da73f/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -37344,11 +37344,11 @@ SOFTWARE.
 
 --------------------------------------------------------------------------------
 Dependency : go.elastic.co/fastjson
-Version: v1.0.0
+Version: v1.1.0
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/go.elastic.co/fastjson@v1.0.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/go.elastic.co/fastjson@v1.1.0/LICENSE:
 
 Copyright 2018 Elasticsearch BV
 
diff --git a/dev-tools/mage/dmgbuilder.go b/dev-tools/mage/dmgbuilder.go
index 06041c74533..47a1125a5a6 100644
--- a/dev-tools/mage/dmgbuilder.go
+++ b/dev-tools/mage/dmgbuilder.go
@@ -109,6 +109,11 @@ func (b *dmgBuilder) buildBeatPkg() error {
 
 	// Copy files into the packaging root and set their mode.
 	for _, f := range b.Files {
+		if f.Symlink {
+			// not supported, handling symlink in post/pre install scripts
+			continue
+		}
+
 		target := filepath.Join(beatPkgRoot, f.Target)
 		if err := Copy(f.Source, target); err != nil {
 			if f.SkipOnMissing && errors.Is(err, os.ErrNotExist) {
diff --git a/dev-tools/mage/pkgtypes.go b/dev-tools/mage/pkgtypes.go
index 5800d5b5c18..b7f7c7bbbee 100644
--- a/dev-tools/mage/pkgtypes.go
+++ b/dev-tools/mage/pkgtypes.go
@@ -108,6 +108,7 @@ type PackageFile struct {
 	Dep           func(PackageSpec) error `yaml:"-" hash:"-" json:"-"`       // Dependency to invoke during Evaluate.
 	Owner         string                  `yaml:"owner,omitempty"`           // File Owner, for user and group name (rpm only).
 	SkipOnMissing bool                    `yaml:"skip_on_missing,omitempty"` // Prevents build failure if the file is missing.
+	Symlink       bool                    `yaml:"symlink"`                   // Symlink marks file as a symlink pointing from target to source.
 }
 
 // OSArchNames defines the names of architectures for use in packages.
@@ -476,6 +477,10 @@ func copyInstallScript(spec PackageSpec, script string, local *string) error {
 		*local = strings.TrimSuffix(*local, ".tmpl")
 	}
 
+	if strings.HasSuffix(*local, "."+spec.Name) {
+		*local = strings.TrimSuffix(*local, "."+spec.Name)
+	}
+
 	if err := spec.ExpandFile(script, createDir(*local)); err != nil {
 		return errors.Wrap(err, "failed to copy install script to package dir")
 	}
@@ -539,6 +544,11 @@ func PackageZip(spec PackageSpec) error {
 
 	// Add files to zip.
 	for _, pkgFile := range spec.Files {
+		if pkgFile.Symlink {
+			// not supported on zip archives
+			continue
+		}
+
 		if err := addFileToZip(w, baseDir, pkgFile); err != nil {
 			p, _ := filepath.Abs(pkgFile.Source)
 			return errors.Wrapf(err, "failed adding file=%+v to zip", p)
@@ -584,11 +594,32 @@ func PackageTarGz(spec PackageSpec) error {
 
 	// Add files to tar.
 	for _, pkgFile := range spec.Files {
+		if pkgFile.Symlink {
+			continue
+		}
+
 		if err := addFileToTar(w, baseDir, pkgFile); err != nil {
 			return errors.Wrapf(err, "failed adding file=%+v to tar", pkgFile)
 		}
 	}
 
+	// same for symlinks so they can point to files in tar
+	for _, pkgFile := range spec.Files {
+		if !pkgFile.Symlink {
+			continue
+		}
+
+		tmpdir, err := ioutil.TempDir("", "TmpSymlinkDropPath")
+		if err != nil {
+			return err
+		}
+		defer os.RemoveAll(tmpdir)
+
+		if err := addSymlinkToTar(tmpdir, w, baseDir, pkgFile); err != nil {
+			return errors.Wrapf(err, "failed adding file=%+v to tar", pkgFile)
+		}
+	}
+
 	if err := w.Close(); err != nil {
 		return err
 	}
@@ -882,6 +913,56 @@ func addFileToTar(ar *tar.Writer, baseDir string, pkgFile PackageFile) error {
 	})
 }
 
+// addSymlinkToTar adds a symlink file  to a tar archive.
+func addSymlinkToTar(tmpdir string, ar *tar.Writer, baseDir string, pkgFile PackageFile) error {
+	// create symlink we can work with later, header will be updated later
+	link := filepath.Join(tmpdir, "link")
+	target := tmpdir
+	if err := os.Symlink(target, link); err != nil {
+		return err
+	}
+
+	return filepath.Walk(link, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			if pkgFile.SkipOnMissing && os.IsNotExist(err) {
+				return nil
+			}
+
+			return err
+		}
+
+		header, err := tar.FileInfoHeader(info, info.Name())
+		if err != nil {
+			return err
+		}
+		header.Uname, header.Gname = "root", "root"
+		header.Uid, header.Gid = 0, 0
+
+		if info.Mode().IsRegular() && pkgFile.Mode > 0 {
+			header.Mode = int64(pkgFile.Mode & os.ModePerm)
+		} else if info.IsDir() {
+			header.Mode = int64(0755)
+		}
+
+		header.Name = filepath.Join(baseDir, pkgFile.Target)
+		if filepath.IsAbs(pkgFile.Target) {
+			header.Name = pkgFile.Target
+		}
+
+		header.Linkname = pkgFile.Source
+		header.Typeflag = tar.TypeSymlink
+
+		if mg.Verbose() {
+			log.Println("Adding", os.FileMode(header.Mode), header.Name)
+		}
+		if err := ar.WriteHeader(header); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
 // PackageDMG packages the Beat into a .dmg file containing an installer pkg
 // and uninstaller app.
 func PackageDMG(spec PackageSpec) error {
diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go
index f7de61e7db0..08a619c3df3 100644
--- a/dev-tools/mage/settings.go
+++ b/dev-tools/mage/settings.go
@@ -85,6 +85,7 @@ var (
 		"beat_doc_branch":   BeatDocBranch,
 		"beat_version":      BeatQualifiedVersion,
 		"commit":            CommitHash,
+		"commit_short":      CommitHashShort,
 		"date":              BuildDate,
 		"elastic_beats_dir": ElasticBeatsDir,
 		"go_version":        GoVersion,
@@ -239,6 +240,15 @@ func CommitHash() (string, error) {
 	return commitHash, err
 }
 
+// CommitHashShort returns the short length git commit hash.
+func CommitHashShort() (string, error) {
+	shortHash, err := CommitHash()
+	if len(shortHash) > 6 {
+		shortHash = shortHash[:6]
+	}
+	return shortHash, err
+}
+
 var (
 	elasticBeatsDirValue string
 	elasticBeatsDirErr   error
diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml
index f4261945233..2dcfba0f8a3 100644
--- a/dev-tools/packaging/packages.yml
+++ b/dev-tools/packaging/packages.yml
@@ -45,6 +45,10 @@ shared:
         source: 'elastic-agent.yml'
         mode: 0600
         config: true
+      /etc/{{.BeatName}}/.elastic-agent.active.commit:
+        content: >
+          {{ commit }}
+        mode: 0644
       /usr/share/{{.BeatName}}/bin/{{.BeatName}}-god:
         source: build/golang-crossbuild/god-{{.GOOS}}-{{.Platform.Arch}}
         mode: 0755
@@ -57,45 +61,48 @@ shared:
       /etc/init.d/{{.BeatServiceName}}:
         template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/{{.PackageType}}/init.sh.tmpl'
         mode: 0755
-      /var/lib/{{.BeatName}}/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/{{.BeatName}}{{.BinaryExt}}:
+        source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+        mode: 0644
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
         source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
-      /var/lib/{{.BeatName}}/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512:
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512:
         source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512'
         mode: 0644
-      /var/lib/{{.BeatName}}/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc:
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc:
         source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
         mode: 0644
         skip_on_missing: true
-      /var/lib/{{.BeatName}}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
         source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
-      /var/lib/{{.BeatName}}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512:
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512:
         source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512'
         mode: 0644
-      /var/lib/{{.BeatName}}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc:
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc:
         source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
         mode: 0644
         skip_on_missing: true
-      /var/lib/{{.BeatName}}/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
         source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
-      /var/lib/{{.BeatName}}/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512:
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512:
         source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512'
         mode: 0644
-      /var/lib/{{.BeatName}}/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc:
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc:
         source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
         mode: 0644
         skip_on_missing: true
-      /var/lib/{{.BeatName}}/downloads/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
         source: '{{.AgentDropPath}}/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
         skip_on_missing: true
-      /var/lib/{{.BeatName}}/downloads/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512:
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512:
         source: '{{.AgentDropPath}}/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512'
         mode: 0644
         skip_on_missing: true
-      /var/lib/{{.BeatName}}/downloads/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc:
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc:
         source: '{{.AgentDropPath}}/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
         mode: 0644
         skip_on_missing: true
@@ -112,7 +119,7 @@ shared:
       identifier: 'co.{{.BeatVendor | tolower}}.beats.{{.BeatName}}'
       install_path: /Library/Application Support
     pre_install_script: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/darwin/scripts/preinstall.tmpl'
-    post_install_script: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/darwin/scripts/postinstall.tmpl'
+    post_install_script: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/darwin/scripts/postinstall.elastic-agent.tmpl'
     files:
       /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/bin/{{.BeatName}}{{.BinaryExt}}:
         source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
@@ -140,47 +147,54 @@ shared:
         source: 'elastic-agent.yml'
         mode: 0600
         config: true
-      /etc/{{.BeatName}}/data/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+      /etc/{{.BeatName}}/.elastic-agent.active.commit:
+        content: >
+          {{ commit }}
+        mode: 0644
+      /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/{{.BeatName}}{{.BinaryExt}}:
+        source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+        mode: 0755
+      /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
         source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
-      /etc/{{.BeatName}}/data/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512:
+      /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512:
         source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512'
         mode: 0644
         skip_on_missing: true
-      /etc/{{.BeatName}}/data/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc:
+      /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc:
         source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
         mode: 0644
         skip_on_missing: true
-      /etc/{{.BeatName}}/data/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+      /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
         source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
-      /etc/{{.BeatName}}/data/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512:
+      /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512:
         source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512'
         mode: 0644
         skip_on_missing: true
-      /etc/{{.BeatName}}/data/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc:
+      /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc:
         source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
         mode: 0644
         skip_on_missing: true
-      /etc/{{.BeatName}}/data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+      /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
         source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
-      /etc/{{.BeatName}}/data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512:
+      /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512:
         source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512'
         mode: 0644
-      /etc/{{.BeatName}}/data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc:
+      /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc:
         source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
         mode: 0644
         skip_on_missing: true
-      /etc/{{.BeatName}}/data/downloads/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+      /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
         source: '{{.AgentDropPath}}/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
         skip_on_missing: true
-      /etc/{{.BeatName}}/data/downloads/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512:
+      /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512:
         source: '{{.AgentDropPath}}/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512'
         mode: 0644
         skip_on_missing: true
-      /etc/{{.BeatName}}/data/downloads/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc:
+      /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc:
         source: '{{.AgentDropPath}}/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
         mode: 0644
         skip_on_missing: true
@@ -189,6 +203,9 @@ shared:
     '{{.BeatName}}{{.BinaryExt}}':
       source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
       mode: 0755
+    'data/{{.BeatName}}-{{ commit_short }}/{{.BeatName}}{{.BinaryExt}}':
+      source: build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+      mode: 0755
     LICENSE.txt:
       source: '{{ repo.RootDir }}/LICENSE.txt'
       mode: 0644
@@ -209,51 +226,55 @@ shared:
       source: 'elastic-agent.yml'
       mode: 0600
       config: true
+    '.elastic-agent.active.commit':
+      content: >
+        {{ commit }}
+      mode: 0644
 
   # Binary package spec (tar.gz for linux/darwin) for community beats.
   - &agent_binary_spec
     <<: *common
     files:
       <<: *agent_binary_files
-      'data/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
         source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
-      'data/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512':
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512':
         source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512'
         mode: 0644
-      'data/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc':
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc':
         source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
         mode: 0644
         skip_on_missing: true
-      'data/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
         source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
-      'data/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512':
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512':
         source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512'
         mode: 0644
-      'data/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc':
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc':
         source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
         mode: 0644
         skip_on_missing: true
-      'data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
         source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
-      'data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512':
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512':
         source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512'
         mode: 0644
-      'data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc':
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc':
         source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
         mode: 0644
         skip_on_missing: true
-      'data/downloads/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
         source: '{{.AgentDropPath}}/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0644
         skip_on_missing: true
-      'data/downloads/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512':
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512':
         source: '{{.AgentDropPath}}/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512'
         mode: 0644
         skip_on_missing: true
-      'data/downloads/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc':
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc':
         source: '{{.AgentDropPath}}/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
         mode: 0644
         skip_on_missing: true
@@ -269,45 +290,45 @@ shared:
       uninstall-service-{{.BeatName}}.ps1:
         template: '{{ elastic_beats_dir }}/dev-tools/packaging/templates/windows/uninstall-service.ps1.tmpl'
         mode: 0755
-      'data/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip':
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip':
         source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip'
         mode: 0644
-      'data/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512':
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512':
         source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512'
         mode: 0644
-      'data/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc':
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc':
         source: '{{.AgentDropPath}}/filebeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc'
         mode: 0644
         skip_on_missing: true
-      'data/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip':
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip':
         source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip'
         mode: 0644
-      'data/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512':
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512':
         source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512'
         mode: 0644
-      'data/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc':
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc':
         source: '{{.AgentDropPath}}/heartbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc'
         mode: 0644
         skip_on_missing: true
-      'data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip':
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip':
         source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip'
         mode: 0644
-      'data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512':
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512':
         source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512'
         mode: 0644
-      'data/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc':
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc':
         source: '{{.AgentDropPath}}/metricbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc'
         mode: 0644
         skip_on_missing: true
-      'data/downloads/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip':
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip':
         source: '{{.AgentDropPath}}/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip'
         mode: 0644
         skip_on_missing: true
-      'data/downloads/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512':
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512':
         source: '{{.AgentDropPath}}/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512'
         mode: 0644
         skip_on_missing: true
-      'data/downloads/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc':
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc':
         source: '{{.AgentDropPath}}/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc'
         mode: 0644
         skip_on_missing: true
@@ -326,6 +347,10 @@ shared:
         source: 'elastic-agent.docker.yml'
         mode: 0600
         config: true
+      '.elastic-agent.active.commit':
+        content: >
+          {{ commit }}
+        mode: 0644
 
   # Deb/RPM spec for community beats.
   - &deb_rpm_spec
@@ -779,7 +804,8 @@ specs:
         <<: *elastic_license_for_binaries
         files:
           '{{.BeatName}}{{.BinaryExt}}':
-            source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+            source: data/{{.BeatName}}-{{ commit_short }}/{{.BeatName}}{{.BinaryExt}}
+            symlink: true
 
     - os: darwin
       types: [dmg]
@@ -789,7 +815,8 @@ specs:
         files:
           /Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/bin/{{.BeatName}}{{.BinaryExt}}:
             mode: 0755
-            source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+            source: /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/{{.BeatName}}{{.BinaryExt}}
+            symlink: true
 
     - os: linux
       types: [tgz]
@@ -798,7 +825,9 @@ specs:
         <<: *elastic_license_for_binaries
         files:
           '{{.BeatName}}{{.BinaryExt}}':
-            source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+            source: data/{{.BeatName}}-{{ commit_short }}/{{.BeatName}}{{.BinaryExt}}
+            symlink: true
+            mode: 0755
 
     - os: linux
       types: [deb, rpm]
@@ -807,7 +836,8 @@ specs:
         <<: *elastic_license_for_deb_rpm
         files:
           /usr/share/{{.BeatName}}/bin/{{.BeatName}}{{.BinaryExt}}:
-            source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+            source: /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/{{.BeatName}}{{.BinaryExt}}
+            symlink: true
 
     - os: linux
       types: [docker]
diff --git a/dev-tools/packaging/templates/darwin/scripts/postinstall.elastic-agent.tmpl b/dev-tools/packaging/templates/darwin/scripts/postinstall.elastic-agent.tmpl
new file mode 100644
index 00000000000..2a9549b1d3e
--- /dev/null
+++ b/dev-tools/packaging/templates/darwin/scripts/postinstall.elastic-agent.tmpl
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+BEAT_NAME="{{.BeatName}}"
+VERSION="{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}"
+SCRIPT="postinstall"
+INSTALL_DIR="{{.install_path}}/{{.BeatVendor}}/{{.BeatName}}"
+IDENTIFIER="{{.identifier}}"
+VERSIONED_EXECUTABLE="/etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/{{.BeatName}}{{.BinaryExt}}"
+EXE_ROOT="/Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/bin"
+EXE_NAME="{{.BeatName}}{{.BinaryExt}}"
+
+log() {
+    LEVEL="$1"; shift
+    syslog -s -l "$LEVEL" "$BEAT_NAME $SCRIPT: $@"
+}
+
+die() {
+    log ERROR "Failed: $@"
+}
+
+log WARN "identifier: $IDENTIFIER"
+log WARN "version: $VERSION"
+log WARN "install_dir: $INSTALL_DIR"
+
+mkdir -p "$EXE_ROOT" || die "Unable to create $BEAT_NAME bin directory"
+ln -s "$VERSIONED_EXECUTABLE" "$EXE_ROOT/$EXE_NAME" || die "Unable to create $BEAT_NAME symlink"
+
+DAEMON_PLIST="/Library/LaunchDaemons/$IDENTIFIER.plist"
+launchctl unload -w "$DAEMON_PLIST"
+rm -f "$DAEMON_PLIST"
+ln -s "$INSTALL_DIR/$IDENTIFIER.plist" "$DAEMON_PLIST" || die "Unable to create $DAEMON_PLIST symlink"
+launchctl load -w "$DAEMON_PLIST" || die "Unable to install launchctl daemon $DAEMON_PLIST"
diff --git a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl
index a7242baa73b..5e6c0fcd6cd 100644
--- a/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl
+++ b/dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl
@@ -8,18 +8,20 @@ FROM {{ .buildFrom }} AS home
 
 COPY beat {{ $beatHome }}
 
-RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/logs && \
+RUN mkdir -p {{ $beatHome }}/data {{ $beatHome }}/data/elastic-agent-{{ commit_short }}/logs && \
     chown -R root:root {{ $beatHome }} && \
     find {{ $beatHome }} -type d -exec chmod 0750 {} \; && \
     find {{ $beatHome }} -type f -exec chmod 0640 {} \; && \
-    chmod 0750 {{ $beatBinary }} && \
+    rm {{ $beatBinary }} && \
+    ln -s {{ $beatHome }}/data/elastic-agent-{{ commit_short }}/elastic-agent {{ $beatBinary }} && \
+    chmod 0750 {{ $beatHome }}/data/elastic-agent-*/elastic-agent && \
 {{- if .linux_capabilities }}
     setcap {{ .linux_capabilities }} {{ $beatBinary }} && \
 {{- end }}
 {{- range $i, $modulesd := .ModulesDirs }}
     chmod 0770 {{ $beatHome}}/{{ $modulesd }} && \
 {{- end }}
-    chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/logs
+    chmod 0770 {{ $beatHome }}/data {{ $beatHome }}/data/elastic-agent-{{ commit_short }}/logs
 
 FROM {{ .from }}
 
diff --git a/filebeat/input/log/input.go b/filebeat/input/log/input.go
index a1837bbd471..365da416ed3 100644
--- a/filebeat/input/log/input.go
+++ b/filebeat/input/log/input.go
@@ -175,7 +175,7 @@ func (p *Input) loadStates(states []file.State) error {
 
 			// In case a input is tried to be started with an unfinished state matching the glob pattern
 			if !state.Finished {
-				return &input.ErrInputNotFinished{State: state.String()}
+				return &common.ErrInputNotFinished{State: state.String()}
 			}
 
 			// Convert state to current identifier if different
diff --git a/filebeat/input/runnerfactory.go b/filebeat/input/runnerfactory.go
index afd67d4e08a..f4973e47948 100644
--- a/filebeat/input/runnerfactory.go
+++ b/filebeat/input/runnerfactory.go
@@ -59,7 +59,7 @@ func (r *RunnerFactory) Create(
 
 func (r *RunnerFactory) CheckConfig(cfg *common.Config) error {
 	_, err := r.Create(pipeline.NewNilPipeline(), cfg)
-	if _, ok := err.(*ErrInputNotFinished); ok {
+	if _, ok := err.(*common.ErrInputNotFinished); ok {
 		// error is related to state, and hence config can be considered valid
 		return nil
 	}
diff --git a/filebeat/tests/system/test_unix.py b/filebeat/tests/system/test_unix.py
index 66d261f3c91..bb9b7f25bd5 100644
--- a/filebeat/tests/system/test_unix.py
+++ b/filebeat/tests/system/test_unix.py
@@ -1,14 +1,16 @@
-from filebeat import BaseTest
 import os
+import platform
 import socket
 import tempfile
 import unittest
+from filebeat import BaseTest
 
 # AF_UNIX support in python isn't available until
 # Python 3.9, see https://bugs.python.org/issue33408
 
 
 @unittest.skipIf(not hasattr(socket, 'AF_UNIX'), "No Windows AF_UNIX support before Python 3.9")
+@unittest.skipIf(platform.system() == 'Darwin', 'Flaky test: https://github.com/elastic/beats/issues/20941')
 class Test(BaseTest):
     """
     Test filebeat UNIX input
diff --git a/go.mod b/go.mod
index b832a3ac892..dbaf85775f5 100644
--- a/go.mod
+++ b/go.mod
@@ -151,7 +151,7 @@ require (
 	github.com/vmware/govmomi v0.0.0-20170802214208-2cad15190b41
 	github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c
 	github.com/yuin/gopher-lua v0.0.0-20170403160031-b402f3114ec7 // indirect
-	go.elastic.co/apm v1.7.2
+	go.elastic.co/apm v1.8.1-0.20200902013556-b34fe04da73f
 	go.elastic.co/apm/module/apmelasticsearch v1.7.2
 	go.elastic.co/apm/module/apmhttp v1.7.2
 	go.elastic.co/ecszap v0.1.1-0.20200424093508-cdd95a104193
diff --git a/go.sum b/go.sum
index 271434f695a..9fec14d5640 100644
--- a/go.sum
+++ b/go.sum
@@ -699,6 +699,8 @@ github.com/yuin/gopher-lua v0.0.0-20170403160031-b402f3114ec7 h1:0gYLpmzecnaDCoe
 github.com/yuin/gopher-lua v0.0.0-20170403160031-b402f3114ec7/go.mod h1:aEV29XrmTYFr3CiRxZeGHpkvbwq+prZduBqMaascyCU=
 go.elastic.co/apm v1.7.2 h1:0nwzVIPp4PDBXSYYtN19+1W5V+sj+C25UjqxDVoKcA8=
 go.elastic.co/apm v1.7.2/go.mod h1:tCw6CkOJgkWnzEthFN9HUP1uL3Gjc/Ur6m7gRPLaoH0=
+go.elastic.co/apm v1.8.1-0.20200902013556-b34fe04da73f h1:Z5e1eChG4ZcP0+jFtztaE5X2dOYRypQcOJGOUB4WgvA=
+go.elastic.co/apm v1.8.1-0.20200902013556-b34fe04da73f/go.mod h1:qoOSi09pnzJDh5fKnfY7bPmQgl8yl2tULdOu03xhui0=
 go.elastic.co/apm/module/apmelasticsearch v1.7.2 h1:5STGHLZLSeAzxordMc+dFVKiyVtMmxADOV+TgRaXXJg=
 go.elastic.co/apm/module/apmelasticsearch v1.7.2/go.mod h1:ZyNFuyWdt42GBZkz0SogoLzDBrBGj4orxpiUuxYeYq8=
 go.elastic.co/apm/module/apmhttp v1.7.2 h1:2mRh7SwBuEVLmJlX+hsMdcSg9xaielCLElaPn/+i34w=
@@ -707,6 +709,8 @@ go.elastic.co/ecszap v0.1.1-0.20200424093508-cdd95a104193 h1:NjYJ/beChqugXSavTkH
 go.elastic.co/ecszap v0.1.1-0.20200424093508-cdd95a104193/go.mod h1:HTUi+QRmr3EuZMqxPX+5fyOdMNfUu5iPebgfhgsTJYQ=
 go.elastic.co/fastjson v1.0.0 h1:ooXV/ABvf+tBul26jcVViPT3sBir0PvXgibYB1IQQzg=
 go.elastic.co/fastjson v1.0.0/go.mod h1:PmeUOMMtLHQr9ZS9J9owrAVg0FkaZDRZJEFTTGHtchs=
+go.elastic.co/fastjson v1.1.0 h1:3MrGBWWVIxe/xvsbpghtkFoPciPhOCmjsR/HfwEeQR4=
+go.elastic.co/fastjson v1.1.0/go.mod h1:boNGISWMjQsUPy/t6yqt2/1Wx4YNPSe+mZjlyw9vKKI=
 go.elastic.co/go-licence-detector v0.4.0 h1:it5dP+6LPxLsosdhtbAqk/zJQxzS0QSSpdNkKVuwKMs=
 go.elastic.co/go-licence-detector v0.4.0/go.mod h1:fSJQU8au4SAgDK+UQFbgUPsXKYNBDv4E/dwWevrMpXU=
 go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg=
diff --git a/libbeat/cfgfile/list.go b/libbeat/cfgfile/list.go
index fc50baa3345..9b62d95f6a9 100644
--- a/libbeat/cfgfile/list.go
+++ b/libbeat/cfgfile/list.go
@@ -92,7 +92,12 @@ func (r *RunnerList) Reload(configs []*reload.ConfigWithMeta) error {
 	for hash, config := range startList {
 		runner, err := createRunner(r.factory, r.pipeline, config)
 		if err != nil {
-			r.logger.Errorf("Error creating runner from config: %s", err)
+			if _, ok := err.(*common.ErrInputNotFinished); ok {
+				// error is related to state, we should not log at error level
+				r.logger.Debugf("Error creating runner from config: %s", err)
+			} else {
+				r.logger.Errorf("Error creating runner from config: %s", err)
+			}
 			errs = append(errs, errors.Wrap(err, "Error creating runner from config"))
 			continue
 		}
diff --git a/filebeat/input/errors.go b/libbeat/common/errors.go
similarity index 98%
rename from filebeat/input/errors.go
rename to libbeat/common/errors.go
index 098156abf91..68fecb8f550 100644
--- a/filebeat/input/errors.go
+++ b/libbeat/common/errors.go
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package input
+package common
 
 import (
 	"fmt"
diff --git a/libbeat/publisher/pipeline/output.go b/libbeat/publisher/pipeline/output.go
index febccdc7067..00c3fc54281 100644
--- a/libbeat/publisher/pipeline/output.go
+++ b/libbeat/publisher/pipeline/output.go
@@ -167,7 +167,7 @@ func (w *netClientWorker) run() {
 
 func (w *netClientWorker) publishBatch(batch publisher.Batch) error {
 	ctx := context.Background()
-	if w.tracer != nil {
+	if w.tracer != nil && w.tracer.Recording() {
 		tx := w.tracer.StartTransaction("publish", "output")
 		defer tx.End()
 		tx.Context.SetLabel("worker", "netclient")
diff --git a/metricbeat/module/elasticsearch/elasticsearch.go b/metricbeat/module/elasticsearch/elasticsearch.go
index 52ee233d88c..a84bf644f3c 100644
--- a/metricbeat/module/elasticsearch/elasticsearch.go
+++ b/metricbeat/module/elasticsearch/elasticsearch.go
@@ -60,17 +60,23 @@ func NewModule(base mb.BaseModule) (mb.Module, error) {
 	return elastic.NewModule(&base, xpackEnabledMetricSets, logp.NewLogger(ModuleName))
 }
 
-// CCRStatsAPIAvailableVersion is the version of Elasticsearch since when the CCR stats API is available.
-var CCRStatsAPIAvailableVersion = common.MustNewVersion("6.5.0")
+var (
+	// CCRStatsAPIAvailableVersion is the version of Elasticsearch since when the CCR stats API is available.
+	CCRStatsAPIAvailableVersion = common.MustNewVersion("6.5.0")
+
+	// EnrichStatsAPIAvailableVersion is the version of Elasticsearch since when the Enrich stats API is available.
+	EnrichStatsAPIAvailableVersion = common.MustNewVersion("7.5.0")
 
-// EnrichStatsAPIAvailableVersion is the version of Elasticsearch since when the Enrich stats API is available.
-var EnrichStatsAPIAvailableVersion = common.MustNewVersion("7.5.0")
+	// BulkStatsAvailableVersion is the version since when bulk indexing stats are available
+	BulkStatsAvailableVersion = common.MustNewVersion("8.0.0")
 
-// BulkStatsAvailableVersion is the version since when bulk indexing stats are available
-var BulkStatsAvailableVersion = common.MustNewVersion("8.0.0")
+	//ExpandWildcardsHiddenAvailableVersion is the version since when the "expand_wildcards" query parameter to
+	// the Indices Stats API can accept "hidden" as a value.
+	ExpandWildcardsHiddenAvailableVersion = common.MustNewVersion("7.7.0")
 
-// Global clusterIdCache. Assumption is that the same node id never can belong to a different cluster id.
-var clusterIDCache = map[string]string{}
+	// Global clusterIdCache. Assumption is that the same node id never can belong to a different cluster id.
+	clusterIDCache = map[string]string{}
+)
 
 // ModuleName is the name of this module.
 const ModuleName = "elasticsearch"
diff --git a/metricbeat/module/elasticsearch/index/index.go b/metricbeat/module/elasticsearch/index/index.go
index 221e78ccfea..69a291aa708 100644
--- a/metricbeat/module/elasticsearch/index/index.go
+++ b/metricbeat/module/elasticsearch/index/index.go
@@ -38,8 +38,12 @@ func init() {
 }
 
 const (
-	statsMetrics = "docs,fielddata,indexing,merge,search,segments,store,refresh,query_cache,request_cache"
-	statsPath    = "/_stats/" + statsMetrics + "?filter_path=indices&expand_wildcards=open,hidden"
+	statsMetrics    = "docs,fielddata,indexing,merge,search,segments,store,refresh,query_cache,request_cache"
+	expandWildcards = "expand_wildcards=open"
+	statsPath       = "/_stats/" + statsMetrics + "?filter_path=indices&" + expandWildcards
+
+	bulkSuffix   = ",bulk"
+	hiddenSuffix = ",hidden"
 )
 
 // MetricSet type defines all fields of the MetricSet
@@ -114,21 +118,18 @@ func (m *MetricSet) updateServicePath(esVersion common.Version) error {
 
 func getServicePath(esVersion common.Version) (string, error) {
 	currPath := statsPath
-	if esVersion.LessThan(elasticsearch.BulkStatsAvailableVersion) {
-		// Can't request bulk stats so don't change service URI
-		return currPath, nil
-	}
-
 	u, err := url.Parse(currPath)
 	if err != nil {
 		return "", err
 	}
 
-	if strings.HasSuffix(u.Path, ",bulk") {
-		// Bulk stats already being requested so don't change service URI
-		return currPath, nil
+	if !esVersion.LessThan(elasticsearch.BulkStatsAvailableVersion) {
+		u.Path += bulkSuffix
+	}
+
+	if !esVersion.LessThan(elasticsearch.ExpandWildcardsHiddenAvailableVersion) {
+		u.RawQuery = strings.Replace(u.RawQuery, expandWildcards, expandWildcards+hiddenSuffix, 1)
 	}
 
-	u.Path += ",bulk"
 	return u.String(), nil
 }
diff --git a/metricbeat/module/elasticsearch/index/index_test.go b/metricbeat/module/elasticsearch/index/index_test.go
index 523a2301e53..fe44dca6ba9 100644
--- a/metricbeat/module/elasticsearch/index/index_test.go
+++ b/metricbeat/module/elasticsearch/index/index_test.go
@@ -27,18 +27,29 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestGetServiceURI(t *testing.T) {
+func TestGetServiceURIExpectedPath(t *testing.T) {
+	path770 := strings.Replace(statsPath, expandWildcards, expandWildcards+hiddenSuffix, 1)
+	path800 := strings.Replace(path770, statsMetrics, statsMetrics+bulkSuffix, 1)
+
 	tests := map[string]struct {
 		esVersion    *common.Version
 		expectedPath string
 	}{
 		"bulk_stats_unavailable": {
-			esVersion:    common.MustNewVersion("7.9.0"),
+			esVersion:    common.MustNewVersion("7.6.0"),
 			expectedPath: statsPath,
 		},
 		"bulk_stats_available": {
 			esVersion:    common.MustNewVersion("8.0.0"),
-			expectedPath: strings.Replace(statsPath, statsMetrics, statsMetrics+",bulk", 1),
+			expectedPath: path800,
+		},
+		"expand_wildcards_hidden_unavailable": {
+			esVersion:    common.MustNewVersion("7.6.0"),
+			expectedPath: statsPath,
+		},
+		"expand_wildcards_hidden_available": {
+			esVersion:    common.MustNewVersion("7.7.0"),
+			expectedPath: path770,
 		},
 	}
 
@@ -52,6 +63,9 @@ func TestGetServiceURI(t *testing.T) {
 }
 
 func TestGetServiceURIMultipleCalls(t *testing.T) {
+	path := strings.Replace(statsPath, expandWildcards, expandWildcards+hiddenSuffix, 1)
+	path = strings.Replace(path, statsMetrics, statsMetrics+bulkSuffix, 1)
+
 	err := quick.Check(func(r uint) bool {
 		numCalls := 2 + (r % 10) // between 2 and 11
 
@@ -64,7 +78,7 @@ func TestGetServiceURIMultipleCalls(t *testing.T) {
 			}
 		}
 
-		return err == nil && uri == strings.Replace(statsPath, statsMetrics, statsMetrics+",bulk", 1)
+		return err == nil && uri == path
 	}, nil)
 	require.NoError(t, err)
 }
diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc
index e4f74663193..efe7ac71d78 100644
--- a/x-pack/elastic-agent/CHANGELOG.next.asciidoc
+++ b/x-pack/elastic-agent/CHANGELOG.next.asciidoc
@@ -17,3 +17,4 @@
 - Add restart CLI cmd {pull}20359[20359]
 - Add new `synthetics/*` inputs to run Heartbeat {pull}20387[20387]
 - Users of the Docker image can now pass `FLEET_ENROLL_INSECURE=1` to include the `--insecure` flag with the `elastic-agent enroll` command {issue}20312[20312] {pull}20713[20713]
+- Add support for dynamic inputs with providers and `{{variable|"default"}}` substitution. {pull}20839[20839]
diff --git a/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/action_example.json b/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/action_example.json
index d76fe58aed2..327b79ed347 100644
--- a/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/action_example.json
+++ b/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/action_example.json
@@ -1,6 +1,5 @@
 {
   "action": "checkin",
-  "success": true,
   "actions": [
     {
       "type": "CONFIG_CHANGE",
diff --git a/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/main.go b/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/main.go
index 3a40641678a..11d90d13773 100644
--- a/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/main.go
+++ b/x-pack/elastic-agent/dev-tools/cmd/fakewebapi/main.go
@@ -27,12 +27,11 @@ var (
 	mutex  sync.Mutex
 
 	pathCheckin     = regexp.MustCompile(`^/api/fleet/agents/(.+)/checkin`)
-	checkinResponse = response{Actions: make([]action, 0), Success: true}
+	checkinResponse = response{Actions: make([]action, 0)}
 )
 
 type response struct {
 	Actions []action `json:"actions"`
-	Success bool     `json:"success"`
 }
 
 type action interface{}
@@ -78,8 +77,7 @@ func handlerEnroll(w http.ResponseWriter, r *http.Request) {
 	}
 
 	response := &fleetapi.EnrollResponse{
-		Action:  "created",
-		Success: true,
+		Action: "created",
 		Item: fleetapi.EnrollItemResponse{
 			ID:                   "a4937110-e53e-11e9-934f-47a8e38a522c",
 			Active:               true,
@@ -147,7 +145,6 @@ func handlerAction(w http.ResponseWriter, r *http.Request) {
 
 	checkinResponse = resp
 	w.WriteHeader(http.StatusCreated)
-	w.Write([]byte(`{ "success": true }`))
 	log.Println("Action request: ", string(c))
 }
 
diff --git a/x-pack/elastic-agent/docs/elastic-agent-command-line.asciidoc b/x-pack/elastic-agent/docs/elastic-agent-command-line.asciidoc
index 611007ad7b5..e102d5b4787 100644
--- a/x-pack/elastic-agent/docs/elastic-agent-command-line.asciidoc
+++ b/x-pack/elastic-agent/docs/elastic-agent-command-line.asciidoc
@@ -150,11 +150,11 @@ accepts additional flags:
 +
 --
 `--output <string>`::
-The name of the output to introspect.
+The name of the output to inspect.
 
 `--program <string>`::
-The type of program to introspect. For example, `filebeat`. This option must be
-combined with `--output`. 
+The type of program to inspect. For example, `filebeat`. This option must be
+combined with `--output`.
 --
 
 `--help`::
diff --git a/x-pack/elastic-agent/pkg/agent/application/application.go b/x-pack/elastic-agent/pkg/agent/application/application.go
index 08bd0f94b8d..d0b16f11f13 100644
--- a/x-pack/elastic-agent/pkg/agent/application/application.go
+++ b/x-pack/elastic-agent/pkg/agent/application/application.go
@@ -26,7 +26,7 @@ func New(log *logger.Logger, pathConfigFile string) (Application, error) {
 	// Load configuration from disk to understand in which mode of operation
 	// we must start the elastic-agent, the mode of operation cannot be changed without restarting the
 	// elastic-agent.
-	rawConfig, err := config.LoadYAML(pathConfigFile)
+	rawConfig, err := LoadConfigFromFile(pathConfigFile)
 	if err != nil {
 		return nil, err
 	}
diff --git a/x-pack/elastic-agent/pkg/agent/application/config.go b/x-pack/elastic-agent/pkg/agent/application/config.go
index 8dfd093e040..ff15ca44074 100644
--- a/x-pack/elastic-agent/pkg/agent/application/config.go
+++ b/x-pack/elastic-agent/pkg/agent/application/config.go
@@ -5,8 +5,15 @@
 package application
 
 import (
+	"io/ioutil"
+
+	"github.com/elastic/go-ucfg"
+
+	"gopkg.in/yaml.v2"
+
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/kibana"
 )
 
@@ -26,3 +33,50 @@ func createFleetConfigFromEnroll(accessAPIKey string, kbn *kibana.Config) (*conf
 	}
 	return cfg, nil
 }
+
+// LoadConfigFromFile loads the Agent configuration from a file.
+//
+// This must be used to load the Agent configuration, so that variables defined in the inputs are not
+// parsed by go-ucfg. Variables from the inputs should be parsed by the transpiler.
+func LoadConfigFromFile(path string) (*config.Config, error) {
+	in, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	var m map[string]interface{}
+	if err := yaml.Unmarshal(in, &m); err != nil {
+		return nil, err
+	}
+	return LoadConfig(m)
+}
+
+// LoadConfig loads the Agent configuration from a map.
+//
+// This must be used to load the Agent configuration, so that variables defined in the inputs are not
+// parsed by go-ucfg. Variables from the inputs should be parsed by the transpiler.
+func LoadConfig(m map[string]interface{}) (*config.Config, error) {
+	inputs, ok := m["inputs"]
+	if ok {
+		// remove the inputs
+		delete(m, "inputs")
+	}
+	cfg, err := config.NewConfigFrom(m)
+	if err != nil {
+		return nil, err
+	}
+	if ok {
+		inputsOnly := map[string]interface{}{
+			"inputs": inputs,
+		}
+		// convert to config without variable substitution
+		inputsCfg, err := config.NewConfigFrom(inputsOnly, ucfg.PathSep("."), ucfg.ResolveNOOP)
+		if err != nil {
+			return nil, err
+		}
+		err = cfg.Merge(inputsCfg, ucfg.PathSep("."), ucfg.ResolveNOOP)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return cfg, err
+}
diff --git a/x-pack/elastic-agent/pkg/agent/application/config_test.go b/x-pack/elastic-agent/pkg/agent/application/config_test.go
index fe9453ac8f4..4d4527a1e60 100644
--- a/x-pack/elastic-agent/pkg/agent/application/config_test.go
+++ b/x-pack/elastic-agent/pkg/agent/application/config_test.go
@@ -5,9 +5,14 @@
 package application
 
 import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
 	"testing"
 	"time"
 
+	"gopkg.in/yaml.v2"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -15,6 +20,44 @@ import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
 )
 
+func TestLoadConfig(t *testing.T) {
+	contents := map[string]interface{}{
+		"outputs": map[string]interface{}{
+			"default": map[string]interface{}{
+				"type":     "elasticsearch",
+				"hosts":    []interface{}{"127.0.0.1:9200"},
+				"username": "elastic",
+				"password": "changeme",
+			},
+		},
+		"inputs": []interface{}{
+			map[string]interface{}{
+				"type": "logfile",
+				"streams": []interface{}{
+					map[string]interface{}{
+						"paths": []interface{}{"/var/log/${host.name}"},
+					},
+				},
+			},
+		},
+	}
+
+	tmp, err := ioutil.TempDir("", "config")
+	require.NoError(t, err)
+	defer os.RemoveAll(tmp)
+
+	cfgPath := filepath.Join(tmp, "config.yml")
+	dumpToYAML(t, cfgPath, contents)
+
+	cfg, err := LoadConfigFromFile(cfgPath)
+	require.NoError(t, err)
+
+	cfgData, err := cfg.ToMapStr()
+	require.NoError(t, err)
+
+	assert.Equal(t, contents, cfgData)
+}
+
 func TestConfig(t *testing.T) {
 	testMgmtMode(t)
 	testLocalConfig(t)
@@ -74,3 +117,9 @@ func mustWithConfigMode(standalone bool) *config.Config {
 		},
 	)
 }
+
+func dumpToYAML(t *testing.T, out string, in interface{}) {
+	b, err := yaml.Marshal(in)
+	require.NoError(t, err)
+	ioutil.WriteFile(out, b, 0600)
+}
diff --git a/x-pack/elastic-agent/pkg/agent/application/emitter.go b/x-pack/elastic-agent/pkg/agent/application/emitter.go
index 249acdd213f..52391b5eff5 100644
--- a/x-pack/elastic-agent/pkg/agent/application/emitter.go
+++ b/x-pack/elastic-agent/pkg/agent/application/emitter.go
@@ -5,11 +5,15 @@
 package application
 
 import (
+	"context"
+	"fmt"
 	"strings"
+	"sync"
 
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/program"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/transpiler"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
 )
@@ -30,54 +34,163 @@ type programsDispatcher interface {
 	Dispatch(id string, grpProg map[routingKey][]program.Program) error
 }
 
-func emitter(log *logger.Logger, router programsDispatcher, modifiers *configModifiers, reloadables ...reloadable) emitterFunc {
-	return func(c *config.Config) error {
-		if err := InjectAgentConfig(c); err != nil {
-			return err
+type emitterController struct {
+	logger      *logger.Logger
+	controller  composable.Controller
+	router      programsDispatcher
+	modifiers   *configModifiers
+	reloadables []reloadable
+
+	// state
+	lock   sync.RWMutex
+	config *config.Config
+	ast    *transpiler.AST
+	vars   []transpiler.Vars
+}
+
+func (e *emitterController) Update(c *config.Config) error {
+	if err := InjectAgentConfig(c); err != nil {
+		return err
+	}
+
+	// perform and verify ast translation
+	m, err := c.ToMapStr()
+	if err != nil {
+		return errors.New(err, "could not create the AST from the configuration", errors.TypeConfig)
+	}
+	rawAst, err := transpiler.NewAST(m)
+	if err != nil {
+		return errors.New(err, "could not create the AST from the configuration", errors.TypeConfig)
+	}
+	for _, filter := range e.modifiers.Filters {
+		if err := filter(e.logger, rawAst); err != nil {
+			return errors.New(err, "failed to filter configuration", errors.TypeConfig)
 		}
+	}
 
-		log.Debug("Transforming configuration into a tree")
-		m, err := c.ToMapStr()
+	// sanitary check that nothing in the config is wrong when it comes to variable syntax
+	ast := rawAst.Clone()
+	inputs, ok := transpiler.Lookup(ast, "inputs")
+	if ok {
+		renderedInputs, err := renderInputs(inputs, []transpiler.Vars{
+			{
+				Mapping: map[string]interface{}{},
+			},
+		})
 		if err != nil {
-			return errors.New(err, "could not create the AST from the configuration", errors.TypeConfig)
+			return err
 		}
-
-		ast, err := transpiler.NewAST(m)
+		err = transpiler.Insert(ast, renderedInputs, "inputs")
 		if err != nil {
-			return errors.New(err, "could not create the AST from the configuration", errors.TypeConfig)
+			return err
 		}
+	}
+
+	programsToRun, err := program.Programs(ast)
+	if err != nil {
+		return err
+	}
 
-		for _, filter := range modifiers.Filters {
-			if err := filter(log, ast); err != nil {
-				return errors.New(err, "failed to filter configuration", errors.TypeConfig)
+	for _, decorator := range e.modifiers.Decorators {
+		for outputType, ptr := range programsToRun {
+			programsToRun[outputType], err = decorator(outputType, ast, ptr)
+			if err != nil {
+				return err
 			}
 		}
+	}
 
-		log.Debugf("Supported programs: %s", strings.Join(program.KnownProgramNames(), ", "))
-		log.Debug("Converting single configuration into specific programs configuration")
+	e.lock.Lock()
+	e.config = c
+	e.ast = rawAst
+	e.lock.Unlock()
+
+	return e.update()
+}
 
-		programsToRun, err := program.Programs(ast)
+func (e *emitterController) Set(vars []transpiler.Vars) {
+	e.lock.Lock()
+	ast := e.ast
+	e.vars = vars
+	e.lock.Unlock()
+
+	if ast != nil {
+		err := e.update()
 		if err != nil {
-			return err
+			e.logger.Errorf("Failed to render configuration with latest context from composable controller: %s", err)
 		}
+	}
+}
 
-		for _, decorator := range modifiers.Decorators {
-			for outputType, ptr := range programsToRun {
-				programsToRun[outputType], err = decorator(outputType, ast, ptr)
-				if err != nil {
-					return err
-				}
-			}
+func (e *emitterController) update() error {
+	e.lock.RLock()
+	cfg := e.config
+	rawAst := e.ast
+	varsArray := e.vars
+	e.lock.RUnlock()
+
+	ast := rawAst.Clone()
+	inputs, ok := transpiler.Lookup(ast, "inputs")
+	if ok {
+		renderedInputs, err := renderInputs(inputs, varsArray)
+		if err != nil {
+			return err
 		}
+		err = transpiler.Insert(ast, renderedInputs, "inputs")
+		if err != nil {
+			return err
+		}
+	}
+
+	e.logger.Debug("Converting single configuration into specific programs configuration")
 
-		for _, r := range reloadables {
-			if err := r.Reload(c); err != nil {
+	programsToRun, err := program.Programs(ast)
+	if err != nil {
+		return err
+	}
+
+	for _, decorator := range e.modifiers.Decorators {
+		for outputType, ptr := range programsToRun {
+			programsToRun[outputType], err = decorator(outputType, ast, ptr)
+			if err != nil {
 				return err
 			}
 		}
+	}
+
+	for _, r := range e.reloadables {
+		if err := r.Reload(cfg); err != nil {
+			return err
+		}
+	}
+
+	return e.router.Dispatch(ast.HashStr(), programsToRun)
+}
+
+func emitter(ctx context.Context, log *logger.Logger, controller composable.Controller, router programsDispatcher, modifiers *configModifiers, reloadables ...reloadable) (emitterFunc, error) {
+	log.Debugf("Supported programs: %s", strings.Join(program.KnownProgramNames(), ", "))
 
-		return router.Dispatch(ast.HashStr(), programsToRun)
+	ctrl := &emitterController{
+		logger:      log,
+		controller:  controller,
+		router:      router,
+		modifiers:   modifiers,
+		reloadables: reloadables,
+		vars: []transpiler.Vars{
+			{
+				Mapping: map[string]interface{}{},
+			},
+		},
 	}
+	err := controller.Run(ctx, func(vars []transpiler.Vars) {
+		ctrl.Set(vars)
+	})
+	if err != nil {
+		return nil, errors.New(err, "failed to start composable controller")
+	}
+	return func(c *config.Config) error {
+		return ctrl.Update(c)
+	}, nil
 }
 
 func readfiles(files []string, emitter emitterFunc) error {
@@ -88,3 +201,80 @@ func readfiles(files []string, emitter emitterFunc) error {
 
 	return emitter(c)
 }
+
+func renderInputs(inputs transpiler.Node, varsArray []transpiler.Vars) (transpiler.Node, error) {
+	l, ok := inputs.Value().(*transpiler.List)
+	if !ok {
+		return nil, fmt.Errorf("inputs must be an array")
+	}
+	nodes := []transpiler.Node{}
+	nodesMap := map[string]*transpiler.Dict{}
+	for _, vars := range varsArray {
+		for _, node := range l.Value().([]transpiler.Node) {
+			dict, ok := node.Clone().(*transpiler.Dict)
+			if !ok {
+				continue
+			}
+			n, err := dict.Apply(vars)
+			if err == transpiler.ErrNoMatch {
+				// has a variable that didn't exist, so we ignore it
+				continue
+			}
+			if err != nil {
+				// another error that needs to be reported
+				return nil, err
+			}
+			dict = n.(*transpiler.Dict)
+			dict = promoteProcessors(dict)
+			hash := string(dict.Hash())
+			_, exists := nodesMap[hash]
+			if !exists {
+				nodesMap[hash] = dict
+				nodes = append(nodes, dict)
+			}
+		}
+	}
+	return transpiler.NewList(nodes), nil
+}
+
+func promoteProcessors(dict *transpiler.Dict) *transpiler.Dict {
+	p := dict.Processors()
+	if p == nil {
+		return dict
+	}
+	current, ok := dict.Find("processors")
+	currentList, isList := current.Value().(*transpiler.List)
+	if !isList {
+		return dict
+	}
+	ast, _ := transpiler.NewAST(map[string]interface{}{
+		"processors": p,
+	})
+	procs, _ := transpiler.Lookup(ast, "processors")
+	nodes := nodesFromList(procs.Value().(*transpiler.List))
+	if ok {
+		nodes = append(nodes, nodesFromList(currentList)...)
+	}
+	dictNodes := dict.Value().([]transpiler.Node)
+	set := false
+	for i, node := range dictNodes {
+		switch n := node.(type) {
+		case *transpiler.Key:
+			if n.Name() == "processors" {
+				dictNodes[i] = transpiler.NewKey("processors", transpiler.NewList(nodes))
+				set = true
+			}
+		}
+		if set {
+			break
+		}
+	}
+	if !set {
+		dictNodes = append(dictNodes, transpiler.NewKey("processors", transpiler.NewList(nodes)))
+	}
+	return transpiler.NewDict(dictNodes)
+}
+
+func nodesFromList(list *transpiler.List) []transpiler.Node {
+	return list.Value().([]transpiler.Node)
+}
diff --git a/x-pack/elastic-agent/pkg/agent/application/emitter_test.go b/x-pack/elastic-agent/pkg/agent/application/emitter_test.go
new file mode 100644
index 00000000000..0c5ba837328
--- /dev/null
+++ b/x-pack/elastic-agent/pkg/agent/application/emitter_test.go
@@ -0,0 +1,537 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package application
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/transpiler"
+)
+
+func TestRenderInputs(t *testing.T) {
+	testcases := map[string]struct {
+		input     transpiler.Node
+		expected  transpiler.Node
+		varsArray []transpiler.Vars
+		err       bool
+	}{
+		"inputs not list": {
+			input: transpiler.NewKey("inputs", transpiler.NewStrVal("not list")),
+			err:   true,
+			varsArray: []transpiler.Vars{
+				{
+					Mapping: map[string]interface{}{},
+				},
+			},
+		},
+		"bad variable error": {
+			input: transpiler.NewKey("inputs", transpiler.NewList([]transpiler.Node{
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("key", transpiler.NewStrVal("${var1.name|'missing ending quote}")),
+				}),
+			})),
+			err: true,
+			varsArray: []transpiler.Vars{
+				{
+					Mapping: map[string]interface{}{
+						"var1": map[string]interface{}{
+							"name": "value1",
+						},
+					},
+				},
+			},
+		},
+		"basic single var": {
+			input: transpiler.NewKey("inputs", transpiler.NewList([]transpiler.Node{
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("key", transpiler.NewStrVal("${var1.name}")),
+				}),
+			})),
+			expected: transpiler.NewList([]transpiler.Node{
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("key", transpiler.NewStrVal("value1")),
+				}),
+			}),
+			varsArray: []transpiler.Vars{
+				{
+					Mapping: map[string]interface{}{
+						"var1": map[string]interface{}{
+							"name": "value1",
+						},
+					},
+				},
+			},
+		},
+		"duplicate result is removed": {
+			input: transpiler.NewKey("inputs", transpiler.NewList([]transpiler.Node{
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("key", transpiler.NewStrVal("${var1.name}")),
+				}),
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("key", transpiler.NewStrVal("${var1.diff}")),
+				}),
+			})),
+			expected: transpiler.NewList([]transpiler.Node{
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("key", transpiler.NewStrVal("value1")),
+				}),
+			}),
+			varsArray: []transpiler.Vars{
+				{
+					Mapping: map[string]interface{}{
+						"var1": map[string]interface{}{
+							"name": "value1",
+							"diff": "value1",
+						},
+					},
+				},
+			},
+		},
+		"missing var removes input": {
+			input: transpiler.NewKey("inputs", transpiler.NewList([]transpiler.Node{
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("key", transpiler.NewStrVal("${var1.name}")),
+				}),
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("key", transpiler.NewStrVal("${var1.missing|var1.diff}")),
+				}),
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("key", transpiler.NewStrVal("${var1.removed}")),
+				}),
+			})),
+			expected: transpiler.NewList([]transpiler.Node{
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("key", transpiler.NewStrVal("value1")),
+				}),
+			}),
+			varsArray: []transpiler.Vars{
+				{
+					Mapping: map[string]interface{}{
+						"var1": map[string]interface{}{
+							"name": "value1",
+							"diff": "value1",
+						},
+					},
+				},
+			},
+		},
+		"duplicate var result but unique input not removed": {
+			input: transpiler.NewKey("inputs", transpiler.NewList([]transpiler.Node{
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("key", transpiler.NewStrVal("${var1.name}")),
+					transpiler.NewKey("unique", transpiler.NewStrVal("0")),
+				}),
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("key", transpiler.NewStrVal("${var1.diff}")),
+					transpiler.NewKey("unique", transpiler.NewStrVal("1")),
+				}),
+			})),
+			expected: transpiler.NewList([]transpiler.Node{
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("key", transpiler.NewStrVal("value1")),
+					transpiler.NewKey("unique", transpiler.NewStrVal("0")),
+				}),
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("key", transpiler.NewStrVal("value1")),
+					transpiler.NewKey("unique", transpiler.NewStrVal("1")),
+				}),
+			}),
+			varsArray: []transpiler.Vars{
+				{
+					Mapping: map[string]interface{}{
+						"var1": map[string]interface{}{
+							"name": "value1",
+							"diff": "value1",
+						},
+					},
+				},
+			},
+		},
+		"duplicates across vars array handled": {
+			input: transpiler.NewKey("inputs", transpiler.NewList([]transpiler.Node{
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("key", transpiler.NewStrVal("${var1.name}")),
+				}),
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("key", transpiler.NewStrVal("${var1.diff}")),
+				}),
+			})),
+			expected: transpiler.NewList([]transpiler.Node{
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("key", transpiler.NewStrVal("value1")),
+				}),
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("key", transpiler.NewStrVal("value2")),
+				}),
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("key", transpiler.NewStrVal("value3")),
+				}),
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("key", transpiler.NewStrVal("value4")),
+				}),
+			}),
+			varsArray: []transpiler.Vars{
+				{
+					Mapping: map[string]interface{}{
+						"var1": map[string]interface{}{
+							"name": "value1",
+							"diff": "value1",
+						},
+					},
+				},
+				{
+					Mapping: map[string]interface{}{
+						"var1": map[string]interface{}{
+							"name": "value1",
+							"diff": "value2",
+						},
+					},
+				},
+				{
+					Mapping: map[string]interface{}{
+						"var1": map[string]interface{}{
+							"name": "value1",
+							"diff": "value3",
+						},
+					},
+				},
+				{
+					Mapping: map[string]interface{}{
+						"var1": map[string]interface{}{
+							"name": "value1",
+							"diff": "value2",
+						},
+					},
+				},
+				{
+					Mapping: map[string]interface{}{
+						"var1": map[string]interface{}{
+							"name": "value1",
+							"diff": "value4",
+						},
+					},
+				},
+			},
+		},
+		"nested in streams": {
+			input: transpiler.NewKey("inputs", transpiler.NewList([]transpiler.Node{
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("type", transpiler.NewStrVal("logfile")),
+					transpiler.NewKey("streams", transpiler.NewList([]transpiler.Node{
+						transpiler.NewDict([]transpiler.Node{
+							transpiler.NewKey("paths", transpiler.NewList([]transpiler.Node{
+								transpiler.NewStrVal("/var/log/${var1.name}.log"),
+							})),
+						}),
+					})),
+				}),
+			})),
+			expected: transpiler.NewList([]transpiler.Node{
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("type", transpiler.NewStrVal("logfile")),
+					transpiler.NewKey("streams", transpiler.NewList([]transpiler.Node{
+						transpiler.NewDict([]transpiler.Node{
+							transpiler.NewKey("paths", transpiler.NewList([]transpiler.Node{
+								transpiler.NewStrVal("/var/log/value1.log"),
+							})),
+						}),
+					})),
+				}),
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("type", transpiler.NewStrVal("logfile")),
+					transpiler.NewKey("streams", transpiler.NewList([]transpiler.Node{
+						transpiler.NewDict([]transpiler.Node{
+							transpiler.NewKey("paths", transpiler.NewList([]transpiler.Node{
+								transpiler.NewStrVal("/var/log/value2.log"),
+							})),
+						}),
+					})),
+				}),
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("type", transpiler.NewStrVal("logfile")),
+					transpiler.NewKey("streams", transpiler.NewList([]transpiler.Node{
+						transpiler.NewDict([]transpiler.Node{
+							transpiler.NewKey("paths", transpiler.NewList([]transpiler.Node{
+								transpiler.NewStrVal("/var/log/value3.log"),
+							})),
+						}),
+					})),
+				}),
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("type", transpiler.NewStrVal("logfile")),
+					transpiler.NewKey("streams", transpiler.NewList([]transpiler.Node{
+						transpiler.NewDict([]transpiler.Node{
+							transpiler.NewKey("paths", transpiler.NewList([]transpiler.Node{
+								transpiler.NewStrVal("/var/log/value4.log"),
+							})),
+						}),
+					})),
+				}),
+			}),
+			varsArray: []transpiler.Vars{
+				{
+					Mapping: map[string]interface{}{
+						"var1": map[string]interface{}{
+							"name": "value1",
+						},
+					},
+				},
+				{
+					Mapping: map[string]interface{}{
+						"var1": map[string]interface{}{
+							"name": "value2",
+						},
+					},
+				},
+				{
+					Mapping: map[string]interface{}{
+						"var1": map[string]interface{}{
+							"name": "value2",
+						},
+					},
+				},
+				{
+					Mapping: map[string]interface{}{
+						"var1": map[string]interface{}{
+							"name": "value3",
+						},
+					},
+				},
+				{
+					Mapping: map[string]interface{}{
+						"var1": map[string]interface{}{
+							"name": "value4",
+						},
+					},
+				},
+				{
+					Mapping: map[string]interface{}{
+						"var1": map[string]interface{}{
+							"missing": "other",
+						},
+					},
+				},
+			},
+		},
+		"inputs with processors": {
+			input: transpiler.NewKey("inputs", transpiler.NewList([]transpiler.Node{
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("type", transpiler.NewStrVal("logfile")),
+					transpiler.NewKey("streams", transpiler.NewList([]transpiler.Node{
+						transpiler.NewDict([]transpiler.Node{
+							transpiler.NewKey("paths", transpiler.NewList([]transpiler.Node{
+								transpiler.NewStrVal("/var/log/${var1.name}.log"),
+							})),
+						}),
+					})),
+					transpiler.NewKey("processors", transpiler.NewList([]transpiler.Node{
+						transpiler.NewDict([]transpiler.Node{
+							transpiler.NewKey("add_fields", transpiler.NewDict([]transpiler.Node{
+								transpiler.NewKey("fields", transpiler.NewDict([]transpiler.Node{
+									transpiler.NewKey("user", transpiler.NewStrVal("user1")),
+								})),
+								transpiler.NewKey("to", transpiler.NewStrVal("user")),
+							})),
+						}),
+					})),
+				}),
+			})),
+			expected: transpiler.NewList([]transpiler.Node{
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("type", transpiler.NewStrVal("logfile")),
+					transpiler.NewKey("streams", transpiler.NewList([]transpiler.Node{
+						transpiler.NewDict([]transpiler.Node{
+							transpiler.NewKey("paths", transpiler.NewList([]transpiler.Node{
+								transpiler.NewStrVal("/var/log/value1.log"),
+							})),
+						}),
+					})),
+					transpiler.NewKey("processors", transpiler.NewList([]transpiler.Node{
+						transpiler.NewDict([]transpiler.Node{
+							transpiler.NewKey("add_fields", transpiler.NewDict([]transpiler.Node{
+								transpiler.NewKey("fields", transpiler.NewDict([]transpiler.Node{
+									transpiler.NewKey("user", transpiler.NewStrVal("user1")),
+								})),
+								transpiler.NewKey("to", transpiler.NewStrVal("user")),
+							})),
+						}),
+					})),
+				}),
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("type", transpiler.NewStrVal("logfile")),
+					transpiler.NewKey("streams", transpiler.NewList([]transpiler.Node{
+						transpiler.NewDict([]transpiler.Node{
+							transpiler.NewKey("paths", transpiler.NewList([]transpiler.Node{
+								transpiler.NewStrVal("/var/log/value2.log"),
+							})),
+						}),
+					})),
+					transpiler.NewKey("processors", transpiler.NewList([]transpiler.Node{
+						transpiler.NewDict([]transpiler.Node{
+							transpiler.NewKey("add_fields", transpiler.NewDict([]transpiler.Node{
+								transpiler.NewKey("fields", transpiler.NewDict([]transpiler.Node{
+									transpiler.NewKey("user", transpiler.NewStrVal("user1")),
+								})),
+								transpiler.NewKey("to", transpiler.NewStrVal("user")),
+							})),
+						}),
+					})),
+				}),
+			}),
+			varsArray: []transpiler.Vars{
+				{
+					Mapping: map[string]interface{}{
+						"var1": map[string]interface{}{
+							"name": "value1",
+						},
+					},
+				},
+				{
+					Mapping: map[string]interface{}{
+						"var1": map[string]interface{}{
+							"name": "value2",
+						},
+					},
+				},
+			},
+		},
+		"vars with processors": {
+			input: transpiler.NewKey("inputs", transpiler.NewList([]transpiler.Node{
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("type", transpiler.NewStrVal("logfile")),
+					transpiler.NewKey("streams", transpiler.NewList([]transpiler.Node{
+						transpiler.NewDict([]transpiler.Node{
+							transpiler.NewKey("paths", transpiler.NewList([]transpiler.Node{
+								transpiler.NewStrVal("/var/log/${var1.name}.log"),
+							})),
+						}),
+					})),
+					transpiler.NewKey("processors", transpiler.NewList([]transpiler.Node{
+						transpiler.NewDict([]transpiler.Node{
+							transpiler.NewKey("add_fields", transpiler.NewDict([]transpiler.Node{
+								transpiler.NewKey("fields", transpiler.NewDict([]transpiler.Node{
+									transpiler.NewKey("user", transpiler.NewStrVal("user1")),
+								})),
+								transpiler.NewKey("to", transpiler.NewStrVal("user")),
+							})),
+						}),
+					})),
+				}),
+			})),
+			expected: transpiler.NewList([]transpiler.Node{
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("type", transpiler.NewStrVal("logfile")),
+					transpiler.NewKey("streams", transpiler.NewList([]transpiler.Node{
+						transpiler.NewDict([]transpiler.Node{
+							transpiler.NewKey("paths", transpiler.NewList([]transpiler.Node{
+								transpiler.NewStrVal("/var/log/value1.log"),
+							})),
+						}),
+					})),
+					transpiler.NewKey("processors", transpiler.NewList([]transpiler.Node{
+						transpiler.NewDict([]transpiler.Node{
+							transpiler.NewKey("add_fields", transpiler.NewDict([]transpiler.Node{
+								transpiler.NewKey("fields", transpiler.NewDict([]transpiler.Node{
+									transpiler.NewKey("custom", transpiler.NewStrVal("value1")),
+								})),
+								transpiler.NewKey("to", transpiler.NewStrVal("dynamic")),
+							})),
+						}),
+						transpiler.NewDict([]transpiler.Node{
+							transpiler.NewKey("add_fields", transpiler.NewDict([]transpiler.Node{
+								transpiler.NewKey("fields", transpiler.NewDict([]transpiler.Node{
+									transpiler.NewKey("user", transpiler.NewStrVal("user1")),
+								})),
+								transpiler.NewKey("to", transpiler.NewStrVal("user")),
+							})),
+						}),
+					})),
+				}),
+				transpiler.NewDict([]transpiler.Node{
+					transpiler.NewKey("type", transpiler.NewStrVal("logfile")),
+					transpiler.NewKey("streams", transpiler.NewList([]transpiler.Node{
+						transpiler.NewDict([]transpiler.Node{
+							transpiler.NewKey("paths", transpiler.NewList([]transpiler.Node{
+								transpiler.NewStrVal("/var/log/value2.log"),
+							})),
+						}),
+					})),
+					transpiler.NewKey("processors", transpiler.NewList([]transpiler.Node{
+						transpiler.NewDict([]transpiler.Node{
+							transpiler.NewKey("add_fields", transpiler.NewDict([]transpiler.Node{
+								transpiler.NewKey("fields", transpiler.NewDict([]transpiler.Node{
+									transpiler.NewKey("custom", transpiler.NewStrVal("value2")),
+								})),
+								transpiler.NewKey("to", transpiler.NewStrVal("dynamic")),
+							})),
+						}),
+						transpiler.NewDict([]transpiler.Node{
+							transpiler.NewKey("add_fields", transpiler.NewDict([]transpiler.Node{
+								transpiler.NewKey("fields", transpiler.NewDict([]transpiler.Node{
+									transpiler.NewKey("user", transpiler.NewStrVal("user1")),
+								})),
+								transpiler.NewKey("to", transpiler.NewStrVal("user")),
+							})),
+						}),
+					})),
+				}),
+			}),
+			varsArray: []transpiler.Vars{
+				{
+					Mapping: map[string]interface{}{
+						"var1": map[string]interface{}{
+							"name": "value1",
+						},
+					},
+					ProcessorsKey: "var1",
+					Processors: []map[string]interface{}{
+						{
+							"add_fields": map[string]interface{}{
+								"fields": map[string]interface{}{
+									"custom": "value1",
+								},
+								"to": "dynamic",
+							},
+						},
+					},
+				},
+				{
+					Mapping: map[string]interface{}{
+						"var1": map[string]interface{}{
+							"name": "value2",
+						},
+					},
+					ProcessorsKey: "var1",
+					Processors: []map[string]interface{}{
+						{
+							"add_fields": map[string]interface{}{
+								"fields": map[string]interface{}{
+									"custom": "value2",
+								},
+								"to": "dynamic",
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for name, test := range testcases {
+		t.Run(name, func(t *testing.T) {
+			v, err := renderInputs(test.input, test.varsArray)
+			if test.err {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+				assert.Equal(t, test.expected.String(), v.String())
+			}
+		})
+	}
+}
diff --git a/x-pack/elastic-agent/pkg/agent/application/enroll_cmd_test.go b/x-pack/elastic-agent/pkg/agent/application/enroll_cmd_test.go
index 97b79e8239e..beab1b253d6 100644
--- a/x-pack/elastic-agent/pkg/agent/application/enroll_cmd_test.go
+++ b/x-pack/elastic-agent/pkg/agent/application/enroll_cmd_test.go
@@ -54,7 +54,6 @@ func TestEnroll(t *testing.T) {
 				w.Write([]byte(`
 {
     "action": "created",
-    "success": true,
     "item": {
        "id": "a9328860-ec54-11e9-93c4-d72ab8a69391",
         "active": true,
@@ -108,7 +107,6 @@ func TestEnroll(t *testing.T) {
 				w.Write([]byte(`
 {
     "action": "created",
-    "success": true,
     "item": {
        "id": "a9328860-ec54-11e9-93c4-d72ab8a69391",
         "active": true,
@@ -170,7 +168,6 @@ func TestEnroll(t *testing.T) {
 				w.Write([]byte(`
 {
     "action": "created",
-    "success": true,
     "item": {
         "id": "a9328860-ec54-11e9-93c4-d72ab8a69391",
         "active": true,
@@ -231,7 +228,6 @@ func TestEnroll(t *testing.T) {
 				w.Write([]byte(`
 {
     "action": "created",
-    "success": true,
     "item": {
         "id": "a9328860-ec54-11e9-93c4-d72ab8a69391",
         "active": true,
diff --git a/x-pack/elastic-agent/pkg/agent/application/fleet_acker_test.go b/x-pack/elastic-agent/pkg/agent/application/fleet_acker_test.go
index 624824e14ec..41e42df7376 100644
--- a/x-pack/elastic-agent/pkg/agent/application/fleet_acker_test.go
+++ b/x-pack/elastic-agent/pkg/agent/application/fleet_acker_test.go
@@ -48,7 +48,7 @@ func TestAcker(t *testing.T) {
 		assert.EqualValues(t, 1, len(cr.Events))
 		assert.EqualValues(t, testID, cr.Events[0].ActionID)
 
-		resp := wrapStrToResp(http.StatusOK, `{ "actions": [], "success": true }`)
+		resp := wrapStrToResp(http.StatusOK, `{ "actions": [] }`)
 		return resp, nil
 	})
 
diff --git a/x-pack/elastic-agent/pkg/agent/application/fleet_gateway.go b/x-pack/elastic-agent/pkg/agent/application/fleet_gateway.go
index cd94380e673..4bb9d2e6280 100644
--- a/x-pack/elastic-agent/pkg/agent/application/fleet_gateway.go
+++ b/x-pack/elastic-agent/pkg/agent/application/fleet_gateway.go
@@ -213,7 +213,6 @@ func (f *fleetGateway) execute(ctx context.Context) (*fleetapi.CheckinResponse,
 			f.log.Warnf("retrieved unauthorized for '%d' times. Unrolling.", f.unauthCounter)
 			return &fleetapi.CheckinResponse{
 				Actions: []fleetapi.Action{&fleetapi.ActionUnenroll{ActionID: "", ActionType: "UNENROLL", IsDetected: true}},
-				Success: true,
 			}, nil
 		}
 
diff --git a/x-pack/elastic-agent/pkg/agent/application/fleet_gateway_test.go b/x-pack/elastic-agent/pkg/agent/application/fleet_gateway_test.go
index 61fd509d995..bd9037416dc 100644
--- a/x-pack/elastic-agent/pkg/agent/application/fleet_gateway_test.go
+++ b/x-pack/elastic-agent/pkg/agent/application/fleet_gateway_test.go
@@ -179,7 +179,7 @@ func TestFleetGateway(t *testing.T) {
 	) {
 		received := ackSeq(
 			client.Answer(func(headers http.Header, body io.Reader) (*http.Response, error) {
-				resp := wrapStrToResp(http.StatusOK, `{ "actions": [], "success": true }`)
+				resp := wrapStrToResp(http.StatusOK, `{ "actions": [] }`)
 				return resp, nil
 			}),
 			dispatcher.Answer(func(actions ...action) error {
@@ -220,8 +220,7 @@ func TestFleetGateway(t *testing.T) {
 			"type": "ANOTHER_ACTION",
 			"id": "id2"
 		}
-	],
-	"success": true
+	]
 }
 `)
 				return resp, nil
@@ -265,7 +264,7 @@ func TestFleetGateway(t *testing.T) {
 		for {
 			received := ackSeq(
 				client.Answer(func(headers http.Header, body io.Reader) (*http.Response, error) {
-					resp := wrapStrToResp(http.StatusOK, `{ "actions": [], "success": true }`)
+					resp := wrapStrToResp(http.StatusOK, `{ "actions": [] }`)
 					return resp, nil
 				}),
 				dispatcher.Answer(func(actions ...action) error {
@@ -305,7 +304,7 @@ func TestFleetGateway(t *testing.T) {
 
 				require.Equal(t, 1, len(cr.Events))
 
-				resp := wrapStrToResp(http.StatusOK, `{ "actions": [], "success": true }`)
+				resp := wrapStrToResp(http.StatusOK, `{ "actions": [] }`)
 				return resp, nil
 			}),
 			dispatcher.Answer(func(actions ...action) error {
@@ -358,7 +357,7 @@ func TestFleetGateway(t *testing.T) {
 
 		// Make sure that all API calls to the checkin API are successfull, the following will happen:
 		ch2 := client.Answer(func(headers http.Header, body io.Reader) (*http.Response, error) {
-			resp := wrapStrToResp(http.StatusOK, `{ "actions": [], "success": true }`)
+			resp := wrapStrToResp(http.StatusOK, `{ "actions": [] }`)
 			return resp, nil
 		})
 
@@ -424,7 +423,7 @@ func TestRetriesOnFailures(t *testing.T) {
 
 					require.Equal(t, 1, len(cr.Events))
 
-					resp := wrapStrToResp(http.StatusOK, `{ "actions": [], "success": true }`)
+					resp := wrapStrToResp(http.StatusOK, `{ "actions": [] }`)
 					return resp, nil
 				}),
 
diff --git a/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change.go b/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change.go
index 996811260ef..34fd5716980 100644
--- a/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change.go
+++ b/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change.go
@@ -9,7 +9,6 @@ import (
 	"fmt"
 
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
-	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi"
 )
@@ -26,7 +25,7 @@ func (h *handlerConfigChange) Handle(ctx context.Context, a action, acker fleetA
 		return fmt.Errorf("invalid type, expected ActionConfigChange and received %T", a)
 	}
 
-	c, err := config.NewConfigFrom(action.Config)
+	c, err := LoadConfig(action.Config)
 	if err != nil {
 		return errors.New(err, "could not parse the configuration from the policy", errors.TypeConfig)
 	}
diff --git a/x-pack/elastic-agent/pkg/agent/application/introspect_config_cmd.go b/x-pack/elastic-agent/pkg/agent/application/inspect_config_cmd.go
similarity index 84%
rename from x-pack/elastic-agent/pkg/agent/application/introspect_config_cmd.go
rename to x-pack/elastic-agent/pkg/agent/application/inspect_config_cmd.go
index 80a21133e08..2c53fc62bf2 100644
--- a/x-pack/elastic-agent/pkg/agent/application/introspect_config_cmd.go
+++ b/x-pack/elastic-agent/pkg/agent/application/inspect_config_cmd.go
@@ -17,25 +17,25 @@ import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi"
 )
 
-// IntrospectConfigCmd is an introspect subcommand that shows configurations of the agent.
-type IntrospectConfigCmd struct {
+// InspectConfigCmd is an inspect subcommand that shows configurations of the agent.
+type InspectConfigCmd struct {
 	cfgPath string
 }
 
-// NewIntrospectConfigCmd creates a new introspect command.
-func NewIntrospectConfigCmd(configPath string,
-) (*IntrospectConfigCmd, error) {
-	return &IntrospectConfigCmd{
+// NewInspectConfigCmd creates a new inspect command.
+func NewInspectConfigCmd(configPath string,
+) (*InspectConfigCmd, error) {
+	return &InspectConfigCmd{
 		cfgPath: configPath,
 	}, nil
 }
 
-// Execute introspects agent configuration.
-func (c *IntrospectConfigCmd) Execute() error {
-	return c.introspectConfig()
+// Execute inspects agent configuration.
+func (c *InspectConfigCmd) Execute() error {
+	return c.inspectConfig()
 }
 
-func (c *IntrospectConfigCmd) introspectConfig() error {
+func (c *InspectConfigCmd) inspectConfig() error {
 	rawConfig, err := loadConfig(c.cfgPath)
 	if err != nil {
 		return err
@@ -61,7 +61,7 @@ func (c *IntrospectConfigCmd) introspectConfig() error {
 }
 
 func loadConfig(configPath string) (*config.Config, error) {
-	rawConfig, err := config.LoadYAML(configPath)
+	rawConfig, err := LoadConfigFromFile(configPath)
 	if err != nil {
 		return nil, err
 	}
diff --git a/x-pack/elastic-agent/pkg/agent/application/introspect_output_cmd.go b/x-pack/elastic-agent/pkg/agent/application/inspect_output_cmd.go
similarity index 72%
rename from x-pack/elastic-agent/pkg/agent/application/introspect_output_cmd.go
rename to x-pack/elastic-agent/pkg/agent/application/inspect_output_cmd.go
index 25a9c744922..ca9983cca22 100644
--- a/x-pack/elastic-agent/pkg/agent/application/introspect_output_cmd.go
+++ b/x-pack/elastic-agent/pkg/agent/application/inspect_output_cmd.go
@@ -5,27 +5,31 @@
 package application
 
 import (
+	"context"
 	"fmt"
 
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/transpiler"
+
 	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/filters"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/program"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/monitoring/noop"
 )
 
-// IntrospectOutputCmd is an introspect subcommand that shows configurations of the agent.
-type IntrospectOutputCmd struct {
+// InspectOutputCmd is an inspect subcommand that shows configurations of the agent.
+type InspectOutputCmd struct {
 	cfgPath string
 	output  string
 	program string
 }
 
-// NewIntrospectOutputCmd creates a new introspect command.
-func NewIntrospectOutputCmd(configPath, output, program string) (*IntrospectOutputCmd, error) {
-	return &IntrospectOutputCmd{
+// NewInspectOutputCmd creates a new inspect command.
+func NewInspectOutputCmd(configPath, output, program string) (*InspectOutputCmd, error) {
+	return &InspectOutputCmd{
 		cfgPath: configPath,
 		output:  output,
 		program: program,
@@ -33,15 +37,15 @@ func NewIntrospectOutputCmd(configPath, output, program string) (*IntrospectOutp
 }
 
 // Execute tries to enroll the agent into Fleet.
-func (c *IntrospectOutputCmd) Execute() error {
+func (c *InspectOutputCmd) Execute() error {
 	if c.output == "" {
-		return c.introspectOutputs()
+		return c.inspectOutputs()
 	}
 
-	return c.introspectOutput()
+	return c.inspectOutput()
 }
 
-func (c *IntrospectOutputCmd) introspectOutputs() error {
+func (c *InspectOutputCmd) inspectOutputs() error {
 	rawConfig, err := loadConfig(c.cfgPath)
 	if err != nil {
 		return err
@@ -94,7 +98,7 @@ func listOutputsFromMap(log *logger.Logger, cfg map[string]interface{}) error {
 	return listOutputsFromConfig(log, c)
 }
 
-func (c *IntrospectOutputCmd) introspectOutput() error {
+func (c *InspectOutputCmd) inspectOutput() error {
 	rawConfig, err := loadConfig(c.cfgPath)
 	if err != nil {
 		return err
@@ -149,7 +153,7 @@ func printOutputFromConfig(log *logger.Logger, output, programName string, cfg *
 		}
 
 		if !programFound {
-			return fmt.Errorf("program '%s' is not recognized within output '%s', try running `elastic-agent introspect output` to find available outputs",
+			return fmt.Errorf("program '%s' is not recognized within output '%s', try running `elastic-agent inspect output` to find available outputs",
 				programName,
 				output)
 		}
@@ -157,7 +161,7 @@ func printOutputFromConfig(log *logger.Logger, output, programName string, cfg *
 		return nil
 	}
 
-	return fmt.Errorf("output '%s' is not recognized, try running `elastic-agent introspect output` to find available outputs", output)
+	return fmt.Errorf("output '%s' is not recognized, try running `elastic-agent inspect output` to find available outputs", output)
 
 }
 
@@ -173,8 +177,17 @@ func printOutputFromMap(log *logger.Logger, output, programName string, cfg map[
 func getProgramsFromConfig(log *logger.Logger, cfg *config.Config) (map[string][]program.Program, error) {
 	monitor := noop.NewMonitor()
 	router := &inmemRouter{}
-	emit := emitter(
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	composableCtrl, err := composable.New(cfg)
+	if err != nil {
+		return nil, err
+	}
+	composableWaiter := newWaitForCompose(composableCtrl)
+	emit, err := emitter(
+		ctx,
 		log,
+		composableWaiter,
 		router,
 		&configModifiers{
 			Decorators: []decoratorFunc{injectMonitoring},
@@ -182,10 +195,14 @@ func getProgramsFromConfig(log *logger.Logger, cfg *config.Config) (map[string][
 		},
 		monitor,
 	)
+	if err != nil {
+		return nil, err
+	}
 
 	if err := emit(cfg); err != nil {
 		return nil, err
 	}
+	composableWaiter.Wait()
 	return router.programs, nil
 }
 
@@ -201,3 +218,27 @@ func (r *inmemRouter) Dispatch(id string, grpProg map[routingKey][]program.Progr
 func newErrorLogger() (*logger.Logger, error) {
 	return logger.NewWithLogpLevel("", logp.ErrorLevel)
 }
+
+type waitForCompose struct {
+	controller composable.Controller
+	done       chan bool
+}
+
+func newWaitForCompose(wrapped composable.Controller) *waitForCompose {
+	return &waitForCompose{
+		controller: wrapped,
+		done:       make(chan bool),
+	}
+}
+
+func (w *waitForCompose) Run(ctx context.Context, cb composable.VarsCallback) error {
+	err := w.controller.Run(ctx, func(vars []transpiler.Vars) {
+		cb(vars)
+		w.done <- true
+	})
+	return err
+}
+
+func (w *waitForCompose) Wait() {
+	<-w.done
+}
diff --git a/x-pack/elastic-agent/pkg/agent/application/lazy_acker_test.go b/x-pack/elastic-agent/pkg/agent/application/lazy_acker_test.go
index aa48e9bd949..24c708c0d91 100644
--- a/x-pack/elastic-agent/pkg/agent/application/lazy_acker_test.go
+++ b/x-pack/elastic-agent/pkg/agent/application/lazy_acker_test.go
@@ -64,7 +64,7 @@ func TestLazyAcker(t *testing.T) {
 			assert.EqualValues(t, 1, len(cr.Events))
 		}
 
-		resp := wrapStrToResp(http.StatusOK, `{ "actions": [], "success": true }`)
+		resp := wrapStrToResp(http.StatusOK, `{ "actions": [] }`)
 		return resp, nil
 	})
 
diff --git a/x-pack/elastic-agent/pkg/agent/application/local_mode.go b/x-pack/elastic-agent/pkg/agent/application/local_mode.go
index df89b97bb96..4b0753af9a8 100644
--- a/x-pack/elastic-agent/pkg/agent/application/local_mode.go
+++ b/x-pack/elastic-agent/pkg/agent/application/local_mode.go
@@ -13,6 +13,7 @@ import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/operation"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/monitoring"
@@ -102,9 +103,16 @@ func newLocal(
 	}
 	localApplication.router = router
 
+	composableCtrl, err := composable.New(rawConfig)
+	if err != nil {
+		return nil, errors.New(err, "failed to initialize composable controller")
+	}
+
 	discover := discoverer(pathConfigFile, cfg.Settings.Path)
-	emit := emitter(
+	emit, err := emitter(
+		localApplication.bgContext,
 		log,
+		composableCtrl,
 		router,
 		&configModifiers{
 			Decorators: []decoratorFunc{injectMonitoring},
@@ -112,6 +120,9 @@ func newLocal(
 		},
 		monitor,
 	)
+	if err != nil {
+		return nil, err
+	}
 
 	var cfgSource source
 	if !cfg.Settings.Reload.Enabled {
diff --git a/x-pack/elastic-agent/pkg/agent/application/managed_mode.go b/x-pack/elastic-agent/pkg/agent/application/managed_mode.go
index f1a21ff528b..8322b1943ea 100644
--- a/x-pack/elastic-agent/pkg/agent/application/managed_mode.go
+++ b/x-pack/elastic-agent/pkg/agent/application/managed_mode.go
@@ -17,6 +17,7 @@ import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/operation"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/storage"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/monitoring"
@@ -147,8 +148,15 @@ func newManaged(
 	}
 	managedApplication.router = router
 
-	emit := emitter(
+	composableCtrl, err := composable.New(rawConfig)
+	if err != nil {
+		return nil, errors.New(err, "failed to initialize composable controller")
+	}
+
+	emit, err := emitter(
+		managedApplication.bgContext,
 		log,
+		composableCtrl,
 		router,
 		&configModifiers{
 			Decorators: []decoratorFunc{injectMonitoring},
@@ -156,6 +164,9 @@ func newManaged(
 		},
 		monitor,
 	)
+	if err != nil {
+		return nil, err
+	}
 	acker, err := newActionAcker(log, agentInfo, client)
 	if err != nil {
 		return nil, err
diff --git a/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go b/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go
index 2958a51f688..efc8608c233 100644
--- a/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go
+++ b/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go
@@ -10,9 +10,11 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/filters"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configrequest"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi"
 )
@@ -26,12 +28,17 @@ func TestManagedModeRouting(t *testing.T) {
 		return m, nil
 	}
 
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
 	log, _ := logger.New("")
 	router, _ := newRouter(log, streamFn)
-	emit := emitter(log, router, &configModifiers{Decorators: []decoratorFunc{injectMonitoring}, Filters: []filterFunc{filters.ConstraintFilter}})
+	composableCtrl, _ := composable.New(nil)
+	emit, err := emitter(ctx, log, composableCtrl, router, &configModifiers{Decorators: []decoratorFunc{injectMonitoring}, Filters: []filterFunc{filters.ConstraintFilter}})
+	require.NoError(t, err)
 
-	actionDispatcher, err := newActionDispatcher(context.Background(), log, &handlerDefault{log: log})
-	assert.NoError(t, err)
+	actionDispatcher, err := newActionDispatcher(ctx, log, &handlerDefault{log: log})
+	require.NoError(t, err)
 
 	actionDispatcher.MustRegister(
 		&fleetapi.ActionConfigChange{},
@@ -42,10 +49,10 @@ func TestManagedModeRouting(t *testing.T) {
 	)
 
 	actions, err := testActions()
-	assert.NoError(t, err)
+	require.NoError(t, err)
 
 	err = actionDispatcher.Dispatch(newNoopAcker(), actions...)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 
 	// has 1 config request for fb, mb and monitoring?
 	assert.Equal(t, 1, len(streams))
@@ -92,7 +99,6 @@ func (m *mockStreamStore) Shutdown() {}
 const fleetResponse = `
 {
 	"action": "checkin",
-	"success": true,
 	"actions": [{
 		"agent_id": "17e93530-7f42-11ea-9330-71e968b29fa4",
 		"type": "CONFIG_CHANGE",
diff --git a/x-pack/elastic-agent/pkg/agent/application/paths/paths.go b/x-pack/elastic-agent/pkg/agent/application/paths/paths.go
index adb5850bfb4..d32be56657d 100644
--- a/x-pack/elastic-agent/pkg/agent/application/paths/paths.go
+++ b/x-pack/elastic-agent/pkg/agent/application/paths/paths.go
@@ -8,23 +8,77 @@ import (
 	"flag"
 	"os"
 	"path/filepath"
+	"runtime"
+
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
 )
 
 var (
-	homePath   string
-	configPath string
-	dataPath   string
-	logsPath   string
+	homePath    string
+	configPath  string
+	dataPath    string
+	logsPath    string
+	serviceName string
 )
 
 func init() {
-	exePath := retrieveExecutablePath()
+	initialHome := initialHome()
+
+	var homePathVar, configPathVar, dataPathVar, logsPathVar string
 
 	fs := flag.CommandLine
-	fs.StringVar(&homePath, "path.home", exePath, "Agent root path")
-	fs.StringVar(&configPath, "path.config", exePath, "Config path is the directory Agent looks for its config file")
-	fs.StringVar(&dataPath, "path.data", filepath.Join(exePath, "data"), "Data path contains Agent managed binaries")
-	fs.StringVar(&logsPath, "path.logs", exePath, "Logs path contains Agent log output")
+	fs.StringVar(&homePathVar, "path.home", initialHome, "Agent root path")
+	fs.StringVar(&configPathVar, "path.config", initialHome, "Config path is the directory Agent looks for its config file")
+	fs.StringVar(&dataPathVar, "path.data", filepath.Join(initialHome, "data"), "Data path contains Agent managed binaries")
+	fs.StringVar(&logsPathVar, "path.logs", initialHome, "Logs path contains Agent log output")
+
+	// avoid rewriting initialized values by flagSet later
+	homePath = homePathVar
+	configPath = configPathVar
+	dataPath = dataPathVar
+	logsPath = logsPathVar
+
+	getOverrides()
+}
+
+// UpdatePaths update paths based on changes in paths file.
+func UpdatePaths() {
+	getOverrides()
+}
+
+func getOverrides() {
+	type paths struct {
+		HomePath    string `config:"path.home" yaml:"path.home"`
+		ConfigPath  string `config:"path.config" yaml:"path.config"`
+		DataPath    string `config:"path.data" yaml:"path.data"`
+		LogsPath    string `config:"path.logs" yaml:"path.logs"`
+		ServiceName string `config:"path.service_name" yaml:"path.service_name"`
+	}
+
+	defaults := &paths{
+		HomePath:   homePath,
+		ConfigPath: configPath,
+		DataPath:   dataPath,
+		LogsPath:   logsPath,
+	}
+
+	pathsFile := filepath.Join(dataPath, "paths.yml")
+	rawConfig, err := config.LoadYAML(pathsFile)
+	if err != nil {
+		return
+	}
+
+	rawConfig.Unpack(defaults)
+	homePath = defaults.HomePath
+	configPath = defaults.ConfigPath
+	dataPath = defaults.DataPath
+	logsPath = defaults.LogsPath
+	serviceName = defaults.ServiceName
+}
+
+// ServiceName return predefined service name if defined by initial call.
+func ServiceName() string {
+	return serviceName
 }
 
 // Home returns a directory where binary lives
@@ -54,5 +108,19 @@ func retrieveExecutablePath() string {
 		panic(err)
 	}
 
-	return filepath.Dir(execPath)
+	evalPath, err := filepath.EvalSymlinks(execPath)
+	if err != nil {
+		panic(err)
+	}
+
+	return filepath.Dir(evalPath)
+}
+
+func initialHome() string {
+	exePath := retrieveExecutablePath()
+	if runtime.GOOS == "windows" {
+		return exePath
+	}
+
+	return filepath.Dir(filepath.Dir(exePath)) // is two level up the executable (symlink evaluated)
 }
diff --git a/x-pack/elastic-agent/pkg/agent/application/reexec/manager.go b/x-pack/elastic-agent/pkg/agent/application/reexec/manager.go
index 4662b1c6230..b21bb9b8c46 100644
--- a/x-pack/elastic-agent/pkg/agent/application/reexec/manager.go
+++ b/x-pack/elastic-agent/pkg/agent/application/reexec/manager.go
@@ -12,7 +12,7 @@ import (
 type ExecManager interface {
 	// ReExec asynchronously re-executes command in the same PID and memory address
 	// as the currently running application.
-	ReExec()
+	ReExec(argOverrides ...string)
 
 	// ShutdownChan returns the shutdown channel the main function should use to
 	// handle shutdown of the current running application.
@@ -42,12 +42,12 @@ func NewManager(log *logger.Logger, exec string) ExecManager {
 	}
 }
 
-func (m *manager) ReExec() {
+func (m *manager) ReExec(argOverrides ...string) {
 	go func() {
 		close(m.trigger)
 		<-m.shutdown
 
-		if err := reexec(m.logger, m.exec); err != nil {
+		if err := reexec(m.logger, m.exec, argOverrides...); err != nil {
 			// panic; because there is no going back, everything is shutdown
 			panic(err)
 		}
diff --git a/x-pack/elastic-agent/pkg/agent/application/reexec/reexec.go b/x-pack/elastic-agent/pkg/agent/application/reexec/reexec.go
index 4f3c24a7aa6..9265ba15266 100644
--- a/x-pack/elastic-agent/pkg/agent/application/reexec/reexec.go
+++ b/x-pack/elastic-agent/pkg/agent/application/reexec/reexec.go
@@ -15,11 +15,12 @@ import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
 )
 
-func reexec(log *logger.Logger, executable string) error {
+func reexec(log *logger.Logger, executable string, argOverrides ...string) error {
 	// force log sync, before re-exec
 	_ = log.Sync()
 
 	args := []string{filepath.Base(executable)}
 	args = append(args, os.Args[1:]...)
+	args = append(args, argOverrides...)
 	return unix.Exec(executable, args, os.Environ())
 }
diff --git a/x-pack/elastic-agent/pkg/agent/application/reexec/reexec_windows.go b/x-pack/elastic-agent/pkg/agent/application/reexec/reexec_windows.go
index b79172226cb..7f2f3230dc5 100644
--- a/x-pack/elastic-agent/pkg/agent/application/reexec/reexec_windows.go
+++ b/x-pack/elastic-agent/pkg/agent/application/reexec/reexec_windows.go
@@ -28,12 +28,13 @@ import (
 //   current process just exits.
 //
 // * Sub-process - As a sub-process a new child is spawned and the current process just exits.
-func reexec(log *logger.Logger, executable string) error {
+func reexec(log *logger.Logger, executable string, argOverrides ...string) error {
 	svc, status, err := getService()
 	if err == nil {
 		// running as a service; spawn re-exec windows sub-process
 		log.Infof("Running as Windows service %s; triggering service restart", svc.Name)
 		args := []string{filepath.Base(executable), "reexec_windows", svc.Name, strconv.Itoa(int(status.ProcessId))}
+		args = append(args, argOverrides...)
 		cmd := exec.Cmd{
 			Path:   executable,
 			Args:   args,
@@ -51,6 +52,7 @@ func reexec(log *logger.Logger, executable string) error {
 		log.Infof("Running as Windows process; spawning new child process")
 		args := []string{filepath.Base(executable)}
 		args = append(args, os.Args[1:]...)
+		args = append(args, argOverrides...)
 		cmd := exec.Cmd{
 			Path:   executable,
 			Args:   args,
diff --git a/x-pack/elastic-agent/pkg/agent/cmd/checks.go b/x-pack/elastic-agent/pkg/agent/cmd/checks.go
new file mode 100644
index 00000000000..4fee7497009
--- /dev/null
+++ b/x-pack/elastic-agent/pkg/agent/cmd/checks.go
@@ -0,0 +1,57 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+// +build !windows
+
+package cmd
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+
+	// import logp flags
+	_ "github.com/elastic/beats/v7/libbeat/logp/configure"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release"
+)
+
+// preRunCheck is noop because
+// - darwin.tar - symlink created during packaging
+// - linux.tar - symlink created during packaging
+// - linux.rpm - symlink created using install script
+// - linux.deb - symlink created using install script
+// - linux.docker - symlink created using Dockerfile
+func preRunCheck(flags *globalFlags) func(cmd *cobra.Command, args []string) error {
+	return func(cmd *cobra.Command, args []string) error {
+		if sn := paths.ServiceName(); sn != "" {
+			// paths were created we're running as child.
+			return nil
+		}
+
+		// get versioned path
+		smallHash := fmt.Sprintf("elastic-agent-%s", smallHash(release.Commit()))
+		commitFilepath := filepath.Join(paths.Config(), commitFile) // use other file in the future
+		if content, err := ioutil.ReadFile(commitFilepath); err == nil {
+			smallHash = hashedDirName(content)
+		}
+
+		origExecPath, err := os.Executable()
+		if err != nil {
+			return err
+		}
+		reexecPath := filepath.Join(paths.Data(), smallHash, filepath.Base(origExecPath))
+
+		// generate paths
+		if err := generatePaths(filepath.Dir(reexecPath), origExecPath); err != nil {
+			return err
+		}
+
+		paths.UpdatePaths()
+		return nil
+	}
+}
diff --git a/x-pack/elastic-agent/pkg/agent/cmd/checks_windows.go b/x-pack/elastic-agent/pkg/agent/cmd/checks_windows.go
new file mode 100644
index 00000000000..36108c8e08b
--- /dev/null
+++ b/x-pack/elastic-agent/pkg/agent/cmd/checks_windows.go
@@ -0,0 +1,114 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+// +build windows
+
+package cmd
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+
+	// import logp flags
+	_ "github.com/elastic/beats/v7/libbeat/logp/configure"
+
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/reexec"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release"
+)
+
+func preRunCheck(flags *globalFlags) func(cmd *cobra.Command, args []string) error {
+	return func(cmd *cobra.Command, args []string) error {
+		if sn := paths.ServiceName(); sn != "" {
+			// replacing with correct service name so we
+			// can talk to service manager.
+			if !filepath.IsAbs(os.Args[0]) {
+				os.Args[0] = sn
+			}
+
+			// paths were created we're running as child.
+			return nil
+		}
+
+		smallHash := fmt.Sprintf("elastic-agent-%s", smallHash(release.Commit()))
+		commitFilepath := filepath.Join(paths.Config(), commitFile)
+		if content, err := ioutil.ReadFile(commitFilepath); err == nil {
+			smallHash = hashedDirName(content)
+		}
+
+		// rename itself
+		origExecPath, err := os.Executable()
+		if err != nil {
+			return err
+		}
+
+		if err := os.Rename(origExecPath, origExecPath+".bak"); err != nil {
+			return err
+		}
+
+		// create symlink to elastic-agent-{hash}
+		reexecPath := filepath.Join(paths.Data(), smallHash, filepath.Base(origExecPath))
+		if err := os.Symlink(reexecPath, origExecPath); err != nil {
+			return err
+		}
+
+		// generate paths
+		if err := generatePaths(filepath.Dir(reexecPath), origExecPath); err != nil {
+			return err
+		}
+
+		paths.UpdatePaths()
+
+		// reexec if running run
+		if cmd.Use == "run" {
+			pathConfigFile := flags.Config()
+			rawConfig, err := config.LoadYAML(pathConfigFile)
+			if err != nil {
+				return errors.New(err,
+					fmt.Sprintf("could not read configuration file %s", pathConfigFile),
+					errors.TypeFilesystem,
+					errors.M(errors.MetaKeyPath, pathConfigFile))
+			}
+
+			cfg, err := configuration.NewFromConfig(rawConfig)
+			if err != nil {
+				return errors.New(err,
+					fmt.Sprintf("could not parse configuration file %s", pathConfigFile),
+					errors.TypeFilesystem,
+					errors.M(errors.MetaKeyPath, pathConfigFile))
+			}
+
+			logger, err := logger.NewFromConfig("", cfg.Settings.LoggingConfig)
+			if err != nil {
+				return err
+			}
+
+			rexLogger := logger.Named("reexec")
+			rm := reexec.NewManager(rexLogger, reexecPath)
+
+			argsOverrides := []string{
+				"--path.data", paths.Data(),
+				"--path.home", filepath.Dir(reexecPath),
+				"--path.config", paths.Config(),
+			}
+			rm.ReExec(argsOverrides...)
+
+			// trigger reexec
+			rm.ShutdownComplete()
+
+			// return without running Run method
+			os.Exit(0)
+		}
+
+		return nil
+	}
+}
diff --git a/x-pack/elastic-agent/pkg/agent/cmd/common.go b/x-pack/elastic-agent/pkg/agent/cmd/common.go
index a9de932bfca..d5c195566bd 100644
--- a/x-pack/elastic-agent/pkg/agent/cmd/common.go
+++ b/x-pack/elastic-agent/pkg/agent/cmd/common.go
@@ -6,10 +6,15 @@ package cmd
 
 import (
 	"flag"
+	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
+	"runtime"
+	"strings"
 
 	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v2"
 
 	// import logp flags
 	_ "github.com/elastic/beats/v7/libbeat/logp/configure"
@@ -19,7 +24,11 @@ import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/cli"
 )
 
-const defaultConfig = "elastic-agent.yml"
+const (
+	defaultConfig = "elastic-agent.yml"
+	hashLen       = 6
+	commitFile    = ".elastic-agent.active.commit"
+)
 
 type globalFlags struct {
 	PathConfigFile string
@@ -64,14 +73,65 @@ func NewCommandWithArgs(args []string, streams *cli.IOStreams) *cobra.Command {
 	cmd.AddCommand(basecmd.NewDefaultCommandsWithArgs(args, streams)...)
 	cmd.AddCommand(run)
 	cmd.AddCommand(newEnrollCommandWithArgs(flags, args, streams))
-	cmd.AddCommand(newIntrospectCommandWithArgs(flags, args, streams))
+	cmd.AddCommand(newInspectCommandWithArgs(flags, args, streams))
 
 	// windows special hidden sub-command (only added on windows)
 	reexec := newReExecWindowsCommand(flags, args, streams)
 	if reexec != nil {
 		cmd.AddCommand(reexec)
 	}
+	cmd.PersistentPreRunE = preRunCheck(flags)
 	cmd.Run = run.Run
 
 	return cmd
 }
+
+func hashedDirName(filecontent []byte) string {
+	s := strings.TrimSpace(string(filecontent))
+	if len(s) == 0 {
+		return "elastic-agent"
+	}
+
+	s = smallHash(s)
+
+	return fmt.Sprintf("elastic-agent-%s", s)
+}
+
+func smallHash(hash string) string {
+	if len(hash) > hashLen {
+		hash = hash[:hashLen]
+	}
+
+	return hash
+}
+
+func generatePaths(dir, origExec string) error {
+	pathsCfg := map[string]interface{}{
+		"path.data":         paths.Data(),
+		"path.home":         dir,
+		"path.config":       paths.Config(),
+		"path.service_name": origExec,
+	}
+
+	pathsCfgPath := filepath.Join(paths.Data(), "paths.yml")
+	pathsContent, err := yaml.Marshal(pathsCfg)
+	if err != nil {
+		return err
+	}
+
+	if err := ioutil.WriteFile(pathsCfgPath, pathsContent, 0740); err != nil {
+		return err
+	}
+
+	if runtime.GOOS == "windows" {
+		// due to two binaries we need to do a path dance
+		// as versioned binary will look for path inside it's own directory
+		versionedPath := filepath.Join(dir, "data", "paths.yml")
+		if err := os.MkdirAll(filepath.Dir(versionedPath), 0700); err != nil {
+			return err
+		}
+		return os.Symlink(pathsCfgPath, versionedPath)
+	}
+
+	return nil
+}
diff --git a/x-pack/elastic-agent/pkg/agent/cmd/enroll.go b/x-pack/elastic-agent/pkg/agent/cmd/enroll.go
index b34e0236782..6749b57b250 100644
--- a/x-pack/elastic-agent/pkg/agent/cmd/enroll.go
+++ b/x-pack/elastic-agent/pkg/agent/cmd/enroll.go
@@ -22,7 +22,6 @@ import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/warn"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/cli"
-	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi"
 )
@@ -56,7 +55,7 @@ func newEnrollCommandWithArgs(flags *globalFlags, _ []string, streams *cli.IOStr
 func enroll(streams *cli.IOStreams, cmd *cobra.Command, flags *globalFlags, args []string) error {
 	warn.PrintNotGA(streams.Out)
 	pathConfigFile := flags.Config()
-	rawConfig, err := config.LoadYAML(pathConfigFile)
+	rawConfig, err := application.LoadConfigFromFile(pathConfigFile)
 	if err != nil {
 		return errors.New(err,
 			fmt.Sprintf("could not read configuration file %s", pathConfigFile),
diff --git a/x-pack/elastic-agent/pkg/agent/cmd/include.go b/x-pack/elastic-agent/pkg/agent/cmd/include.go
new file mode 100644
index 00000000000..a28d47490d5
--- /dev/null
+++ b/x-pack/elastic-agent/pkg/agent/cmd/include.go
@@ -0,0 +1,15 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package cmd
+
+import (
+	// include the composable providers
+	_ "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable/providers/agent"
+	_ "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable/providers/env"
+	_ "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable/providers/host"
+	_ "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable/providers/local"
+	_ "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable/providers/localdynamic"
+	_ "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable/providers/path"
+)
diff --git a/x-pack/elastic-agent/pkg/agent/cmd/introspect.go b/x-pack/elastic-agent/pkg/agent/cmd/inspect.go
similarity index 72%
rename from x-pack/elastic-agent/pkg/agent/cmd/introspect.go
rename to x-pack/elastic-agent/pkg/agent/cmd/inspect.go
index f6cb40e1894..bf6d3009f10 100644
--- a/x-pack/elastic-agent/pkg/agent/cmd/introspect.go
+++ b/x-pack/elastic-agent/pkg/agent/cmd/inspect.go
@@ -14,14 +14,14 @@ import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/cli"
 )
 
-func newIntrospectCommandWithArgs(flags *globalFlags, s []string, streams *cli.IOStreams) *cobra.Command {
+func newInspectCommandWithArgs(flags *globalFlags, s []string, streams *cli.IOStreams) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "inspect",
 		Short: "Shows configuration of the agent",
 		Long:  "Shows current configuration of the agent",
 		Args:  cobra.ExactArgs(0),
 		Run: func(c *cobra.Command, args []string) {
-			command, err := application.NewIntrospectConfigCmd(flags.Config())
+			command, err := application.NewInspectConfigCmd(flags.Config())
 			if err != nil {
 				fmt.Fprintf(streams.Err, "%v\n", err)
 				os.Exit(1)
@@ -34,12 +34,12 @@ func newIntrospectCommandWithArgs(flags *globalFlags, s []string, streams *cli.I
 		},
 	}
 
-	cmd.AddCommand(newIntrospectOutputCommandWithArgs(flags, s, streams))
+	cmd.AddCommand(newInspectOutputCommandWithArgs(flags, s, streams))
 
 	return cmd
 }
 
-func newIntrospectOutputCommandWithArgs(flags *globalFlags, _ []string, streams *cli.IOStreams) *cobra.Command {
+func newInspectOutputCommandWithArgs(flags *globalFlags, _ []string, streams *cli.IOStreams) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "output",
 		Short: "Displays configuration generated for output",
@@ -49,7 +49,7 @@ func newIntrospectOutputCommandWithArgs(flags *globalFlags, _ []string, streams
 			outName, _ := c.Flags().GetString("output")
 			program, _ := c.Flags().GetString("program")
 
-			command, err := application.NewIntrospectOutputCmd(flags.Config(), outName, program)
+			command, err := application.NewInspectOutputCmd(flags.Config(), outName, program)
 			if err != nil {
 				fmt.Fprintf(streams.Err, "%v\n", err)
 				os.Exit(1)
@@ -62,8 +62,8 @@ func newIntrospectOutputCommandWithArgs(flags *globalFlags, _ []string, streams
 		},
 	}
 
-	cmd.Flags().StringP("output", "o", "", "name of the output to be introspected")
-	cmd.Flags().StringP("program", "p", "", "type of program to introspect, needs to be combined with output. e.g filebeat")
+	cmd.Flags().StringP("output", "o", "", "name of the output to be inspected")
+	cmd.Flags().StringP("program", "p", "", "type of program to inspect, needs to be combined with output. e.g filebeat")
 
 	return cmd
 }
diff --git a/x-pack/elastic-agent/pkg/agent/cmd/run.go b/x-pack/elastic-agent/pkg/agent/cmd/run.go
index f502ef9cf49..a60c8a87c93 100644
--- a/x-pack/elastic-agent/pkg/agent/cmd/run.go
+++ b/x-pack/elastic-agent/pkg/agent/cmd/run.go
@@ -22,7 +22,6 @@ import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control/server"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/cli"
-	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
 )
 
@@ -39,9 +38,30 @@ func newRunCommandWithArgs(flags *globalFlags, _ []string, streams *cli.IOStream
 	}
 }
 
-func run(flags *globalFlags, streams *cli.IOStreams) error {
+func run(flags *globalFlags, streams *cli.IOStreams) error { // Windows: Mark service as stopped.
+	// After this is run, the service is considered by the OS to be stopped.
+	// This must be the first deferred cleanup task (last to execute).
+	defer service.NotifyTermination()
+
+	locker := application.NewAppLocker(paths.Data())
+	if err := locker.TryLock(); err != nil {
+		return err
+	}
+	defer locker.Unlock()
+
+	service.BeforeRun()
+	defer service.Cleanup()
+
+	// register as a service
+	stop := make(chan bool)
+	_, cancel := context.WithCancel(context.Background())
+	var stopBeat = func() {
+		close(stop)
+	}
+	service.HandleSignals(stopBeat, cancel)
+
 	pathConfigFile := flags.Config()
-	rawConfig, err := config.LoadYAML(pathConfigFile)
+	rawConfig, err := application.LoadConfigFromFile(pathConfigFile)
 	if err != nil {
 		return errors.New(err,
 			fmt.Sprintf("could not read configuration file %s", pathConfigFile),
@@ -62,20 +82,6 @@ func run(flags *globalFlags, streams *cli.IOStreams) error {
 		return err
 	}
 
-	// Windows: Mark service as stopped.
-	// After this is run, the service is considered by the OS to be stopped.
-	// This must be the first deferred cleanup task (last to execute).
-	defer service.NotifyTermination()
-
-	locker := application.NewAppLocker(paths.Data())
-	if err := locker.TryLock(); err != nil {
-		return err
-	}
-	defer locker.Unlock()
-
-	service.BeforeRun()
-	defer service.Cleanup()
-
 	execPath, err := os.Executable()
 	if err != nil {
 		return err
@@ -99,14 +105,6 @@ func run(flags *globalFlags, streams *cli.IOStreams) error {
 		return err
 	}
 
-	// register as a service
-	stop := make(chan bool)
-	_, cancel := context.WithCancel(context.Background())
-	var stopBeat = func() {
-		close(stop)
-	}
-	service.HandleSignals(stopBeat, cancel)
-
 	// listen for signals
 	signals := make(chan os.Signal, 1)
 	signal.Notify(signals, syscall.SIGINT, syscall.SIGKILL, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGHUP)
@@ -128,12 +126,16 @@ func run(flags *globalFlags, streams *cli.IOStreams) error {
 			}
 		}
 		if breakout {
+			if !reexecing {
+				logger.Info("Shutting down Elastic Agent and sending last events...")
+			}
 			break
 		}
 	}
 
 	err = app.Stop()
 	if !reexecing {
+		logger.Info("Shutting down completed.")
 		return err
 	}
 	rex.ShutdownComplete()
diff --git a/x-pack/elastic-agent/pkg/agent/control/server/listener.go b/x-pack/elastic-agent/pkg/agent/control/server/listener.go
index bf03f54e2da..7edfc7b8ee9 100644
--- a/x-pack/elastic-agent/pkg/agent/control/server/listener.go
+++ b/x-pack/elastic-agent/pkg/agent/control/server/listener.go
@@ -23,7 +23,7 @@ func createListener(log *logger.Logger) (net.Listener, error) {
 	path := strings.TrimPrefix(control.Address(), "unix://")
 	if _, err := os.Stat(path); !os.IsNotExist(err) {
 		err = os.Remove(path)
-		if err != nil {
+		if err != nil && !os.IsNotExist(err) {
 			log.Errorf("%s", errors.New(err, fmt.Sprintf("Failed to cleanup %s", path), errors.TypeFilesystem, errors.M("path", path)))
 		}
 	}
diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go
index cd9d3d95a2c..fe33de852d1 100644
--- a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go
+++ b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go
@@ -184,7 +184,7 @@ func (o *Operator) getMonitoringFilebeatConfig(output interface{}) (map[string]i
 				"message_key":     "message",
 			},
 			"paths": []string{
-				filepath.Join(paths.Data(), "logs", "elastic-agent-json.log"),
+				filepath.Join(paths.Home(), "logs", "elastic-agent-json.log"),
 			},
 			"index": "logs-elastic.agent-default",
 			"processors": []map[string]interface{}{
diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/ast.go b/x-pack/elastic-agent/pkg/agent/transpiler/ast.go
index 498c32b15d7..14ebba556eb 100644
--- a/x-pack/elastic-agent/pkg/agent/transpiler/ast.go
+++ b/x-pack/elastic-agent/pkg/agent/transpiler/ast.go
@@ -14,6 +14,8 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+
+	"github.com/elastic/go-ucfg"
 )
 
 const selectorSep = "."
@@ -28,6 +30,9 @@ var (
 	falseVal = []byte{0}
 )
 
+// Processors represent an attached list of processors.
+type Processors []map[string]interface{}
+
 // Node represents a node in the configuration Tree a Node can point to one or multiples children
 // nodes.
 type Node interface {
@@ -44,6 +49,12 @@ type Node interface {
 
 	// Hash compute a sha256 hash of the current node and recursively call any children.
 	Hash() []byte
+
+	// Apply apply the current vars, returning the new value for the node.
+	Apply(Vars) (Node, error)
+
+	// Processors returns any attached processors, because of variable substitution.
+	Processors() Processors
 }
 
 // AST represents a raw configuration which is purely data, only primitives are currently supported,
@@ -61,12 +72,18 @@ func (a *AST) String() string {
 // Dict represents a dictionary in the Tree, where each key is a entry into an array. The Dict will
 // keep the ordering.
 type Dict struct {
-	value []Node
+	value      []Node
+	processors []map[string]interface{}
 }
 
 // NewDict creates a new dict with provided nodes.
 func NewDict(nodes []Node) *Dict {
-	return &Dict{nodes}
+	return NewDictWithProcessors(nodes, nil)
+}
+
+// NewDictWithProcessors creates a new dict with provided nodes and attached processors.
+func NewDictWithProcessors(nodes []Node, processors Processors) *Dict {
+	return &Dict{nodes, processors}
 }
 
 // Find takes a string which is a key and try to find the elements in the associated K/V.
@@ -115,6 +132,32 @@ func (d *Dict) Hash() []byte {
 	return h.Sum(nil)
 }
 
+// Apply applies the vars to all the nodes in the dictionary.
+func (d *Dict) Apply(vars Vars) (Node, error) {
+	nodes := make([]Node, len(d.value))
+	for i, v := range d.value {
+		n, err := v.Apply(vars)
+		if err != nil {
+			return nil, err
+		}
+		nodes[i] = n
+	}
+	return &Dict{nodes, nil}, nil
+}
+
+// Processors returns any attached processors, because of variable substitution.
+func (d *Dict) Processors() Processors {
+	if d.processors != nil {
+		return d.processors
+	}
+	for _, v := range d.value {
+		if p := v.Processors(); p != nil {
+			return p
+		}
+	}
+	return nil
+}
+
 // sort sorts the keys in the dictionary
 func (d *Dict) sort() {
 	sort.Slice(d.value, func(i, j int) bool {
@@ -157,6 +200,11 @@ func (k *Key) Find(key string) (Node, bool) {
 	}
 }
 
+// Name returns the name for the key.
+func (k *Key) Name() string {
+	return k.name
+}
+
 // Value returns the raw value.
 func (k *Key) Value() interface{} {
 	return k.value
@@ -181,26 +229,52 @@ func (k *Key) Hash() []byte {
 	return h.Sum(nil)
 }
 
+// Apply applies the vars to the value.
+func (k *Key) Apply(vars Vars) (Node, error) {
+	if k.value == nil {
+		return k, nil
+	}
+	v, err := k.value.Apply(vars)
+	if err != nil {
+		return nil, err
+	}
+	return &Key{k.name, v}, nil
+}
+
+// Processors returns any attached processors, because of variable substitution.
+func (k *Key) Processors() Processors {
+	if k.value != nil {
+		return k.value.Processors()
+	}
+	return nil
+}
+
 // List represents a slice in our Tree.
 type List struct {
-	value []Node
+	value      []Node
+	processors Processors
 }
 
 // NewList creates a new list with provided nodes.
 func NewList(nodes []Node) *List {
-	return &List{nodes}
+	return NewListWithProcessors(nodes, nil)
+}
+
+// NewListWithProcessors creates a new list with provided nodes with processors attached.
+func NewListWithProcessors(nodes []Node, processors Processors) *List {
+	return &List{nodes, processors}
 }
 
 func (l *List) String() string {
 	var sb strings.Builder
+	sb.WriteString("[")
 	for i := 0; i < len(l.value); i++ {
-		sb.WriteString("[")
 		sb.WriteString(l.value[i].String())
-		sb.WriteString("]")
 		if i < len(l.value)-1 {
 			sb.WriteString(",")
 		}
 	}
+	sb.WriteString("]")
 	return sb.String()
 }
 
@@ -244,14 +318,46 @@ func (l *List) Clone() Node {
 	return &List{value: nodes}
 }
 
+// Apply applies the vars to all nodes in the list.
+func (l *List) Apply(vars Vars) (Node, error) {
+	nodes := make([]Node, len(l.value))
+	for i, v := range l.value {
+		n, err := v.Apply(vars)
+		if err != nil {
+			return nil, err
+		}
+		nodes[i] = n
+	}
+	return NewList(nodes), nil
+}
+
+// Processors returns any attached processors, because of variable substitution.
+func (l *List) Processors() Processors {
+	if l.processors != nil {
+		return l.processors
+	}
+	for _, v := range l.value {
+		if p := v.Processors(); p != nil {
+			return p
+		}
+	}
+	return nil
+}
+
 // StrVal represents a string.
 type StrVal struct {
-	value string
+	value      string
+	processors Processors
 }
 
 // NewStrVal creates a new string value node with provided value.
 func NewStrVal(val string) *StrVal {
-	return &StrVal{val}
+	return NewStrValWithProcessors(val, nil)
+}
+
+// NewStrValWithProcessors creates a new string value node with provided value and processors.
+func NewStrValWithProcessors(val string, processors Processors) *StrVal {
+	return &StrVal{val, processors}
 }
 
 // Find receive a key and return false since the node is not a List or Dict.
@@ -279,14 +385,30 @@ func (s *StrVal) Hash() []byte {
 	return []byte(s.value)
 }
 
+// Apply applies the vars to the string value.
+func (s *StrVal) Apply(vars Vars) (Node, error) {
+	return vars.Replace(s.value)
+}
+
+// Processors returns any linked processors that are now connected because of Apply.
+func (s *StrVal) Processors() Processors {
+	return s.processors
+}
+
 // IntVal represents an int.
 type IntVal struct {
-	value int
+	value      int
+	processors Processors
 }
 
 // NewIntVal creates a new int value node with provided value.
 func NewIntVal(val int) *IntVal {
-	return &IntVal{val}
+	return NewIntValWithProcessors(val, nil)
+}
+
+// NewIntValWithProcessors creates a new int value node with provided value and attached processors.
+func NewIntValWithProcessors(val int, processors Processors) *IntVal {
+	return &IntVal{val, processors}
 }
 
 // Find receive a key and return false since the node is not a List or Dict.
@@ -309,19 +431,35 @@ func (s *IntVal) Clone() Node {
 	return &k
 }
 
+// Apply does nothing.
+func (s *IntVal) Apply(_ Vars) (Node, error) {
+	return s, nil
+}
+
 // Hash we convert the value into a string and return the byte slice.
 func (s *IntVal) Hash() []byte {
 	return []byte(s.String())
 }
 
+// Processors returns any linked processors that are now connected because of Apply.
+func (s *IntVal) Processors() Processors {
+	return s.processors
+}
+
 // UIntVal represents an int.
 type UIntVal struct {
-	value uint64
+	value      uint64
+	processors Processors
 }
 
 // NewUIntVal creates a new uint value node with provided value.
 func NewUIntVal(val uint64) *UIntVal {
-	return &UIntVal{val}
+	return NewUIntValWithProcessors(val, nil)
+}
+
+// NewUIntValWithProcessors creates a new uint value node with provided value with processors attached.
+func NewUIntValWithProcessors(val uint64, processors Processors) *UIntVal {
+	return &UIntVal{val, processors}
 }
 
 // Find receive a key and return false since the node is not a List or Dict.
@@ -349,15 +487,31 @@ func (s *UIntVal) Hash() []byte {
 	return []byte(s.String())
 }
 
+// Apply does nothing.
+func (s *UIntVal) Apply(_ Vars) (Node, error) {
+	return s, nil
+}
+
+// Processors returns any linked processors that are now connected because of Apply.
+func (s *UIntVal) Processors() Processors {
+	return s.processors
+}
+
 // FloatVal represents a float.
 // NOTE: We will convert float32 to a float64.
 type FloatVal struct {
-	value float64
+	value      float64
+	processors Processors
 }
 
 // NewFloatVal creates a new float value node with provided value.
 func NewFloatVal(val float64) *FloatVal {
-	return &FloatVal{val}
+	return NewFloatValWithProcessors(val, nil)
+}
+
+// NewFloatValWithProcessors creates a new float value node with provided value with processors attached.
+func NewFloatValWithProcessors(val float64, processors Processors) *FloatVal {
+	return &FloatVal{val, processors}
 }
 
 // Find receive a key and return false since the node is not a List or Dict.
@@ -385,14 +539,30 @@ func (s *FloatVal) Hash() []byte {
 	return []byte(strconv.FormatFloat(s.value, 'f', -1, 64))
 }
 
+// Apply does nothing.
+func (s *FloatVal) Apply(_ Vars) (Node, error) {
+	return s, nil
+}
+
+// Processors returns any linked processors that are now connected because of Apply.
+func (s *FloatVal) Processors() Processors {
+	return s.processors
+}
+
 // BoolVal represents a boolean in our Tree.
 type BoolVal struct {
-	value bool
+	value      bool
+	processors Processors
 }
 
 // NewBoolVal creates a new bool value node with provided value.
 func NewBoolVal(val bool) *BoolVal {
-	return &BoolVal{val}
+	return NewBoolValWithProcessors(val, nil)
+}
+
+// NewBoolValWithProcessors creates a new bool value node with provided value with processors attached.
+func NewBoolValWithProcessors(val bool, processors Processors) *BoolVal {
+	return &BoolVal{val, processors}
 }
 
 // Find receive a key and return false since the node is not a List or Dict.
@@ -426,13 +596,22 @@ func (s *BoolVal) Hash() []byte {
 	return falseVal
 }
 
+// Apply does nothing.
+func (s *BoolVal) Apply(_ Vars) (Node, error) {
+	return s, nil
+}
+
+// Processors returns any linked processors that are now connected because of Apply.
+func (s *BoolVal) Processors() Processors {
+	return s.processors
+}
+
 // NewAST takes a map and convert it to an internal Tree, allowing us to executes rules on the
 // data to shape it in a different way or to filter some of the information.
 func NewAST(m map[string]interface{}) (*AST, error) {
-	val := reflect.ValueOf(m)
-	root, err := load(val)
+	root, err := loadForNew(m)
 	if err != nil {
-		return nil, fmt.Errorf("could not parse configuration into a tree, error: %+v", err)
+		return nil, err
 	}
 	return &AST{root: root}, nil
 }
@@ -446,6 +625,40 @@ func MustNewAST(m map[string]interface{}) *AST {
 	return v
 }
 
+// NewASTFromConfig takes a config and converts it to an internal Tree, allowing us to executes rules on the
+// data to shape it in a different way or to filter some of the information.
+func NewASTFromConfig(cfg *ucfg.Config) (*AST, error) {
+	var v interface{}
+	if cfg.IsDict() {
+		var m map[string]interface{}
+		if err := cfg.Unpack(&m); err != nil {
+			return nil, err
+		}
+		v = m
+	} else if cfg.IsArray() {
+		var l []string
+		if err := cfg.Unpack(&l); err != nil {
+			return nil, err
+		}
+		v = l
+	} else {
+		return nil, fmt.Errorf("cannot create AST from none dict or array type")
+	}
+	root, err := loadForNew(v)
+	if err != nil {
+		return nil, err
+	}
+	return &AST{root: root}, nil
+}
+
+func loadForNew(val interface{}) (Node, error) {
+	root, err := load(reflect.ValueOf(val))
+	if err != nil {
+		return nil, fmt.Errorf("could not parse configuration into a tree, error: %+v", err)
+	}
+	return root, nil
+}
+
 func load(val reflect.Value) (Node, error) {
 	val = lookupVal(val)
 
@@ -557,6 +770,16 @@ func (a *AST) MarshalJSON() ([]byte, error) {
 	return b, nil
 }
 
+// Apply applies the variables to the replacement in the AST.
+func (a *AST) Apply(vars Vars) error {
+	n, err := a.root.Apply(vars)
+	if err != nil {
+		return err
+	}
+	a.root = n
+	return nil
+}
+
 func splitPath(s Selector) []string {
 	if s == "" {
 		return nil
@@ -666,6 +889,26 @@ func lookupVal(val reflect.Value) reflect.Value {
 	return val
 }
 
+func attachProcessors(node Node, processors Processors) Node {
+	switch n := node.(type) {
+	case *Dict:
+		n.processors = processors
+	case *List:
+		n.processors = processors
+	case *StrVal:
+		n.processors = processors
+	case *IntVal:
+		n.processors = processors
+	case *UIntVal:
+		n.processors = processors
+	case *FloatVal:
+		n.processors = processors
+	case *BoolVal:
+		n.processors = processors
+	}
+	return node
+}
+
 // Select takes an AST and a selector and will return a sub AST based on the selector path, will
 // return false if the path could not be found.
 func Select(a *AST, selector Selector) (*AST, bool) {
@@ -763,7 +1006,7 @@ func Insert(a *AST, node Node, to Selector) error {
 	case *List:
 		d.value = node
 	default:
-		d.value = &Dict{[]Node{node}}
+		d.value = &Dict{[]Node{node}, nil}
 	}
 	return nil
 }
diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/ast_test.go b/x-pack/elastic-agent/pkg/agent/transpiler/ast_test.go
index 80c7a1d7401..742850bdeb4 100644
--- a/x-pack/elastic-agent/pkg/agent/transpiler/ast_test.go
+++ b/x-pack/elastic-agent/pkg/agent/transpiler/ast_test.go
@@ -112,13 +112,13 @@ func TestAST(t *testing.T) {
 					value: []Node{
 						&Key{
 							name: "range",
-							value: &List{
+							value: NewList(
 								[]Node{
 									&IntVal{value: 20},
 									&IntVal{value: 30},
 									&IntVal{value: 40},
 								},
-							},
+							),
 						},
 						&Key{name: "timeout", value: &IntVal{value: 12}},
 					},
@@ -135,13 +135,13 @@ func TestAST(t *testing.T) {
 					value: []Node{
 						&Key{
 							name: "range",
-							value: &List{
+							value: NewList(
 								[]Node{
 									&UIntVal{value: uint64(20)},
 									&UIntVal{value: uint64(30)},
 									&UIntVal{value: uint64(40)},
 								},
-							},
+							),
 						},
 						&Key{name: "timeout", value: &IntVal{value: 12}},
 					},
@@ -159,23 +159,23 @@ func TestAST(t *testing.T) {
 					value: []Node{
 						&Key{
 							name: "range32",
-							value: &List{
+							value: NewList(
 								[]Node{
 									&FloatVal{value: 20.0},
 									&FloatVal{value: 30.0},
 									&FloatVal{value: 40.0},
 								},
-							},
+							),
 						},
 						&Key{
 							name: "range64",
-							value: &List{
+							value: NewList(
 								[]Node{
 									&FloatVal{value: 20.0},
 									&FloatVal{value: 30.0},
 									&FloatVal{value: 40.0},
 								},
-							},
+							),
 						},
 						&Key{name: "ratio", value: &FloatVal{value: 0.5}},
 					},
@@ -195,7 +195,7 @@ func TestAST(t *testing.T) {
 					value: []Node{
 						&Key{
 							name: "inputs",
-							value: &Dict{
+							value: NewDict(
 								[]Node{
 									&Key{name: "ignore_older", value: &StrVal{value: "20s"}},
 									&Key{name: "paths", value: &List{value: []Node{
@@ -203,8 +203,8 @@ func TestAST(t *testing.T) {
 										&StrVal{value: "/var/log/log2"},
 									}}},
 									&Key{name: "type", value: &StrVal{value: "log/docker"}},
-								},
-							}},
+								}),
+						},
 					},
 				},
 			},
@@ -225,11 +225,11 @@ func TestAST(t *testing.T) {
 				},
 			},
 			ast: &AST{
-				root: &Dict{
-					value: []Node{
+				root: NewDict(
+					[]Node{
 						&Key{
 							name: "inputs",
-							value: &Dict{
+							value: NewDict(
 								[]Node{
 									&Key{name: "ignore_older", value: &StrVal{value: "20s"}},
 									&Key{name: "paths", value: &List{value: []Node{
@@ -237,42 +237,37 @@ func TestAST(t *testing.T) {
 										&StrVal{value: "/var/log/log2"},
 									}}},
 									&Key{name: "type", value: &StrVal{value: "log/docker"}},
-								},
-							}},
+								}),
+						},
 						&Key{
 							name: "outputs",
-							value: &Dict{
+							value: NewDict(
 								[]Node{
 									&Key{
 										name: "elasticsearch",
-										value: &Dict{
+										value: NewDict(
 											[]Node{
 												&Key{
 													name: "ssl",
-													value: &Dict{
+													value: NewDict(
 														[]Node{
 															&Key{name: "certificates_authorities",
-																value: &List{
+																value: NewList(
 																	[]Node{
 																		&StrVal{value: "abc1"},
 																		&StrVal{value: "abc2"},
 																	},
-																},
+																),
 															},
-														},
-													},
+														}),
 												},
-											},
-										},
+											}),
 									},
-								},
-							},
+								}),
 						},
-					},
-				},
+					}),
 			},
 		},
-
 		"Keys with multiple levels of deeps with compact keys": {
 			hashmap: map[string]interface{}{
 				"inputs": map[string]interface{}{
@@ -305,7 +300,7 @@ func TestAST(t *testing.T) {
 					value: []Node{
 						&Key{
 							name: "inputs",
-							value: &Dict{
+							value: NewDict(
 								[]Node{
 									&Key{name: "ignore_older", value: &StrVal{value: "20s"}},
 									&Key{name: "paths", value: &List{value: []Node{
@@ -313,36 +308,33 @@ func TestAST(t *testing.T) {
 										&StrVal{value: "/var/log/log2"},
 									}}},
 									&Key{name: "type", value: &StrVal{value: "log/docker"}},
-								},
-							}},
+								}),
+						},
 						&Key{
 							name: "outputs",
-							value: &Dict{
+							value: NewDict(
 								[]Node{
 									&Key{
 										name: "elasticsearch",
-										value: &Dict{
+										value: NewDict(
 											[]Node{
 												&Key{
 													name: "ssl",
-													value: &Dict{
+													value: NewDict(
 														[]Node{
 															&Key{name: "certificates_authorities",
-																value: &List{
+																value: NewList(
 																	[]Node{
 																		&StrVal{value: "abc1"},
 																		&StrVal{value: "abc2"},
 																	},
-																},
+																),
 															},
-														},
-													},
+														}),
 												},
-											},
-										},
+											}),
 									},
-								},
-							},
+								}),
 						},
 					},
 				},
@@ -404,11 +396,11 @@ func TestSelector(t *testing.T) {
 					value: []Node{
 						&Key{
 							name: "inputs",
-							value: &Dict{
+							value: NewDict(
 								[]Node{
 									&Key{name: "type", value: &StrVal{value: "log/docker"}},
-								},
-							}},
+								}),
+						},
 					},
 				},
 			},
@@ -431,17 +423,18 @@ func TestSelector(t *testing.T) {
 					value: []Node{
 						&Key{
 							name: "inputs",
-							value: &Dict{
+							value: NewDict(
 								[]Node{
-									&Key{name: "ssl", value: &Dict{
+									&Key{name: "ssl", value: NewDict(
 										[]Node{
 											&Key{name: "ca", value: &List{
 												value: []Node{&StrVal{value: "ca1"}, &StrVal{value: "ca2"}},
 											}},
 											&Key{name: "certificate", value: &StrVal{value: "/etc/ssl/my.crt"}},
-										}}},
-								},
-							}},
+										}),
+									},
+								}),
+						},
 					},
 				},
 			},
@@ -471,11 +464,11 @@ func TestSelector(t *testing.T) {
 					value: []Node{
 						&Key{
 							name: "inputs",
-							value: &Dict{
+							value: NewDict(
 								[]Node{
 									&Key{name: "1", value: &StrVal{value: "log/docker"}},
-								},
-							}},
+								}),
+						},
 					},
 				},
 			},
@@ -495,21 +488,20 @@ func TestSelector(t *testing.T) {
 					value: []Node{
 						&Key{
 							name: "inputs",
-							value: &Dict{
+							value: NewDict(
 								[]Node{
-									&Key{name: "x", value: &Dict{
+									&Key{name: "x", value: NewDict(
 										[]Node{
-											&Key{name: "ssl", value: &Dict{
+											&Key{name: "ssl", value: NewDict(
 												[]Node{
 													&Key{name: "ca", value: &List{
 														value: []Node{&StrVal{value: "ca1"}, &StrVal{value: "ca2"}},
 													}},
 													&Key{name: "certificate", value: &StrVal{value: "/etc/ssl/my.crt"}},
-												}}},
-										},
-									}},
-								},
-							},
+												})},
+										}),
+									},
+								}),
 						},
 					},
 				},
@@ -536,32 +528,31 @@ func TestSelector(t *testing.T) {
 					value: []Node{
 						&Key{
 							name: "inputs",
-							value: &Dict{
+							value: NewDict(
 								[]Node{
-									&Key{name: "x", value: &Dict{
+									&Key{name: "x", value: NewDict(
 										[]Node{
-											&Key{name: "ssl", value: &Dict{
+											&Key{name: "ssl", value: NewDict(
 												[]Node{
 													&Key{name: "ca", value: &List{
 														value: []Node{&StrVal{value: "ca1"}, &StrVal{value: "ca2"}},
 													}},
 													&Key{name: "certificate", value: &StrVal{value: "/etc/ssl/my.crt"}},
-												}}},
-										},
-									}},
-									&Key{name: "y", value: &Dict{
+												})},
+										}),
+									},
+									&Key{name: "y", value: NewDict(
 										[]Node{
-											&Key{name: "ssl", value: &Dict{
+											&Key{name: "ssl", value: NewDict(
 												[]Node{
 													&Key{name: "ca", value: &List{
 														value: []Node{&StrVal{value: "ca1"}, &StrVal{value: "ca2"}},
 													}},
 													&Key{name: "certificate", value: &StrVal{value: "/etc/ssl/my.crt"}},
-												}}},
-										},
-									}},
-								},
-							},
+												})},
+										}),
+									},
+								}),
 						},
 					},
 				},
@@ -588,21 +579,20 @@ func TestSelector(t *testing.T) {
 					value: []Node{
 						&Key{
 							name: "inputs",
-							value: &Dict{
+							value: NewDict(
 								[]Node{
-									&Key{name: "x", value: &Dict{
+									&Key{name: "x", value: NewDict(
 										[]Node{
-											&Key{name: "ssl", value: &Dict{
+											&Key{name: "ssl", value: NewDict(
 												[]Node{
 													&Key{name: "ca", value: &List{
 														value: []Node{&StrVal{value: "ca1"}, &StrVal{value: "ca2"}},
 													}},
 													&Key{name: "certificate", value: &StrVal{value: "/etc/ssl/my.crt"}},
-												}}},
-										},
-									}},
-								},
-							},
+												})},
+										}),
+									},
+								}),
 						},
 					},
 				},
@@ -623,21 +613,20 @@ func TestSelector(t *testing.T) {
 					value: []Node{
 						&Key{
 							name: "inputs",
-							value: &Dict{
+							value: NewDict(
 								[]Node{
-									&Key{name: "x", value: &Dict{
+									&Key{name: "x", value: NewDict(
 										[]Node{
-											&Key{name: "ssl", value: &Dict{
+											&Key{name: "ssl", value: NewDict(
 												[]Node{
 													&Key{name: "ca", value: &List{
 														value: []Node{&StrVal{value: "ca1"}, &StrVal{value: "ca2"}},
 													}},
 													&Key{name: "certificate", value: &StrVal{value: "/etc/ssl/my.crt"}},
-												}}},
-										},
-									}},
-								},
-							},
+												})},
+										}),
+									},
+								}),
 						},
 					},
 				},
@@ -662,21 +651,20 @@ func TestSelector(t *testing.T) {
 					value: []Node{
 						&Key{
 							name: "inputs",
-							value: &Dict{
+							value: NewDict(
 								[]Node{
-									&Key{name: "x", value: &Dict{
+									&Key{name: "x", value: NewDict(
 										[]Node{
-											&Key{name: "ssl", value: &Dict{
+											&Key{name: "ssl", value: NewDict(
 												[]Node{
 													&Key{name: "ca", value: &List{
 														value: []Node{&StrVal{value: "ca1"}, &StrVal{value: "ca2"}},
 													}},
 													&Key{name: "certificate", value: &StrVal{value: "/etc/ssl/my.crt"}},
-												}}},
-										},
-									}},
-								},
-							},
+												})},
+										}),
+									},
+								}),
 						},
 					},
 				},
@@ -704,6 +692,196 @@ func TestSelector(t *testing.T) {
 	}
 }
 
+func TestAST_Apply(t *testing.T) {
+	testcases := map[string]struct {
+		input    map[string]interface{}
+		expected *AST
+		vars     Vars
+		matchErr bool
+	}{
+		//"2 vars missing with default": {
+		//	input: map[string]interface{}{
+		//		"inputs": map[string]interface{}{
+		//			"type":  "log/docker",
+		//			"paths": []string{"/var/log/${var1.key1}", "/var/log/${var1.missing|'other'}"},
+		//		},
+		//	},
+		//	expected: &AST{
+		//		root: &Dict{
+		//			value: []Node{
+		//				&Key{
+		//					name: "inputs",
+		//					value: NewDict(
+		//						[]Node{
+		//							&Key{
+		//								name: "paths",
+		//								value: &List{
+		//									value: []Node{
+		//										&StrVal{value: "/var/log/value1"},
+		//										&StrVal{value: "/var/log/other"},
+		//									},
+		//								},
+		//							},
+		//							&Key{name: "type", value: &StrVal{value: "log/docker"}},
+		//						}),
+		//				},
+		//			},
+		//		},
+		//	},
+		//	vars: Vars{
+		//		Mapping: map[string]interface{}{
+		//			"var1": map[string]interface{}{
+		//				"key1": "value1",
+		//			},
+		//		},
+		//	},
+		//},
+		//"2 vars missing no default": {
+		//	input: map[string]interface{}{
+		//		"inputs": map[string]interface{}{
+		//			"type":  "log/docker",
+		//			"paths": []string{"/var/log/${var1.key1}", "/var/log/${var1.missing}"},
+		//		},
+		//	},
+		//	vars: Vars{
+		//		Mapping: map[string]interface{}{
+		//			"var1": map[string]interface{}{
+		//				"key1": "value1",
+		//			},
+		//		},
+		//	},
+		//	matchErr: true,
+		//},
+		//"vars not string": {
+		//	input: map[string]interface{}{
+		//		"inputs": map[string]interface{}{
+		//			"type":  "log/docker",
+		//			"paths": []string{"/var/log/${var1.key1}"},
+		//		},
+		//	},
+		//	expected: &AST{
+		//		root: &Dict{
+		//			value: []Node{
+		//				&Key{
+		//					name: "inputs",
+		//					value: NewDict(
+		//						[]Node{
+		//							&Key{
+		//								name: "paths",
+		//								value: &List{
+		//									value: []Node{
+		//										&StrVal{value: "/var/log/1"},
+		//									},
+		//								},
+		//							},
+		//							&Key{name: "type", value: &StrVal{value: "log/docker"}},
+		//						}),
+		//				},
+		//			},
+		//		},
+		//	},
+		//	vars: Vars{
+		//		Mapping: map[string]interface{}{
+		//			"var1": map[string]interface{}{
+		//				"key1": 1,
+		//			},
+		//		},
+		//	},
+		//},
+		"vars replace with object": {
+			input: map[string]interface{}{
+				"inputs": map[string]interface{}{
+					"type":  "logfile",
+					"paths": []string{"/var/log/syslog"},
+					"processors": []map[string]interface{}{
+						{
+							"add_fields": map[string]interface{}{
+								"labels": "${host.labels}",
+							},
+						},
+					},
+				},
+			},
+			expected: &AST{
+				root: &Dict{
+					value: []Node{
+						&Key{
+							name: "inputs",
+							value: NewDict(
+								[]Node{
+									&Key{
+										name: "paths",
+										value: &List{
+											value: []Node{
+												&StrVal{value: "/var/log/syslog"},
+											},
+										},
+									},
+									&Key{
+										name: "processors",
+										value: &List{
+											value: []Node{
+												NewDict(
+													[]Node{
+														&Key{
+															name: "add_fields",
+															value: NewDict(
+																[]Node{
+																	&Key{
+																		name: "labels",
+																		value: &List{
+																			value: []Node{
+																				&StrVal{value: "label1"},
+																				&StrVal{value: "label2"},
+																			},
+																		},
+																	},
+																},
+															),
+														},
+													},
+												),
+											},
+										},
+									},
+									&Key{name: "type", value: &StrVal{value: "logfile"}},
+								}),
+						},
+					},
+				},
+			},
+			vars: Vars{
+				Mapping: map[string]interface{}{
+					"host": map[string]interface{}{
+						"labels": []string{
+							"label1",
+							"label2",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for name, test := range testcases {
+		t.Run(name, func(t *testing.T) {
+			v, err := NewAST(test.input)
+			require.NoError(t, err)
+			err = v.Apply(test.vars)
+			if test.matchErr {
+				require.Equal(t, ErrNoMatch, err)
+			} else {
+				require.NoError(t, err)
+				if !assert.True(t, reflect.DeepEqual(test.expected, v)) {
+					t.Logf(
+						`received: %+v
+					 expected: %+v`, v, test.expected)
+				}
+			}
+		})
+	}
+}
+
 func TestCount(t *testing.T) {
 	ast := &AST{
 		root: &Dict{
diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/vars.go b/x-pack/elastic-agent/pkg/agent/transpiler/vars.go
new file mode 100644
index 00000000000..f5b7b9922d3
--- /dev/null
+++ b/x-pack/elastic-agent/pkg/agent/transpiler/vars.go
@@ -0,0 +1,199 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package transpiler
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+	"unicode"
+
+	"github.com/elastic/go-ucfg"
+)
+
+var varsRegex = regexp.MustCompile(`\${([\p{L}\d\s\\\-_|.'"]*)}`)
+
+// ErrNoMatch is return when the replace didn't fail, just that no vars match to perform the replace.
+var ErrNoMatch = fmt.Errorf("no matching vars")
+
+// Vars is a context of variables that also contain a list of processors that go with the mapping.
+type Vars struct {
+	Mapping map[string]interface{}
+
+	ProcessorsKey string
+	Processors    Processors
+}
+
+// Replace returns a new value based on variable replacement.
+func (v *Vars) Replace(value string) (Node, error) {
+	var processors []map[string]interface{}
+	c, err := ucfg.NewFrom(v.Mapping, ucfg.PathSep("."))
+	if err != nil {
+		return nil, err
+	}
+	matchIdxs := varsRegex.FindAllSubmatchIndex([]byte(value), -1)
+	if !validBrackets(value, matchIdxs) {
+		return nil, fmt.Errorf("starting ${ is missing ending }")
+	}
+
+	result := ""
+	lastIndex := 0
+	for _, r := range matchIdxs {
+		for i := 0; i < len(r); i += 4 {
+			vars, err := extractVars(value[r[i+2]:r[i+3]])
+			if err != nil {
+				return nil, fmt.Errorf(`error parsing variable "%s": %s`, value[r[i]:r[i+1]], err)
+			}
+			set := false
+			for _, val := range vars {
+				switch val.(type) {
+				case *constString:
+					result += value[lastIndex:r[0]] + val.Value()
+					set = true
+				case *varString:
+					if r[i] == 0 && r[i+1] == len(value) {
+						// possible for complete replacement of object, because the variable
+						// is not inside of a string
+						child, err := c.Child(val.Value(), -1, ucfg.PathSep("."))
+						if err == nil {
+							ast, err := NewASTFromConfig(child)
+							if err == nil {
+								if v.ProcessorsKey != "" && varPrefixMatched(val.Value(), v.ProcessorsKey) {
+									processors = v.Processors
+								}
+								return attachProcessors(ast.root, processors), nil
+							}
+						}
+					}
+					replace, err := c.String(val.Value(), -1, ucfg.PathSep("."))
+					if err == nil {
+						result += value[lastIndex:r[0]] + replace
+						set = true
+						if v.ProcessorsKey != "" && varPrefixMatched(val.Value(), v.ProcessorsKey) {
+							processors = v.Processors
+						}
+					}
+				}
+				if set {
+					break
+				}
+			}
+			if !set {
+				return NewStrVal(""), ErrNoMatch
+			}
+			lastIndex = r[1]
+		}
+	}
+	return NewStrValWithProcessors(result+value[lastIndex:], processors), nil
+}
+
+// validBrackets returns true when all starting {$ have a matching ending }.
+func validBrackets(s string, matchIdxs [][]int) bool {
+	result := ""
+	lastIndex := 0
+	match := false
+	for _, r := range matchIdxs {
+		match = true
+		for i := 0; i < len(r); i += 4 {
+			result += s[lastIndex:r[0]]
+			lastIndex = r[1]
+		}
+	}
+	if !match {
+		return !strings.Contains(s, "${")
+	}
+	return !strings.Contains(result, "${")
+}
+
+type varI interface {
+	Value() string
+}
+
+type varString struct {
+	value string
+}
+
+func (v *varString) Value() string {
+	return v.value
+}
+
+type constString struct {
+	value string
+}
+
+func (v *constString) Value() string {
+	return v.value
+}
+
+func extractVars(i string) ([]varI, error) {
+	const out = rune(0)
+
+	quote := out
+	constant := false
+	escape := false
+	is := make([]rune, 0, len(i))
+	res := make([]varI, 0)
+	for _, r := range i {
+		if r == '|' {
+			if escape {
+				return nil, fmt.Errorf(`variable pipe cannot be escaped; remove \ before |`)
+			}
+			if quote == out {
+				if constant {
+					res = append(res, &constString{string(is)})
+				} else if len(is) > 0 {
+					if is[len(is)-1] == '.' {
+						return nil, fmt.Errorf("variable cannot end with '.'")
+					}
+					res = append(res, &varString{string(is)})
+				}
+				is = is[:0] // slice to zero length; to keep allocated memory
+				constant = false
+			} else {
+				is = append(is, r)
+			}
+			continue
+		}
+		if !escape && (r == '"' || r == '\'') {
+			if quote == out {
+				// start of unescaped quote
+				quote = r
+				constant = true
+			} else if quote == r {
+				// end of unescaped quote
+				quote = out
+			} else {
+				is = append(is, r)
+			}
+			continue
+		}
+		// escape because of backslash (\); except when it is the second backslash of a pair
+		escape = !escape && r == '\\'
+		if r == '\\' {
+			if !escape {
+				is = append(is, r)
+			}
+		} else if quote != out || !unicode.IsSpace(r) {
+			is = append(is, r)
+		}
+	}
+	if quote != out {
+		return nil, fmt.Errorf(`starting %s is missing ending %s`, string(quote), string(quote))
+	}
+	if constant {
+		res = append(res, &constString{string(is)})
+	} else if len(is) > 0 {
+		if is[len(is)-1] == '.' {
+			return nil, fmt.Errorf("variable cannot end with '.'")
+		}
+		res = append(res, &varString{string(is)})
+	}
+	return res, nil
+}
+
+func varPrefixMatched(val string, key string) bool {
+	s := strings.SplitN(val, ".", 2)
+	return s[0] == key
+}
diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/vars_test.go b/x-pack/elastic-agent/pkg/agent/transpiler/vars_test.go
new file mode 100644
index 00000000000..31249316099
--- /dev/null
+++ b/x-pack/elastic-agent/pkg/agent/transpiler/vars_test.go
@@ -0,0 +1,250 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package transpiler
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestVars_Replace(t *testing.T) {
+	vars := &Vars{
+		Mapping: map[string]interface{}{
+			"un-der_score": map[string]interface{}{
+				"key1": "data1",
+				"key2": "data2",
+				"list": []string{
+					"array1",
+					"array2",
+				},
+				"dict": map[string]interface{}{
+					"key1": "value1",
+					"key2": "value2",
+				},
+			},
+			"other": map[string]interface{}{
+				"data": "info",
+			},
+		},
+	}
+	tests := []struct {
+		Input   string
+		Result  Node
+		Error   bool
+		NoMatch bool
+	}{
+		{
+			"${un-der_score.key1}",
+			NewStrVal("data1"),
+			false,
+			false,
+		},
+		{
+			"${un-der_score.missing}",
+			NewStrVal(""),
+			false,
+			true,
+		},
+		{
+			"${un-der_score.missing|un-der_score.key2}",
+			NewStrVal("data2"),
+			false,
+			false,
+		},
+		{
+			"${un-der_score.missing|un-der_score.missing2|other.data}",
+			NewStrVal("info"),
+			false,
+			false,
+		},
+		{
+			"${un-der_score.missing|'fallback'}",
+			NewStrVal("fallback"),
+			false,
+			false,
+		},
+		{
+			`${un-der_score.missing|||||||||"fallback"}`,
+			NewStrVal("fallback"),
+			false,
+			false,
+		},
+		{
+			`${"direct"}`,
+			NewStrVal("direct"),
+			false,
+			false,
+		},
+		{
+			`${un-der_score.}`,
+			NewStrVal(""),
+			true,
+			false,
+		},
+		{
+			`${un-der_score.missing|"oth}`,
+			NewStrVal(""),
+			true,
+			false,
+		},
+		{
+			`${un-der_score.missing`,
+			NewStrVal(""),
+			true,
+			false,
+		},
+		{
+			`${un-der_score.missing  ${other}`,
+			NewStrVal(""),
+			true,
+			false,
+		},
+		{
+			`${}`,
+			NewStrVal(""),
+			true,
+			false,
+		},
+		{
+			"around ${un-der_score.key1} the var",
+			NewStrVal("around data1 the var"),
+			false,
+			false,
+		},
+		{
+			"multi ${un-der_score.key1} var ${ un-der_score.missing |     un-der_score.key2      } around",
+			NewStrVal("multi data1 var data2 around"),
+			false,
+			false,
+		},
+		{
+			`multi ${un-der_score.key1} var ${  un-der_score.missing|  'other"s with space'  } around`,
+			NewStrVal(`multi data1 var other"s with space around`),
+			false,
+			false,
+		},
+		{
+			`start ${  un-der_score.missing|  'others | with space'  } end`,
+			NewStrVal(`start others | with space end`),
+			false,
+			false,
+		},
+		{
+			`start ${  un-der_score.missing|  'other\'s with space'  } end`,
+			NewStrVal(`start other's with space end`),
+			false,
+			false,
+		},
+		{
+			`${un-der_score.list}`,
+			NewList([]Node{
+				NewStrVal("array1"),
+				NewStrVal("array2"),
+			}),
+			false,
+			false,
+		},
+		{
+			`list inside string ${un-der_score.list} causes no match`,
+			NewList([]Node{
+				NewStrVal("array1"),
+				NewStrVal("array2"),
+			}),
+			false,
+			true,
+		},
+		{
+			`${un-der_score.dict}`,
+			NewDict([]Node{
+				NewKey("key1", NewStrVal("value1")),
+				NewKey("key2", NewStrVal("value2")),
+			}),
+			false,
+			false,
+		},
+		{
+			`dict inside string ${un-der_score.dict} causes no match`,
+			NewDict([]Node{
+				NewKey("key1", NewStrVal("value1")),
+				NewKey("key2", NewStrVal("value2")),
+			}),
+			false,
+			true,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.Input, func(t *testing.T) {
+			res, err := vars.Replace(test.Input)
+			if test.Error {
+				assert.Error(t, err)
+			} else if test.NoMatch {
+				assert.Error(t, ErrNoMatch, err)
+			} else {
+				require.NoError(t, err)
+				assert.Equal(t, test.Result, res)
+			}
+		})
+	}
+}
+
+func TestVars_ReplaceWithProcessors(t *testing.T) {
+	processers := Processors{
+		{
+			"add_fields": map[string]interface{}{
+				"dynamic": "added",
+			},
+		},
+	}
+	vars := &Vars{
+		Mapping: map[string]interface{}{
+			"testing": map[string]interface{}{
+				"key1": "data1",
+			},
+			"dynamic": map[string]interface{}{
+				"key1": "dynamic1",
+				"list": []string{
+					"array1",
+					"array2",
+				},
+				"dict": map[string]string{
+					"key1": "value1",
+					"key2": "value2",
+				},
+			},
+		},
+		ProcessorsKey: "dynamic",
+		Processors:    processers,
+	}
+
+	res, err := vars.Replace("${testing.key1}")
+	require.NoError(t, err)
+	assert.Equal(t, NewStrVal("data1"), res)
+
+	res, err = vars.Replace("${dynamic.key1}")
+	require.NoError(t, err)
+	assert.Equal(t, NewStrValWithProcessors("dynamic1", processers), res)
+
+	res, err = vars.Replace("${other.key1|dynamic.key1}")
+	require.NoError(t, err)
+	assert.Equal(t, NewStrValWithProcessors("dynamic1", processers), res)
+
+	res, err = vars.Replace("${dynamic.list}")
+	require.NoError(t, err)
+	assert.Equal(t, processers, res.Processors())
+	assert.Equal(t, NewListWithProcessors([]Node{
+		NewStrVal("array1"),
+		NewStrVal("array2"),
+	}, processers), res)
+
+	res, err = vars.Replace("${dynamic.dict}")
+	require.NoError(t, err)
+	assert.Equal(t, processers, res.Processors())
+	assert.Equal(t, NewDictWithProcessors([]Node{
+		NewKey("key1", NewStrVal("value1")),
+		NewKey("key2", NewStrVal("value2")),
+	}, processers), res)
+}
diff --git a/x-pack/elastic-agent/pkg/artifact/config.go b/x-pack/elastic-agent/pkg/artifact/config.go
index a8a09de8e48..6faa9861710 100644
--- a/x-pack/elastic-agent/pkg/artifact/config.go
+++ b/x-pack/elastic-agent/pkg/artifact/config.go
@@ -47,13 +47,14 @@ type Config struct {
 
 // DefaultConfig creates a config with pre-set default values.
 func DefaultConfig() *Config {
+	homePath := paths.Home()
 	dataPath := paths.Data()
 	return &Config{
 		SourceURI:       "https://artifacts.elastic.co/downloads/",
-		TargetDirectory: filepath.Join(dataPath, "downloads"),
+		TargetDirectory: filepath.Join(homePath, "downloads"),
 		Timeout:         30 * time.Second,
 		PgpFile:         filepath.Join(dataPath, "elastic.pgp"),
-		InstallPath:     filepath.Join(dataPath, "install"),
+		InstallPath:     filepath.Join(homePath, "install"),
 	}
 }
 
diff --git a/x-pack/elastic-agent/pkg/artifact/download/fs/downloader.go b/x-pack/elastic-agent/pkg/artifact/download/fs/downloader.go
index df289ae03ad..04f4c667e02 100644
--- a/x-pack/elastic-agent/pkg/artifact/download/fs/downloader.go
+++ b/x-pack/elastic-agent/pkg/artifact/download/fs/downloader.go
@@ -11,6 +11,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact"
 )
@@ -19,10 +20,6 @@ const (
 	packagePermissions = 0660
 )
 
-var (
-	defaultDropSubdir = filepath.Join("data", "downloads")
-)
-
 // Downloader is a downloader able to fetch artifacts from elastic.co web page.
 type Downloader struct {
 	dropPath string
@@ -117,13 +114,13 @@ func (e *Downloader) downloadFile(filename, fullPath string) (string, error) {
 func getDropPath(cfg *artifact.Config) string {
 	// if drop path is not provided fallback to beats subfolder
 	if cfg == nil || cfg.DropPath == "" {
-		return defaultDropSubdir
+		return filepath.Join(paths.Home(), "downloads")
 	}
 
 	// if droppath does not exist fallback to beats subfolder
 	stat, err := os.Stat(cfg.DropPath)
 	if err != nil || !stat.IsDir() {
-		return defaultDropSubdir
+		return filepath.Join(paths.Home(), "downloads")
 	}
 
 	return cfg.DropPath
diff --git a/x-pack/elastic-agent/pkg/artifact/download/fs/verifier.go b/x-pack/elastic-agent/pkg/artifact/download/fs/verifier.go
index 942a412efdf..20bff381a39 100644
--- a/x-pack/elastic-agent/pkg/artifact/download/fs/verifier.go
+++ b/x-pack/elastic-agent/pkg/artifact/download/fs/verifier.go
@@ -18,6 +18,7 @@ import (
 
 	"golang.org/x/crypto/openpgp"
 
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact"
 )
@@ -144,7 +145,7 @@ func (v *Verifier) verifyAsc(filename, fullPath string) (bool, error) {
 
 func (v *Verifier) getPublicAsc(filename string) ([]byte, error) {
 	ascFile := fmt.Sprintf("%s%s", filename, ascSuffix)
-	fullPath := filepath.Join(defaultDropSubdir, ascFile)
+	fullPath := filepath.Join(paths.Home(), "downloads", ascFile)
 
 	b, err := ioutil.ReadFile(fullPath)
 	if err != nil {
diff --git a/x-pack/elastic-agent/pkg/composable/controller.go b/x-pack/elastic-agent/pkg/composable/controller.go
index e11f416cbba..9ff0013e82d 100644
--- a/x-pack/elastic-agent/pkg/composable/controller.go
+++ b/x-pack/elastic-agent/pkg/composable/controller.go
@@ -8,38 +8,49 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+
 	"reflect"
 	"sort"
 	"sync"
 	"time"
 
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/transpiler"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
 )
 
-// Vars is a context of variables that also contain a list of processors that go with the mapping.
-type Vars struct {
-	Mapping map[string]interface{}
-
-	ProcessorsKey string
-	Processors    []map[string]interface{}
-}
-
 // VarsCallback is callback called when the current vars state changes.
-type VarsCallback func([]Vars)
+type VarsCallback func([]transpiler.Vars)
 
 // Controller manages the state of the providers current context.
-type Controller struct {
+type Controller interface {
+	// Run runs the controller.
+	//
+	// Cancelling the context stops the controller.
+	Run(ctx context.Context, cb VarsCallback) error
+}
+
+// controller manages the state of the providers current context.
+type controller struct {
 	contextProviders map[string]*contextProviderState
 	dynamicProviders map[string]*dynamicProviderState
 }
 
 // New creates a new controller.
-func New(c *config.Config) (*Controller, error) {
-	var providersCfg Config
-	err := c.Unpack(&providersCfg)
+func New(c *config.Config) (Controller, error) {
+	l, err := logger.New("composable")
 	if err != nil {
-		return nil, errors.New(err, "failed to unpack providers config", errors.TypeConfig)
+		return nil, err
+	}
+	l.Info("EXPERIMENTAL - Inputs with variables are currently experimental and should not be used in production")
+
+	var providersCfg Config
+	if c != nil {
+		err := c.Unpack(&providersCfg)
+		if err != nil {
+			return nil, errors.New(err, "failed to unpack providers config", errors.TypeConfig)
+		}
 	}
 
 	// build all the context providers
@@ -73,18 +84,18 @@ func New(c *config.Config) (*Controller, error) {
 		}
 		dynamicProviders[name] = &dynamicProviderState{
 			provider: provider,
-			mappings: map[string]Vars{},
+			mappings: map[string]transpiler.Vars{},
 		}
 	}
 
-	return &Controller{
+	return &controller{
 		contextProviders: contextProviders,
 		dynamicProviders: dynamicProviders,
 	}, nil
 }
 
 // Run runs the controller.
-func (c *Controller) Run(ctx context.Context, cb VarsCallback) error {
+func (c *controller) Run(ctx context.Context, cb VarsCallback) error {
 	// large number not to block performing Run on the provided providers
 	notify := make(chan bool, 5000)
 	localCtx, cancel := context.WithCancel(ctx)
@@ -136,12 +147,12 @@ func (c *Controller) Run(ctx context.Context, cb VarsCallback) error {
 			}
 
 			// build the vars list of mappings
-			vars := make([]Vars, 1)
+			vars := make([]transpiler.Vars, 1)
 			mapping := map[string]interface{}{}
 			for name, state := range c.contextProviders {
 				mapping[name] = state.Current()
 			}
-			vars[0] = Vars{
+			vars[0] = transpiler.Vars{
 				Mapping: mapping,
 			}
 
@@ -150,7 +161,7 @@ func (c *Controller) Run(ctx context.Context, cb VarsCallback) error {
 				for _, mappings := range state.Mappings() {
 					local, _ := cloneMap(mapping) // will not fail; already been successfully cloned once
 					local[name] = mappings.Mapping
-					vars = append(vars, Vars{
+					vars = append(vars, transpiler.Vars{
 						Mapping:       local,
 						ProcessorsKey: name,
 						Processors:    mappings.Processors,
@@ -207,7 +218,7 @@ type dynamicProviderState struct {
 
 	provider DynamicProvider
 	lock     sync.RWMutex
-	mappings map[string]Vars
+	mappings map[string]transpiler.Vars
 	signal   chan bool
 }
 
@@ -230,7 +241,7 @@ func (c *dynamicProviderState) AddOrUpdate(id string, mapping map[string]interfa
 		// same mapping; no need to update and signal
 		return nil
 	}
-	c.mappings[id] = Vars{
+	c.mappings[id] = transpiler.Vars{
 		Mapping:    mapping,
 		Processors: processors,
 	}
@@ -251,11 +262,11 @@ func (c *dynamicProviderState) Remove(id string) {
 }
 
 // Mappings returns the current mappings.
-func (c *dynamicProviderState) Mappings() []Vars {
+func (c *dynamicProviderState) Mappings() []transpiler.Vars {
 	c.lock.RLock()
 	defer c.lock.RUnlock()
 
-	mappings := make([]Vars, 0)
+	mappings := make([]transpiler.Vars, 0)
 	ids := make([]string, 0)
 	for name := range c.mappings {
 		ids = append(ids, name)
diff --git a/x-pack/elastic-agent/pkg/composable/controller_test.go b/x-pack/elastic-agent/pkg/composable/controller_test.go
index 43c2889a92a..2c1e2e15f3c 100644
--- a/x-pack/elastic-agent/pkg/composable/controller_test.go
+++ b/x-pack/elastic-agent/pkg/composable/controller_test.go
@@ -9,6 +9,8 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/transpiler"
+
 	"github.com/stretchr/testify/assert"
 
 	"github.com/stretchr/testify/require"
@@ -34,12 +36,36 @@ func TestController(t *testing.T) {
 				},
 			},
 			"local_dynamic": map[string]interface{}{
-				"vars": []map[string]interface{}{
+				"items": []map[string]interface{}{
 					{
-						"key1": "value1",
+						"vars": map[string]interface{}{
+							"key1": "value1",
+						},
+						"processors": []map[string]interface{}{
+							{
+								"add_fields": map[string]interface{}{
+									"fields": map[string]interface{}{
+										"add": "value1",
+									},
+									"to": "dynamic",
+								},
+							},
+						},
 					},
 					{
-						"key1": "value2",
+						"vars": map[string]interface{}{
+							"key1": "value2",
+						},
+						"processors": []map[string]interface{}{
+							{
+								"add_fields": map[string]interface{}{
+									"fields": map[string]interface{}{
+										"add": "value2",
+									},
+									"to": "dynamic",
+								},
+							},
+						},
 					},
 				},
 			},
@@ -54,8 +80,8 @@ func TestController(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	wg.Add(1)
-	var setVars []composable.Vars
-	err = c.Run(ctx, func(vars []composable.Vars) {
+	var setVars []transpiler.Vars
+	err = c.Run(ctx, func(vars []transpiler.Vars) {
 		setVars = vars
 		wg.Done()
 	})
@@ -76,10 +102,10 @@ func TestController(t *testing.T) {
 	localMap = setVars[1].Mapping["local_dynamic"].(map[string]interface{})
 	assert.Equal(t, "value1", localMap["key1"])
 	assert.Equal(t, "local_dynamic", setVars[1].ProcessorsKey)
-	assert.Nil(t, setVars[1].Processors)
+	assert.Len(t, setVars[1].Processors, 1)
 
 	localMap = setVars[2].Mapping["local_dynamic"].(map[string]interface{})
 	assert.Equal(t, "value2", localMap["key1"])
 	assert.Equal(t, "local_dynamic", setVars[2].ProcessorsKey)
-	assert.Nil(t, setVars[2].Processors)
+	assert.Len(t, setVars[2].Processors, 1)
 }
diff --git a/x-pack/elastic-agent/pkg/composable/providers/localdynamic/localdynamic.go b/x-pack/elastic-agent/pkg/composable/providers/localdynamic/localdynamic.go
index f622b13410e..0e5297a9e10 100644
--- a/x-pack/elastic-agent/pkg/composable/providers/localdynamic/localdynamic.go
+++ b/x-pack/elastic-agent/pkg/composable/providers/localdynamic/localdynamic.go
@@ -17,14 +17,19 @@ func init() {
 	composable.Providers.AddDynamicProvider("local_dynamic", DynamicProviderBuilder)
 }
 
+type dynamicItem struct {
+	Mapping    map[string]interface{}   `config:"vars"`
+	Processors []map[string]interface{} `config:"processors"`
+}
+
 type dynamicProvider struct {
-	Mappings []map[string]interface{} `config:"vars"`
+	Items []dynamicItem `config:"items"`
 }
 
 // Run runs the environment context provider.
 func (c *dynamicProvider) Run(comm composable.DynamicProviderComm) error {
-	for i, mapping := range c.Mappings {
-		if err := comm.AddOrUpdate(strconv.Itoa(i), mapping, nil); err != nil {
+	for i, item := range c.Items {
+		if err := comm.AddOrUpdate(strconv.Itoa(i), item.Mapping, item.Processors); err != nil {
 			return errors.New(err, fmt.Sprintf("failed to add mapping for index %d", i), errors.TypeUnexpected)
 		}
 	}
@@ -40,8 +45,8 @@ func DynamicProviderBuilder(c *config.Config) (composable.DynamicProvider, error
 			return nil, fmt.Errorf("failed to unpack vars: %s", err)
 		}
 	}
-	if p.Mappings == nil {
-		p.Mappings = []map[string]interface{}{}
+	if p.Items == nil {
+		p.Items = []dynamicItem{}
 	}
 	return p, nil
 }
diff --git a/x-pack/elastic-agent/pkg/composable/providers/localdynamic/localdynamic_test.go b/x-pack/elastic-agent/pkg/composable/providers/localdynamic/localdynamic_test.go
index 68dc676dc7c..79c107372e3 100644
--- a/x-pack/elastic-agent/pkg/composable/providers/localdynamic/localdynamic_test.go
+++ b/x-pack/elastic-agent/pkg/composable/providers/localdynamic/localdynamic_test.go
@@ -21,13 +21,42 @@ func TestContextProvider(t *testing.T) {
 		"key1": "value1",
 		"key2": "value2",
 	}
+	processors1 := []map[string]interface{}{
+		{
+			"add_fields": map[string]interface{}{
+				"fields": map[string]interface{}{
+					"add": "value1",
+				},
+				"to": "dynamic",
+			},
+		},
+	}
 	mapping2 := map[string]interface{}{
 		"key1": "value12",
 		"key2": "value22",
 	}
-	mapping := []map[string]interface{}{mapping1, mapping2}
+	processors2 := []map[string]interface{}{
+		{
+			"add_fields": map[string]interface{}{
+				"fields": map[string]interface{}{
+					"add": "value12",
+				},
+				"to": "dynamic",
+			},
+		},
+	}
+	mapping := []map[string]interface{}{
+		{
+			"vars":       mapping1,
+			"processors": processors1,
+		},
+		{
+			"vars":       mapping2,
+			"processors": processors2,
+		},
+	}
 	cfg, err := config.NewConfigFrom(map[string]interface{}{
-		"vars": mapping,
+		"items": mapping,
 	})
 	require.NoError(t, err)
 	builder, _ := composable.Providers.GetDynamicProvider("local_dynamic")
@@ -41,8 +70,10 @@ func TestContextProvider(t *testing.T) {
 	curr1, ok1 := comm.Current("0")
 	assert.True(t, ok1)
 	assert.Equal(t, mapping1, curr1.Mapping)
+	assert.Equal(t, processors1, curr1.Processors)
 
 	curr2, ok2 := comm.Current("1")
 	assert.True(t, ok2)
 	assert.Equal(t, mapping2, curr2.Mapping)
+	assert.Equal(t, processors2, curr2.Processors)
 }
diff --git a/x-pack/elastic-agent/pkg/composable/providers/path/path.go b/x-pack/elastic-agent/pkg/composable/providers/path/path.go
new file mode 100644
index 00000000000..41f18b91deb
--- /dev/null
+++ b/x-pack/elastic-agent/pkg/composable/providers/path/path.go
@@ -0,0 +1,37 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package path
+
+import (
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
+)
+
+func init() {
+	composable.Providers.AddContextProvider("path", ContextProviderBuilder)
+}
+
+type contextProvider struct{}
+
+// Run runs the Agent context provider.
+func (*contextProvider) Run(comm composable.ContextProviderComm) error {
+	err := comm.Set(map[string]interface{}{
+		"home":   paths.Home(),
+		"data":   paths.Data(),
+		"config": paths.Config(),
+		"logs":   paths.Logs(),
+	})
+	if err != nil {
+		return errors.New(err, "failed to set mapping", errors.TypeUnexpected)
+	}
+	return nil
+}
+
+// ContextProviderBuilder builds the context provider.
+func ContextProviderBuilder(_ *config.Config) (composable.ContextProvider, error) {
+	return &contextProvider{}, nil
+}
diff --git a/x-pack/elastic-agent/pkg/composable/providers/path/path_test.go b/x-pack/elastic-agent/pkg/composable/providers/path/path_test.go
new file mode 100644
index 00000000000..46d5006cebd
--- /dev/null
+++ b/x-pack/elastic-agent/pkg/composable/providers/path/path_test.go
@@ -0,0 +1,33 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package path
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable"
+	ctesting "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable/testing"
+)
+
+func TestContextProvider(t *testing.T) {
+	builder, _ := composable.Providers.GetContextProvider("path")
+	provider, err := builder(nil)
+	require.NoError(t, err)
+
+	comm := ctesting.NewContextComm(context.Background())
+	err = provider.Run(comm)
+	require.NoError(t, err)
+
+	current := comm.Current()
+	assert.Equal(t, paths.Home(), current["home"])
+	assert.Equal(t, paths.Data(), current["data"])
+	assert.Equal(t, paths.Config(), current["config"])
+	assert.Equal(t, paths.Logs(), current["logs"])
+}
diff --git a/x-pack/elastic-agent/pkg/config/config.go b/x-pack/elastic-agent/pkg/config/config.go
index 2968dd618e0..35bdac74aae 100644
--- a/x-pack/elastic-agent/pkg/config/config.go
+++ b/x-pack/elastic-agent/pkg/config/config.go
@@ -24,11 +24,6 @@ var DefaultOptions = []ucfg.Option{
 // Config custom type over a ucfg.Config to add new methods on the object.
 type Config ucfg.Config
 
-// ReadFile reads a configuration from disk.
-func ReadFile(file string) (*Config, error) {
-	return nil, nil
-}
-
 // LoadYAML takes YAML configuration and return a concrete Config or any errors.
 func LoadYAML(path string, opts ...ucfg.Option) (*Config, error) {
 	if len(opts) == 0 {
@@ -42,9 +37,13 @@ func LoadYAML(path string, opts ...ucfg.Option) (*Config, error) {
 }
 
 // NewConfigFrom takes a interface and read the configuration like it was YAML.
-func NewConfigFrom(from interface{}) (*Config, error) {
+func NewConfigFrom(from interface{}, opts ...ucfg.Option) (*Config, error) {
+	if len(opts) == 0 {
+		opts = DefaultOptions
+	}
+
 	if str, ok := from.(string); ok {
-		c, err := yaml.NewConfig([]byte(str), DefaultOptions...)
+		c, err := yaml.NewConfig([]byte(str), opts...)
 		return newConfigFrom(c), err
 	}
 
@@ -57,11 +56,11 @@ func NewConfigFrom(from interface{}) (*Config, error) {
 		if err != nil {
 			return nil, err
 		}
-		c, err := yaml.NewConfig(content, DefaultOptions...)
+		c, err := yaml.NewConfig(content, opts...)
 		return newConfigFrom(c), err
 	}
 
-	c, err := ucfg.NewFrom(from, DefaultOptions...)
+	c, err := ucfg.NewFrom(from, opts...)
 	return newConfigFrom(c), err
 }
 
@@ -89,8 +88,11 @@ func (c *Config) access() *ucfg.Config {
 }
 
 // Merge merges two configuration together.
-func (c *Config) Merge(from interface{}) error {
-	return c.access().Merge(from, DefaultOptions...)
+func (c *Config) Merge(from interface{}, opts ...ucfg.Option) error {
+	if len(opts) == 0 {
+		opts = DefaultOptions
+	}
+	return c.access().Merge(from, opts...)
 }
 
 // ToMapStr takes the config and transform it into a map[string]interface{}
diff --git a/x-pack/elastic-agent/pkg/core/logger/logger.go b/x-pack/elastic-agent/pkg/core/logger/logger.go
index 0a73f36f08f..a2886ccf28e 100644
--- a/x-pack/elastic-agent/pkg/core/logger/logger.go
+++ b/x-pack/elastic-agent/pkg/core/logger/logger.go
@@ -100,7 +100,7 @@ func makeInternalFileOutput() (zapcore.Core, error) {
 	// defaultCfg is used to set the defaults for the file rotation of the internal logging
 	// these settings cannot be changed by a user configuration
 	defaultCfg := logp.DefaultConfig(logp.DefaultEnvironment)
-	filename := filepath.Join(paths.Data(), "logs", fmt.Sprintf("%s-json.log", agentName))
+	filename := filepath.Join(paths.Home(), "logs", fmt.Sprintf("%s-json.log", agentName))
 
 	rotator, err := file.NewFileRotator(filename,
 		file.MaxSizeBytes(defaultCfg.Files.MaxSize),
diff --git a/x-pack/elastic-agent/pkg/core/monitoring/beats/monitoring.go b/x-pack/elastic-agent/pkg/core/monitoring/beats/monitoring.go
index d38f7d5843c..f75108425a0 100644
--- a/x-pack/elastic-agent/pkg/core/monitoring/beats/monitoring.go
+++ b/x-pack/elastic-agent/pkg/core/monitoring/beats/monitoring.go
@@ -32,8 +32,8 @@ func getMonitoringEndpoint(program, operatingSystem, pipelineID string) string {
 
 func getLoggingFile(program, operatingSystem, installPath, pipelineID string) string {
 	if operatingSystem == "windows" {
-		return fmt.Sprintf(logFileFormatWin, paths.Data(), pipelineID, program)
+		return fmt.Sprintf(logFileFormatWin, paths.Home(), pipelineID, program)
 	}
 
-	return fmt.Sprintf(logFileFormat, paths.Data(), pipelineID, program)
+	return fmt.Sprintf(logFileFormat, paths.Home(), pipelineID, program)
 }
diff --git a/x-pack/elastic-agent/pkg/core/plugin/process/start.go b/x-pack/elastic-agent/pkg/core/plugin/process/start.go
index 3b675417970..c5f1ffb3842 100644
--- a/x-pack/elastic-agent/pkg/core/plugin/process/start.go
+++ b/x-pack/elastic-agent/pkg/core/plugin/process/start.go
@@ -152,6 +152,6 @@ func injectLogLevel(logLevel string, args []string) []string {
 }
 
 func injectDataPath(args []string, pipelineID, id string) []string {
-	dataPath := filepath.Join(paths.Data(), "run", pipelineID, id)
+	dataPath := filepath.Join(paths.Home(), "run", pipelineID, id)
 	return append(args, "-E", "path.data="+dataPath)
 }
diff --git a/x-pack/elastic-agent/pkg/fleetapi/ack_cmd.go b/x-pack/elastic-agent/pkg/fleetapi/ack_cmd.go
index a1b1a245114..ac568c884d1 100644
--- a/x-pack/elastic-agent/pkg/fleetapi/ack_cmd.go
+++ b/x-pack/elastic-agent/pkg/fleetapi/ack_cmd.go
@@ -45,12 +45,10 @@ func (e *AckRequest) Validate() error {
 // AckResponse is the response send back from the server.
 // 200
 // {
-// 	 "action": "acks",
-// 	 "success": true
+// 	 "action": "acks"
 // }
 type AckResponse struct {
-	Action  string `json:"action"`
-	Success bool   `json:"success"`
+	Action string `json:"action"`
 }
 
 // Validate validates the response send from the server.
diff --git a/x-pack/elastic-agent/pkg/fleetapi/ack_cmd_test.go b/x-pack/elastic-agent/pkg/fleetapi/ack_cmd_test.go
index 945cce72a40..a9e3aebc25b 100644
--- a/x-pack/elastic-agent/pkg/fleetapi/ack_cmd_test.go
+++ b/x-pack/elastic-agent/pkg/fleetapi/ack_cmd_test.go
@@ -20,12 +20,7 @@ func TestAck(t *testing.T) {
 
 	t.Run("Test ack roundtrip", withServerWithAuthClient(
 		func(t *testing.T) *http.ServeMux {
-			raw := `
-{
-    "action": "ack",
-    "success": true
-}
-`
+			raw := `{"action": "ack"}`
 			mux := http.NewServeMux()
 			path := fmt.Sprintf("/api/ingest_manager/fleet/agents/%s/acks", agentInfo.AgentID())
 			mux.HandleFunc(path, authHandler(func(w http.ResponseWriter, r *http.Request) {
@@ -63,7 +58,7 @@ func TestAck(t *testing.T) {
 
 			request := AckRequest{
 				Events: []AckEvent{
-					AckEvent{
+					{
 						EventType: "ACTION_RESULT",
 						SubType:   "ACKNOWLEDGED",
 						ActionID:  action.ID(),
@@ -73,7 +68,6 @@ func TestAck(t *testing.T) {
 
 			r, err := cmd.Execute(context.Background(), &request)
 			require.NoError(t, err)
-			require.True(t, r.Success)
 			require.Equal(t, "ack", r.Action)
 		},
 	))
diff --git a/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd.go b/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd.go
index 22a09f5e892..58c80b0adf1 100644
--- a/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd.go
+++ b/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd.go
@@ -49,7 +49,6 @@ func (e *CheckinRequest) Validate() error {
 // need to be executed or proxy to running processes.
 type CheckinResponse struct {
 	Actions Actions `json:"actions"`
-	Success bool    `json:"success"`
 }
 
 // Validate validates the response send from the server.
diff --git a/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd_test.go b/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd_test.go
index 43501c7fac7..953b86a260e 100644
--- a/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd_test.go
+++ b/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd_test.go
@@ -79,8 +79,7 @@ func TestCheckin(t *testing.T) {
 					}]
 				}
 			}
-		}],
-		"success": true
+		}]
 	}
 	`
 			mux := http.NewServeMux()
@@ -98,7 +97,6 @@ func TestCheckin(t *testing.T) {
 
 			r, err := cmd.Execute(ctx, &request)
 			require.NoError(t, err)
-			require.True(t, r.Success)
 
 			require.Equal(t, 1, len(r.Actions))
 
@@ -142,8 +140,7 @@ func TestCheckin(t *testing.T) {
 	            "type": "WHAT_TO_DO_WITH_IT",
 	            "id": "id2"
 	        }
-	    ],
-	    "success": true
+	    ]
 	}
 	`
 			mux := http.NewServeMux()
@@ -161,7 +158,6 @@ func TestCheckin(t *testing.T) {
 
 			r, err := cmd.Execute(ctx, &request)
 			require.NoError(t, err)
-			require.True(t, r.Success)
 
 			require.Equal(t, 2, len(r.Actions))
 
@@ -178,12 +174,7 @@ func TestCheckin(t *testing.T) {
 
 	t.Run("When we receive no action", withServerWithAuthClient(
 		func(t *testing.T) *http.ServeMux {
-			raw := `
-	{
-	  "actions": [],
-		"success": true
-	}
-	`
+			raw := `{ "actions": [] }`
 			mux := http.NewServeMux()
 			path := fmt.Sprintf("/api/ingest_manager/fleet/agents/%s/checkin", agentInfo.AgentID())
 			mux.HandleFunc(path, authHandler(func(w http.ResponseWriter, r *http.Request) {
@@ -199,7 +190,6 @@ func TestCheckin(t *testing.T) {
 
 			r, err := cmd.Execute(ctx, &request)
 			require.NoError(t, err)
-			require.True(t, r.Success)
 
 			require.Equal(t, 0, len(r.Actions))
 		},
@@ -207,12 +197,7 @@ func TestCheckin(t *testing.T) {
 
 	t.Run("Meta are sent", withServerWithAuthClient(
 		func(t *testing.T) *http.ServeMux {
-			raw := `
-{
-  "actions": [],
-	"success": true
-}
-`
+			raw := `{"actions": []}`
 			mux := http.NewServeMux()
 			path := fmt.Sprintf("/api/ingest_manager/fleet/agents/%s/checkin", agentInfo.AgentID())
 			mux.HandleFunc(path, authHandler(func(w http.ResponseWriter, r *http.Request) {
@@ -239,7 +224,6 @@ func TestCheckin(t *testing.T) {
 
 			r, err := cmd.Execute(ctx, &request)
 			require.NoError(t, err)
-			require.True(t, r.Success)
 
 			require.Equal(t, 0, len(r.Actions))
 		},
@@ -247,12 +231,7 @@ func TestCheckin(t *testing.T) {
 
 	t.Run("No meta are sent when not provided", withServerWithAuthClient(
 		func(t *testing.T) *http.ServeMux {
-			raw := `
-	{
-	  "actions": [],
-		"success": true
-	}
-	`
+			raw := `{"actions": []}`
 			mux := http.NewServeMux()
 			path := fmt.Sprintf("/api/ingest_manager/fleet/agents/%s/checkin", agentInfo.AgentID())
 			mux.HandleFunc(path, authHandler(func(w http.ResponseWriter, r *http.Request) {
@@ -279,7 +258,6 @@ func TestCheckin(t *testing.T) {
 
 			r, err := cmd.Execute(ctx, &request)
 			require.NoError(t, err)
-			require.True(t, r.Success)
 
 			require.Equal(t, 0, len(r.Actions))
 		},
diff --git a/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd.go b/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd.go
index 55955f3edd5..3e6336fbc81 100644
--- a/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd.go
+++ b/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd.go
@@ -112,7 +112,6 @@ func (e *EnrollRequest) Validate() error {
 // Example:
 // {
 //   "action": "created",
-//   "success": true,
 //   "item": {
 //     "id": "a4937110-e53e-11e9-934f-47a8e38a522c",
 //     "active": true,
@@ -126,9 +125,8 @@ func (e *EnrollRequest) Validate() error {
 //   }
 // }
 type EnrollResponse struct {
-	Action  string             `json:"action"`
-	Success bool               `json:"success"`
-	Item    EnrollItemResponse `json:"item"`
+	Action string             `json:"action"`
+	Item   EnrollItemResponse `json:"item"`
 }
 
 // EnrollItemResponse item response.
diff --git a/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd_test.go b/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd_test.go
index df341b0110e..6533ad6643e 100644
--- a/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd_test.go
+++ b/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd_test.go
@@ -43,8 +43,7 @@ func TestEnroll(t *testing.T) {
 				require.Equal(t, "linux", req.Metadata.Local.OS.Name)
 
 				response := &EnrollResponse{
-					Action:  "created",
-					Success: true,
+					Action: "created",
 					Item: EnrollItemResponse{
 						ID:                   "a4937110-e53e-11e9-934f-47a8e38a522c",
 						Active:               true,
@@ -87,7 +86,6 @@ func TestEnroll(t *testing.T) {
 
 			require.Equal(t, "my-access-api-key", resp.Item.AccessAPIKey)
 			require.Equal(t, "created", resp.Action)
-			require.True(t, resp.Success)
 		},
 	))
 
diff --git a/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml
index 4f5fd4851bc..ad8edd5392f 100644
--- a/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml
+++ b/x-pack/filebeat/module/zeek/ssl/ingest/pipeline.yml
@@ -76,26 +76,50 @@ processors:
     field: zeek.ssl.server.issuer.C
     target_field: zeek.ssl.server.issuer.country
     ignore_missing: true
+- set:
+    field: tls.server.x509.issuer.country
+    value: '{{zeek.ssl.server.issuer.country}}'
+    ignore_empty_value: true
 - rename:
     field: zeek.ssl.server.issuer.CN
     target_field: zeek.ssl.server.issuer.common_name
     ignore_missing: true
+- set:
+    field: tls.server.x509.issuer.common_name
+    value: '{{zeek.ssl.server.issuer.common_name}}'
+    ignore_empty_value: true
 - rename:
     field: zeek.ssl.server.issuer.L
     target_field: zeek.ssl.server.issuer.locality
     ignore_missing: true
+- set:
+    field: tls.server.x509.issuer.locality
+    value: '{{zeek.ssl.server.issuer.locality}}'
+    ignore_empty_value: true
 - rename:
     field: zeek.ssl.server.issuer.O
     target_field: zeek.ssl.server.issuer.organization
     ignore_missing: true
+- set:
+    field: tls.server.x509.issuer.organization
+    value: '{{zeek.ssl.server.issuer.organization}}'
+    ignore_empty_value: true
 - rename:
     field: zeek.ssl.server.issuer.OU
     target_field: zeek.ssl.server.issuer.organizational_unit
     ignore_missing: true
+- set:
+    field: tls.server.x509.issuer.organizational_unit
+    value: '{{zeek.ssl.server.issuer.organizational_unit}}'
+    ignore_empty_value: true
 - rename:
     field: zeek.ssl.server.issuer.ST
     target_field: zeek.ssl.server.issuer.state
     ignore_missing: true
+- set:
+    field: tls.server.x509.issuer.state_or_province
+    value: '{{zeek.ssl.server.issuer.state}}'
+    ignore_empty_value: true
 - gsub:
     field: zeek.ssl.subject
     pattern: \\,
@@ -114,26 +138,50 @@ processors:
     field: zeek.ssl.server.subject.C
     target_field: zeek.ssl.server.subject.country
     ignore_missing: true
+- set:
+    field: tls.server.x509.subject.country
+    value: '{{zeek.ssl.server.subject.country}}'
+    ignore_empty_value: true
 - rename:
     field: zeek.ssl.server.subject.CN
     target_field: zeek.ssl.server.subject.common_name
     ignore_missing: true
+- set:
+    field: tls.server.x509.subject.common_name
+    value: '{{zeek.ssl.server.subject.common_name}}'
+    ignore_empty_value: true
 - rename:
     field: zeek.ssl.server.subject.L
     target_field: zeek.ssl.server.subject.locality
     ignore_missing: true
+- set:
+    field: tls.server.x509.subject.locality
+    value: '{{zeek.ssl.server.subject.locality}}'
+    ignore_empty_value: true
 - rename:
     field: zeek.ssl.server.subject.O
     target_field: zeek.ssl.server.subject.organization
     ignore_missing: true
+- set:
+    field: tls.server.x509.subject.organization
+    value: '{{zeek.ssl.server.subject.organization}}'
+    ignore_empty_value: true
 - rename:
     field: zeek.ssl.server.subject.OU
     target_field: zeek.ssl.server.subject.organizational_unit
     ignore_missing: true
+- set:
+    field: tls.server.x509.subject.organizational_unit
+    value: '{{zeek.ssl.server.subject.organizational_unit}}'
+    ignore_empty_value: true
 - rename:
     field: zeek.ssl.server.subject.ST
     target_field: zeek.ssl.server.subject.state
     ignore_missing: true
+- set:
+    field: tls.server.x509.subject.state_or_province
+    value: '{{zeek.ssl.server.subject.state}}'
+    ignore_empty_value: true
 - gsub:
     field: zeek.ssl.client_issuer
     pattern: \\,
@@ -153,26 +201,50 @@ processors:
     field: zeek.ssl.client.issuer.C
     target_field: zeek.ssl.client.issuer.country
     ignore_missing: true
+- set:
+    field: tls.client.x509.issuer.country
+    value: '{{zeek.ssl.client.issuer.country}}'
+    ignore_empty_value: true
 - rename:
     field: zeek.ssl.client.issuer.CN
     target_field: zeek.ssl.client.issuer.common_name
     ignore_missing: true
+- set:
+    field: tls.client.x509.issuer.common_name
+    value: '{{zeek.ssl.client.issuer.common_name}}'
+    ignore_empty_value: true
 - rename:
     field: zeek.ssl.client.issuer.L
     target_field: zeek.ssl.client.issuer.locality
     ignore_missing: true
+- set:
+    field: tls.client.x509.issuer.locality
+    value: '{{zeek.ssl.client.issuer.locality}}'
+    ignore_empty_value: true
 - rename:
     field: zeek.ssl.client.issuer.O
     target_field: zeek.ssl.client.issuer.organization
     ignore_missing: true
+- set:
+    field: tls.client.x509.issuer.organization
+    value: '{{zeek.ssl.client.issuer.organization}}'
+    ignore_empty_value: true
 - rename:
     field: zeek.ssl.client.issuer.OU
     target_field: zeek.ssl.client.issuer.organizational_unit
     ignore_missing: true
+- set:
+    field: tls.client.x509.issuer.organizational_unit
+    value: '{{zeek.ssl.client.issuer.organizational_unit}}'
+    ignore_empty_value: true
 - rename:
     field: zeek.ssl.client.issuer.ST
     target_field: zeek.ssl.client.issuer.state
     ignore_missing: true
+- set:
+    field: tls.client.x509.issuer.state_or_province
+    value: '{{zeek.ssl.client.issuer.state}}'
+    ignore_empty_value: true
 - gsub:
     field: zeek.ssl.client_subject
     pattern: \\,
@@ -191,26 +263,50 @@ processors:
     field: zeek.ssl.client.subject.C
     target_field: zeek.ssl.client.subject.country
     ignore_missing: true
+- set:
+    field: tls.client.x509.subject.country
+    value: '{{zeek.ssl.client.subject.country}}'
+    ignore_empty_value: true
 - rename:
     field: zeek.ssl.client.subject.CN
     target_field: zeek.ssl.client.subject.common_name
     ignore_missing: true
+- set:
+    field: tls.client.x509.subject.common_name
+    value: '{{zeek.ssl.client.subject.common_name}}'
+    ignore_empty_value: true
 - rename:
     field: zeek.ssl.client.subject.L
     target_field: zeek.ssl.client.subject.locality
     ignore_missing: true
+- set:
+    field: tls.client.x509.subject.locality
+    value: '{{zeek.ssl.client.subject.locality}}'
+    ignore_empty_value: true
 - rename:
     field: zeek.ssl.client.subject.O
     target_field: zeek.ssl.client.subject.organization
     ignore_missing: true
+- set:
+    field: tls.client.x509.subject.organization
+    value: '{{zeek.ssl.client.subject.organization}}'
+    ignore_empty_value: true
 - rename:
     field: zeek.ssl.client.subject.OU
     target_field: zeek.ssl.client.subject.organizational_unit
     ignore_missing: true
+- set:
+    field: tls.client.x509.subject.organizational_unit
+    value: '{{zeek.ssl.client.subject.organizational_unit}}'
+    ignore_empty_value: true
 - rename:
     field: zeek.ssl.client.subject.ST
     target_field: zeek.ssl.client.subject.state
     ignore_missing: true
+- set:
+    field: tls.client.x509.subject.state_or_province
+    value: '{{zeek.ssl.client.subject.state}}'
+    ignore_empty_value: true
 - set:
     field: tls.cipher
     value: '{{zeek.ssl.cipher}}'
diff --git a/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json b/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json
index 2897b7df9f2..805d20d2a54 100644
--- a/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json
+++ b/x-pack/filebeat/module/zeek/ssl/test/ssl-json.log-expected.json
@@ -47,6 +47,14 @@
         "tls.established": true,
         "tls.resumed": false,
         "tls.server.issuer": "CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US",
+        "tls.server.x509.issuer.common_name": "DigiCert SHA2 Secure Server CA",
+        "tls.server.x509.issuer.country": "US",
+        "tls.server.x509.issuer.organization": "DigiCert Inc",
+        "tls.server.x509.subject.common_name": "*.gcp.cloud.es.io",
+        "tls.server.x509.subject.country": "US",
+        "tls.server.x509.subject.locality": "Mountain View",
+        "tls.server.x509.subject.organization": "Elasticsearch Inc.",
+        "tls.server.x509.subject.state_or_province": "California",
         "tls.version": "1.2",
         "tls.version_protocol": "tls",
         "zeek.session_id": "CAOvs1BMFCX2Eh0Y3",
@@ -119,6 +127,14 @@
         "tls.established": true,
         "tls.resumed": false,
         "tls.server.issuer": "CN=DigiCert SHA2 Secure Server CA,O=DigiCert Inc,C=US",
+        "tls.server.x509.issuer.common_name": "DigiCert SHA2 Secure Server CA",
+        "tls.server.x509.issuer.country": "US",
+        "tls.server.x509.issuer.organization": "DigiCert Inc",
+        "tls.server.x509.subject.common_name": "*.gcp.cloud.es.io",
+        "tls.server.x509.subject.country": "US",
+        "tls.server.x509.subject.locality": "Mountain View",
+        "tls.server.x509.subject.organization": "Elasticsearch Inc.",
+        "tls.server.x509.subject.state_or_province": "California",
         "tls.version": "1.2",
         "tls.version_protocol": "tls",
         "zeek.session_id": "C3mki91FnnNtm0u1ok",
diff --git a/x-pack/heartbeat/magefile.go b/x-pack/heartbeat/magefile.go
index 85aeae0c98f..83f3593c117 100644
--- a/x-pack/heartbeat/magefile.go
+++ b/x-pack/heartbeat/magefile.go
@@ -8,6 +8,7 @@ package main
 
 import (
 	"fmt"
+	"os"
 	"time"
 
 	"github.com/magefile/mage/mg"
@@ -48,7 +49,12 @@ func Package() {
 	start := time.Now()
 	defer func() { fmt.Println("package ran for", time.Since(start)) }()
 
-	devtools.UseElasticBeatXPackPackaging()
+	if v, found := os.LookupEnv("AGENT_PACKAGING"); found && v != "" {
+		devtools.UseElasticBeatXPackReducedPackaging()
+	} else {
+		devtools.UseElasticBeatXPackPackaging()
+	}
+
 	devtools.PackageKibanaDashboardsFromBuildDir()
 	heartbeat.CustomizePackaging()
 
diff --git a/x-pack/metricbeat/module/aws/ec2/ec2_integration_test.go b/x-pack/metricbeat/module/aws/ec2/ec2_integration_test.go
index baaf6e563e1..121df878b27 100644
--- a/x-pack/metricbeat/module/aws/ec2/ec2_integration_test.go
+++ b/x-pack/metricbeat/module/aws/ec2/ec2_integration_test.go
@@ -17,6 +17,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("flaky test: https://github.com/elastic/beats/issues/20951")
 	config := mtest.GetConfigForTest(t, "ec2", "300s")
 
 	metricSet := mbtest.NewReportingMetricSetV2Error(t, config)