Merge remote-tracking branch 'upstream/7.x' into mergify/bp/7.x/pr-28481

* upstream/7.x: chore: use same branching schema for e2e (#28493) [Automation] Update elastic stack version to 7.16.0-8bf0b9b1 for testing (#28484) chore(ci): remove not used param when triggering e2e tests (#27823) (#27828) Add deprecation warning for filebeat disabled modules (#28393) Run Python tests in libbeat (#28438) (#28485) Register `storage_account` as second name for `storage` metricset (#28447) (#28470) [CI] Add E2E for x-pack/elastic-agent (fleet) (#24112) (#28465)
elastic · Oct 18, 2021 · c995d4f · c995d4f
2 parents bdff58c + 055e859
commit c995d4f
Show file tree

Hide file tree

Showing 18 changed files with 226 additions and 231 deletions.
diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy
@@ -417,7 +417,7 @@ def runE2ETests(){
 def triggerE2ETests(String suite) {
   echo("Triggering E2E tests for PR-${env.CHANGE_ID}. Test suites: ${suite}.")
 
-  def branchName = isPR() ? "${env.CHANGE_TARGET}" : "${env.JOB_BASE_NAME}.x"
+  def branchName = isPR() ? "${env.CHANGE_TARGET}" : "${env.JOB_BASE_NAME}"
   def e2eTestsPipeline = "e2e-tests/e2e-testing-mbp/${branchName}"
   def beatVersion = "${env.BEAT_VERSION}-SNAPSHOT"
 
@@ -426,7 +426,6 @@ def triggerE2ETests(String suite) {
     booleanParam(name: 'forceSkipPresubmit', value: true),
     booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
     string(name: 'BEAT_VERSION', value: beatVersion),
-    booleanParam(name: 'BEATS_USE_CI_SNAPSHOTS', value: true),
     string(name: 'runTestsSuites', value: suite),
     string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),
     string(name: 'GITHUB_CHECK_REPO', value: env.REPO),

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -509,6 +509,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix release state of kubernetes metricsets. {pull}26864[26864]
 - Add `gke` metricset collection to `gcp` module {pull}26824[26824]
 - Added a new beta `enterprisesearch` module for Elastic Enterprise Search {pull}27549[27549]
+- Register additional name for `storage` metricset in the azure module. {pull}28447[28447]
 
 *Packetbeat*
 

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -512,8 +512,6 @@ def getBeatsName(baseDir) {
 }
 
 /**
-<<<<<<< HEAD
-=======
 * This method runs the end 2 end testing
 */
 def e2e(Map args = [:]) {
@@ -531,15 +529,12 @@ def e2e(Map args = [:]) {
 }
 
 /**
->>>>>>> d1bcb503b7 (CI: disable e2e for elastic-agent on tags/branches (#28481))
 * This method runs the end 2 end testing in the same worker where the packages have been
 * generated, this should help to speed up the things
 */
-def e2e(Map args = [:]) {
-  def enabled = args.e2e?.get('enabled', false)
+def e2e_with_entrypoint(Map args = [:]) {
   def entrypoint = args.e2e?.get('entrypoint')
   def dockerLogFile = "docker_logs_${entrypoint}.log"
-  if (!enabled) { return }
   dir("${env.WORKSPACE}/src/github.com/elastic/e2e-testing") {
     // TBC with the target branch if running on a PR basis.
     git(branch: 'master', credentialsId: '2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken', url: 'https://github.com/elastic/e2e-testing.git')
@@ -553,19 +548,46 @@ def e2e(Map args = [:]) {
               "LOG_LEVEL=TRACE"]) {
       def status = 0
       filebeat(output: dockerLogFile){
-        status = sh(script: ".ci/scripts/${entrypoint}",
-                    label: "Run functional tests ${entrypoint}",
-                    returnStatus: true)
-      }
-      junit(allowEmptyResults: true, keepLongStdio: true, testResults: "outputs/TEST-*.xml")
-      archiveArtifacts allowEmptyArchive: true, artifacts: "outputs/TEST-*.xml"
-      if (status != 0) {
-        error("ERROR: functional tests for ${args?.directory?.trim()} has failed. See the test report and ${dockerLogFile}.")
+        try {
+          sh(script: ".ci/scripts/${entrypoint}", label: "Run functional tests ${entrypoint}")
+        } finally {
+          junit(allowEmptyResults: true, keepLongStdio: true, testResults: "outputs/TEST-*.xml")
+          archiveArtifacts allowEmptyArchive: true, artifacts: "outputs/TEST-*.xml"
+        }
       }
     }
   }
 }
 
+/**
+* This method triggers the end 2 end testing job.
+*/
+def e2e_with_job(Map args = [:]) {
+  def jobName = args.e2e?.get('job')
+  def testMatrixFile = args.e2e?.get('testMatrixFile', '')
+  def notifyContext = "e2e-${args.context}"
+  def e2eTestsPipeline = "${jobName}/${isPR() ? "${env.CHANGE_TARGET}" : "${env.JOB_BASE_NAME}"}"
+
+  def parameters = [
+    booleanParam(name: 'forceSkipGitChecks', value: true),
+    booleanParam(name: 'forceSkipPresubmit', value: true),
+    booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
+    string(name: 'BEAT_VERSION', value: "${env.VERSION}-SNAPSHOT"),
+    string(name: 'testMatrixFile', value: testMatrixFile),
+    string(name: 'GITHUB_CHECK_NAME', value: notifyContext),
+    string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
+    string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT),
+  ]
+
+  build(job: "${e2eTestsPipeline}",
+    parameters: parameters,
+    propagate: false,
+    wait: false
+  )
+
+  githubNotify(context: "${notifyContext}", description: "${notifyContext} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${e2eTestsPipeline.replaceAll('/','+')}")
+}
+
 /**
 * This method runs the given command supporting two kind of scenarios:
 *  - make -C <folder> then the dir(location) is not required, aka by disaling isMage: false
@@ -598,19 +620,15 @@ def target(Map args = [:]) {
             cmd(label: "${args.id?.trim() ? args.id : env.STAGE_NAME} - ${command}", script: "${command}")
           }
         }
-        // TODO:
-        // Packaging should happen only after the e2e?
+        // Publish packages should happen always to easily consume those artifacts if the
+        // e2e were triggered and failed.
         if (isPackaging) {
           publishPackages("${directory}")
+          pushCIDockerImages(beatsFolder: "${directory}", arch: dockerArch)
         }
         if(isE2E) {
           e2e(args)
         }
-        // TODO:
-        // push docker images should happen only after the e2e?
-        if (isPackaging) {
-          pushCIDockerImages(beatsFolder: "${directory}", arch: dockerArch)
-        }
       }
     }
   }

diff --git a/filebeat/beater/filebeat.go b/filebeat/beater/filebeat.go
@@ -116,6 +116,29 @@ func newBeater(b *beat.Beat, plugins PluginFactory, rawConfig *common.Config) (b
 	}
 	if !moduleRegistry.Empty() {
 		logp.Info("Enabled modules/filesets: %s", moduleRegistry.InfoString())
+
+		// Deprecation warning logic for v8.0 (https://github.com/elastic/beats/pull/27526)
+		for _, mod := range moduleRegistry.ModuleNames() {
+			if mod == "" {
+				continue
+			}
+
+			loadedFilesets, err := moduleRegistry.ModuleFilesets(mod)
+			if err != nil {
+				logp.Err("Error retrieving module filesets: %+v", err)
+				return nil, err
+			}
+
+			configuredFilesets := moduleRegistry.ModuleConfiguredFilesets(mod)
+			if len(configuredFilesets) != len(loadedFilesets) {
+				for _, loadedFileset := range loadedFilesets {
+					if _, ok := configuredFilesets[loadedFileset]; !ok {
+						logp.Warn("Fileset `%s` for module `%s` is loaded but was not explicitly defined in the config. "+
+							"Starting from v8.0 this fileset won't be loaded unless explicitly defined.", loadedFileset, mod)
+					}
+				}
+			}
+		}
 	}
 
 	moduleInputs, err := moduleRegistry.GetInputConfigs()

diff --git a/filebeat/fileset/modules.go b/filebeat/fileset/modules.go
@@ -44,19 +44,22 @@ var availableMLModules = map[string]string{
 const logName = "modules"
 
 type ModuleRegistry struct {
-	registry map[string]map[string]*Fileset // module -> fileset -> Fileset
-	log      *logp.Logger
+	configuredFilesets map[string]map[string]struct{} // module -> fileset -> struct{}
+	registry           map[string]map[string]*Fileset // module -> fileset -> Fileset
+	log                *logp.Logger
 }
 
 // newModuleRegistry reads and loads the configured module into the registry.
 func newModuleRegistry(modulesPath string,
 	moduleConfigs []*ModuleConfig,
 	overrides *ModuleOverrides,
+	configuredFilesets map[string]map[string]struct{},
 	beatInfo beat.Info,
 ) (*ModuleRegistry, error) {
 	reg := ModuleRegistry{
-		registry: map[string]map[string]*Fileset{},
-		log:      logp.NewLogger(logName),
+		configuredFilesets: configuredFilesets,
+		registry:           map[string]map[string]*Fileset{},
+		log:                logp.NewLogger(logName),
 	}
 
 	for _, mcfg := range moduleConfigs {
@@ -77,15 +80,23 @@ func newModuleRegistry(modulesPath string,
 		}
 
 		for _, filesetName := range moduleFilesets {
+			var hasOverride bool
 			fcfg, exists := mcfg.Filesets[filesetName]
 			if !exists {
 				fcfg = &FilesetConfig{}
 			}
 
-			fcfg, err = applyOverrides(fcfg, mcfg.Module, filesetName, overrides)
+			fcfg, hasOverride, err = applyOverrides(fcfg, mcfg.Module, filesetName, overrides)
 			if err != nil {
 				return nil, fmt.Errorf("error applying overrides on fileset %s/%s: %v", mcfg.Module, filesetName, err)
 			}
+			if hasOverride {
+				if _, ok := reg.configuredFilesets[mcfg.Module]; !ok {
+					reg.configuredFilesets[mcfg.Module] = map[string]struct{}{}
+				}
+
+				reg.configuredFilesets[mcfg.Module][filesetName] = struct{}{}
+			}
 
 			if fcfg.Enabled != nil && !(*fcfg.Enabled) {
 				continue
@@ -141,16 +152,19 @@ func NewModuleRegistry(moduleConfigs []*common.Config, beatInfo beat.Info, init
 		}
 	}
 	var mcfgs []*ModuleConfig
+	configuredFilesets := map[string]map[string]struct{}{}
 	for _, cfg := range moduleConfigs {
 		cfg, err = mergePathDefaults(cfg)
 		if err != nil {
 			return nil, err
 		}
 
-		moduleConfig, err := mcfgFromConfig(cfg)
+		moduleConfig, moduleConfiguredFilests, err := mcfgFromConfig(cfg)
 		if err != nil {
 			return nil, errors.Wrap(err, "error unpacking module config")
 		}
+
+		configuredFilesets[moduleConfig.Module] = moduleConfiguredFilests
 		mcfgs = append(mcfgs, moduleConfig)
 	}
 
@@ -159,22 +173,22 @@ func NewModuleRegistry(moduleConfigs []*common.Config, beatInfo beat.Info, init
 		return nil, err
 	}
 
-	return newModuleRegistry(modulesPath, mcfgs, modulesOverrides, beatInfo)
+	return newModuleRegistry(modulesPath, mcfgs, modulesOverrides, configuredFilesets, beatInfo)
 }
 
-func mcfgFromConfig(cfg *common.Config) (*ModuleConfig, error) {
+func mcfgFromConfig(cfg *common.Config) (*ModuleConfig, map[string]struct{}, error) {
 	var mcfg ModuleConfig
 
 	err := cfg.Unpack(&mcfg)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	var dict map[string]interface{}
 
 	err = cfg.Unpack(&dict)
 	if err != nil {
-		return nil, fmt.Errorf("error unpacking module %s in a dict: %v", mcfg.Module, err)
+		return nil, nil, fmt.Errorf("error unpacking module %s in a dict: %v", mcfg.Module, err)
 	}
 
 	mcfg.Filesets = map[string]*FilesetConfig{}
@@ -185,17 +199,31 @@ func mcfgFromConfig(cfg *common.Config) (*ModuleConfig, error) {
 
 		tmpCfg, err := common.NewConfigFrom(filesetConfig)
 		if err != nil {
-			return nil, fmt.Errorf("error creating config from fileset %s/%s: %v", mcfg.Module, name, err)
+			return nil, nil, fmt.Errorf("error creating config from fileset %s/%s: %v", mcfg.Module, name, err)
 		}
 
 		fcfg, err := NewFilesetConfig(tmpCfg)
 		if err != nil {
-			return nil, fmt.Errorf("error creating config from fileset %s/%s: %v", mcfg.Module, name, err)
+			return nil, nil, fmt.Errorf("error creating config from fileset %s/%s: %v", mcfg.Module, name, err)
 		}
 		mcfg.Filesets[name] = fcfg
 	}
 
-	return &mcfg, nil
+	// This calls cfg.GetFields() instead of iterating over `dict` keys
+	// because cfg.Unpack above doesn't return keys that map to a nil value,
+	// but GetFields() returns all keys. We need to observe filesets that
+	// don't contain any configuration (all default values).
+	configuredFilets := map[string]struct{}{}
+
+	for _, name := range cfg.GetFields() {
+		if name == "module" || name == "enabled" || name == "path" {
+			continue
+		}
+
+		configuredFilets[name] = struct{}{}
+	}
+
+	return &mcfg, configuredFilets, nil
 }
 
 func getCurrentModuleName(modulePath, module string) (string, bool) {
@@ -239,36 +267,36 @@ func getModuleFilesets(modulePath, module string) ([]string, error) {
 
 func applyOverrides(fcfg *FilesetConfig,
 	module, fileset string,
-	overrides *ModuleOverrides) (*FilesetConfig, error) {
+	overrides *ModuleOverrides) (*FilesetConfig, bool, error) {
 
 	if overrides == nil {
-		return fcfg, nil
+		return fcfg, false, nil
 	}
 
 	overridesConfigs := overrides.Get(module, fileset)
 	if len(overridesConfigs) == 0 {
-		return fcfg, nil
+		return fcfg, false, nil
 	}
 
 	config, err := common.NewConfigFrom(fcfg)
 	if err != nil {
-		return nil, fmt.Errorf("error creating vars config object: %v", err)
+		return nil, false, fmt.Errorf("error creating vars config object: %v", err)
 	}
 
 	toMerge := []*common.Config{config}
 	toMerge = append(toMerge, overridesConfigs...)
 
 	resultConfig, err := common.MergeConfigs(toMerge...)
 	if err != nil {
-		return nil, fmt.Errorf("error merging configs: %v", err)
+		return nil, false, fmt.Errorf("error merging configs: %v", err)
 	}
 
 	res, err := NewFilesetConfig(resultConfig)
 	if err != nil {
-		return nil, fmt.Errorf("error unpacking configs: %v", err)
+		return nil, false, fmt.Errorf("error unpacking configs: %v", err)
 	}
 
-	return res, nil
+	return res, true, nil
 }
 
 // appendWithoutDuplicates appends basic module configuration for each module in the
@@ -485,3 +513,13 @@ func (reg *ModuleRegistry) ModuleFilesets(module string) ([]string, error) {
 	modulesPath := paths.Resolve(paths.Home, "module")
 	return getModuleFilesets(modulesPath, module)
 }
+
+// ModuleConfiguredFilesets return the map of configured filesets for the given module
+// it returns an empty map if the module doesn't exist
+func (reg *ModuleRegistry) ModuleConfiguredFilesets(module string) map[string]struct{} {
+	if _, ok := reg.configuredFilesets[module]; ok {
+		return reg.configuredFilesets[module]
+	}
+
+	return map[string]struct{}{}
+}
diff --git a/filebeat/fileset/modules_integration_test.go b/filebeat/fileset/modules_integration_test.go
@@ -109,7 +109,7 @@ func TestSetupNginx(t *testing.T) {
 		{Module: "nginx"},
 	}
 
-	reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("5.2.0"))
+	reg, err := newModuleRegistry(modulesPath, configs, nil, nil, makeTestInfo("5.2.0"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -188,7 +188,7 @@ func TestLoadMultiplePipelines(t *testing.T) {
 		&ModuleConfig{"foo", &enabled, filesetConfigs},
 	}
 
-	reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("6.6.0"))
+	reg, err := newModuleRegistry(modulesPath, configs, nil, nil, makeTestInfo("6.6.0"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -233,7 +233,7 @@ func TestLoadMultiplePipelinesWithRollback(t *testing.T) {
 		{"foo", &enabled, filesetConfigs},
 	}
 
-	reg, err := newModuleRegistry(modulesPath, configs, nil, makeTestInfo("6.6.0"))
+	reg, err := newModuleRegistry(modulesPath, configs, nil, nil, makeTestInfo("6.6.0"))
 	if err != nil {
 		t.Fatal(err)
 	}