diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 5d212875154..9a247e10ba4 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -40,6 +40,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...main[Check the HEAD dif - Fix a logging bug when `ssl.verification_mode` was set to `full` or `certificate`, the command `test output` incorrectly logged that TLS was disabled. - Fix the ability for subcommands to be ran properly from the beats containers. {pull}30452[30452] - Update docker/distribution dependency library to fix a security issues concerning OCI Manifest Type Confusion Issue. {pull}30462[30462] +- Log errors when parsing and applying config blocks and if the input is disabled. {pull}30534[30534] *Auditbeat* diff --git a/filebeat/beater/crawler.go b/filebeat/beater/crawler.go index 7e5ce969f5f..b7c40d39c63 100644 --- a/filebeat/beater/crawler.go +++ b/filebeat/beater/crawler.go @@ -68,30 +68,28 @@ func (c *crawler) Start( ) error { log := c.log - log.Infof("Loading Inputs: %v", len(c.inputConfigs)) + log.Infof("Loading Inputs: %d", len(c.inputConfigs)) // Prospect the globs/paths given on the command line and launch harvesters for _, inputConfig := range c.inputConfigs { err := c.startInput(pipeline, inputConfig) if err != nil { - return fmt.Errorf("starting input failed: %+v", err) + return fmt.Errorf("starting input failed: %w", err) } } if configInputs.Enabled() { c.inputReloader = cfgfile.NewReloader(pipeline, configInputs) if err := c.inputReloader.Check(c.inputsFactory); err != nil { - return fmt.Errorf("creating input reloader failed: %+v", err) + return fmt.Errorf("creating input reloader failed: %w", err) } - } if configModules.Enabled() { c.modulesReloader = cfgfile.NewReloader(pipeline, configModules) if err := c.modulesReloader.Check(c.modulesFactory); err != nil { - return fmt.Errorf("creating module reloader failed: %+v", err) + return fmt.Errorf("creating module reloader failed: %w", err) } - } if c.inputReloader != nil { @@ -105,7 +103,7 @@ func (c *crawler) Start( }() } - log.Infof("Loading and starting Inputs completed. Enabled inputs: %v", len(c.inputs)) + log.Infof("Loading and starting Inputs completed. Enabled inputs: %d", len(c.inputs)) return nil } @@ -114,23 +112,32 @@ func (c *crawler) startInput( pipeline beat.PipelineConnector, config *common.Config, ) error { + // TODO: Either use debug or remove it after https://github.com/elastic/beats/pull/30534 + // is fixed. + c.log.Infof("starting input, keys present on the config: %v", + config.FlattenedKeys()) + if !config.Enabled() { + c.log.Infof("input disabled, skipping it") return nil } var h map[string]interface{} - config.Unpack(&h) + err := config.Unpack(&h) + if err != nil { + return fmt.Errorf("could not unpack config: %w", err) + } id, err := hashstructure.Hash(h, nil) if err != nil { - return fmt.Errorf("can not compute id from configuration: %v", err) + return fmt.Errorf("can not compute id from configuration: %w", err) } if _, ok := c.inputs[id]; ok { - return fmt.Errorf("input with same ID already exists: %v", id) + return fmt.Errorf("input with same ID already exists: %d", id) } runner, err := c.inputsFactory.Create(pipeline, config) if err != nil { - return fmt.Errorf("Error while initializing input: %+v", err) + return fmt.Errorf("error while initializing input: %w", err) } if inputRunner, ok := runner.(*input.Runner); ok { inputRunner.Once = c.once @@ -155,7 +162,7 @@ func (c *crawler) Stop() { }() } - logp.Info("Stopping %v inputs", len(c.inputs)) + logp.Info("Stopping %d inputs", len(c.inputs)) // Stop inputs in parallel for id, p := range c.inputs { id, p := id, p diff --git a/filebeat/input/log/harvester.go b/filebeat/input/log/harvester.go index 34bf1e221d6..1c0bc07bd75 100644 --- a/filebeat/input/log/harvester.go +++ b/filebeat/input/log/harvester.go @@ -306,7 +306,7 @@ func (h *Harvester) Run() error { } }(h.state.Source) - logger.Info("Harvester started for file.") + logger.Infof("Harvester started for paths: %v", h.config.Paths) h.doneWg.Add(1) go func() { diff --git a/filebeat/tests/system/test_stdin.py b/filebeat/tests/system/test_stdin.py index a8283925ab8..02234a060c2 100644 --- a/filebeat/tests/system/test_stdin.py +++ b/filebeat/tests/system/test_stdin.py @@ -22,10 +22,11 @@ def test_stdin(self): proc = self.start_beat() + msg = "Harvester started" self.wait_until( - lambda: self.log_contains( - "Harvester started for file."), - max_timeout=10) + lambda: self.log_contains(msg), + max_timeout=10, + err_msg=f"did not find '{msg}' in the logs") iterations1 = 5 for n in range(0, iterations1): @@ -102,4 +103,5 @@ def test_stdin_is_exclusive(self): filebeat = self.start_beat() filebeat.check_wait(exit_code=1) - assert self.log_contains("Exiting: stdin requires to be run in exclusive mode, configured inputs: stdin, udp") + msg = "Exiting: stdin requires to be run in exclusive mode, configured inputs: stdin, udp" + assert self.log_contains(msg), f"did not find '{msg}' in the logs" diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index 5819a670cd7..ca9bcd19f55 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -348,7 +348,7 @@ def setUp(self): # running tests in parallel pass - def wait_until(self, cond, max_timeout=10, poll_interval=0.1, name="cond"): + def wait_until(self, cond, max_timeout=10, poll_interval=0.1, name="cond", err_msg=""): """ Waits until the cond function returns true, or until the max_timeout is reached. Calls the cond @@ -360,8 +360,8 @@ def wait_until(self, cond, max_timeout=10, poll_interval=0.1, name="cond"): start = datetime.now() while not cond(): if datetime.now() - start > timedelta(seconds=max_timeout): - raise TimeoutError("Timeout waiting for '{}' to be true. ".format(name) + - "Waited {} seconds.".format(max_timeout)) + raise WaitTimeoutError( + f"Timeout waiting for condition '{name}'. Waited {max_timeout} seconds: {err_msg}") time.sleep(poll_interval) def get_log(self, logfile=None): diff --git a/x-pack/libbeat/management/manager.go b/x-pack/libbeat/management/manager.go index 5ae65a69523..e52ebc4eeb7 100644 --- a/x-pack/libbeat/management/manager.go +++ b/x-pack/libbeat/management/manager.go @@ -195,6 +195,7 @@ func (cm *Manager) OnConfig(s string) { if errs := cm.apply(blocks); errs != nil { // `cm.apply` already logs the errors; currently allow beat to run degraded cm.updateStatusWithError(err) + cm.logger.Errorf("failed applying config blocks: %v", err) return } @@ -256,8 +257,8 @@ func (cm *Manager) apply(blocks ConfigBlocks) error { } // Unset missing configs - for name := range missing { - if missing[name] { + for name, isMissing := range missing { + if isMissing { if err := cm.reload(name, []*ConfigBlock{}); err != nil { errors = multierror.Append(errors, err) } @@ -319,6 +320,7 @@ func (cm *Manager) toConfigBlocks(cfg common.MapStr) (ConfigBlocks, error) { for _, regName := range cm.registry.GetRegisteredNames() { iBlock, err := cfg.GetValue(regName) if err != nil { + cm.logger.Errorf("failed to get '%s' from config: %v. Continuing to next one", regName, err) continue }