Add container input, deprecate docker in favor of it (elastic#12162)

Add `container` input, deprecate `docker` in favor of it

This change adds a new container input for better support of CRI based
scenarios.

`docker` input was acting as a catch all input for all container related
cases, but its config options were very opinionated towards Docker, with
some issues:

 * `containers.ids` setting was good to abstract logs path, but we have
 seen many cases were logs are not under default location, or follow a
 different path pattern (ie CRI logs).
 * `containers.*` settings have shown counter intuitive for many users,
 in many cases we have seen people writing `container.*` instead, ending
 up in a config error.
 * Some existing settings (`combine_partials`, `cri.parse_flags`) were
 introduced as a way to offer a backwards compatible upgrades, but it
 doesn't really make sense to disable them, as they handle actual
 format behaviors.

This new `container` input offers the same wrapper to read log files
from containers with the following changes:

 * It exposes `paths` as the `log` input, instead of `containers.ids`
 and `containers.path`.
 * `parse_flags` and `combine_partials` are hardcoded, as there is no
 good reason to disable them.
 * `stream` selector is still available, under root settings.
 * It allows to select the log format (also atodetect it), giving room
 for future format changes. `format` can be `auto` (default), `docker`
    and `CRI`.

Example configurations:

Get Docker logs:

```
filebeat.inputs:
- type: container
  paths:
    - /var/lib/docker/containers/*/*.log
```

Get Kubernetes logs:

```
filebeat.inputs:
- type: container
  paths:
    - /var/log/pods/*/*/*.log
    # this could also be used:
    #- /var/log/containers/*.log
```

Previous `docker` input is deprecated in favor of this, to be removed in 8.0

CHANGELOG.next.asciidoc

@@ -180,6 +180,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support to new MongoDB additional diagnostic information {pull}11952[11952]
 - New module `palo_alto` for Palo Alto Networks PAN-OS logs. {pull}11999[11999]
 - Add RabbitMQ module. {pull}12032[12032]
+- Add new `container` input. {pull}12162[12162]
 
 *Heartbeat*
 
@@ -220,6 +221,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Filebeat*
 
+- `docker` input is deprecated in favour `container`. {pull}12162[12162]
+
 *Heartbeat*
 
 *Journalbeat*

filebeat/autodiscover/builder/hints/config.go

@@ -26,7 +26,7 @@ type config struct {
 
 func defaultConfig() config {
  rawCfg := map[string]interface{}{
- "type": "docker",
+ "type": "container",
  "containers": map[string]interface{}{
  "paths": []string{
  // To be able to use this builder with CRI-O replace paths with:

filebeat/docs/filebeat-options.asciidoc

@@ -44,6 +44,7 @@ You can configure {beatname_uc} to use the following inputs:
 
 * <<{beatname_lc}-input-log>>
 * <<{beatname_lc}-input-stdin>>
+* <<{beatname_lc}-input-container>>
 * <<{beatname_lc}-input-redis>>
 * <<{beatname_lc}-input-udp>>
 * <<{beatname_lc}-input-docker>>
@@ -57,6 +58,8 @@ include::inputs/input-log.asciidoc[]
 
 include::inputs/input-stdin.asciidoc[]
 
+include::inputs/input-container.asciidoc[]
+
 include::inputs/input-redis.asciidoc[]
 
 include::inputs/input-udp.asciidoc[]

filebeat/docs/inputs/input-container.asciidoc

@@ -0,0 +1,64 @@
+:type: container
+
+[id="{beatname_lc}-input-{type}"]
+=== Container input
+
+++++
+<titleabbrev>Container</titleabbrev>
+++++
+
+Use the `container` input to read containers log files.
+
+This input searches for container logs under the given path, and parse them into
+common message lines, extracting timestamps too. Everything happens before line
+filtering, multiline, and JSON decoding, so this input can be used in
+combination with those settings.
+
+Example configuration:
+
+["source","yaml",subs="attributes"]
+----
+{beatname_lc}.inputs:
+- type: container
+ paths: <1>
+ - '/var/lib/docker/containers/*/*.log'
+----
+
+<1> `paths` is required. All other settings are optional.
+
+==== Configuration options
+
+The `container` input supports the following configuration options plus the
+<<{beatname_lc}-input-{type}-common-options>> described later.
+
+===== `stream`
+
+Reads from the specified streams only: `all`, `stdout` or `stderr`. The default
+is `all`.
+
+===== `format`
+
+Use the given format when reading the log file: `auto`, `docker` or `cri`. The
+default is `auto`, it will automatically detect the format. To disable
+autodetection set any of the other options.
+
+
+The following input configures {beatname_uc} to read the `stdout` stream from
+all containers under the default Kubernetes logs path:
+
+[source,yaml]
+----
+- type: container
+ stream: stdout
+ paths:
+ - "/var/log/containers/*.log"
+----
+
+include::../inputs/input-common-harvester-options.asciidoc[]
+
+include::../inputs/input-common-file-options.asciidoc[]
+
+[id="{beatname_lc}-input-{type}-common-options"]
+include::../inputs/input-common-options.asciidoc[]
+
+:type!:

filebeat/docs/inputs/input-docker.asciidoc

@@ -7,6 +7,8 @@
 <titleabbrev>Docker</titleabbrev>
 ++++
 
+deprecated[7.2.0, Use `container` input instead.]
+
 Use the `docker` input to read logs from Docker containers.
 
 This input searches for container logs under its path, and parse them into
@@ -103,4 +105,4 @@ include::../inputs/input-common-file-options.asciidoc[]
 [id="{beatname_lc}-input-{type}-common-options"]
 include::../inputs/input-common-options.asciidoc[]
 
-:type!:
+:type!:

filebeat/harvester/util.go

@@ -21,11 +21,12 @@ import "github.com/elastic/beats/libbeat/common/match"
 
 // Contains available input types
 const (
- LogType = "log"
- StdinType = "stdin"
- RedisType = "redis"
- UdpType = "udp"
- DockerType = "docker"
+ LogType = "log"
+ StdinType = "stdin"
+ RedisType = "redis"
+ UdpType = "udp"
+ DockerType = "docker"
+ ContainerType = "container"
 )
 
 // MatchAny checks if the text matches any of the regular expressions

filebeat/input/container/config.go

@@ -0,0 +1,58 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package container
+
+import (
+ "fmt"
+ "strings"
+)
+
+var defaultConfig = config{
+ Stream: "all",
+ Format: "auto",
+}
+
+type config struct {
+ // Stream can be all, stdout or stderr
+ Stream string `config:"stream"`
+
+ // Format can be auto, cri, json-file
+ Format string `config:"format"`
+}
+
+// Validate validates the config.
+func (c *config) Validate() error {
+ if !stringInSlice(c.Stream, []string{"all", "stdout", "stderr"}) {
+ return fmt.Errorf("invalid value for stream: %s, supported values are: all, stdout, stderr", c.Stream)
+ }
+
+ if !stringInSlice(strings.ToLower(c.Format), []string{"auto", "docker", "cri"}) {
+ return fmt.Errorf("invalid value for format: %s, supported values are: auto, docker, cri", c.Format)
+ }
+
+ return nil
+}
+
+func stringInSlice(str string, list []string) bool {
+ for _, v := range list {
+ if v == str {
+ return true
+ }
+ }
+ return false
+}

filebeat/input/container/input.go

@@ -0,0 +1,74 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package container
+
+import (
+ "github.com/elastic/beats/filebeat/channel"
+ "github.com/elastic/beats/filebeat/input"
+ "github.com/elastic/beats/filebeat/input/log"
+ "github.com/elastic/beats/libbeat/common"
+
+ "github.com/pkg/errors"
+)
+
+func init() {
+ err := input.Register("container", NewInput)
+ if err != nil {
+ panic(err)
+ }
+}
+
+// NewInput creates a new container input
+func NewInput(
+ cfg *common.Config,
+ outletFactory channel.Connector,
+ context input.Context,
+) (input.Input, error) {
+ // Wrap log input with custom docker settings
+ config := defaultConfig
+ if err := cfg.Unpack(&config); err != nil {
+ return nil, errors.Wrap(err, "reading container input config")
+ }
+
+ err := cfg.Merge(common.MapStr{
+ "docker-json.partial": true,
+ "docker-json.cri_flags": true,
+
+ // Allow stream selection (stdout/stderr/all)
+ "docker-json.stream": config.Stream,
+
+ // Select file format (auto/cri/docker)
+ "docker-json.format": config.Format,
+
+ // Set symlinks to true as CRI-O paths could point to symlinks instead of the actual path.
+ "symlinks": true,
+ })
+ if err != nil {
+ return nil, errors.Wrap(err, "update input config")
+ }
+
+ // Add stream to meta to ensure different state per stream
+ if config.Stream != "all" {
+ if context.Meta == nil {
+ context.Meta = map[string]string{}
+ }
+ context.Meta["stream"] = config.Stream
+ }
+
+ return log.NewInput(cfg, outletFactory, context)
+}

filebeat/input/docker/config.go

@@ -44,8 +44,6 @@ type containers struct {
  IDs []string `config:"ids"`
  Path string `config:"path"`
 
- Paths []string `config:"paths"`
-
  // Stream can be all, stdout or stderr
  Stream string `config:"stream"`
 }

filebeat/input/docker/input.go

@@ -25,6 +25,7 @@ import (
  "github.com/elastic/beats/filebeat/input"
  "github.com/elastic/beats/filebeat/input/log"
  "github.com/elastic/beats/libbeat/common"
+ "github.com/elastic/beats/libbeat/common/cfgwarn"
  "github.com/elastic/beats/libbeat/logp"
 
  "github.com/pkg/errors"
@@ -45,25 +46,18 @@ func NewInput(
 ) (input.Input, error) {
  logger := logp.NewLogger("docker")
 
+ cfgwarn.Deprecate("8.0.0", "'docker' input deprecated. Use 'container' input instead.")
+
  // Wrap log input with custom docker settings
  config := defaultConfig
  if err := cfg.Unpack(&config); err != nil {
  return nil, errors.Wrap(err, "reading docker input config")
  }
 
- // Docker input should make sure that no callers should ever pass empty strings as container IDs or paths
+ // Docker input should make sure that no callers should ever pass empty strings as container IDs
  // Hence we explicitly make sure that we catch such things and print stack traces in the event of
  // an invocation so that it can be fixed.
- var ids, paths []string
- for _, p := range config.Containers.Paths {
- if p != "" {
- paths = append(paths, p)
- } else {
- logger.Error("Docker paths can't be empty for Docker input config")
- logger.Debugw("Empty path for Docker logfile was received", logp.Stack("stacktrace"))
- }
- }
-
+ var ids []string
  for _, containerID := range config.Containers.IDs {
  if containerID != "" {
  ids = append(ids, containerID)
@@ -73,23 +67,12 @@ func NewInput(
  }
  }
 
- if len(ids) == 0 && len(paths) == 0 {
- return nil, errors.New("Docker input requires at least one entry under 'containers.ids' or 'containers.paths'")
- }
-
- // IDs + Path and Paths are mutually exclusive. Ensure that only one of them are set in a given configuration
- if len(ids) != 0 && len(paths) != 0 {
- return nil, errors.New("can not provide both 'containers.ids' and 'containers.paths' in the same input config")
+ if len(ids) == 0 {
+ return nil, errors.New("Docker input requires at least one entry under 'containers.ids''")
  }
 
- if len(ids) != 0 {
- for idx, containerID := range ids {
- cfg.SetString("paths", idx, path.Join(config.Containers.Path, containerID, "*.log"))
- }
- } else {
- for idx, p := range paths {
- cfg.SetString("paths", idx, p)
- }
+ for idx, containerID := range ids {
+ cfg.SetString("paths", idx, path.Join(config.Containers.Path, containerID, "*.log"))
  }
 
  if err := checkStream(config.Containers.Stream); err != nil {
@@ -108,13 +91,8 @@ func NewInput(
  return nil, errors.Wrap(err, "update input config")
  }
 
- if err := cfg.SetBool("docker-json.force_cri_logs", -1, config.CRIForce); err != nil {
- return nil, errors.Wrap(err, "update input config")
- }
-
- if len(paths) != 0 {
- // Set symlinks to true as CRI-O paths could point to symlinks instead of the actual path.
- if err := cfg.SetBool("symlinks", -1, true); err != nil {
+ if config.CRIForce {
+ if err := cfg.SetString("docker-json.format", -1, "cri"); err != nil {
  return nil, errors.Wrap(err, "update input config")
  }
  }

filebeat/input/log/config.go

@@ -109,7 +109,7 @@ type config struct {
  DockerJSON *struct {
  Stream string `config:"stream"`
  Partial bool `config:"partial"`
- ForceCRI bool  `config:"force_cri_logs"`
+ Format string `config:"format"`
  CRIFlags bool `config:"cri_flags"`
  } `config:"docker-json"`
 }