From 04c538c7c3e95bbc4e40ca7075f1f46177fc56b4 Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Mon, 19 Oct 2020 10:30:32 -0400 Subject: [PATCH 1/9] Create packetbeat x-pack magefile --- packetbeat/magefile.go | 319 +---------------------------- packetbeat/scripts/mage/config.go | 27 +++ packetbeat/scripts/mage/package.go | 72 +++++++ packetbeat/scripts/mage/pcap.go | 274 +++++++++++++++++++++++++ x-pack/packetbeat/magefile.go | 100 +++++++++ 5 files changed, 477 insertions(+), 315 deletions(-) create mode 100644 packetbeat/scripts/mage/package.go create mode 100644 packetbeat/scripts/mage/pcap.go create mode 100644 x-pack/packetbeat/magefile.go diff --git a/packetbeat/magefile.go b/packetbeat/magefile.go index fd00c932ab46..66e467ab6dda 100644 --- a/packetbeat/magefile.go +++ b/packetbeat/magefile.go @@ -21,13 +21,9 @@ package main import ( "fmt" - "log" - "strings" "time" "github.com/magefile/mage/mg" - "github.com/magefile/mage/sh" - "github.com/pkg/errors" devtools "github.com/elastic/beats/v7/dev-tools/mage" packetbeat "github.com/elastic/beats/v7/packetbeat/scripts/mage" @@ -44,7 +40,7 @@ import ( func init() { common.RegisterCheckDeps(Update) - unittest.RegisterPythonTestDeps(fieldsYML, Dashboards) + unittest.RegisterPythonTestDeps(packetbeat.FieldsYML, Dashboards) devtools.BeatDescription = "Packetbeat analyzes network traffic and sends the data to Elasticsearch." } @@ -57,21 +53,7 @@ func Build() error { // GolangCrossBuild build the Beat binary inside of the golang-builder. // Do not use directly, use crossBuild instead. func GolangCrossBuild() error { - if dep, found := crossBuildDeps[devtools.Platform.Name]; found { - mg.Deps(dep) - } - - params := devtools.DefaultGolangCrossBuildArgs() - if flags, found := libpcapLDFLAGS[devtools.Platform.Name]; found { - params.Env = map[string]string{ - "CGO_LDFLAGS": flags, - } - } - if flags, found := libpcapCFLAGS[devtools.Platform.Name]; found { - params.Env["CGO_CFLAGS"] = flags - } - - return devtools.GolangCrossBuild(params) + return packetbeat.GolangCrossBuild() } // BuildGoDaemon builds the go-daemon binary (use crossBuildGoDaemon). @@ -120,7 +102,7 @@ func Package() { devtools.UseElasticBeatPackaging() devtools.PackageKibanaDashboardsFromBuildDir() - customizePackaging() + packetbeat.CustomizePackaging() mg.Deps(Update) mg.Deps(CrossBuild, CrossBuildXPack, CrossBuildGoDaemon) @@ -151,27 +133,7 @@ func includeList() error { // Fields generates fields.yml and fields.go files for the Beat. func Fields() { - mg.Deps(libbeatAndPacketbeatCommonFieldsGo, protosFieldsGo) - mg.Deps(fieldsYML) -} - -// libbeatAndPacketbeatCommonFieldsGo generates a fields.go containing both -// libbeat and packetbeat's common fields. -func libbeatAndPacketbeatCommonFieldsGo() error { - if err := devtools.GenerateFieldsYAML(); err != nil { - return err - } - return devtools.GenerateAllInOneFieldsGo() -} - -// protosFieldsGo generates a fields.go for each protocol. -func protosFieldsGo() error { - return devtools.GenerateModuleFieldsGo("protos") -} - -// fieldsYML generates the fields.yml file containing all fields. -func fieldsYML() error { - return devtools.GenerateFieldsYAML("protos") + packetbeat.Fields() } func fieldDocs() error { @@ -182,276 +144,3 @@ func fieldDocs() error { func Dashboards() error { return devtools.KibanaDashboards("protos") } - -// ----------------------------------------------------------------------------- -// Customizations specific to Packetbeat. -// - Config file contains an OS specific device name (affects darwin, windows). -// - Must compile libpcap or winpcap during cross-compilation. -// - On Linux libpcap is statically linked. Darwin and Windows are dynamic. - -const ( - libpcapURL = "https://s3.amazonaws.com/beats-files/deps/libpcap-1.8.1.tar.gz" - libpcapSHA256 = "673dbc69fdc3f5a86fb5759ab19899039a8e5e6c631749e48dcd9c6f0c83541e" -) - -const ( - linuxPcapLDFLAGS = "-L/libpcap/libpcap-1.8.1 -lpcap" - linuxPcapCFLAGS = "-I /libpcap/libpcap-1.8.1" -) - -var libpcapLDFLAGS = map[string]string{ - "linux/386": linuxPcapLDFLAGS, - "linux/amd64": linuxPcapLDFLAGS, - "linux/arm64": linuxPcapLDFLAGS, - "linux/armv5": linuxPcapLDFLAGS, - "linux/armv6": linuxPcapLDFLAGS, - "linux/armv7": linuxPcapLDFLAGS, - "linux/mips": linuxPcapLDFLAGS, - "linux/mipsle": linuxPcapLDFLAGS, - "linux/mips64": linuxPcapLDFLAGS, - "linux/mips64le": linuxPcapLDFLAGS, - "linux/ppc64le": linuxPcapLDFLAGS, - "linux/s390x": linuxPcapLDFLAGS, - "darwin/amd64": "-lpcap", - "windows/amd64": "-L /libpcap/win/WpdPack/Lib/x64 -lwpcap", - "windows/386": "-L /libpcap/win/WpdPack/Lib -lwpcap", -} - -var libpcapCFLAGS = map[string]string{ - "linux/386": linuxPcapCFLAGS, - "linux/amd64": linuxPcapCFLAGS, - "linux/arm64": linuxPcapCFLAGS, - "linux/armv5": linuxPcapCFLAGS, - "linux/armv6": linuxPcapCFLAGS, - "linux/armv7": linuxPcapCFLAGS, - "linux/mips": linuxPcapCFLAGS, - "linux/mipsle": linuxPcapCFLAGS, - "linux/mips64": linuxPcapCFLAGS, - "linux/mips64le": linuxPcapCFLAGS, - "linux/ppc64le": linuxPcapCFLAGS, - "linux/s390x": linuxPcapCFLAGS, - "windows/amd64": "-I /libpcap/win/WpdPack/Include", - "windows/386": "-I /libpcap/win/WpdPack/Include", -} - -var crossBuildDeps = map[string]func() error{ - "linux/386": buildLibpcapLinux386, - "linux/amd64": buildLibpcapLinuxAMD64, - "linux/arm64": buildLibpcapLinuxARM64, - "linux/armv5": buildLibpcapLinuxARMv5, - "linux/armv6": buildLibpcapLinuxARMv6, - "linux/armv7": buildLibpcapLinuxARMv7, - "linux/mips": buildLibpcapLinuxMIPS, - "linux/mipsle": buildLibpcapLinuxMIPSLE, - "linux/mips64": buildLibpcapLinuxMIPS64, - "linux/mips64le": buildLibpcapLinuxMIPS64LE, - "linux/ppc64le": buildLibpcapLinuxPPC64LE, - "linux/s390x": buildLibpcapLinuxS390x, - "windows/amd64": installLibpcapWindowsAMD64, - "windows/386": installLibpcapWindows386, -} - -// buildLibpcapFromSource builds libpcap from source because the library needs -// to be compiled with -fPIC. -// See https://github.com/elastic/beats/v7/pull/4217. -func buildLibpcapFromSource(params map[string]string) error { - tarFile, err := devtools.DownloadFile(libpcapURL, "/libpcap") - if err != nil { - return errors.Wrap(err, "failed to download libpcap source") - } - - if err = devtools.VerifySHA256(tarFile, libpcapSHA256); err != nil { - return err - } - - if err = devtools.Extract(tarFile, "/libpcap"); err != nil { - return errors.Wrap(err, "failed to extract libpcap") - } - - var configureArgs []string - for k, v := range params { - if strings.HasPrefix(k, "-") { - delete(params, k) - configureArgs = append(configureArgs, k+"="+v) - } - } - - // Use sh -c here because sh.Run does not expose a way to change the CWD. - // This command only runs in Linux so this is fine. - return sh.RunWith(params, "sh", "-c", - "cd /libpcap/libpcap-1.8.1 && "+ - "./configure --enable-usb=no --enable-bluetooth=no --enable-dbus=no "+strings.Join(configureArgs, " ")+"&& "+ - "make") -} - -func buildLibpcapLinux386() error { - return buildLibpcapFromSource(map[string]string{ - "CFLAGS": "-m32", - "LDFLAGS": "-m32", - }) -} - -func buildLibpcapLinuxAMD64() error { - return buildLibpcapFromSource(map[string]string{}) -} - -func buildLibpcapLinuxARM64() error { - return buildLibpcapFromSource(map[string]string{ - "--host": "aarch64-unknown-linux-gnu", - "--with-pcap": "linux", - }) -} - -func buildLibpcapLinuxARMv5() error { - return buildLibpcapFromSource(map[string]string{ - "--host": "arm-linux-gnueabi", - "--with-pcap": "linux", - }) -} - -func buildLibpcapLinuxARMv6() error { - return buildLibpcapFromSource(map[string]string{ - "--host": "arm-linux-gnueabi", - "--with-pcap": "linux", - }) -} - -func buildLibpcapLinuxARMv7() error { - return buildLibpcapFromSource(map[string]string{ - "--host": "arm-linux-gnueabihf", - "--with-pcap": "linux", - }) -} - -func buildLibpcapLinuxMIPS() error { - return buildLibpcapFromSource(map[string]string{ - "--host": "mips-unknown-linux-gnu", - "--with-pcap": "linux", - }) -} - -func buildLibpcapLinuxMIPSLE() error { - return buildLibpcapFromSource(map[string]string{ - "--host": "mipsle-unknown-linux-gnu", - "--with-pcap": "linux", - }) -} - -func buildLibpcapLinuxMIPS64() error { - return buildLibpcapFromSource(map[string]string{ - "--host": "mips64-unknown-linux-gnu", - "--with-pcap": "linux", - }) -} - -func buildLibpcapLinuxMIPS64LE() error { - return buildLibpcapFromSource(map[string]string{ - "--host": "mips64le-unknown-linux-gnu", - "--with-pcap": "linux", - }) -} - -func buildLibpcapLinuxPPC64LE() error { - return buildLibpcapFromSource(map[string]string{ - "--host": "powerpc64le-linux-gnu", - "--with-pcap": "linux", - }) -} - -func buildLibpcapLinuxS390x() error { - return buildLibpcapFromSource(map[string]string{ - "--host": "s390x-ibm-linux-gnu", - "--with-pcap": "linux", - }) -} - -func installLibpcapWindowsAMD64() error { - mg.SerialDeps(installWinpcap, generateWin64StaticWinpcap) - return nil -} - -func installLibpcapWindows386() error { - return installWinpcap() -} - -func installWinpcap() error { - log.Println("Install Winpcap") - const wpdpackURL = "https://www.winpcap.org/install/bin/WpdPack_4_1_2.zip" - - winpcapZip, err := devtools.DownloadFile(wpdpackURL, "/") - if err != nil { - return err - } - - if err = devtools.Extract(winpcapZip, "/libpcap/win"); err != nil { - return err - } - - return nil -} - -func generateWin64StaticWinpcap() error { - log.Println(">> Generating 64-bit winpcap static lib") - - // Notes: We are using absolute path to make sure the files - // are available for x-pack build. - // Ref: https://github.com/elastic/beats/v7/issues/1259 - defer devtools.DockerChown(devtools.MustExpand("{{elastic_beats_dir}}/{{.BeatName}}/lib")) - return devtools.RunCmds( - // Requires mingw-w64-tools. - []string{"gendef", devtools.MustExpand("{{elastic_beats_dir}}/{{.BeatName}}/lib/windows-64/wpcap.dll")}, - []string{"mv", "wpcap.def", devtools.MustExpand("{{ elastic_beats_dir}}/{{.BeatName}}/lib/windows-64/wpcap.def")}, - []string{"x86_64-w64-mingw32-dlltool", "--as-flags=--64", - "-m", "i386:x86-64", "-k", - "--output-lib", "/libpcap/win/WpdPack/Lib/x64/libwpcap.a", - "--input-def", devtools.MustExpand("{{elastic_beats_dir}}/{{.BeatName}}/lib/windows-64/wpcap.def")}, - ) -} - -// customizePackaging modifies the device in the configuration files based on -// the target OS. -func customizePackaging() { - var ( - configYml = devtools.PackageFile{ - Mode: 0600, - Source: "{{.PackageDir}}/{{.BeatName}}.yml", - Config: true, - Dep: func(spec devtools.PackageSpec) error { - c := packetbeat.ConfigFileParams() - c.ExtraVars["GOOS"] = spec.OS - c.ExtraVars["GOARCH"] = spec.MustExpand("{{.GOARCH}}") - return devtools.Config(devtools.ShortConfigType, c, spec.MustExpand("{{.PackageDir}}")) - }, - } - referenceConfigYml = devtools.PackageFile{ - Mode: 0644, - Source: "{{.PackageDir}}/{{.BeatName}}.reference.yml", - Dep: func(spec devtools.PackageSpec) error { - c := packetbeat.ConfigFileParams() - c.ExtraVars["GOOS"] = spec.OS - c.ExtraVars["GOARCH"] = spec.MustExpand("{{.GOARCH}}") - return devtools.Config(devtools.ReferenceConfigType, c, spec.MustExpand("{{.PackageDir}}")) - }, - } - ) - - for _, args := range devtools.Packages { - for _, pkgType := range args.Types { - switch pkgType { - case devtools.TarGz, devtools.Zip: - args.Spec.ReplaceFile("{{.BeatName}}.yml", configYml) - args.Spec.ReplaceFile("{{.BeatName}}.reference.yml", referenceConfigYml) - case devtools.Deb, devtools.RPM, devtools.DMG: - args.Spec.ReplaceFile("/etc/{{.BeatName}}/{{.BeatName}}.yml", configYml) - args.Spec.ReplaceFile("/etc/{{.BeatName}}/{{.BeatName}}.reference.yml", referenceConfigYml) - case devtools.Docker: - args.Spec.ExtraVar("linux_capabilities", "cap_net_raw,cap_net_admin=eip") - default: - panic(errors.Errorf("unhandled package type: %v", pkgType)) - } - - // Match the first package type then continue. - break - } - } -} diff --git a/packetbeat/scripts/mage/config.go b/packetbeat/scripts/mage/config.go index a143cda22e71..c8b3d7242a40 100644 --- a/packetbeat/scripts/mage/config.go +++ b/packetbeat/scripts/mage/config.go @@ -18,6 +18,8 @@ package mage import ( + "github.com/magefile/mage/mg" + devtools "github.com/elastic/beats/v7/dev-tools/mage" ) @@ -50,3 +52,28 @@ func ConfigFileParams() devtools.ConfigFileParams { } return p } + +// Fields generates fields.yml and fields.go files for the Beat. +func Fields() { + mg.Deps(libbeatAndPacketbeatCommonFieldsGo, protosFieldsGo) + mg.Deps(FieldsYML) +} + +// libbeatAndPacketbeatCommonFieldsGo generates a fields.go containing both +// libbeat and packetbeat's common fields. +func libbeatAndPacketbeatCommonFieldsGo() error { + if err := devtools.GenerateFieldsYAML(); err != nil { + return err + } + return devtools.GenerateAllInOneFieldsGo() +} + +// protosFieldsGo generates a fields.go for each protocol. +func protosFieldsGo() error { + return devtools.GenerateModuleFieldsGo(devtools.OSSBeatDir("protos")) +} + +// FieldsYML generates the fields.yml file containing all fields. +func FieldsYML() error { + return devtools.GenerateFieldsYAML(devtools.OSSBeatDir("protos")) +} diff --git a/packetbeat/scripts/mage/package.go b/packetbeat/scripts/mage/package.go new file mode 100644 index 000000000000..e81176615ba6 --- /dev/null +++ b/packetbeat/scripts/mage/package.go @@ -0,0 +1,72 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mage + +import ( + "github.com/pkg/errors" + + devtools "github.com/elastic/beats/v7/dev-tools/mage" +) + +// CustomizePackaging modifies the device in the configuration files based on +// the target OS. +func CustomizePackaging() { + var ( + configYml = devtools.PackageFile{ + Mode: 0600, + Source: "{{.PackageDir}}/{{.BeatName}}.yml", + Config: true, + Dep: func(spec devtools.PackageSpec) error { + c := ConfigFileParams() + c.ExtraVars["GOOS"] = spec.OS + c.ExtraVars["GOARCH"] = spec.MustExpand("{{.GOARCH}}") + return devtools.Config(devtools.ShortConfigType, c, spec.MustExpand("{{.PackageDir}}")) + }, + } + referenceConfigYml = devtools.PackageFile{ + Mode: 0644, + Source: "{{.PackageDir}}/{{.BeatName}}.reference.yml", + Dep: func(spec devtools.PackageSpec) error { + c := ConfigFileParams() + c.ExtraVars["GOOS"] = spec.OS + c.ExtraVars["GOARCH"] = spec.MustExpand("{{.GOARCH}}") + return devtools.Config(devtools.ReferenceConfigType, c, spec.MustExpand("{{.PackageDir}}")) + }, + } + ) + + for _, args := range devtools.Packages { + for _, pkgType := range args.Types { + switch pkgType { + case devtools.TarGz, devtools.Zip: + args.Spec.ReplaceFile("{{.BeatName}}.yml", configYml) + args.Spec.ReplaceFile("{{.BeatName}}.reference.yml", referenceConfigYml) + case devtools.Deb, devtools.RPM, devtools.DMG: + args.Spec.ReplaceFile("/etc/{{.BeatName}}/{{.BeatName}}.yml", configYml) + args.Spec.ReplaceFile("/etc/{{.BeatName}}/{{.BeatName}}.reference.yml", referenceConfigYml) + case devtools.Docker: + args.Spec.ExtraVar("linux_capabilities", "cap_net_raw,cap_net_admin=eip") + default: + panic(errors.Errorf("unhandled package type: %v", pkgType)) + } + + // Match the first package type then continue. + break + } + } +} diff --git a/packetbeat/scripts/mage/pcap.go b/packetbeat/scripts/mage/pcap.go new file mode 100644 index 000000000000..5f0ce0dc5754 --- /dev/null +++ b/packetbeat/scripts/mage/pcap.go @@ -0,0 +1,274 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mage + +import ( + "log" + "strings" + + devtools "github.com/elastic/beats/v7/dev-tools/mage" + + "github.com/magefile/mage/mg" + "github.com/magefile/mage/sh" + "github.com/pkg/errors" +) + +// GolangCrossBuild build the Beat binary inside of the golang-builder. +// Do not use directly, use crossBuild instead. +func GolangCrossBuild() error { + if dep, found := crossBuildDeps[devtools.Platform.Name]; found { + mg.Deps(dep) + } + + params := devtools.DefaultGolangCrossBuildArgs() + if flags, found := libpcapLDFLAGS[devtools.Platform.Name]; found { + params.Env = map[string]string{ + "CGO_LDFLAGS": flags, + } + } + if flags, found := libpcapCFLAGS[devtools.Platform.Name]; found { + params.Env["CGO_CFLAGS"] = flags + } + + return devtools.GolangCrossBuild(params) +} + +// ----------------------------------------------------------------------------- +// Customizations specific to Packetbeat. +// - Config file contains an OS specific device name (affects darwin, windows). +// - Must compile libpcap or winpcap during cross-compilation. +// - On Linux libpcap is statically linked. Darwin and Windows are dynamic. + +const ( + libpcapURL = "https://s3.amazonaws.com/beats-files/deps/libpcap-1.8.1.tar.gz" + libpcapSHA256 = "673dbc69fdc3f5a86fb5759ab19899039a8e5e6c631749e48dcd9c6f0c83541e" +) + +const ( + linuxPcapLDFLAGS = "-L/libpcap/libpcap-1.8.1 -lpcap" + linuxPcapCFLAGS = "-I /libpcap/libpcap-1.8.1" +) + +var libpcapLDFLAGS = map[string]string{ + "linux/386": linuxPcapLDFLAGS, + "linux/amd64": linuxPcapLDFLAGS, + "linux/arm64": linuxPcapLDFLAGS, + "linux/armv5": linuxPcapLDFLAGS, + "linux/armv6": linuxPcapLDFLAGS, + "linux/armv7": linuxPcapLDFLAGS, + "linux/mips": linuxPcapLDFLAGS, + "linux/mipsle": linuxPcapLDFLAGS, + "linux/mips64": linuxPcapLDFLAGS, + "linux/mips64le": linuxPcapLDFLAGS, + "linux/ppc64le": linuxPcapLDFLAGS, + "linux/s390x": linuxPcapLDFLAGS, + "darwin/amd64": "-lpcap", + "windows/amd64": "-L /libpcap/win/WpdPack/Lib/x64 -lwpcap", + "windows/386": "-L /libpcap/win/WpdPack/Lib -lwpcap", +} + +var libpcapCFLAGS = map[string]string{ + "linux/386": linuxPcapCFLAGS, + "linux/amd64": linuxPcapCFLAGS, + "linux/arm64": linuxPcapCFLAGS, + "linux/armv5": linuxPcapCFLAGS, + "linux/armv6": linuxPcapCFLAGS, + "linux/armv7": linuxPcapCFLAGS, + "linux/mips": linuxPcapCFLAGS, + "linux/mipsle": linuxPcapCFLAGS, + "linux/mips64": linuxPcapCFLAGS, + "linux/mips64le": linuxPcapCFLAGS, + "linux/ppc64le": linuxPcapCFLAGS, + "linux/s390x": linuxPcapCFLAGS, + "windows/amd64": "-I /libpcap/win/WpdPack/Include", + "windows/386": "-I /libpcap/win/WpdPack/Include", +} + +var crossBuildDeps = map[string]func() error{ + "linux/386": buildLibpcapLinux386, + "linux/amd64": buildLibpcapLinuxAMD64, + "linux/arm64": buildLibpcapLinuxARM64, + "linux/armv5": buildLibpcapLinuxARMv5, + "linux/armv6": buildLibpcapLinuxARMv6, + "linux/armv7": buildLibpcapLinuxARMv7, + "linux/mips": buildLibpcapLinuxMIPS, + "linux/mipsle": buildLibpcapLinuxMIPSLE, + "linux/mips64": buildLibpcapLinuxMIPS64, + "linux/mips64le": buildLibpcapLinuxMIPS64LE, + "linux/ppc64le": buildLibpcapLinuxPPC64LE, + "linux/s390x": buildLibpcapLinuxS390x, + "windows/amd64": installLibpcapWindowsAMD64, + "windows/386": installLibpcapWindows386, +} + +// buildLibpcapFromSource builds libpcap from source because the library needs +// to be compiled with -fPIC. +// See https://github.com/elastic/beats/v7/pull/4217. +func buildLibpcapFromSource(params map[string]string) error { + tarFile, err := devtools.DownloadFile(libpcapURL, "/libpcap") + if err != nil { + return errors.Wrap(err, "failed to download libpcap source") + } + + if err = devtools.VerifySHA256(tarFile, libpcapSHA256); err != nil { + return err + } + + if err = devtools.Extract(tarFile, "/libpcap"); err != nil { + return errors.Wrap(err, "failed to extract libpcap") + } + + var configureArgs []string + for k, v := range params { + if strings.HasPrefix(k, "-") { + delete(params, k) + configureArgs = append(configureArgs, k+"="+v) + } + } + + // Use sh -c here because sh.Run does not expose a way to change the CWD. + // This command only runs in Linux so this is fine. + return sh.RunWith(params, "sh", "-c", + "cd /libpcap/libpcap-1.8.1 && "+ + "./configure --enable-usb=no --enable-bluetooth=no --enable-dbus=no "+strings.Join(configureArgs, " ")+"&& "+ + "make") +} + +func buildLibpcapLinux386() error { + return buildLibpcapFromSource(map[string]string{ + "CFLAGS": "-m32", + "LDFLAGS": "-m32", + }) +} + +func buildLibpcapLinuxAMD64() error { + return buildLibpcapFromSource(map[string]string{}) +} + +func buildLibpcapLinuxARM64() error { + return buildLibpcapFromSource(map[string]string{ + "--host": "aarch64-unknown-linux-gnu", + "--with-pcap": "linux", + }) +} + +func buildLibpcapLinuxARMv5() error { + return buildLibpcapFromSource(map[string]string{ + "--host": "arm-linux-gnueabi", + "--with-pcap": "linux", + }) +} + +func buildLibpcapLinuxARMv6() error { + return buildLibpcapFromSource(map[string]string{ + "--host": "arm-linux-gnueabi", + "--with-pcap": "linux", + }) +} + +func buildLibpcapLinuxARMv7() error { + return buildLibpcapFromSource(map[string]string{ + "--host": "arm-linux-gnueabihf", + "--with-pcap": "linux", + }) +} + +func buildLibpcapLinuxMIPS() error { + return buildLibpcapFromSource(map[string]string{ + "--host": "mips-unknown-linux-gnu", + "--with-pcap": "linux", + }) +} + +func buildLibpcapLinuxMIPSLE() error { + return buildLibpcapFromSource(map[string]string{ + "--host": "mipsle-unknown-linux-gnu", + "--with-pcap": "linux", + }) +} + +func buildLibpcapLinuxMIPS64() error { + return buildLibpcapFromSource(map[string]string{ + "--host": "mips64-unknown-linux-gnu", + "--with-pcap": "linux", + }) +} + +func buildLibpcapLinuxMIPS64LE() error { + return buildLibpcapFromSource(map[string]string{ + "--host": "mips64le-unknown-linux-gnu", + "--with-pcap": "linux", + }) +} + +func buildLibpcapLinuxPPC64LE() error { + return buildLibpcapFromSource(map[string]string{ + "--host": "powerpc64le-linux-gnu", + "--with-pcap": "linux", + }) +} + +func buildLibpcapLinuxS390x() error { + return buildLibpcapFromSource(map[string]string{ + "--host": "s390x-ibm-linux-gnu", + "--with-pcap": "linux", + }) +} + +func installLibpcapWindowsAMD64() error { + mg.SerialDeps(installWinpcap, generateWin64StaticWinpcap) + return nil +} + +func installLibpcapWindows386() error { + return installWinpcap() +} + +func installWinpcap() error { + log.Println("Install Winpcap") + const wpdpackURL = "https://www.winpcap.org/install/bin/WpdPack_4_1_2.zip" + + winpcapZip, err := devtools.DownloadFile(wpdpackURL, "/") + if err != nil { + return err + } + + if err = devtools.Extract(winpcapZip, "/libpcap/win"); err != nil { + return err + } + + return nil +} + +func generateWin64StaticWinpcap() error { + log.Println(">> Generating 64-bit winpcap static lib") + + // Notes: We are using absolute path to make sure the files + // are available for x-pack build. + // Ref: https://github.com/elastic/beats/v7/issues/1259 + defer devtools.DockerChown(devtools.MustExpand("{{elastic_beats_dir}}/{{.BeatName}}/lib")) + return devtools.RunCmds( + // Requires mingw-w64-tools. + []string{"gendef", devtools.MustExpand("{{elastic_beats_dir}}/{{.BeatName}}/lib/windows-64/wpcap.dll")}, + []string{"mv", "wpcap.def", devtools.MustExpand("{{ elastic_beats_dir}}/{{.BeatName}}/lib/windows-64/wpcap.def")}, + []string{"x86_64-w64-mingw32-dlltool", "--as-flags=--64", + "-m", "i386:x86-64", "-k", + "--output-lib", "/libpcap/win/WpdPack/Lib/x64/libwpcap.a", + "--input-def", devtools.MustExpand("{{elastic_beats_dir}}/{{.BeatName}}/lib/windows-64/wpcap.def")}, + ) +} diff --git a/x-pack/packetbeat/magefile.go b/x-pack/packetbeat/magefile.go new file mode 100644 index 000000000000..9c86b3007e11 --- /dev/null +++ b/x-pack/packetbeat/magefile.go @@ -0,0 +1,100 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build mage + +package main + +import ( + "fmt" + "os" + "time" + + "github.com/magefile/mage/mg" + + devtools "github.com/elastic/beats/v7/dev-tools/mage" + packetbeat "github.com/elastic/beats/v7/packetbeat/scripts/mage" + + // mage:import + "github.com/elastic/beats/v7/dev-tools/mage/target/common" + // mage:import + _ "github.com/elastic/beats/v7/dev-tools/mage/target/compose" + // mage:import + _ "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" + // mage:import +) + +func init() { + common.RegisterCheckDeps(Update) + + devtools.BeatDescription = "Packetbeat analyzes network traffic and sends the data to Elasticsearch." + devtools.BeatLicense = "Elastic License" +} + +// Update updates the generated files. +func Update() {} + +// Build builds the Beat binary. +func Build() error { + return devtools.Build(devtools.DefaultBuildArgs()) +} + +// GolangCrossBuild build the Beat binary inside of the golang-builder. +// Do not use directly, use crossBuild instead. +func GolangCrossBuild() error { + return packetbeat.GolangCrossBuild() +} + +// CrossBuild cross-builds the beat for all target platforms. +func CrossBuild() error { + return devtools.CrossBuild() +} + +// BuildGoDaemon builds the go-daemon binary (use crossBuildGoDaemon). +func BuildGoDaemon() error { + return devtools.BuildGoDaemon() +} + +// CrossBuildGoDaemon cross-builds the go-daemon binary using Docker. +func CrossBuildGoDaemon() error { + return devtools.CrossBuildGoDaemon() +} + +// Package packages the Beat for distribution. +// Use SNAPSHOT=true to build snapshots. +// Use PLATFORMS to control the target platforms. +// Use VERSION_QUALIFIER to control the version qualifier. +func Package() { + start := time.Now() + defer func() { fmt.Println("package ran for", time.Since(start)) }() + + if v, found := os.LookupEnv("AGENT_PACKAGING"); found && v != "" { + devtools.UseElasticBeatXPackReducedPackaging() + } else { + devtools.UseElasticBeatXPackPackaging() + } + + devtools.PackageKibanaDashboardsFromBuildDir() + packetbeat.CustomizePackaging() + + mg.Deps(Update) + mg.Deps(CrossBuild, CrossBuildGoDaemon) + mg.SerialDeps(devtools.Package, TestPackages) +} + +// TestPackages tests the generated packages (i.e. file modes, owners, groups). +func TestPackages() error { + return devtools.TestPackages() +} + +func configYML() error { + return devtools.Config(devtools.AllConfigTypes, packetbeat.ConfigFileParams(), ".") +} + +func includeList() error { + options := devtools.DefaultIncludeListOptions() + options.ImportDirs = []string{"protos/*"} + options.ModuleDirs = nil + return devtools.GenerateIncludeListGo(options) +} From 037bbe4eac02617b30e09f2be34f6b0ae919f7ed Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Mon, 19 Oct 2020 10:43:36 -0400 Subject: [PATCH 2/9] add changelog entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 51255305f42c..651e69de1526 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -770,6 +770,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add ECS fields for x509 certs, event categorization, and related IP info. {pull}19167[19167] - Add 100-continue support {issue}15830[15830] {pull}19349[19349] - Add initial SIP protocol support {pull}21221[21221] +- Change build process for x-pack distribution {pull}21979[21979] *Functionbeat* From aa77abc1921fe7723db4b0c70af2510d7f1faecb Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Fri, 23 Oct 2020 16:21:27 -0400 Subject: [PATCH 3/9] Fix up packetbeat magefile --- Makefile | 2 +- packetbeat/magefile.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index c21c0a7346ee..c3c14245f8de 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ XPACK_SUFFIX=x-pack/ # PROJECTS_XPACK_PKG is a list of Beats that have independent packaging support # in the x-pack directory (rather than having the OSS build produce both sets # of artifacts). This will be removed once we complete the transition. -PROJECTS_XPACK_PKG=x-pack/auditbeat x-pack/dockerlogbeat x-pack/filebeat x-pack/heartbeat x-pack/metricbeat x-pack/winlogbeat +PROJECTS_XPACK_PKG=x-pack/auditbeat x-pack/dockerlogbeat x-pack/filebeat x-pack/heartbeat x-pack/metricbeat x-pack/winlogbeat x-pack/packetbeat # PROJECTS_XPACK_MAGE is a list of Beats whose primary build logic is based in # Mage. For compatibility with CI testing these projects support a subset of the # makefile targets. After all Beats converge to primarily using Mage we can diff --git a/packetbeat/magefile.go b/packetbeat/magefile.go index 66e467ab6dda..e8973334621d 100644 --- a/packetbeat/magefile.go +++ b/packetbeat/magefile.go @@ -100,12 +100,12 @@ func Package() { start := time.Now() defer func() { fmt.Println("package ran for", time.Since(start)) }() - devtools.UseElasticBeatPackaging() + devtools.UseElasticBeatOSSPackaging() devtools.PackageKibanaDashboardsFromBuildDir() packetbeat.CustomizePackaging() mg.Deps(Update) - mg.Deps(CrossBuild, CrossBuildXPack, CrossBuildGoDaemon) + mg.Deps(CrossBuild, CrossBuildGoDaemon) mg.SerialDeps(devtools.Package, TestPackages) } From 32389b718e823dbeda611c9358343fb7de8344c5 Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Fri, 23 Oct 2020 16:58:47 -0400 Subject: [PATCH 4/9] Add makefile for x-pack directory --- x-pack/packetbeat/Makefile | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 x-pack/packetbeat/Makefile diff --git a/x-pack/packetbeat/Makefile b/x-pack/packetbeat/Makefile new file mode 100644 index 000000000000..019d3b9309a8 --- /dev/null +++ b/x-pack/packetbeat/Makefile @@ -0,0 +1,3 @@ +ES_BEATS ?= ../.. + +include $(ES_BEATS)/dev-tools/make/mage.mk From 06af2ef74f7c32ea3298fd68de7e1c75c777475a Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Mon, 26 Oct 2020 10:03:37 -0400 Subject: [PATCH 5/9] Uncomment packaging --- .ci/packaging.groovy | 2 +- x-pack/packetbeat/Jenkinsfile.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy index 4145ee6bdd13..ca60c8dba3f3 100644 --- a/.ci/packaging.groovy +++ b/.ci/packaging.groovy @@ -91,7 +91,7 @@ pipeline { 'x-pack/heartbeat', // 'x-pack/journalbeat', 'x-pack/metricbeat', - // 'x-pack/packetbeat', + 'x-pack/packetbeat', 'x-pack/winlogbeat' ) } diff --git a/x-pack/packetbeat/Jenkinsfile.yml b/x-pack/packetbeat/Jenkinsfile.yml index e3fa9ad0324b..c3bb6c83025f 100644 --- a/x-pack/packetbeat/Jenkinsfile.yml +++ b/x-pack/packetbeat/Jenkinsfile.yml @@ -24,7 +24,7 @@ stages: - "windows-2016" when: ## Override the top-level when. comments: - - "/test x-pack/winlogbeat for windows-2016" + - "/test x-pack/packetbeat for windows-2016" labels: - "windows-2016" branches: true ## for all the branches @@ -35,7 +35,7 @@ stages: - "windows-2012-r2" when: ## Override the top-level when. comments: - - "/test x-pack/winlogbeat for windows-2012" + - "/test x-pack/packetbeat for windows-2012" labels: - "windows-2012" branches: true ## for all the branches From 27297613629af9573e15ed6e279f22645277cfd7 Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Mon, 26 Oct 2020 10:26:45 -0400 Subject: [PATCH 6/9] Add desired platforms to Jenkinsfile --- x-pack/packetbeat/Jenkinsfile.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/x-pack/packetbeat/Jenkinsfile.yml b/x-pack/packetbeat/Jenkinsfile.yml index c3bb6c83025f..3aa791a34b0f 100644 --- a/x-pack/packetbeat/Jenkinsfile.yml +++ b/x-pack/packetbeat/Jenkinsfile.yml @@ -13,6 +13,34 @@ when: tags: true ## for all the tags platform: "linux && ubuntu-18" ## default label for all the stages stages: + arm: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "arm" + when: ## Override the top-level when. + comments: + - "/test x-pack/packetbeat for arm" + labels: + - "arm" + parameters: + - "armTest" + branches: true ## for all the branches + tags: true ## for all the tags + build: + mage: "mage build test" + macos: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "macosx" + when: ## Override the top-level when. + comments: + - "/test x-pack/packetbeat for macos" + labels: + - "macOS" + parameters: + - "macosTest" + branches: true ## for all the branches + tags: true ## for all the tags windows: mage: "mage build unitTest" withModule: true From 2afdb559953563b65aba4f352d1b8207705ceacc Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Mon, 26 Oct 2020 14:14:19 -0400 Subject: [PATCH 7/9] Add back in fields yml and kibana dashboards on package --- x-pack/packetbeat/magefile.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/x-pack/packetbeat/magefile.go b/x-pack/packetbeat/magefile.go index 9c86b3007e11..ca98507d1862 100644 --- a/x-pack/packetbeat/magefile.go +++ b/x-pack/packetbeat/magefile.go @@ -33,7 +33,14 @@ func init() { } // Update updates the generated files. -func Update() {} +func Update() { + mg.SerialDeps(packetbeat.FieldsYML, Dashboards) +} + +// Dashboards packages kibana dashboards +func Dashboards() error { + return devtools.KibanaDashboards("protos") +} // Build builds the Beat binary. func Build() error { From 7eda37f9f06d03ce3205aaad7d38d21ed86c1b13 Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Mon, 26 Oct 2020 16:03:32 -0400 Subject: [PATCH 8/9] Add generated configs in x-pack/packetbeat --- x-pack/packetbeat/magefile.go | 7 +- x-pack/packetbeat/packetbeat.docker.yml | 50 + x-pack/packetbeat/packetbeat.reference.yml | 2045 ++++++++++++++++++++ x-pack/packetbeat/packetbeat.yml | 277 +++ 4 files changed, 2378 insertions(+), 1 deletion(-) create mode 100644 x-pack/packetbeat/packetbeat.docker.yml create mode 100644 x-pack/packetbeat/packetbeat.reference.yml create mode 100644 x-pack/packetbeat/packetbeat.yml diff --git a/x-pack/packetbeat/magefile.go b/x-pack/packetbeat/magefile.go index ca98507d1862..fc54d9954da3 100644 --- a/x-pack/packetbeat/magefile.go +++ b/x-pack/packetbeat/magefile.go @@ -34,7 +34,12 @@ func init() { // Update updates the generated files. func Update() { - mg.SerialDeps(packetbeat.FieldsYML, Dashboards) + mg.SerialDeps(packetbeat.FieldsYML, Dashboards, Config) +} + +// Config generates the config files. +func Config() error { + return devtools.Config(devtools.AllConfigTypes, packetbeat.ConfigFileParams(), ".") } // Dashboards packages kibana dashboards diff --git a/x-pack/packetbeat/packetbeat.docker.yml b/x-pack/packetbeat/packetbeat.docker.yml new file mode 100644 index 000000000000..4cf9016a9261 --- /dev/null +++ b/x-pack/packetbeat/packetbeat.docker.yml @@ -0,0 +1,50 @@ +packetbeat.interfaces.device: any +packetbeat.interfaces.snaplen: 1514 +packetbeat.interfaces.type: af_packet +packetbeat.interfaces.buffer_size_mb: 100 + +packetbeat.flows: + timeout: 30s + period: 10s + +packetbeat.protocols.dns: + ports: [53] + +packetbeat.protocols.http: + ports: [80, 5601, 9200, 8080, 8081, 5000, 8002] + +packetbeat.protocols.memcache: + ports: [11211] + +packetbeat.protocols.mysql: + ports: [3306] + +packetbeat.protocols.pgsql: + ports: [5432] + +packetbeat.protocols.redis: + ports: [6379] + +packetbeat.protocols.thrift: + ports: [9090] + +packetbeat.protocols.mongodb: + ports: [27017] + +packetbeat.protocols.cassandra: + ports: [9042] + +packetbeat.protocols.tls: + ports: [443, 993, 995, 5223, 8443, 8883, 9243] + +packetbeat.protocols.sip: + ports: [5060] + +processors: + - add_cloud_metadata: ~ + - add_docker_metadata: ~ + +output.elasticsearch: + hosts: '${ELASTICSEARCH_HOSTS:elasticsearch:9200}' + username: '${ELASTICSEARCH_USERNAME:}' + password: '${ELASTICSEARCH_PASSWORD:}' diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml new file mode 100644 index 000000000000..6b936240bbb2 --- /dev/null +++ b/x-pack/packetbeat/packetbeat.reference.yml @@ -0,0 +1,2045 @@ +###################### Packetbeat Configuration Example ####################### + +# This file is a full configuration example documenting all non-deprecated +# options in comments. For a shorter configuration example, that contains only +# the most common options, please see packetbeat.yml in the same directory. +# +# You can find the full configuration reference here: +# https://www.elastic.co/guide/en/beats/packetbeat/index.html + +# =============================== Network device =============================== + +# Select the network interface to sniff the data. You can use the "any" +# keyword to sniff on all connected interfaces. +packetbeat.interfaces.device: any + +# Packetbeat supports three sniffer types: +# * pcap, which uses the libpcap library and works on most platforms, but it's +# not the fastest option. +# * af_packet, which uses memory-mapped sniffing. This option is faster than +# libpcap and doesn't require a kernel module, but it's Linux-specific. +#packetbeat.interfaces.type: pcap + +# The maximum size of the packets to capture. The default is 65535, which is +# large enough for almost all networks and interface types. If you sniff on a +# physical network interface, the optimal setting is the MTU size. On virtual +# interfaces, however, it's safer to accept the default value. +#packetbeat.interfaces.snaplen: 65535 + +# The maximum size of the shared memory buffer to use between the kernel and +# user space. A bigger buffer usually results in lower CPU usage, but consumes +# more memory. This setting is only available for the af_packet sniffer type. +# The default is 30 MB. +#packetbeat.interfaces.buffer_size_mb: 30 + +# Packetbeat automatically generates a BPF for capturing only the traffic on +# ports where it expects to find known protocols. Use this settings to tell +# Packetbeat to generate a BPF filter that accepts VLAN tags. +#packetbeat.interfaces.with_vlans: true + +# Use this setting to override the automatically generated BPF filter. +#packetbeat.interfaces.bpf_filter: + +# With `auto_promisc_mode` Packetbeat puts interface in promiscuous mode automatically on startup. +# This option does not work with `any` interface device. +# The default option is false and requires manual set-up of promiscuous mode. +# Warning: under some circumstances (e.g beat crash) promiscuous mode +# can stay enabled even after beat is shut down. +#packetbeat.interfaces.auto_promisc_mode: true + +# =================================== Flows ==================================== + +packetbeat.flows: + # Enable Network flows. Default: true + #enabled: true + + # Set network flow timeout. Flow is killed if no packet is received before being + # timed out. + timeout: 30s + + # Configure reporting period. If set to -1, only killed flows will be reported + period: 10s + + # Set to true to publish fields with null values in events. + #keep_null: false + +# =========================== Transaction protocols ============================ + +packetbeat.protocols: +- type: icmp + # Enable ICMPv4 and ICMPv6 monitoring. Default: true + #enabled: true + + # Set to true to publish fields with null values in events. + #keep_null: false + +- type: amqp + # Enable AMQP monitoring. Default: true + #enabled: true + + # Configure the ports where to listen for AMQP traffic. You can disable + # the AMQP protocol by commenting out the list of ports. + ports: [5672] + # Truncate messages that are published and avoid huge messages being + # indexed. + # Default: 1000 + #max_body_length: 1000 + + # Hide the header fields in header frames. + # Default: false + #parse_headers: false + + # Hide the additional arguments of method frames. + # Default: false + #parse_arguments: false + + # Hide all methods relative to connection negotiation between server and + # client. + # Default: true + #hide_connection_information: true + + # If this option is enabled, the raw message of the request (`request` field) + # is sent to Elasticsearch. The default is false. + #send_request: false + + # If this option is enabled, the raw message of the response (`response` + # field) is sent to Elasticsearch. The default is false. + #send_response: false + + # Set to true to publish fields with null values in events. + #keep_null: false + + # Transaction timeout. Expired transactions will no longer be correlated to + # incoming responses, but sent to Elasticsearch immediately. + #transaction_timeout: 10s + +- type: cassandra + #Cassandra port for traffic monitoring. + ports: [9042] + + # If this option is enabled, the raw message of the request (`cassandra_request` field) + # is included in published events. The default is true. + #send_request: true + + # If this option is enabled, the raw message of the response (`cassandra_request.request_headers` field) + # is included in published events. The default is true. enable `send_request` first before enable this option. + #send_request_header: true + + # If this option is enabled, the raw message of the response (`cassandra_response` field) + # is included in published events. The default is true. + #send_response: true + + # If this option is enabled, the raw message of the response (`cassandra_response.response_headers` field) + # is included in published events. The default is true. enable `send_response` first before enable this option. + #send_response_header: true + + # Set to true to publish fields with null values in events. + #keep_null: false + + # Configures the default compression algorithm being used to uncompress compressed frames by name. Currently only `snappy` is can be configured. + # By default no compressor is configured. + #compressor: "snappy" + + # This option indicates which Operator/Operators will be ignored. + #ignored_ops: ["SUPPORTED","OPTIONS"] + +- type: dhcpv4 + # Configure the DHCP for IPv4 ports. + ports: [67, 68] + + # Set to true to publish fields with null values in events. + #keep_null: false + +- type: dns + # Enable DNS monitoring. Default: true + #enabled: true + + # Configure the ports where to listen for DNS traffic. You can disable + # the DNS protocol by commenting out the list of ports. + ports: [53] + + # include_authorities controls whether or not the dns.authorities field + # (authority resource records) is added to messages. + # Default: false + include_authorities: true + # include_additionals controls whether or not the dns.additionals field + # (additional resource records) is added to messages. + # Default: false + include_additionals: true + + # send_request and send_response control whether or not the stringified DNS + # request and response message are added to the result. + # Nearly all data about the request/response is available in the dns.* + # fields, but this can be useful if you need visibility specifically + # into the request or the response. + # Default: false + # send_request: true + # send_response: true + + # Set to true to publish fields with null values in events. + #keep_null: false + + # Transaction timeout. Expired transactions will no longer be correlated to + # incoming responses, but sent to Elasticsearch immediately. + #transaction_timeout: 10s + +- type: http + # Enable HTTP monitoring. Default: true + #enabled: true + + # Configure the ports where to listen for HTTP traffic. You can disable + # the HTTP protocol by commenting out the list of ports. + ports: [80, 8080, 8000, 5000, 8002] + + # Uncomment the following to hide certain parameters in URL or forms attached + # to HTTP requests. The names of the parameters are case insensitive. + # The value of the parameters will be replaced with the 'xxxxx' string. + # This is generally useful for avoiding storing user passwords or other + # sensitive information. + # Only query parameters and top level form parameters are replaced. + # hide_keywords: ['pass', 'password', 'passwd'] + + # A list of header names to capture and send to Elasticsearch. These headers + # are placed under the `headers` dictionary in the resulting JSON. + #send_headers: false + + # Instead of sending a white list of headers to Elasticsearch, you can send + # all headers by setting this option to true. The default is false. + #send_all_headers: false + + # A list of headers to redact if present in the HTTP request. This will keep + # the header field present, but will redact it's value to show the headers + # presence. + #redact_headers: [] + + # The list of content types for which Packetbeat includes the full HTTP + # payload. If the request's or response's Content-Type matches any on this + # list, the full body will be included under the request or response field. + #include_body_for: [] + + # The list of content types for which Packetbeat includes the full HTTP + # request payload. + #include_request_body_for: [] + + # The list of content types for which Packetbeat includes the full HTTP + # response payload. + #include_response_body_for: [] + + # Whether the body of a request must be decoded when a content-encoding + # or transfer-encoding has been applied. + #decode_body: true + + # If the Cookie or Set-Cookie headers are sent, this option controls whether + # they are split into individual values. + #split_cookie: false + + # The header field to extract the real IP from. This setting is useful when + # you want to capture traffic behind a reverse proxy, but you want to get the + # geo-location information. + #real_ip_header: + + # If this option is enabled, the raw message of the request (`request` field) + # is sent to Elasticsearch. The default is false. + #send_request: false + + # If this option is enabled, the raw message of the response (`response` + # field) is sent to Elasticsearch. The default is false. + #send_response: false + + # Set to true to publish fields with null values in events. + #keep_null: false + + # Transaction timeout. Expired transactions will no longer be correlated to + # incoming responses, but sent to Elasticsearch immediately. + #transaction_timeout: 10s + + # Maximum message size. If an HTTP message is larger than this, it will + # be trimmed to this size. Default is 10 MB. + #max_message_size: 10485760 + +- type: memcache + # Enable memcache monitoring. Default: true + #enabled: true + + # Configure the ports where to listen for memcache traffic. You can disable + # the Memcache protocol by commenting out the list of ports. + ports: [11211] + + # Uncomment the parseunknown option to force the memcache text protocol parser + # to accept unknown commands. + # Note: All unknown commands MUST not contain any data parts! + # Default: false + # parseunknown: true + + # Update the maxvalue option to store the values - base64 encoded - in the + # json output. + # possible values: + # maxvalue: -1 # store all values (text based protocol multi-get) + # maxvalue: 0 # store no values at all + # maxvalue: N # store up to N values + # Default: 0 + # maxvalues: -1 + + # Use maxbytespervalue to limit the number of bytes to be copied per value element. + # Note: Values will be base64 encoded, so actual size in json document + # will be 4 times maxbytespervalue. + # Default: unlimited + # maxbytespervalue: 100 + + # UDP transaction timeout in milliseconds. + # Note: Quiet messages in UDP binary protocol will get response only in error case. + # The memcached analyzer will wait for udptransactiontimeout milliseconds + # before publishing quiet messages. Non quiet messages or quiet requests with + # error response will not have to wait for the timeout. + # Default: 200 + # udptransactiontimeout: 1000 + + # If this option is enabled, the raw message of the request (`request` field) + # is sent to Elasticsearch. The default is false. + #send_request: false + + # If this option is enabled, the raw message of the response (`response` + # field) is sent to Elasticsearch. The default is false. + #send_response: false + + # Set to true to publish fields with null values in events. + #keep_null: false + + # Transaction timeout. Expired transactions will no longer be correlated to + # incoming responses, but sent to Elasticsearch immediately. + #transaction_timeout: 10s + +- type: mysql + # Enable mysql monitoring. Default: true + #enabled: true + + # Configure the ports where to listen for MySQL traffic. You can disable + # the MySQL protocol by commenting out the list of ports. + ports: [3306,3307] + + # If this option is enabled, the raw message of the request (`request` field) + # is sent to Elasticsearch. The default is false. + #send_request: false + + # If this option is enabled, the raw message of the response (`response` + # field) is sent to Elasticsearch. The default is false. + #send_response: false + + # Set to true to publish fields with null values in events. + #keep_null: false + + # Transaction timeout. Expired transactions will no longer be correlated to + # incoming responses, but sent to Elasticsearch immediately. + #transaction_timeout: 10s + +- type: pgsql + # Enable pgsql monitoring. Default: true + #enabled: true + + # Configure the ports where to listen for Pgsql traffic. You can disable + # the Pgsql protocol by commenting out the list of ports. + ports: [5432] + + # If this option is enabled, the raw message of the request (`request` field) + # is sent to Elasticsearch. The default is false. + #send_request: false + + # If this option is enabled, the raw message of the response (`response` + # field) is sent to Elasticsearch. The default is false. + #send_response: false + + # Set to true to publish fields with null values in events. + #keep_null: false + + # Transaction timeout. Expired transactions will no longer be correlated to + # incoming responses, but sent to Elasticsearch immediately. + #transaction_timeout: 10s + +- type: redis + # Enable redis monitoring. Default: true + #enabled: true + + # Configure the ports where to listen for Redis traffic. You can disable + # the Redis protocol by commenting out the list of ports. + ports: [6379] + + # If this option is enabled, the raw message of the request (`request` field) + # is sent to Elasticsearch. The default is false. + #send_request: false + + # If this option is enabled, the raw message of the response (`response` + # field) is sent to Elasticsearch. The default is false. + #send_response: false + + # Set to true to publish fields with null values in events. + #keep_null: false + + # Transaction timeout. Expired transactions will no longer be correlated to + # incoming responses, but sent to Elasticsearch immediately. + #transaction_timeout: 10s + + # Max size for per-session message queue. This places a limit on the memory + # that can be used to buffer requests and responses for correlation. + #queue_max_bytes: 1048576 + + # Max number of messages for per-session message queue. This limits the number + # of requests or responses that can be buffered for correlation. Set a value + # large enough to allow for pipelining. + #queue_max_messages: 20000 + +- type: thrift + # Enable thrift monitoring. Default: true + #enabled: true + + # Configure the ports where to listen for Thrift-RPC traffic. You can disable + # the Thrift-RPC protocol by commenting out the list of ports. + ports: [9090] + + # The Thrift transport type. Currently this option accepts the values socket + # for TSocket, which is the default Thrift transport, and framed for the + # TFramed Thrift transport. The default is socket. + #transport_type: socket + + # The Thrift protocol type. Currently the only accepted value is binary for + # the TBinary protocol, which is the default Thrift protocol. + #protocol_type: binary + + # The Thrift interface description language (IDL) files for the service that + # Packetbeat is monitoring. Providing the IDL enables Packetbeat to include + # parameter and exception names. + #idl_files: [] + + # The maximum length for strings in parameters or return values. If a string + # is longer than this value, the string is automatically truncated to this + # length. + #string_max_size: 200 + + # The maximum number of elements in a Thrift list, set, map, or structure. + #collection_max_size: 15 + + # If this option is set to false, Packetbeat decodes the method name from the + # reply and simply skips the rest of the response message. + #capture_reply: true + + # If this option is set to true, Packetbeat replaces all strings found in + # method parameters, return codes, or exception structures with the "*" + # string. + #obfuscate_strings: false + + # The maximum number of fields that a structure can have before Packetbeat + # ignores the whole transaction. + #drop_after_n_struct_fields: 500 + + # If this option is enabled, the raw message of the request (`request` field) + # is sent to Elasticsearch. The default is false. + #send_request: false + + # If this option is enabled, the raw message of the response (`response` + # field) is sent to Elasticsearch. The default is false. + #send_response: false + + # Set to true to publish fields with null values in events. + #keep_null: false + + # Transaction timeout. Expired transactions will no longer be correlated to + # incoming responses, but sent to Elasticsearch immediately. + #transaction_timeout: 10s + +- type: mongodb + # Enable mongodb monitoring. Default: true + #enabled: true + + # Configure the ports where to listen for MongoDB traffic. You can disable + # the MongoDB protocol by commenting out the list of ports. + ports: [27017] + + + # The maximum number of documents from the response to index in the `response` + # field. The default is 10. + #max_docs: 10 + + # The maximum number of characters in a single document indexed in the + # `response` field. The default is 5000. You can set this to 0 to index an + # unlimited number of characters per document. + #max_doc_length: 5000 + + # If this option is enabled, the raw message of the request (`request` field) + # is sent to Elasticsearch. The default is false. + #send_request: false + + # If this option is enabled, the raw message of the response (`response` + # field) is sent to Elasticsearch. The default is false. + #send_response: false + + # Set to true to publish fields with null values in events. + #keep_null: false + + # Transaction timeout. Expired transactions will no longer be correlated to + # incoming responses, but sent to Elasticsearch immediately. + #transaction_timeout: 10s + +- type: nfs + # Enable NFS monitoring. Default: true + #enabled: true + + # Configure the ports where to listen for NFS traffic. You can disable + # the NFS protocol by commenting out the list of ports. + ports: [2049] + + # If this option is enabled, the raw message of the request (`request` field) + # is sent to Elasticsearch. The default is false. + #send_request: false + + # If this option is enabled, the raw message of the response (`response` + # field) is sent to Elasticsearch. The default is false. + #send_response: false + + # Set to true to publish fields with null values in events. + #keep_null: false + + # Transaction timeout. Expired transactions will no longer be correlated to + # incoming responses, but sent to Elasticsearch immediately. + #transaction_timeout: 10s + +- type: tls + # Enable TLS monitoring. Default: true + #enabled: true + + # Configure the ports where to listen for TLS traffic. You can disable + # the TLS protocol by commenting out the list of ports. + ports: + - 443 # HTTPS + - 993 # IMAPS + - 995 # POP3S + - 5223 # XMPP over SSL + - 8443 + - 8883 # Secure MQTT + - 9243 # Elasticsearch + + # List of hash algorithms to use to calculate certificates' fingerprints. + # Valid values are `sha1`, `sha256` and `md5`. + #fingerprints: [sha1] + + # If this option is enabled, the client and server certificates and + # certificate chains are sent to Elasticsearch. The default is true. + #send_certificates: true + + # If this option is enabled, the raw certificates will be stored + # in PEM format under the `raw` key. The default is false. + #include_raw_certificates: false + + # Set to true to publish fields with null values in events. + #keep_null: false + +- type: sip + # Configure the ports where to listen for SIP traffic. You can disable the SIP protocol by commenting out the list of ports. + ports: [5060] + + # Parse the authorization headers + parse_authorization: true + + # Parse body contents (only when body is SDP) + parse_body: true + + # Preserve original contents in event.original + keep_original: true + +# ============================ Monitored processes ============================= + +# Packetbeat can enrich events with information about the process associated +# the socket that sent or received the packet if Packetbeat is monitoring +# traffic from the host machine. By default process enrichment is disabled. +# This feature works on Linux and Windows. +packetbeat.procs.enabled: false + +# If you want to ignore transactions created by the server on which the shipper +# is installed you can enable this option. This option is useful to remove +# duplicates if shippers are installed on multiple servers. Default value is +# false. +packetbeat.ignore_outgoing: false + +# ================================== General =================================== + +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +# If this options is not defined, the hostname is used. +#name: + +# The tags of the shipper are included in their own field with each +# transaction published. Tags make it easy to group servers by different +# logical properties. +#tags: ["service-X", "web-tier"] + +# Optional fields that you can specify to add additional information to the +# output. Fields can be scalar values, arrays, dictionaries, or any nested +# combination of these. +#fields: +# env: staging + +# If this option is set to true, the custom fields are stored as top-level +# fields in the output document instead of being grouped under a fields +# sub-dictionary. Default is false. +#fields_under_root: false + +# Internal queue configuration for buffering events to be published. +#queue: + # Queue type by name (default 'mem') + # The memory queue will present all available events (up to the outputs + # bulk_max_size) to the output, the moment the output is ready to server + # another batch of events. + #mem: + # Max number of events the queue can buffer. + #events: 4096 + + # Hints the minimum number of events stored in the queue, + # before providing a batch of events to the outputs. + # The default value is set to 2048. + # A value of 0 ensures events are immediately available + # to be sent to the outputs. + #flush.min_events: 2048 + + # Maximum duration after which events are available to the outputs, + # if the number of events stored in the queue is < `flush.min_events`. + #flush.timeout: 1s + + # The spool queue will store events in a local spool file, before + # forwarding the events to the outputs. + # + # Beta: spooling to disk is currently a beta feature. Use with care. + # + # The spool file is a circular buffer, which blocks once the file/buffer is full. + # Events are put into a write buffer and flushed once the write buffer + # is full or the flush_timeout is triggered. + # Once ACKed by the output, events are removed immediately from the queue, + # making space for new events to be persisted. + #spool: + # The file namespace configures the file path and the file creation settings. + # Once the file exists, the `size`, `page_size` and `prealloc` settings + # will have no more effect. + #file: + # Location of spool file. The default value is ${path.data}/spool.dat. + #path: "${path.data}/spool.dat" + + # Configure file permissions if file is created. The default value is 0600. + #permissions: 0600 + + # File size hint. The spool blocks, once this limit is reached. The default value is 100 MiB. + #size: 100MiB + + # The files page size. A file is split into multiple pages of the same size. The default value is 4KiB. + #page_size: 4KiB + + # If prealloc is set, the required space for the file is reserved using + # truncate. The default value is true. + #prealloc: true + + # Spool writer settings + # Events are serialized into a write buffer. The write buffer is flushed if: + # - The buffer limit has been reached. + # - The configured limit of buffered events is reached. + # - The flush timeout is triggered. + #write: + # Sets the write buffer size. + #buffer_size: 1MiB + + # Maximum duration after which events are flushed if the write buffer + # is not full yet. The default value is 1s. + #flush.timeout: 1s + + # Number of maximum buffered events. The write buffer is flushed once the + # limit is reached. + #flush.events: 16384 + + # Configure the on-disk event encoding. The encoding can be changed + # between restarts. + # Valid encodings are: json, ubjson, and cbor. + #codec: cbor + #read: + # Reader flush timeout, waiting for more events to become available, so + # to fill a complete batch as required by the outputs. + # If flush_timeout is 0, all available events are forwarded to the + # outputs immediately. + # The default value is 0s. + #flush.timeout: 0s + +# Sets the maximum number of CPUs that can be executing simultaneously. The +# default is the number of logical CPUs available in the system. +#max_procs: + +# ================================= Processors ================================= + +# Processors are used to reduce the number of fields in the exported event or to +# enhance the event with external metadata. This section defines a list of +# processors that are applied one by one and the first one receives the initial +# event: +# +# event -> filter1 -> event1 -> filter2 ->event2 ... +# +# The supported processors are drop_fields, drop_event, include_fields, +# decode_json_fields, and add_cloud_metadata. +# +# For example, you can use the following processors to keep the fields that +# contain CPU load percentages, but remove the fields that contain CPU ticks +# values: +# +#processors: +# - include_fields: +# fields: ["cpu"] +# - drop_fields: +# fields: ["cpu.user", "cpu.system"] +# +# The following example drops the events that have the HTTP response code 200: +# +#processors: +# - drop_event: +# when: +# equals: +# http.code: 200 +# +# The following example renames the field a to b: +# +#processors: +# - rename: +# fields: +# - from: "a" +# to: "b" +# +# The following example tokenizes the string into fields: +# +#processors: +# - dissect: +# tokenizer: "%{key1} - %{key2}" +# field: "message" +# target_prefix: "dissect" +# +# The following example enriches each event with metadata from the cloud +# provider about the host machine. It works on EC2, GCE, DigitalOcean, +# Tencent Cloud, and Alibaba Cloud. +# +#processors: +# - add_cloud_metadata: ~ +# +# The following example enriches each event with the machine's local time zone +# offset from UTC. +# +#processors: +# - add_locale: +# format: offset +# +# The following example enriches each event with docker metadata, it matches +# given fields to an existing container id and adds info from that container: +# +#processors: +# - add_docker_metadata: +# host: "unix:///var/run/docker.sock" +# match_fields: ["system.process.cgroup.id"] +# match_pids: ["process.pid", "process.ppid"] +# match_source: true +# match_source_index: 4 +# match_short_id: false +# cleanup_timeout: 60 +# labels.dedot: false +# # To connect to Docker over TLS you must specify a client and CA certificate. +# #ssl: +# # certificate_authority: "/etc/pki/root/ca.pem" +# # certificate: "/etc/pki/client/cert.pem" +# # key: "/etc/pki/client/cert.key" +# +# The following example enriches each event with docker metadata, it matches +# container id from log path available in `source` field (by default it expects +# it to be /var/lib/docker/containers/*/*.log). +# +#processors: +# - add_docker_metadata: ~ +# +# The following example enriches each event with host metadata. +# +#processors: +# - add_host_metadata: ~ +# +# The following example enriches each event with process metadata using +# process IDs included in the event. +# +#processors: +# - add_process_metadata: +# match_pids: ["system.process.ppid"] +# target: system.process.parent +# +# The following example decodes fields containing JSON strings +# and replaces the strings with valid JSON objects. +# +#processors: +# - decode_json_fields: +# fields: ["field1", "field2", ...] +# process_array: false +# max_depth: 1 +# target: "" +# overwrite_keys: false +# +#processors: +# - decompress_gzip_field: +# from: "field1" +# to: "field2" +# ignore_missing: false +# fail_on_error: true +# +# The following example copies the value of message to message_copied +# +#processors: +# - copy_fields: +# fields: +# - from: message +# to: message_copied +# fail_on_error: true +# ignore_missing: false +# +# The following example truncates the value of message to 1024 bytes +# +#processors: +# - truncate_fields: +# fields: +# - message +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true +# +# The following example preserves the raw message under event.original +# +#processors: +# - copy_fields: +# fields: +# - from: message +# to: event.original +# fail_on_error: false +# ignore_missing: true +# - truncate_fields: +# fields: +# - event.original +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true +# +# The following example URL-decodes the value of field1 to field2 +# +#processors: +# - urldecode: +# fields: +# - from: "field1" +# to: "field2" +# ignore_missing: false +# fail_on_error: true + +# =============================== Elastic Cloud ================================ + +# These settings simplify using Packetbeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + +# ================================== Outputs =================================== + +# Configure what output to use when sending the data collected by the beat. + +# ---------------------------- Elasticsearch Output ---------------------------- +output.elasticsearch: + # Boolean flag to enable or disable the output module. + #enabled: true + + # Array of hosts to connect to. + # Scheme and port can be left out and will be set to the default (http and 9200) + # In case you specify and additional path, the scheme is required: http://localhost:9200/path + # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 + hosts: ["localhost:9200"] + + # Set gzip compression level. + #compression_level: 0 + + # Configure escaping HTML symbols in strings. + #escape_html: false + + # Protocol - either `http` (default) or `https`. + #protocol: "https" + + # Authentication credentials - either API key or username/password. + #api_key: "id:api_key" + #username: "elastic" + #password: "changeme" + + # Dictionary of HTTP parameters to pass within the URL with index operations. + #parameters: + #param1: value1 + #param2: value2 + + # Number of workers per Elasticsearch host. + #worker: 1 + + # Optional index name. The default is "packetbeat" plus date + # and generates [packetbeat-]YYYY.MM.DD keys. + # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly. + #index: "packetbeat-%{[agent.version]}-%{+yyyy.MM.dd}" + + # Optional ingest node pipeline. By default no pipeline will be used. + #pipeline: "" + + # Optional HTTP path + #path: "/elasticsearch" + + # Custom HTTP headers to add to each request + #headers: + # X-My-Header: Contents of the header + + # Proxy server URL + #proxy_url: http://proxy:3128 + + # Whether to disable proxy settings for outgoing connections. If true, this + # takes precedence over both the proxy_url field and any environment settings + # (HTTP_PROXY, HTTPS_PROXY). The default is false. + #proxy_disable: false + + # The number of times a particular Elasticsearch index operation is attempted. If + # the indexing operation doesn't succeed after this many retries, the events are + # dropped. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Elasticsearch bulk API index request. + # The default is 50. + #bulk_max_size: 50 + + # The number of seconds to wait before trying to reconnect to Elasticsearch + # after a network error. After waiting backoff.init seconds, the Beat + # tries to reconnect. If the attempt fails, the backoff timer is increased + # exponentially up to backoff.max. After a successful connection, the backoff + # timer is reset. The default is 1s. + #backoff.init: 1s + + # The maximum number of seconds to wait before attempting to connect to + # Elasticsearch after a network error. The default is 60s. + #backoff.max: 60s + + # Configure HTTP request timeout before failing a request to Elasticsearch. + #timeout: 90 + + # Use SSL settings for HTTPS. + #ssl.enabled: true + + # Controls the verification of certificates. Valid values are: + # * full, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. + # * none, which performs no verification of the server's certificate. This + # mode disables many of the security benefits of SSL/TLS and should only be used + # after very careful consideration. It is primarily intended as a temporary + # diagnostic mechanism when attempting to resolve TLS errors; its use in + # production environments is strongly discouraged. + # The default value is full. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions from 1.1 + # up to 1.3 are enabled. + #ssl.supported_protocols: [TLSv1.1, TLSv1.2, TLSv1.3] + + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client certificate key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the certificate key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE-based cipher suites + #ssl.curve_types: [] + + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256: "" + + # Enable Kerberos support. Kerberos is automatically enabled if any Kerberos setting is set. + #kerberos.enabled: true + + # Authentication type to use with Kerberos. Available options: keytab, password. + #kerberos.auth_type: password + + # Path to the keytab file. It is used when auth_type is set to keytab. + #kerberos.keytab: /etc/elastic.keytab + + # Path to the Kerberos configuration. + #kerberos.config_path: /etc/krb5.conf + + # Name of the Kerberos user. + #kerberos.username: elastic + + # Password of the Kerberos user. It is used when auth_type is set to password. + #kerberos.password: changeme + + # Kerberos realm. + #kerberos.realm: ELASTIC + +# ------------------------------ Logstash Output ------------------------------- +#output.logstash: + # Boolean flag to enable or disable the output module. + #enabled: true + + # The Logstash hosts + #hosts: ["localhost:5044"] + + # Number of workers per Logstash host. + #worker: 1 + + # Set gzip compression level. + #compression_level: 3 + + # Configure escaping HTML symbols in strings. + #escape_html: false + + # Optional maximum time to live for a connection to Logstash, after which the + # connection will be re-established. A value of `0s` (the default) will + # disable this feature. + # + # Not yet supported for async connections (i.e. with the "pipelining" option set) + #ttl: 30s + + # Optionally load-balance events between Logstash hosts. Default is false. + #loadbalance: false + + # Number of batches to be sent asynchronously to Logstash while processing + # new batches. + #pipelining: 2 + + # If enabled only a subset of events in a batch of events is transferred per + # transaction. The number of events to be sent increases up to `bulk_max_size` + # if no error is encountered. + #slow_start: false + + # The number of seconds to wait before trying to reconnect to Logstash + # after a network error. After waiting backoff.init seconds, the Beat + # tries to reconnect. If the attempt fails, the backoff timer is increased + # exponentially up to backoff.max. After a successful connection, the backoff + # timer is reset. The default is 1s. + #backoff.init: 1s + + # The maximum number of seconds to wait before attempting to connect to + # Logstash after a network error. The default is 60s. + #backoff.max: 60s + + # Optional index name. The default index name is set to packetbeat + # in all lowercase. + #index: 'packetbeat' + + # SOCKS5 proxy server URL + #proxy_url: socks5://user:password@socks5-server:2233 + + # Resolve names locally when using a proxy server. Defaults to false. + #proxy_use_local_resolver: false + + # Use SSL settings for HTTPS. + #ssl.enabled: true + + # Controls the verification of certificates. Valid values are: + # * full, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. + # * none, which performs no verification of the server's certificate. This + # mode disables many of the security benefits of SSL/TLS and should only be used + # after very careful consideration. It is primarily intended as a temporary + # diagnostic mechanism when attempting to resolve TLS errors; its use in + # production environments is strongly discouraged. + # The default value is full. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions from 1.1 + # up to 1.3 are enabled. + #ssl.supported_protocols: [TLSv1.1, TLSv1.2, TLSv1.3] + + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client certificate key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the certificate key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE-based cipher suites + #ssl.curve_types: [] + + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256: "" + + # The number of times to retry publishing an event after a publishing failure. + # After the specified number of retries, the events are typically dropped. + # Some Beats, such as Filebeat and Winlogbeat, ignore the max_retries setting + # and retry until all events are published. Set max_retries to a value less + # than 0 to retry until all events are published. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Logstash request. The + # default is 2048. + #bulk_max_size: 2048 + + # The number of seconds to wait for responses from the Logstash server before + # timing out. The default is 30s. + #timeout: 30s + +# -------------------------------- Kafka Output -------------------------------- +#output.kafka: + # Boolean flag to enable or disable the output module. + #enabled: true + + # The list of Kafka broker addresses from which to fetch the cluster metadata. + # The cluster metadata contain the actual Kafka brokers events are published + # to. + #hosts: ["localhost:9092"] + + # The Kafka topic used for produced events. The setting can be a format string + # using any event field. To set the topic from document type use `%{[type]}`. + #topic: beats + + # The Kafka event key setting. Use format string to create a unique event key. + # By default no event key will be generated. + #key: '' + + # The Kafka event partitioning strategy. Default hashing strategy is `hash` + # using the `output.kafka.key` setting or randomly distributes events if + # `output.kafka.key` is not configured. + #partition.hash: + # If enabled, events will only be published to partitions with reachable + # leaders. Default is false. + #reachable_only: false + + # Configure alternative event field names used to compute the hash value. + # If empty `output.kafka.key` setting will be used. + # Default value is empty list. + #hash: [] + + # Authentication details. Password is required if username is set. + #username: '' + #password: '' + + # Kafka version Packetbeat is assumed to run against. Defaults to the "1.0.0". + #version: '1.0.0' + + # Configure JSON encoding + #codec.json: + # Pretty-print JSON event + #pretty: false + + # Configure escaping HTML symbols in strings. + #escape_html: false + + # Metadata update configuration. Metadata contains leader information + # used to decide which broker to use when publishing. + #metadata: + # Max metadata request retry attempts when cluster is in middle of leader + # election. Defaults to 3 retries. + #retry.max: 3 + + # Wait time between retries during leader elections. Default is 250ms. + #retry.backoff: 250ms + + # Refresh metadata interval. Defaults to every 10 minutes. + #refresh_frequency: 10m + + # Strategy for fetching the topics metadata from the broker. Default is false. + #full: false + + # The number of concurrent load-balanced Kafka output workers. + #worker: 1 + + # The number of times to retry publishing an event after a publishing failure. + # After the specified number of retries, events are typically dropped. + # Some Beats, such as Filebeat, ignore the max_retries setting and retry until + # all events are published. Set max_retries to a value less than 0 to retry + # until all events are published. The default is 3. + #max_retries: 3 + + # The number of seconds to wait before trying to republish to Kafka + # after a network error. After waiting backoff.init seconds, the Beat + # tries to republish. If the attempt fails, the backoff timer is increased + # exponentially up to backoff.max. After a successful publish, the backoff + # timer is reset. The default is 1s. + #backoff.init: 1s + + # The maximum number of seconds to wait before attempting to republish to + # Kafka after a network error. The default is 60s. + #backoff.max: 60s + + # The maximum number of events to bulk in a single Kafka request. The default + # is 2048. + #bulk_max_size: 2048 + + # Duration to wait before sending bulk Kafka request. 0 is no delay. The default + # is 0. + #bulk_flush_frequency: 0s + + # The number of seconds to wait for responses from the Kafka brokers before + # timing out. The default is 30s. + #timeout: 30s + + # The maximum duration a broker will wait for number of required ACKs. The + # default is 10s. + #broker_timeout: 10s + + # The number of messages buffered for each Kafka broker. The default is 256. + #channel_buffer_size: 256 + + # The keep-alive period for an active network connection. If 0s, keep-alives + # are disabled. The default is 0 seconds. + #keep_alive: 0 + + # Sets the output compression codec. Must be one of none, snappy and gzip. The + # default is gzip. + #compression: gzip + + # Set the compression level. Currently only gzip provides a compression level + # between 0 and 9. The default value is chosen by the compression algorithm. + #compression_level: 4 + + # The maximum permitted size of JSON-encoded messages. Bigger messages will be + # dropped. The default value is 1000000 (bytes). This value should be equal to + # or less than the broker's message.max.bytes. + #max_message_bytes: 1000000 + + # The ACK reliability level required from broker. 0=no response, 1=wait for + # local commit, -1=wait for all replicas to commit. The default is 1. Note: + # If set to 0, no ACKs are returned by Kafka. Messages might be lost silently + # on error. + #required_acks: 1 + + # The configurable ClientID used for logging, debugging, and auditing + # purposes. The default is "beats". + #client_id: beats + + # Use SSL settings for HTTPS. + #ssl.enabled: true + + # Controls the verification of certificates. Valid values are: + # * full, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. + # * none, which performs no verification of the server's certificate. This + # mode disables many of the security benefits of SSL/TLS and should only be used + # after very careful consideration. It is primarily intended as a temporary + # diagnostic mechanism when attempting to resolve TLS errors; its use in + # production environments is strongly discouraged. + # The default value is full. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions from 1.1 + # up to 1.3 are enabled. + #ssl.supported_protocols: [TLSv1.1, TLSv1.2, TLSv1.3] + + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client certificate key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the certificate key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE-based cipher suites + #ssl.curve_types: [] + + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256: "" + + # Enable Kerberos support. Kerberos is automatically enabled if any Kerberos setting is set. + #kerberos.enabled: true + + # Authentication type to use with Kerberos. Available options: keytab, password. + #kerberos.auth_type: password + + # Path to the keytab file. It is used when auth_type is set to keytab. + #kerberos.keytab: /etc/security/keytabs/kafka.keytab + + # Path to the Kerberos configuration. + #kerberos.config_path: /etc/krb5.conf + + # The service name. Service principal name is contructed from + # service_name/hostname@realm. + #kerberos.service_name: kafka + + # Name of the Kerberos user. + #kerberos.username: elastic + + # Password of the Kerberos user. It is used when auth_type is set to password. + #kerberos.password: changeme + + # Kerberos realm. + #kerberos.realm: ELASTIC + +# -------------------------------- Redis Output -------------------------------- +#output.redis: + # Boolean flag to enable or disable the output module. + #enabled: true + + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping HTML symbols in strings. + #escape_html: false + + # The list of Redis servers to connect to. If load-balancing is enabled, the + # events are distributed to the servers in the list. If one server becomes + # unreachable, the events are distributed to the reachable servers only. + # The hosts setting supports redis and rediss urls with custom password like + # redis://:password@localhost:6379. + #hosts: ["localhost:6379"] + + # The name of the Redis list or channel the events are published to. The + # default is packetbeat. + #key: packetbeat + + # The password to authenticate to Redis with. The default is no authentication. + #password: + + # The Redis database number where the events are published. The default is 0. + #db: 0 + + # The Redis data type to use for publishing events. If the data type is list, + # the Redis RPUSH command is used. If the data type is channel, the Redis + # PUBLISH command is used. The default value is list. + #datatype: list + + # The number of workers to use for each host configured to publish events to + # Redis. Use this setting along with the loadbalance option. For example, if + # you have 2 hosts and 3 workers, in total 6 workers are started (3 for each + # host). + #worker: 1 + + # If set to true and multiple hosts or workers are configured, the output + # plugin load balances published events onto all Redis hosts. If set to false, + # the output plugin sends all events to only one host (determined at random) + # and will switch to another host if the currently selected one becomes + # unreachable. The default value is true. + #loadbalance: true + + # The Redis connection timeout in seconds. The default is 5 seconds. + #timeout: 5s + + # The number of times to retry publishing an event after a publishing failure. + # After the specified number of retries, the events are typically dropped. + # Some Beats, such as Filebeat, ignore the max_retries setting and retry until + # all events are published. Set max_retries to a value less than 0 to retry + # until all events are published. The default is 3. + #max_retries: 3 + + # The number of seconds to wait before trying to reconnect to Redis + # after a network error. After waiting backoff.init seconds, the Beat + # tries to reconnect. If the attempt fails, the backoff timer is increased + # exponentially up to backoff.max. After a successful connection, the backoff + # timer is reset. The default is 1s. + #backoff.init: 1s + + # The maximum number of seconds to wait before attempting to connect to + # Redis after a network error. The default is 60s. + #backoff.max: 60s + + # The maximum number of events to bulk in a single Redis request or pipeline. + # The default is 2048. + #bulk_max_size: 2048 + + # The URL of the SOCKS5 proxy to use when connecting to the Redis servers. The + # value must be a URL with a scheme of socks5://. + #proxy_url: + + # This option determines whether Redis hostnames are resolved locally when + # using a proxy. The default value is false, which means that name resolution + # occurs on the proxy server. + #proxy_use_local_resolver: false + + # Use SSL settings for HTTPS. + #ssl.enabled: true + + # Controls the verification of certificates. Valid values are: + # * full, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. + # * none, which performs no verification of the server's certificate. This + # mode disables many of the security benefits of SSL/TLS and should only be used + # after very careful consideration. It is primarily intended as a temporary + # diagnostic mechanism when attempting to resolve TLS errors; its use in + # production environments is strongly discouraged. + # The default value is full. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions from 1.1 + # up to 1.3 are enabled. + #ssl.supported_protocols: [TLSv1.1, TLSv1.2, TLSv1.3] + + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client certificate key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the certificate key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE-based cipher suites + #ssl.curve_types: [] + + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256: "" + + +# -------------------------------- File Output --------------------------------- +#output.file: + # Boolean flag to enable or disable the output module. + #enabled: true + + # Configure JSON encoding + #codec.json: + # Pretty-print JSON event + #pretty: false + + # Configure escaping HTML symbols in strings. + #escape_html: false + + # Path to the directory where to save the generated files. The option is + # mandatory. + #path: "/tmp/packetbeat" + + # Name of the generated files. The default is `packetbeat` and it generates + # files: `packetbeat`, `packetbeat.1`, `packetbeat.2`, etc. + #filename: packetbeat + + # Maximum size in kilobytes of each file. When this size is reached, and on + # every Packetbeat restart, the files are rotated. The default value is 10240 + # kB. + #rotate_every_kb: 10000 + + # Maximum number of files under path. When this number of files is reached, + # the oldest file is deleted and the rest are shifted from last to first. The + # default is 7 files. + #number_of_files: 7 + + # Permissions to use for file creation. The default is 0600. + #permissions: 0600 + +# ------------------------------- Console Output ------------------------------- +#output.console: + # Boolean flag to enable or disable the output module. + #enabled: true + + # Configure JSON encoding + #codec.json: + # Pretty-print JSON event + #pretty: false + + # Configure escaping HTML symbols in strings. + #escape_html: false + +# =================================== Paths ==================================== + +# The home path for the Packetbeat installation. This is the default base path +# for all other path settings and for miscellaneous files that come with the +# distribution (for example, the sample dashboards). +# If not set by a CLI flag or in the configuration file, the default for the +# home path is the location of the binary. +#path.home: + +# The configuration path for the Packetbeat installation. This is the default +# base path for configuration files, including the main YAML configuration file +# and the Elasticsearch template file. If not set by a CLI flag or in the +# configuration file, the default for the configuration path is the home path. +#path.config: ${path.home} + +# The data path for the Packetbeat installation. This is the default base path +# for all the files in which Packetbeat needs to store its data. If not set by a +# CLI flag or in the configuration file, the default for the data path is a data +# subdirectory inside the home path. +#path.data: ${path.home}/data + +# The logs path for a Packetbeat installation. This is the default location for +# the Beat's log files. If not set by a CLI flag or in the configuration file, +# the default for the logs path is a logs subdirectory inside the home path. +#path.logs: ${path.home}/logs + +# ================================== Keystore ================================== + +# Location of the Keystore containing the keys and their sensitive values. +#keystore.path: "${path.config}/beats.keystore" + +# ================================= Dashboards ================================= + +# These settings control loading the sample dashboards to the Kibana index. Loading +# the dashboards are disabled by default and can be enabled either by setting the +# options here, or by using the `-setup` CLI flag or the `setup` command. +#setup.dashboards.enabled: false + +# The directory from where to read the dashboards. The default is the `kibana` +# folder in the home path. +#setup.dashboards.directory: ${path.home}/kibana + +# The URL from where to download the dashboards archive. It is used instead of +# the directory if it has a value. +#setup.dashboards.url: + +# The file archive (zip file) from where to read the dashboards. It is used instead +# of the directory when it has a value. +#setup.dashboards.file: + +# In case the archive contains the dashboards from multiple Beats, this lets you +# select which one to load. You can load all the dashboards in the archive by +# setting this to the empty string. +#setup.dashboards.beat: packetbeat + +# The name of the Kibana index to use for setting the configuration. Default is ".kibana" +#setup.dashboards.kibana_index: .kibana + +# The Elasticsearch index name. This overwrites the index name defined in the +# dashboards and index pattern. Example: testbeat-* +#setup.dashboards.index: + +# Always use the Kibana API for loading the dashboards instead of autodetecting +# how to install the dashboards by first querying Elasticsearch. +#setup.dashboards.always_kibana: false + +# If true and Kibana is not reachable at the time when dashboards are loaded, +# it will retry to reconnect to Kibana instead of exiting with an error. +#setup.dashboards.retry.enabled: false + +# Duration interval between Kibana connection retries. +#setup.dashboards.retry.interval: 1s + +# Maximum number of retries before exiting with an error, 0 for unlimited retrying. +#setup.dashboards.retry.maximum: 0 + +# ================================== Template ================================== + +# A template is used to set the mapping in Elasticsearch +# By default template loading is enabled and the template is loaded. +# These settings can be adjusted to load your own template or overwrite existing ones. + +# Set to false to disable template loading. +#setup.template.enabled: true + +# Select the kind of index template. From Elasticsearch 7.8, it is possible to +# use component templates. Available options: legacy, component, index. +# By default packetbeat uses the legacy index templates. +#setup.template.type: legacy + +# Template name. By default the template name is "packetbeat-%{[agent.version]}" +# The template name and pattern has to be set in case the Elasticsearch index pattern is modified. +#setup.template.name: "packetbeat-%{[agent.version]}" + +# Template pattern. By default the template pattern is "-%{[agent.version]}-*" to apply to the default index settings. +# The first part is the version of the beat and then -* is used to match all daily indices. +# The template name and pattern has to be set in case the Elasticsearch index pattern is modified. +#setup.template.pattern: "packetbeat-%{[agent.version]}-*" + +# Path to fields.yml file to generate the template +#setup.template.fields: "${path.config}/fields.yml" + +# A list of fields to be added to the template and Kibana index pattern. Also +# specify setup.template.overwrite: true to overwrite the existing template. +#setup.template.append_fields: +#- name: field_name +# type: field_type + +# Enable JSON template loading. If this is enabled, the fields.yml is ignored. +#setup.template.json.enabled: false + +# Path to the JSON template file +#setup.template.json.path: "${path.config}/template.json" + +# Name under which the template is stored in Elasticsearch +#setup.template.json.name: "" + +# Overwrite existing template +#setup.template.overwrite: false + +# Elasticsearch template settings +setup.template.settings: + + # A dictionary of settings to place into the settings.index dictionary + # of the Elasticsearch template. For more details, please check + # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html + #index: + #number_of_shards: 1 + #codec: best_compression + + # A dictionary of settings for the _source field. For more details, please check + # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html + #_source: + #enabled: false + +# ====================== Index Lifecycle Management (ILM) ====================== + +# Configure index lifecycle management (ILM). These settings create a write +# alias and add additional settings to the index template. When ILM is enabled, +# output.elasticsearch.index is ignored, and the write alias is used to set the +# index name. + +# Enable ILM support. Valid values are true, false, and auto. When set to auto +# (the default), the Beat uses index lifecycle management when it connects to a +# cluster that supports ILM; otherwise, it creates daily indices. +#setup.ilm.enabled: auto + +# Set the prefix used in the index lifecycle write alias name. The default alias +# name is 'packetbeat-%{[agent.version]}'. +#setup.ilm.rollover_alias: 'packetbeat' + +# Set the rollover index pattern. The default is "%{now/d}-000001". +#setup.ilm.pattern: "{now/d}-000001" + +# Set the lifecycle policy name. The default policy name is +# 'beatname'. +#setup.ilm.policy_name: "mypolicy" + +# The path to a JSON file that contains a lifecycle policy configuration. Used +# to load your own lifecycle policy. +#setup.ilm.policy_file: + +# Disable the check for an existing lifecycle policy. The default is true. If +# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy +# can be installed. +#setup.ilm.check_exists: true + +# Overwrite the lifecycle policy at startup. The default is false. +#setup.ilm.overwrite: false + +# =================================== Kibana =================================== + +# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. +# This requires a Kibana endpoint configuration. +setup.kibana: + + # Kibana Host + # Scheme and port can be left out and will be set to the default (http and 5601) + # In case you specify and additional path, the scheme is required: http://localhost:5601/path + # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 + #host: "localhost:5601" + + # Optional protocol and basic auth credentials. + #protocol: "https" + #username: "elastic" + #password: "changeme" + + # Optional HTTP path + #path: "" + + # Optional Kibana space ID. + #space.id: "" + + # Custom HTTP headers to add to each request + #headers: + # X-My-Header: Contents of the header + + # Use SSL settings for HTTPS. + #ssl.enabled: true + + # Controls the verification of certificates. Valid values are: + # * full, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. + # * none, which performs no verification of the server's certificate. This + # mode disables many of the security benefits of SSL/TLS and should only be used + # after very careful consideration. It is primarily intended as a temporary + # diagnostic mechanism when attempting to resolve TLS errors; its use in + # production environments is strongly discouraged. + # The default value is full. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions from 1.1 + # up to 1.3 are enabled. + #ssl.supported_protocols: [TLSv1.1, TLSv1.2, TLSv1.3] + + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client certificate key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the certificate key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE-based cipher suites + #ssl.curve_types: [] + + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256: "" + + +# ================================== Logging =================================== + +# There are four options for the log output: file, stderr, syslog, eventlog +# The file output is the default. + +# Sets log level. The default log level is info. +# Available log levels are: error, warning, info, debug +#logging.level: info + +# Enable debug output for selected components. To enable all selectors use ["*"] +# Other available selectors are "beat", "publish", "service" +# Multiple selectors can be chained. +#logging.selectors: [ ] + +# Send all logging output to stderr. The default is false. +#logging.to_stderr: false + +# Send all logging output to syslog. The default is false. +#logging.to_syslog: false + +# Send all logging output to Windows Event Logs. The default is false. +#logging.to_eventlog: false + +# If enabled, Packetbeat periodically logs its internal metrics that have changed +# in the last period. For each metric that changed, the delta from the value at +# the beginning of the period is logged. Also, the total values for +# all non-zero internal metrics are logged on shutdown. The default is true. +#logging.metrics.enabled: true + +# The period after which to log the internal metrics. The default is 30s. +#logging.metrics.period: 30s + +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +logging.to_files: true +logging.files: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/packetbeat + + # The name of the files where the logs are written to. + #name: packetbeat + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated + #rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + #keepfiles: 7 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # Unix epoch. Defaults to disabled. + #interval: 0 + + # Rotate existing logs on startup rather than appending to the existing + # file. Defaults to true. + # rotateonstartup: true + +# Set to true to log messages in JSON format. +#logging.json: false + +# Set to true, to log messages with minimal required Elastic Common Schema (ECS) +# information. Recommended to use in combination with `logging.json=true` +# Defaults to false. +#logging.ecs: false + +# ============================= X-Pack Monitoring ============================== +# Packetbeat can export internal metrics to a central Elasticsearch monitoring +# cluster. This requires xpack monitoring to be enabled in Elasticsearch. The +# reporting is disabled by default. + +# Set to true to enable the monitoring reporter. +#monitoring.enabled: false + +# Sets the UUID of the Elasticsearch cluster under which monitoring data for this +# Packetbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch +# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch. +#monitoring.cluster_uuid: + +# Uncomment to send the metrics to Elasticsearch. Most settings from the +# Elasticsearch output are accepted here as well. +# Note that the settings should point to your Elasticsearch *monitoring* cluster. +# Any setting that is not set is automatically inherited from the Elasticsearch +# output configuration, so if you have the Elasticsearch output configured such +# that it is pointing to your Elasticsearch monitoring cluster, you can simply +# uncomment the following line. +#monitoring.elasticsearch: + + # Array of hosts to connect to. + # Scheme and port can be left out and will be set to the default (http and 9200) + # In case you specify and additional path, the scheme is required: http://localhost:9200/path + # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 + #hosts: ["localhost:9200"] + + # Set gzip compression level. + #compression_level: 0 + + # Protocol - either `http` (default) or `https`. + #protocol: "https" + + # Authentication credentials - either API key or username/password. + #api_key: "id:api_key" + #username: "beats_system" + #password: "changeme" + + # Dictionary of HTTP parameters to pass within the URL with index operations. + #parameters: + #param1: value1 + #param2: value2 + + # Custom HTTP headers to add to each request + #headers: + # X-My-Header: Contents of the header + + # Proxy server url + #proxy_url: http://proxy:3128 + + # The number of times a particular Elasticsearch index operation is attempted. If + # the indexing operation doesn't succeed after this many retries, the events are + # dropped. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Elasticsearch bulk API index request. + # The default is 50. + #bulk_max_size: 50 + + # The number of seconds to wait before trying to reconnect to Elasticsearch + # after a network error. After waiting backoff.init seconds, the Beat + # tries to reconnect. If the attempt fails, the backoff timer is increased + # exponentially up to backoff.max. After a successful connection, the backoff + # timer is reset. The default is 1s. + #backoff.init: 1s + + # The maximum number of seconds to wait before attempting to connect to + # Elasticsearch after a network error. The default is 60s. + #backoff.max: 60s + + # Configure HTTP request timeout before failing an request to Elasticsearch. + #timeout: 90 + + # Use SSL settings for HTTPS. + #ssl.enabled: true + + # Controls the verification of certificates. Valid values are: + # * full, which verifies that the provided certificate is signed by a trusted + # authority (CA) and also verifies that the server's hostname (or IP address) + # matches the names identified within the certificate. + # * certificate, which verifies that the provided certificate is signed by a + # trusted authority (CA), but does not perform any hostname verification. + # * none, which performs no verification of the server's certificate. This + # mode disables many of the security benefits of SSL/TLS and should only be used + # after very careful consideration. It is primarily intended as a temporary + # diagnostic mechanism when attempting to resolve TLS errors; its use in + # production environments is strongly discouraged. + # The default value is full. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions from 1.1 + # up to 1.3 are enabled. + #ssl.supported_protocols: [TLSv1.1, TLSv1.2, TLSv1.3] + + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client certificate key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the certificate key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE-based cipher suites + #ssl.curve_types: [] + + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + + # Configure a pin that can be used to do extra validation of the verified certificate chain, + # this allow you to ensure that a specific certificate is used to validate the chain of trust. + # + # The pin is a base64 encoded string of the SHA-256 fingerprint. + #ssl.ca_sha256: "" + + # Enable Kerberos support. Kerberos is automatically enabled if any Kerberos setting is set. + #kerberos.enabled: true + + # Authentication type to use with Kerberos. Available options: keytab, password. + #kerberos.auth_type: password + + # Path to the keytab file. It is used when auth_type is set to keytab. + #kerberos.keytab: /etc/elastic.keytab + + # Path to the Kerberos configuration. + #kerberos.config_path: /etc/krb5.conf + + # Name of the Kerberos user. + #kerberos.username: elastic + + # Password of the Kerberos user. It is used when auth_type is set to password. + #kerberos.password: changeme + + # Kerberos realm. + #kerberos.realm: ELASTIC + + #metrics.period: 10s + #state.period: 1m + +# The `monitoring.cloud.id` setting overwrites the `monitoring.elasticsearch.hosts` +# setting. You can find the value for this setting in the Elastic Cloud web UI. +#monitoring.cloud.id: + +# The `monitoring.cloud.auth` setting overwrites the `monitoring.elasticsearch.username` +# and `monitoring.elasticsearch.password` settings. The format is `:`. +#monitoring.cloud.auth: + +# =============================== HTTP Endpoint ================================ + +# Each beat can expose internal metrics through a HTTP endpoint. For security +# reasons the endpoint is disabled by default. This feature is currently experimental. +# Stats can be access through http://localhost:5066/stats . For pretty JSON output +# append ?pretty to the URL. + +# Defines if the HTTP endpoint is enabled. +#http.enabled: false + +# The HTTP endpoint will bind to this hostname, IP address, unix socket or named pipe. +# When using IP addresses, it is recommended to only use localhost. +#http.host: localhost + +# Port on which the HTTP endpoint will bind. Default is 5066. +#http.port: 5066 + +# Define which user should be owning the named pipe. +#http.named_pipe.user: + +# Define which the permissions that should be applied to the named pipe, use the Security +# Descriptor Definition Language (SDDL) to define the permission. This option cannot be used with +# `http.user`. +#http.named_pipe.security_descriptor: + +# ============================== Process Security ============================== + +# Enable or disable seccomp system call filtering on Linux. Default is enabled. +#seccomp.enabled: true + +# ============================== Instrumentation =============================== + +# Instrumentation support for the packetbeat. +#instrumentation: + # Set to true to enable instrumentation of packetbeat. + #enabled: false + + # Environment in which packetbeat is running on (eg: staging, production, etc.) + #environment: "" + + # APM Server hosts to report instrumentation results to. + #hosts: + # - http://localhost:8200 + + # API Key for the APM Server(s). + # If api_key is set then secret_token will be ignored. + #api_key: + + # Secret token for the APM Server(s). + #secret_token: + + # Enable profiling of the server, recording profile samples as events. + # + # This feature is experimental. + #profiling: + #cpu: + # Set to true to enable CPU profiling. + #enabled: false + #interval: 60s + #duration: 10s + #heap: + # Set to true to enable heap profiling. + #enabled: false + #interval: 60s + +# ================================= Migration ================================== + +# This allows to enable 6.7 migration aliases +#migration.6_to_7.enabled: false + diff --git a/x-pack/packetbeat/packetbeat.yml b/x-pack/packetbeat/packetbeat.yml new file mode 100644 index 000000000000..31c229b1ef76 --- /dev/null +++ b/x-pack/packetbeat/packetbeat.yml @@ -0,0 +1,277 @@ +#################### Packetbeat Configuration Example ######################### + +# This file is an example configuration file highlighting only the most common +# options. The packetbeat.reference.yml file from the same directory contains all the +# supported options with more comments. You can use it as a reference. +# +# You can find the full configuration reference here: +# https://www.elastic.co/guide/en/beats/packetbeat/index.html + +# =============================== Network device =============================== + +# Select the network interface to sniff the data. On Linux, you can use the +# "any" keyword to sniff on all connected interfaces. +packetbeat.interfaces.device: any + +# =================================== Flows ==================================== + +# Set `enabled: false` or comment out all options to disable flows reporting. +packetbeat.flows: + # Set network flow timeout. Flow is killed if no packet is received before being + # timed out. + timeout: 30s + + # Configure reporting period. If set to -1, only killed flows will be reported + period: 10s + +# =========================== Transaction protocols ============================ + +packetbeat.protocols: +- type: icmp + # Enable ICMPv4 and ICMPv6 monitoring. Default: false + enabled: true + +- type: amqp + # Configure the ports where to listen for AMQP traffic. You can disable + # the AMQP protocol by commenting out the list of ports. + ports: [5672] + +- type: cassandra + #Cassandra port for traffic monitoring. + ports: [9042] + +- type: dhcpv4 + # Configure the DHCP for IPv4 ports. + ports: [67, 68] + +- type: dns + # Configure the ports where to listen for DNS traffic. You can disable + # the DNS protocol by commenting out the list of ports. + ports: [53] + +- type: http + # Configure the ports where to listen for HTTP traffic. You can disable + # the HTTP protocol by commenting out the list of ports. + ports: [80, 8080, 8000, 5000, 8002] + +- type: memcache + # Configure the ports where to listen for memcache traffic. You can disable + # the Memcache protocol by commenting out the list of ports. + ports: [11211] + +- type: mysql + # Configure the ports where to listen for MySQL traffic. You can disable + # the MySQL protocol by commenting out the list of ports. + ports: [3306,3307] + +- type: pgsql + # Configure the ports where to listen for Pgsql traffic. You can disable + # the Pgsql protocol by commenting out the list of ports. + ports: [5432] + +- type: redis + # Configure the ports where to listen for Redis traffic. You can disable + # the Redis protocol by commenting out the list of ports. + ports: [6379] + +- type: thrift + # Configure the ports where to listen for Thrift-RPC traffic. You can disable + # the Thrift-RPC protocol by commenting out the list of ports. + ports: [9090] + +- type: mongodb + # Configure the ports where to listen for MongoDB traffic. You can disable + # the MongoDB protocol by commenting out the list of ports. + ports: [27017] + +- type: nfs + # Configure the ports where to listen for NFS traffic. You can disable + # the NFS protocol by commenting out the list of ports. + ports: [2049] + +- type: tls + # Configure the ports where to listen for TLS traffic. You can disable + # the TLS protocol by commenting out the list of ports. + ports: + - 443 # HTTPS + - 993 # IMAPS + - 995 # POP3S + - 5223 # XMPP over SSL + - 8443 + - 8883 # Secure MQTT + - 9243 # Elasticsearch + +- type: sip + # Configure the ports where to listen for SIP traffic. You can disable the SIP protocol by commenting out the list of ports. + ports: [5060] + +# ======================= Elasticsearch template setting ======================= + +setup.template.settings: + index.number_of_shards: 1 + #index.codec: best_compression + #_source.enabled: false + +# ================================== General =================================== + +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +#name: + +# A list of tags to include in every event. In the default configuration file +# the forwarded tag causes Packetbeat to not add any host fields. If you are +# monitoring a network tap or mirror port then add the forwarded tag. +#tags: [forwarded] + +# Optional fields that you can specify to add additional information to the +# output. +#fields: +# env: staging + +# ================================= Dashboards ================================= +# These settings control loading the sample dashboards to the Kibana index. Loading +# the dashboards is disabled by default and can be enabled either by setting the +# options here or by using the `setup` command. +#setup.dashboards.enabled: false + +# The URL from where to download the dashboards archive. By default this URL +# has a value which is computed based on the Beat name and version. For released +# versions, this URL points to the dashboard archive on the artifacts.elastic.co +# website. +#setup.dashboards.url: + +# =================================== Kibana =================================== + +# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. +# This requires a Kibana endpoint configuration. +setup.kibana: + + # Kibana Host + # Scheme and port can be left out and will be set to the default (http and 5601) + # In case you specify and additional path, the scheme is required: http://localhost:5601/path + # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 + #host: "localhost:5601" + + # Kibana Space ID + # ID of the Kibana Space into which the dashboards should be loaded. By default, + # the Default Space will be used. + #space.id: + +# =============================== Elastic Cloud ================================ + +# These settings simplify using Packetbeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + +# ================================== Outputs =================================== + +# Configure what output to use when sending the data collected by the beat. + +# ---------------------------- Elasticsearch Output ---------------------------- +output.elasticsearch: + # Array of hosts to connect to. + hosts: ["localhost:9200"] + + # Protocol - either `http` (default) or `https`. + #protocol: "https" + + # Authentication credentials - either API key or username/password. + #api_key: "id:api_key" + #username: "elastic" + #password: "changeme" + +# ------------------------------ Logstash Output ------------------------------- +#output.logstash: + # The Logstash hosts + #hosts: ["localhost:5044"] + + # Optional SSL. By default is off. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + +# ================================= Processors ================================= + +processors: + - # Add forwarded to tags when processing data from a network tap or mirror. + if.contains.tags: forwarded + then: + - drop_fields: + fields: [host] + else: + - add_host_metadata: ~ + - add_cloud_metadata: ~ + - add_docker_metadata: ~ + +# ================================== Logging =================================== + +# Sets log level. The default log level is info. +# Available log levels are: error, warning, info, debug +#logging.level: debug + +# At debug level, you can selectively enable logging only for some components. +# To enable all selectors use ["*"]. Examples of other selectors are "beat", +# "publish", "service". +#logging.selectors: ["*"] + +# ============================= X-Pack Monitoring ============================== +# Packetbeat can export internal metrics to a central Elasticsearch monitoring +# cluster. This requires xpack monitoring to be enabled in Elasticsearch. The +# reporting is disabled by default. + +# Set to true to enable the monitoring reporter. +#monitoring.enabled: false + +# Sets the UUID of the Elasticsearch cluster under which monitoring data for this +# Packetbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch +# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch. +#monitoring.cluster_uuid: + +# Uncomment to send the metrics to Elasticsearch. Most settings from the +# Elasticsearch output are accepted here as well. +# Note that the settings should point to your Elasticsearch *monitoring* cluster. +# Any setting that is not set is automatically inherited from the Elasticsearch +# output configuration, so if you have the Elasticsearch output configured such +# that it is pointing to your Elasticsearch monitoring cluster, you can simply +# uncomment the following line. +#monitoring.elasticsearch: + +# ============================== Instrumentation =============================== + +# Instrumentation support for the packetbeat. +#instrumentation: + # Set to true to enable instrumentation of packetbeat. + #enabled: false + + # Environment in which packetbeat is running on (eg: staging, production, etc.) + #environment: "" + + # APM Server hosts to report instrumentation results to. + #hosts: + # - http://localhost:8200 + + # API Key for the APM Server(s). + # If api_key is set then secret_token will be ignored. + #api_key: + + # Secret token for the APM Server(s). + #secret_token: + + +# ================================= Migration ================================== + +# This allows to enable 6.7 migration aliases +#migration.6_to_7.enabled: true + From 1a6a5b951e57ad74a8f72cc05786d3f297586b08 Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Mon, 26 Oct 2020 17:17:26 -0400 Subject: [PATCH 9/9] Suggested changes --- x-pack/packetbeat/magefile.go | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/x-pack/packetbeat/magefile.go b/x-pack/packetbeat/magefile.go index fc54d9954da3..0cdeba15dcd3 100644 --- a/x-pack/packetbeat/magefile.go +++ b/x-pack/packetbeat/magefile.go @@ -23,6 +23,7 @@ import ( // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" // mage:import + _ "github.com/elastic/beats/v7/dev-tools/mage/target/test" ) func init() { @@ -44,7 +45,7 @@ func Config() error { // Dashboards packages kibana dashboards func Dashboards() error { - return devtools.KibanaDashboards("protos") + return devtools.KibanaDashboards(devtools.OSSBeatDir("protos")) } // Build builds the Beat binary. @@ -99,14 +100,3 @@ func Package() { func TestPackages() error { return devtools.TestPackages() } - -func configYML() error { - return devtools.Config(devtools.AllConfigTypes, packetbeat.ConfigFileParams(), ".") -} - -func includeList() error { - options := devtools.DefaultIncludeListOptions() - options.ImportDirs = []string{"protos/*"} - options.ModuleDirs = nil - return devtools.GenerateIncludeListGo(options) -}