New Filebeat subcommand: generate

filebeat/cmd/generate.go

@@ -0,0 +1,121 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package cmd
+
+import (
+ "fmt"
+ "os"
+
+ "github.com/spf13/cobra"
+
+ "github.com/elastic/beats/filebeat/generator/fields"
+ "github.com/elastic/beats/filebeat/generator/fileset"
+ "github.com/elastic/beats/filebeat/generator/module"
+ "github.com/elastic/beats/libbeat/common/cli"
+ "github.com/elastic/beats/libbeat/paths"
+)
+
+var (
+ defaultHomePath = paths.Resolve(paths.Home, "")
+)
+
+func genGenerateCmd() *cobra.Command {
+ generateCmd := cobra.Command{
+ Use: "generate",
+ Short: "Generate Filebeat modules, filesets and fields.yml",
+ }
+ generateCmd.AddCommand(genGenerateModuleCmd())
+ generateCmd.AddCommand(genGenerateFilesetCmd())
+ generateCmd.AddCommand(genGenerateFieldsCmd())
+
+ return &generateCmd
+}
+
+func genGenerateModuleCmd() *cobra.Command {
+ genModuleCmd := &cobra.Command{
+ Use: "module [module]",
+ Short: "Generates a new module",
+ Run: cli.RunWith(func(cmd *cobra.Command, args []string) error {
+ modulesPath, _ := cmd.Flags().GetString("modules-path")
+ esBeatsPath, _ := cmd.Flags().GetString("es-beats")
+
+ if len(args) != 1 {
+ fmt.Fprintf(os.Stderr, "Exactly one parameter is required: module name\n")
+ os.Exit(1)
+ }
+ name := args[0]
+
+ return module.Generate(name, modulesPath, esBeatsPath)
+ }),
+ }
+
+ genModuleCmd.Flags().String("modules-path", defaultHomePath, "Path to modules directory")
+ genModuleCmd.Flags().String("es-beats", defaultHomePath, "Path to Elastic Beats")
+
+ return genModuleCmd
+}
+
+func genGenerateFilesetCmd() *cobra.Command {
+ genFilesetCmd := &cobra.Command{
+ Use: "fileset [module] [fileset]",
+ Short: "Generates a new fileset",
+ Run: cli.RunWith(func(cmd *cobra.Command, args []string) error {
+ modulesPath, _ := cmd.Flags().GetString("modules-path")
+ esBeatsPath, _ := cmd.Flags().GetString("es-beats")
+
+ if len(args) != 2 {
+ fmt.Fprintf(os.Stderr, "Two parameters are required: module name, fileset name\n")
+ os.Exit(1)
+ }
+ moduleName := args[0]
+ filesetName := args[1]
+
+ return fileset.Generate(moduleName, filesetName, modulesPath, esBeatsPath)
+ }),
+ }
+
+ genFilesetCmd.Flags().String("modules-path", defaultHomePath, "Path to modules directory")
+ genFilesetCmd.Flags().String("es-beats", defaultHomePath, "Path to Elastic Beats")
+
+ return genFilesetCmd
+}
+
+func genGenerateFieldsCmd() *cobra.Command {
+ genFieldsCmd := &cobra.Command{
+ Use: "fields [module] [fileset]",
+ Short: "Generates a new fields.yml file for fileset",
+ Run: cli.RunWith(func(cmd *cobra.Command, args []string) error {
+ esBeatsPath, _ := cmd.Flags().GetString("es-beats")
+ noDoc, _ := cmd.Flags().GetBool("without-documentation")
+
+ if len(args) != 2 {
+ fmt.Fprintf(os.Stderr, "Two parameters are required: module name, fileset name\n")
+ os.Exit(1)
+ }
+ moduleName := args[0]
+ filesetName := args[1]
+
+ return fields.Generate(esBeatsPath, moduleName, filesetName, noDoc)
+ }),
+ }
+
+ genFieldsCmd.Flags().String("es-beats", defaultHomePath, "Path to Elastic Beats")
+ genFieldsCmd.Flags().Bool("without-documentation", false, "Do not add description fields")
+
+ return genFieldsCmd
+}

filebeat/cmd/root.go

@@ -43,4 +43,5 @@ func init() {
  RootCmd.TestCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("modules"))
  RootCmd.SetupCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("modules"))
  RootCmd.AddCommand(cmd.GenModulesCmd(Name, "", buildModulesManager))
+ RootCmd.AddCommand(genGenerateCmd())
 }

filebeat/tests/system/input/my-module-pipeline.json

@@ -0,0 +1,27 @@
+{
+ "description": "Pipeline for parsing PostgreSQL logs.",
+ "processors": [
+ {
+ "grok": {
+ "field": "message",
+ "ignore_missing": true,
+ "patterns": [
+ "^%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:postgresql.log.timezone} \\[%{NUMBER:postgresql.log.thread_id}\\] ((\\[%{USERNAME:postgresql.log.user}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\]|%{USERNAME:postgresql.log.user}@%{POSTGRESQL_DB_NAME:postgresql.log.database}) )?%{WORD:postgresql.log.level}: (duration: %{NUMBER:postgresql.log.duration} ms statement: %{GREEDYDATA:postgresql.log.query}|%{GREEDYDATA:postgresql.log.message})"
+ ],
+ "pattern_definitions": {
+ "LOCALDATETIME": "[-0-9]+ %{TIME}",
+ "GREEDYDATA": "(.|\n|\t)*",
+ "POSTGRESQL_DB_NAME": "[a-zA-Z0-9_]+[a-zA-Z0-9_\\$]*"
+ }
+ }
+ }
+ ],
+ "on_failure": [
+ {
+ "set": {
+ "field": "error.message",
+ "value": "{{ _ingest.on_failure_message }}"
+ }
+ }
+ ]
+}

filebeat/tests/system/test_generate.py

@@ -0,0 +1,127 @@
+import os
+import shutil
+import filebeat
+
+
+class Test(filebeat.BaseTest):
+ def test_generate_module(self):
+ """
+ Test filebeat generate module my_module generates a new module
+ """
+
+ self._create_clean_test_modules()
+ exit_code = self.run_beat(
+ extra_args=["generate", "module", "my_module", "-modules-path", "test_modules", "-es-beats", self.beat_path])
+ assert exit_code == 0
+
+ module_root = os.path.join("test_modules", "module", "my_module")
+ module_meta_root = os.path.join(module_root, "_meta")
+ self._assert_required_module_directories_are_created(module_root, module_meta_root)
+ self._assert_required_module_files_are_created_and_substitution_is_done(module_root, module_meta_root)
+
+ shutil.rmtree("test_modules")
+
+ def _assert_required_module_directories_are_created(self, module_root, module_meta_root):
+ expected_created_directories = [
+ module_root,
+ module_meta_root,
+ ]
+ for expected_dir in expected_created_directories:
+ assert os.path.isdir(expected_dir)
+
+ def _assert_required_module_files_are_created_and_substitution_is_done(self, module_root, module_meta_root):
+ expected_created_template_files = [
+ os.path.join(module_root, "module.yml"),
+ os.path.join(module_meta_root, "config.yml"),
+ os.path.join(module_meta_root, "docs.asciidoc"),
+ os.path.join(module_meta_root, "fields.yml"),
+ ]
+ for template_file in expected_created_template_files:
+ assert os.path.isfile(template_file)
+ assert '{module}' not in open(template_file)
+
+ def test_generate_fileset(self):
+ """
+ Test filebeat generate fileset my_module my_fileset generates a new fileset
+ """
+
+ self._create_clean_test_modules()
+ exit_code = self.run_beat(
+ extra_args=["generate", "module", "my_module", "-modules-path", "test_modules", "-es-beats", self.beat_path])
+ assert exit_code == 0
+
+ exit_code = self.run_beat(
+ extra_args=["generate", "fileset", "my_module", "my_fileset", "-modules-path", "test_modules", "-es-beats", self.beat_path])
+ assert exit_code == 0
+
+ fileset_root = os.path.join("test_modules", "module", "my_module", "my_fileset")
+ self._assert_required_fileset_directories_are_created(fileset_root)
+ self._assert_required_fileset_files_are_created_and_substitution_is_done(fileset_root)
+
+ shutil.rmtree("test_modules")
+
+ def _assert_required_fileset_directories_are_created(self, fileset_root):
+ expected_created_directories = [
+ fileset_root,
+ os.path.join(fileset_root, "config"),
+ os.path.join(fileset_root, "ingest"),
+ os.path.join(fileset_root, "_meta"),
+ os.path.join(fileset_root, "test"),
+ ]
+ for expected_dir in expected_created_directories:
+ assert os.path.isdir(expected_dir)
+
+ def _assert_required_fileset_files_are_created_and_substitution_is_done(self, fileset_root):
+ expected_created_template_files = [
+ os.path.join(fileset_root, "config", "my_fileset.yml"),
+ os.path.join(fileset_root, "ingest", "pipeline.json"),
+ os.path.join(fileset_root, "manifest.yml"),
+ ]
+ for template_file in expected_created_template_files:
+ assert os.path.isfile(template_file)
+ assert '{fileset}' not in open(template_file)
+
+ def test_generate_fields_yml(self):
+ """
+ Test filebeat generate fields my_module my_fileset generates a new fields.yml for my_module/my_fileset
+ """
+
+ self._create_clean_test_modules()
+ exit_code = self.run_beat(
+ extra_args=["generate", "module", "my_module", "-modules-path", "test_modules", "-es-beats", self.beat_path])
+ assert exit_code == 0
+
+ exit_code = self.run_beat(
+ extra_args=["generate", "fileset", "my_module", "my_fileset", "-modules-path", "test_modules", "-es-beats", self.beat_path])
+ assert exit_code == 0
+
+ test_pipeline_path = os.path.join(self.beat_path, "tests", "system", "input", "my-module-pipeline.json")
+ fileset_pipeline = os.path.join("test_modules", "module",
+ "my_module", "my_fileset", "ingest", "pipeline.json")
+
+ print(os.path.isdir("test_modules"))
+ print(os.path.isdir(os.path.join("test_modules", "module")))
+ print(os.path.isdir(os.path.join("test_modules", "module", "my_module")))
+ print(os.path.isdir(os.path.join("test_modules", "module", "my_module", "my_fileset")))
+ print(os.path.isdir(os.path.join("test_modules", "module", "my_module", "my_fileset", "ingest")))
+ print(os.path.isdir(os.path.join("test_modules", "module", "my_module", "my_fileset", "ingest")))
+ print(os.path.exists(os.path.join("test_modules", "module", "my_module", "my_fileset", "ingest", "pipeline.json")))
+ print(os.path.exists(fileset_pipeline))
+ print(os.path.exists(test_pipeline_path))
+ shutil.copyfile(test_pipeline_path, fileset_pipeline)
+
+ print(fileset_pipeline)
+ print(os.path.abspath(fileset_pipeline))
+ exit_code = self.run_beat(
+ extra_args=["generate", "fields", "my_module", "my_fileset", "-es-beats", "test_modules", "-without-documentation"])
+ assert exit_code == 0
+
+ fields_yml_path = os.path.join("test_modules", "module", "my_module", "my_fileset", "_meta", "fields.yml")
+ assert os.path.isfile(fields_yml_path)
+
+ shutil.rmtree("test_modules")
+
+ def _create_clean_test_modules(self):
+ if os.path.isdir("test_modules"):
+ shutil.rmtree("test_modules")
+ os.mkdir("test_modules")