Adding support in for custom analyzers for text and overriding the de…

…fault template settings for index templates (elastic#1737)
huntresslabs · Feb 9, 2022 · d76ae49 · d76ae49
1 parent ea0550d
commit d76ae49
Show file tree

Hide file tree

Showing 8 changed files with 130 additions and 56 deletions.
diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md
@@ -14,8 +14,6 @@ Thanks, you're awesome :-) -->
 
 #### Bugfixes
 
-* Fixed the `default_field` flag for root fields in Beats generator. #1711
-
 #### Added
 
 #### Improvements
@@ -101,6 +99,10 @@ Thanks, you're awesome :-) -->
 * Remove `log.original` field. #1580
 * Remove `process.ppid` field. #1596
 
+#### Bugfixes
+
+* Fixed the `default_field` flag for root fields in Beats generator. #1711
+
 #### Added
 
 * Added `faas.*` field set as beta. #1628, #1755
@@ -120,13 +122,15 @@ Thanks, you're awesome :-) -->
 * Remove Go code generator. #1567
 * Remove template generation for ES6. #1680
 * Update folder structure for generated ES artifacts. #1700
+* Updated support for overridable composable settings template. #1737
 
 #### Improvements
 
 * Align input options for --include and --subset arguments #1519
 * Remove remaining Go deps after removing Go code generator. #1585
 * Add explicit `default_field: true` for Beats artifacts. #1633
 * Reorganize docs directory structure. #1679
+* Added support for `analyzer` definitions for text fields. #1737
 
 <!-- All empty sections:
 

diff --git a/USAGE.md b/USAGE.md
@@ -49,6 +49,7 @@ python scripts/generator.py --ref v1.6.0 \
   --subset            ../my-project/fields/subset.yml \
   --include           ../my-project/fields/custom/ \
   --out               ../my-project/ \
+  --template-settings-legacy ../my-project/fields/template-settings-legacy.json \
   --template-settings ../my-project/fields/template-settings.json \
   --mapping-settings  ../my-project/fields/mapping-settings.json
 ```
@@ -339,13 +340,16 @@ The command above will produce artifacts based on:
 
 #### Mapping & Template Settings
 
-The `--template-settings` and `--mapping-settings` arguments allow overriding the default template and mapping settings, respectively, in the generated Elasticsearch template artifacts. Both artifacts expect a JSON file which contains custom settings defined.
+The `--template-settings-legacy` / `--template-settings` and `--mapping-settings` arguments allow overriding the default template and mapping settings, respectively, in the generated Elasticsearch template artifacts. Both artifacts expect a JSON file which contains custom settings defined.
 
 ```
-$ python scripts/generator.py --template-settings ../myproject/es-overrides/template.json --mapping-settings ../myproject/es-overrides/mappings.json
+$ python scripts/generator.py --template-settings-legacy ../myproject/es-overrides/template.json --mapping-settings ../myproject/es-overrides/mappings.json
 ```
 
-The `--template-settings` argument defines [index level settings](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules.html#index-modules-settings) that will be applied to the index template in the generated artifacts. This is an example `template.json` to be passed with `--template-setting`:
+The `--template-settings-legacy` argument defines [index level settings](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-modules.html#index-modules-settings) that will be applied to the legacy index template in the generated artifacts. The `--template-settings` argument now defines those same settings, but for the composable template in the generated artifacts.
+
+
+This is an example `template.json` to be passed with `--template-setting-legacy`:
 
 ```json
 {

diff --git a/scripts/generator.py b/scripts/generator.py
@@ -70,8 +70,9 @@ def main():
         exit()
 
     csv_generator.generate(flat, ecs_generated_version, out_dir)
-    es_template.generate(nested, ecs_generated_version, out_dir, args.mapping_settings)
-    es_template.generate_legacy(flat, ecs_generated_version, out_dir, args.template_settings, args.mapping_settings)
+    es_template.generate(nested, ecs_generated_version, out_dir, args.mapping_settings, args.template_settings)
+    es_template.generate_legacy(flat, ecs_generated_version, out_dir,
+                                args.mapping_settings, args.template_settings_legacy)
     beats.generate(nested, ecs_generated_version, out_dir)
     if args.include or args.subset or args.exclude:
         exit()
@@ -93,6 +94,8 @@ def argument_parser():
     parser.add_argument('--out', action='store', help='directory to output the generated files')
     parser.add_argument('--template-settings', action='store',
                         help='index template settings to use when generating elasticsearch template')
+    parser.add_argument('--template-settings-legacy', action='store',
+                        help='legacy index template settings to use when generating elasticsearch template')
     parser.add_argument('--mapping-settings', action='store',
                         help='mapping settings to use when generating elasticsearch template')
     parser.add_argument('--strict', action='store_true',

diff --git a/scripts/generators/es_template.py b/scripts/generators/es_template.py
@@ -25,37 +25,17 @@
 # Composable Template
 
 
-def generate(ecs_nested, ecs_version, out_dir, mapping_settings_file):
+def generate(ecs_nested, ecs_version, out_dir, mapping_settings_file, template_settings_file):
     """This generates all artifacts for the composable template approach"""
     all_component_templates(ecs_nested, ecs_version, out_dir)
     component_names = component_name_convention(ecs_version, ecs_nested)
-    save_composable_template(ecs_version, component_names, out_dir, mapping_settings_file)
+    save_composable_template(ecs_version, component_names, out_dir, mapping_settings_file, template_settings_file)
 
 
-def save_composable_template(ecs_version, component_names, out_dir, mapping_settings_file):
-    """Generate the master sample composable template"""
-    template = {
-        "index_patterns": ["try-ecs-*"],
-        "composed_of": component_names,
-        "priority": 1,  # Very low, as this is a sample template
-        "_meta": {
-            "ecs_version": ecs_version,
-            "description": "Sample composable template that includes all ECS fields"
-        },
-        "template": {
-            "settings": {
-                "index": {
-                    "codec": "best_compression",
-                    "mapping": {
-                        "total_fields": {
-                            "limit": 2000
-                        }
-                    }
-                }
-            },
-            "mappings": mapping_settings(mapping_settings_file)
-        }
-    }
+def save_composable_template(ecs_version, component_names, out_dir, mapping_settings_file, template_settings_file):
+    mappings_section = mapping_settings(mapping_settings_file)
+    template = template_settings(ecs_version, mappings_section, template_settings_file, component_names=component_names)
+
     filename = join(out_dir, "elasticsearch/composable/template.json")
     save_json(filename, template)
 
@@ -114,7 +94,7 @@ def candidate_components(ecs_nested):
 # Legacy template
 
 
-def generate_legacy(ecs_flat, ecs_version, out_dir, template_settings_file, mapping_settings_file):
+def generate_legacy(ecs_flat, ecs_version, out_dir, mapping_settings_file, template_settings_file,):
     """Generate the legacy index template"""
     field_mappings = {}
     for flat_name in sorted(ecs_flat):
@@ -125,12 +105,12 @@ def generate_legacy(ecs_flat, ecs_version, out_dir, template_settings_file, mapp
     mappings_section = mapping_settings(mapping_settings_file)
     mappings_section['properties'] = field_mappings
 
-    generate_legacy_template_version(7, ecs_version, mappings_section, out_dir, template_settings_file)
+    generate_legacy_template_version(ecs_version, mappings_section, out_dir, template_settings_file)
 
 
-def generate_legacy_template_version(es_version, ecs_version, mappings_section, out_dir, template_settings_file):
+def generate_legacy_template_version(ecs_version, mappings_section, out_dir, template_settings_file):
     ecs_helpers.make_dirs(join(out_dir, 'elasticsearch', "legacy"))
-    template = template_settings(es_version, ecs_version, mappings_section, template_settings_file)
+    template = template_settings(ecs_version, mappings_section, template_settings_file, isLegacy=True)
 
     filename = join(out_dir, "elasticsearch/legacy/template.json")
     save_json(filename, template)
@@ -186,7 +166,7 @@ def entry_for(field):
                 if mf_type == 'keyword':
                     ecs_helpers.dict_copy_existing_keys(mf, mf_entry, ['normalizer', 'ignore_above'])
                 elif mf_type == 'text':
-                    ecs_helpers.dict_copy_existing_keys(mf, mf_entry, ['norms'])
+                    ecs_helpers.dict_copy_existing_keys(mf, mf_entry, ['norms', 'analyzer'])
                 field_entry['fields'][mf['name']] = mf_entry
 
     except KeyError as ex:
@@ -204,23 +184,40 @@ def mapping_settings(mapping_settings_file):
     return mappings
 
 
-def template_settings(es_version, ecs_version, mappings_section, template_settings_file):
+def template_settings(ecs_version, mappings_section, template_settings_file, isLegacy=False, component_names=None):
     if template_settings_file:
         with open(template_settings_file) as f:
             template = json.load(f)
     else:
-        template = default_template_settings(ecs_version)
-
-    template['mappings'] = mappings_section
+        if isLegacy:
+            template = default_legacy_template_settings(ecs_version)
+        else:
+            template = default_template_settings(ecs_version)
 
-    # _meta can't be at template root in legacy templates, so moving back to mappings section
-    # if present
-    if '_meta' in template:
-        mappings_section['_meta'] = template.pop('_meta')
+    finalize_template(template, ecs_version, isLegacy, mappings_section, component_names)
 
     return template
 
 
+def finalize_template(template, ecs_version, isLegacy, mappings_section, component_names):
+    if isLegacy:
+        if mappings_section:
+            template['mappings'] = mappings_section
+
+            # _meta can't be at template root in legacy templates, so moving back to mappings section
+            # if present
+            if '_meta' in template:
+                mappings_section['_meta'] = template.pop('_meta')
+
+    else:
+        template['template']['mappings'] = mappings_section
+        template['composed_of'] = component_names
+        template['_meta'] = {
+            "ecs_version": ecs_version,
+            "description": "Sample composable template that includes all ECS fields"
+        }
+
+
 def save_json(file, data):
     open_mode = "wb"
     if sys.version_info >= (3, 0):
@@ -231,6 +228,29 @@ def save_json(file, data):
 
 
 def default_template_settings(ecs_version):
+    return {
+        "index_patterns": ["try-ecs-*"],
+        "_meta": {
+            "ecs_version": ecs_version,
+            "description": "Sample composable template that includes all ECS fields"
+        },
+        "priority": 1,  # Very low, as this is a sample template
+        "template": {
+            "settings": {
+                "index": {
+                    "codec": "best_compression",
+                    "mapping": {
+                        "total_fields": {
+                            "limit": 2000
+                        }
+                    }
+                }
+            },
+        }
+    }
+
+
+def default_legacy_template_settings(ecs_version):
     return {
         "index_patterns": ["try-ecs-*"],
         "_meta": {"version": ecs_version},

diff --git a/scripts/tests/test_es_template.py b/scripts/tests/test_es_template.py
@@ -185,6 +185,31 @@ def test_component_composable_template_name(self):
         exp = ["ecs_{}_acme".format(version)]
         self.assertEqual(es_template.component_name_convention(version, test_map), exp)
 
+    def test_legacy_template_settings_override(self):
+        ecs_version = 100
+        default = es_template.default_legacy_template_settings(ecs_version)
+
+        generated_template = es_template.template_settings(ecs_version, None, None, isLegacy=True)
+        self.assertEqual(generated_template, default)
+
+        generated_template = es_template.template_settings(
+            ecs_version, None, './usage-example/fields/template-settings-legacy.json', isLegacy=True)
+        self.assertNotEqual(generated_template, default)
+
+    def test_default_composable_template_settings(self):
+        ecs_version = 100
+        default = es_template.default_template_settings(ecs_version)
+        # Setting these to empty since we aren't testing this piece
+        default['template']['mappings'] = None
+        default['composed_of'] = None
+
+        generated_template = es_template.template_settings(ecs_version, None, None)
+        self.assertEqual(generated_template, default)
+
+        generated_template = es_template.template_settings(
+            ecs_version, None, './usage-example/fields/template-settings.json', isLegacy=True)
+        self.assertNotEqual(generated_template, default)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/usage-example/README.md b/usage-example/README.md
@@ -12,7 +12,8 @@ python scripts/generator.py --ref v1.6.0 \
   --subset            usage-example/fields/subset.yml \
   --include           usage-example/fields/custom/ \
   --out               usage-example/ \
-  --template-settings usage-example/fields/template-settings.json \
+  --template-settings-legacy ../my-project/fields/template-settings-legacy.json \
+  --template-settings ../my-project/fields/template-settings.json \
   --mapping-settings  usage-example/fields/mapping-settings.json
 ```
 

diff --git a/usage-example/fields/template-settings-legacy.json b/usage-example/fields/template-settings-legacy.json
@@ -0,0 +1,16 @@
+{
+  "index_patterns": ["acme-weblogs-*"],
+  "order": 1,
+  "settings": {
+    "index": {
+      "codec" : "best_compression",
+      "mapping": {
+        "total_fields": {
+          "limit": 1000
+        }
+      },
+      "refresh_interval": "2s"
+    }
+  }
+}
+
diff --git a/usage-example/fields/template-settings.json b/usage-example/fields/template-settings.json
@@ -1,15 +1,16 @@
 {
   "index_patterns": ["acme-weblogs-*"],
-  "order": 1,
-  "settings": {
-    "index": {
-      "codec" : "best_compression",
-      "mapping": {
-        "total_fields": {
-          "limit": 1000
+  "priority": 1,
+  "template": {
+    "settings": {
+      "index": {
+        "codec" : "best_compression",
+        "mapping": {
+          "total_fields": {
+            "limit": 2000
+          }
         }
-      },
-      "refresh_interval": "2s"
+      }
     }
   }
 }