Skip to content

Commit

Permalink
[Fleet] Add support for Runtime Fields (#161129)
Browse files Browse the repository at this point in the history 
## Summary
Closes #155255
Closes elastic/package-spec#39

Add support in Fleet for Runtime fields, based on these docs:
- Defining runtime fields:
-
https://www.elastic.co/guide/en/elasticsearch/reference/8.8/runtime-mapping-fields.html
-
https://www.elastic.co/guide/en/elasticsearch/reference/8.8/runtime-retrieving-fields.html
- Mapping runtime fields in dynamic templates:
-
https://www.elastic.co/guide/en/elasticsearch/reference/current/dynamic-templates.html#dynamic-mapping-runtime-fields
- Adding runtime fields under groups

Given these field definitions in packages:
```yaml
- name: bar
  type: boolean
- name: uptime
  type: keyword
- name: runtime_boolean
  type: boolean
  runtime: true
- name: runtime.day
  type: keyword
  runtime: >-
    emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))
- name: to_be_long
  type: long
  runtime: true
- name: runtime.date
  type: date
  date_format: 'yyyy-MM-dd'
  runtime: >-
    emit(doc['@timestamp'].value.toEpochMilli())
- name: runtime.epoch_milli
  type: long
  runtime: >-
    emit(doc['@timestamp'].value.toEpochMilli())
- name: lowercase
  type: keyword
  runtime: >-
    emit(doc['uppercase'].value.toLowerCase())
- name: labels.*
  type: long
  object_type_mapping_type: double
  runtime: true
- name: responses
  type: group
  fields:
    - name: runtime_group_boolean
      type: boolean
      runtime: true
    - name: foo
      type: boolean
```
and this definition in the manifest
```yaml
elasticsearch:
  index_template:
    mappings:
      runtime:
        day_of_week_two:
          type: keyword
          script:
            source: "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
```

This PR adds the required fields into the `mappings` key when installing
the package. For this example, the resulting mappings are (just showing
the relevant data for these changes):

```json
{
  ".ds-logs-runtime_fields.foo-default-2023.07.10-000001": {
    "mappings": {
      "dynamic_templates": [
        {
          "labels.*": {
            "path_match": "labels.*",
            "match_mapping_type": "double",
            "runtime": {
              "type": "long"
            }
          }
        }
      ],
      "runtime": {
        "day_of_week_two": {
          "type": "keyword",
          "script": {
            "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))",
            "lang": "painless"
          }
        },
        "labels.a": {
          "type": "long"
        },
        "labels.b": {
          "type": "long"
        },
        "lowercase": {
          "type": "keyword",
          "script": {
            "source": "emit(doc['uppercase'].value.toLowerCase())",
            "lang": "painless"
          }
        },
        "responses.runtime_group_boolean": {
          "type": "boolean"
        },
        "runtime.date": {
          "type": "date",
          "script": {
            "source": "emit(doc['@timestamp'].value.toEpochMilli())",
            "lang": "painless"
          },
          "format": "yyyy-MM-dd"
        },
        "runtime.day": {
          "type": "keyword",
          "script": {
            "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))",
            "lang": "painless"
          }
        },
        "runtime.epoch_milli": {
          "type": "long",
          "script": {
            "source": "emit(doc['@timestamp'].value.toEpochMilli())",
            "lang": "painless"
          }
        },
        "runtime_boolean": {
          "type": "boolean"
        },
        "to_be_long": {
          "type": "long"
        }
      },
      "properties": {
        "@timestamp": {
          "type": "date",
          "ignore_malformed": false
        },
        "bar": {
          "type": "boolean"
        },
        "data_stream": {
          "properties": {
            "dataset": {
              "type": "constant_keyword"
            },
            "namespace": {
              "type": "constant_keyword"
            },
            "type": {
              "type": "constant_keyword"
            }
          }
        },
        "labels": {
          "type": "object"
        },
        "message": {
          "type": "keyword",
          "ignore_above": 1024
        },
        "responses": {
          "properties": {
            "foo": {
              "type": "boolean"
            }
          }
        },
        "uppercase": {
          "type": "keyword",
          "ignore_above": 1024
        },
        "user": {
          "properties": {
            "id": {
              "type": "keyword",
              "ignore_above": 1024
            }
          }
        }
      }
    }
  }
}
```

Tested manually installing a package containing runtime field
definitions as the example above.

Tested also indexing some documents and retrieving the runtime fields:
- Indexing documents:
```json
POST /logs-runtime_fields.foo-default/_doc/
{
  "@timestamp": "2023-07-07T13:32:09.000Z",
  "datastream": {
    "dataset": "logs-runtime_fields.foo",
    "namespace": "default",
    "type": "logs"
  },
  "user": {
    "id": "8a4f500d"
  },
  "message": "Login successful",
  "labels": {
    "a": 1.6,
    "b": 2.5
  },
  "uppercase": "SOMETHING",
  "to_be_long": 1.6,
  "runtime_boolean": true,
  "responses.runtime_group_boolean": false
}
```
- Retrieving runtime fields (`_source` disabled):
```json
GET logs-runtime_fields.foo-default/_search
{
  "fields": [
    "@timestamp",
    "runtime_boolean",
    "responses.runtime_group_boolean",
    "runtime.day",
    "runtime.date",
    "runtime.epoch_milli",
    "labels.*",
    "uppercase",
    "lowercase",
    "to_be_long"
  ],
  "_source": false
}
```
- Output:
```json
...
    "hits": [
      {
        "_index": ".ds-logs-runtime_fields.foo-default-2023.07.10-000001",
        "_id": "_7p1P4kBtEvrlGnsxiFN",
        "_score": 1,
        "fields": {
          "uppercase": [
            "SOMETHING"
          ],
          "runtime.date": [
            "2023-07-10"
          ],
          "@timestamp": [
            "2023-07-10T09:33:09.000Z"
          ],
          "lowercase": [
            "something"
          ],
          "to_be_long": [
            1
          ],
          "runtime_boolean": [
            true
          ],
          "runtime.day": [
            "Monday"
          ],
          "labels.a": [
            1
          ],
          "labels.b": [
            2
          ],
          "responses.runtime_group_boolean": [
            false
          ],
          "runtime.epoch_milli": [
            1688981589000
          ]
        }
      }
    ]
...
```


### Checklist

Delete any items that are not applicable to this PR.

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

(https://chrome.google.com/webstore/detail/axe-web-accessibility-tes/lhdoppojpmngadmnindnejefpokejbdd?hl=en-US))

---------

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
  • Loading branch information
@mrodm @kibanamachine
mrodm and kibanamachine authored Jul 12, 2023
1 parent 2e2da69 commit 9a7cc5a
Show file tree
Hide file tree
Showing 19 changed files with 689 additions and 11 deletions.
1 change: 1 addition & 0 deletions x-pack/plugins/fleet/common/types/models/epm.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -582,6 +582,7 @@ export interface PackageAssetReference {
export interface IndexTemplateMappings {
properties: any;
dynamic_templates?: any;
runtime?: any;
}

// This is an index template v2, see https://github.com/elastic/elasticsearch/issues/53101
Expand Down
55 changes: 54 additions & 1 deletion x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -320,11 +320,64 @@ describe('EPM index template install', () => {
const packageTemplate = componentTemplates['logs-package.dataset@package'].template;

if (!('settings' in packageTemplate)) {
throw new Error('no mappings on package template');
throw new Error('no settings on package template');
}

expect(packageTemplate.settings?.index?.mapping).toEqual(
expect.objectContaining({ ignored_malformed: true })
);
});

it('test prepareTemplate to set a runtime field in index_template.mappings', () => {
const dataStream = {
type: 'logs',
dataset: 'package.dataset',
title: 'test data stream',
release: 'experimental',
package: 'package',
path: 'path',
ingest_pipeline: 'default',
elasticsearch: {
'index_template.mappings': {
runtime: {
day_of_week: {
type: 'keyword',
script: {
source:
"emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))",
},
},
},
},
},
} as RegistryDataStream;

const pkg = {
name: 'package',
version: '0.0.1',
};

const { componentTemplates } = prepareTemplate({
pkg,
dataStream,
});

const packageTemplate = componentTemplates['logs-package.dataset@package'].template;

if (!('mappings' in packageTemplate)) {
throw new Error('no mappings on package template');
}

expect(packageTemplate.mappings?.runtime).toEqual(
expect.objectContaining({
day_of_week: {
type: 'keyword',
script: {
source:
"emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))",
},
},
})
);
});
});
7 changes: 6 additions & 1 deletion x-pack/plugins/fleet/server/services/epm/elasticsearch/template/install.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,8 @@ export function buildComponentTemplates(params: {
(dynampingTemplate) => Object.keys(dynampingTemplate)[0]
);

const mappingsRuntimeFields = merge(mappings.runtime, indexTemplateMappings.runtime ?? {});

const isTimeSeriesEnabledByDefault = registryElasticsearch?.index_mode === 'time_series';
const isSyntheticSourceEnabledByDefault = registryElasticsearch?.source_mode === 'synthetic';

Expand Down Expand Up @@ -359,8 +361,11 @@ export function buildComponentTemplates(params: {
},
mappings: {
properties: mappingsProperties,
...(Object.keys(mappingsRuntimeFields).length > 0
? { runtime: mappingsRuntimeFields }
: {}),
dynamic_templates: mappingsDynamicTemplates.length ? mappingsDynamicTemplates : undefined,
...omit(indexTemplateMappings, 'properties', 'dynamic_templates', '_source'),
...omit(indexTemplateMappings, 'properties', 'dynamic_templates', '_source', 'runtime'),
...(indexTemplateMappings?._source || sourceModeSynthetic
? {
_source: {
Expand Down
101 changes: 101 additions & 0 deletions x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1021,6 +1021,107 @@ describe('EPM template', () => {
expect(JSON.stringify(mappings)).toEqual(JSON.stringify(fieldMapping));
});

it('tests processing runtime fields without script', () => {
const textWithRuntimeFieldsLiteralYml = `
- name: runtime_field
type: boolean
runtime: true
`;
const runtimeFieldMapping = {
properties: {},
runtime: {
runtime_field: {
type: 'boolean',
},
},
};
const fields: Field[] = safeLoad(textWithRuntimeFieldsLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(mappings).toEqual(runtimeFieldMapping);
});

it('tests processing runtime fields with painless script', () => {
const textWithRuntimeFieldsLiteralYml = `
- name: day_of_week
type: date
runtime: |
emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))
`;
const runtimeFieldMapping = {
properties: {},
runtime: {
day_of_week: {
type: 'date',
script: {
source:
"emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))",
},
},
},
};
const fields: Field[] = safeLoad(textWithRuntimeFieldsLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(mappings).toEqual(runtimeFieldMapping);
});

it('tests processing runtime fields defined in a group', () => {
const textWithRuntimeFieldsLiteralYml = `
- name: responses
type: group
fields:
- name: day_of_week
type: date
date_format: date_optional_time
runtime: |
emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))
`;
const runtimeFieldMapping = {
properties: {},
runtime: {
'responses.day_of_week': {
type: 'date',
format: 'date_optional_time',
script: {
source:
"emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))",
},
},
},
};
const fields: Field[] = safeLoad(textWithRuntimeFieldsLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(mappings).toEqual(runtimeFieldMapping);
});

it('tests processing runtime fields in a dynamic template', () => {
const textWithRuntimeFieldsLiteralYml = `
- name: labels.*
type: keyword
runtime: true
`;
const runtimeFieldMapping = {
properties: {},
dynamic_templates: [
{
'labels.*': {
match_mapping_type: 'string',
path_match: 'labels.*',
runtime: {
type: 'keyword',
},
},
},
],
};
const fields: Field[] = safeLoad(textWithRuntimeFieldsLiteralYml);
const processedFields = processFields(fields);
const mappings = generateMappings(processedFields);
expect(mappings).toEqual(runtimeFieldMapping);
});

it('tests priority and index pattern for data stream without dataset_is_prefix', () => {
const dataStreamDatasetIsPrefixUnset = {
type: 'metrics',
Expand Down
107 changes: 98 additions & 9 deletions x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,10 @@ interface MultiFields {
[key: string]: object;
}

interface RuntimeFields {
[key: string]: any;
}

export interface IndexTemplateMapping {
[key: string]: any;
}
Expand Down Expand Up @@ -115,22 +119,27 @@ export function getTemplate({
export function generateMappings(fields: Field[]): IndexTemplateMappings {
const dynamicTemplates: Array<Record<string, Properties>> = [];
const dynamicTemplateNames = new Set<string>();
const runtimeFields: RuntimeFields = {};

const { properties } = _generateMappings(fields, {
addDynamicMapping: (dynamicMapping: {
path: string;
matchingType: string;
pathMatch: string;
properties: string;
runtimeProperties?: string;
}) => {
const name = dynamicMapping.path;
if (dynamicTemplateNames.has(name)) {
return;
}

const dynamicTemplate: Properties = {
mapping: dynamicMapping.properties,
};
const dynamicTemplate: Properties = {};
if (dynamicMapping.runtimeProperties !== undefined) {
dynamicTemplate.runtime = dynamicMapping.runtimeProperties;
} else {
dynamicTemplate.mapping = dynamicMapping.properties;
}

if (dynamicMapping.matchingType) {
dynamicTemplate.match_mapping_type = dynamicMapping.matchingType;
Expand All @@ -139,17 +148,23 @@ export function generateMappings(fields: Field[]): IndexTemplateMappings {
if (dynamicMapping.pathMatch) {
dynamicTemplate.path_match = dynamicMapping.pathMatch;
}

dynamicTemplateNames.add(name);
dynamicTemplates.push({ [dynamicMapping.path]: dynamicTemplate });
},
addRuntimeField: (runtimeField: { path: string; properties: Properties }) => {
runtimeFields[`${runtimeField.path}`] = runtimeField.properties;
},
});

return dynamicTemplates.length
? {
properties,
dynamic_templates: dynamicTemplates,
}
: { properties };
const indexTemplateMappings: IndexTemplateMappings = { properties };
if (dynamicTemplates.length > 0) {
indexTemplateMappings.dynamic_templates = dynamicTemplates;
}
if (Object.keys(runtimeFields).length > 0) {
indexTemplateMappings.runtime = runtimeFields;
}
return indexTemplateMappings;
}

/**
Expand All @@ -164,6 +179,7 @@ function _generateMappings(
fields: Field[],
ctx: {
addDynamicMapping: any;
addRuntimeField: any;
groupFieldName?: string;
}
): {
Expand All @@ -179,6 +195,55 @@ function _generateMappings(
// If type is not defined, assume keyword
const type = field.type || 'keyword';

if (field.runtime !== undefined) {
const path = ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name;
let runtimeFieldProps: Properties = getDefaultProperties(field);

// Is it a dynamic template?
if (type === 'object' && field.object_type) {
const pathMatch = path.includes('*') ? path : `${path}.*`;

let dynProperties: Properties = getDefaultProperties(field);
let matchingType: string | undefined;
switch (field.object_type) {
case 'keyword':
dynProperties.type = field.object_type;
matchingType = field.object_type_mapping_type ?? 'string';
break;
case 'double':
case 'long':
case 'boolean':
dynProperties = {
type: field.object_type,
time_series_metric: field.metric_type,
};
matchingType = field.object_type_mapping_type ?? field.object_type;
default:
break;
}

// get the runtime properies of this field assuming type equals to object_type
const _field = { ...field, type: field.object_type };
const fieldProps = generateRuntimeFieldProps(_field);

if (dynProperties && matchingType) {
ctx.addDynamicMapping({
path,
pathMatch,
matchingType,
properties: dynProperties,
runtimeProperties: fieldProps,
});
}
return;
}
const fieldProps = generateRuntimeFieldProps(field);
runtimeFieldProps = { ...runtimeFieldProps, ...fieldProps };

ctx.addRuntimeField({ path, properties: runtimeFieldProps });
return; // runtime fields should not be added as a property
}

if (type === 'object' && field.object_type) {
const path = ctx.groupFieldName ? `${ctx.groupFieldName}.${field.name}` : field.name;
const pathMatch = path.includes('*') ? path : `${path}.*`;
Expand Down Expand Up @@ -435,6 +500,30 @@ function generateDateMapping(field: Field): IndexTemplateMapping {
return mapping;
}

function generateRuntimeFieldProps(field: Field): IndexTemplateMapping {
let mapping: IndexTemplateMapping = {};
const type = field.type || keyword;
switch (type) {
case 'integer':
mapping.type = 'long';
break;
case 'date':
const dateMappings = generateDateMapping(field);
mapping = { ...mapping, ...dateMappings, type: 'date' };
break;
default:
mapping.type = type;
}

if (typeof field.runtime === 'string') {
const scriptObject = {
source: field.runtime.trim(),
};
mapping.script = scriptObject;
}
return mapping;
}

/**
* Generates the template name out of the given information
*/
Expand Down
1 change: 1 addition & 0 deletions x-pack/plugins/fleet/server/services/epm/fields/field.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ export interface Field {
null_value?: string;
dimension?: boolean;
default_field?: boolean;
runtime?: boolean | string;

// Fields specific of the aggregate_metric_double type
metrics?: string[];
Expand Down
1 change: 1 addition & 0 deletions x-pack/test/fleet_api_integration/apis/epm/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,5 +43,6 @@ export default function loadTests({ loadTestFile, getService }) {
loadTestFile(require.resolve('./install_hidden_datastreams'));
loadTestFile(require.resolve('./bulk_get_assets'));
loadTestFile(require.resolve('./install_dynamic_template_metric'));
loadTestFile(require.resolve('./install_runtime_field'));
});
}
Loading

0 comments on commit 9a7cc5a

Please sign in to comment.