diff --git a/.ci/Jenkinsfile_flaky b/.ci/Jenkinsfile_flaky index 7eafc66465bc7..8121405e5ae24 100644 --- a/.ci/Jenkinsfile_flaky +++ b/.ci/Jenkinsfile_flaky @@ -73,11 +73,7 @@ def agentProcess(Map params = [:]) { ]) { task { if (config.needBuild) { - if (!config.isXpack) { - kibanaPipeline.buildOss() - } else { - kibanaPipeline.buildXpack() - } + kibanaPipeline.buildKibana() } for(def i = 0; i < config.agentExecutions; i++) { diff --git a/.ci/Jenkinsfile_security_cypress b/.ci/Jenkinsfile_security_cypress index 811af44d1ca56..d48b9965919dc 100644 --- a/.ci/Jenkinsfile_security_cypress +++ b/.ci/Jenkinsfile_security_cypress @@ -16,7 +16,7 @@ kibanaPipeline(timeoutMinutes: 180) { def job = 'xpack-securityCypress' workers.ci(name: job, size: 'l', ramDisk: true) { - kibanaPipeline.bash('test/scripts/jenkins_xpack_build_kibana.sh', 'Build Default Distributable') + kibanaPipeline.bash('test/scripts/jenkins_build_kibana.sh', 'Build Distributable') kibanaPipeline.functionalTestProcess(job, 'test/scripts/jenkins_security_solution_cypress_chrome.sh')() // Temporarily disabled to figure out test flake // kibanaPipeline.functionalTestProcess(job, 'test/scripts/jenkins_security_solution_cypress_firefox.sh')() diff --git a/.ci/es-snapshots/Jenkinsfile_verify_es b/.ci/es-snapshots/Jenkinsfile_verify_es index d56ec61314ac7..dc3a3cde7d658 100644 --- a/.ci/es-snapshots/Jenkinsfile_verify_es +++ b/.ci/es-snapshots/Jenkinsfile_verify_es @@ -37,12 +37,8 @@ kibanaPipeline(timeoutMinutes: 210) { ]) task { - kibanaPipeline.buildOss(6) + kibanaPipeline.buildKibana(16) tasks.ossCiGroups() - } - - task { - kibanaPipeline.buildXpack(10, true) tasks.xpackCiGroups() tasks.xpackCiGroupDocker() } diff --git a/STYLEGUIDE.mdx b/STYLEGUIDE.mdx index 628a7d157dfcc..afe00476640b3 100644 --- a/STYLEGUIDE.mdx +++ b/STYLEGUIDE.mdx @@ -1,7 +1,7 @@ --- id: kibStyleGuide slug: /kibana-dev-docs/styleguide -title: StyleGuide +title: Style Guide summary: JavaScript/TypeScript styleguide. date: 2021-05-06 tags: ['kibana', 'onboarding', 'dev', 'styleguide', 'typescript', 'javascript'] @@ -680,7 +680,7 @@ Using `react-component` means adding a bunch of components into angular, while ` ### Action function names and prop function names -Name action functions in the form of a strong verb and passed properties in the form of on. E.g: +Name action functions in the form of a strong verb and passed properties in the form of `on`. E.g: ```jsx diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index d334d7979ed59..60af851ff2dc6 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -10,15 +10,15 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") # Fetch Node.js rules http_archive( name = "build_bazel_rules_nodejs", - sha256 = "65067dcad93a61deb593be7d3d9a32a4577d09665536d8da536d731da5cd15e2", - urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/3.4.2/rules_nodejs-3.4.2.tar.gz"], + sha256 = "10f534e1c80f795cffe1f2822becd4897754d18564612510c59b3c73544ae7c6", + urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/3.5.0/rules_nodejs-3.5.0.tar.gz"], ) # Now that we have the rules let's import from them to complete the work load("@build_bazel_rules_nodejs//:index.bzl", "check_rules_nodejs_version", "node_repositories", "yarn_install") # Assure we have at least a given rules_nodejs version -check_rules_nodejs_version(minimum_version_string = "3.4.2") +check_rules_nodejs_version(minimum_version_string = "3.5.0") # Setup the Node.js toolchain for the architectures we want to support # diff --git a/api_docs/spaces_oss.json b/api_docs/spaces_oss.json index bce557db8e516..a0ed4297ddc39 100644 --- a/api_docs/spaces_oss.json +++ b/api_docs/spaces_oss.json @@ -577,15 +577,11 @@ "deprecated": false, "children": [ { - "parentPluginId": "spacesOss", - "id": "def-public.SpacesApi.activeSpace$", - "type": "Object", - "tags": [], - "label": "activeSpace$", - "description": [ - "\nObservable representing the currently active space.\nThe details of the space can change without a full page reload (such as display name, color, etc.)" - ], + "id": "def-public.SpacesApi.getActiveSpace$", + "type": "Function", + "label": "getActiveSpace$", "signature": [ + "() => ", "Observable", "<", { @@ -597,11 +593,16 @@ }, ">" ], + "description": [ + "\nObservable representing the currently active space.\nThe details of the space can change without a full page reload (such as display name, color, etc.)" + ], + "children": [], + "tags": [], + "returnComment": [], "source": { "path": "src/plugins/spaces_oss/public/api.ts", "lineNumber": 22 - }, - "deprecated": false + } }, { "parentPluginId": "spacesOss", diff --git a/docs/concepts/index-patterns.asciidoc b/docs/concepts/index-patterns.asciidoc index 158fa6282e6fa..03bad72a317c6 100644 --- a/docs/concepts/index-patterns.asciidoc +++ b/docs/concepts/index-patterns.asciidoc @@ -10,10 +10,9 @@ or all indices that contain your data. It can also point to a You’ll learn how to: -* Create an index pattern -* Explore and configure the data fields +* Create index patterns * Set the default index pattern -* Delete an index pattern +* Delete index patterns [float] [[index-patterns-read-only-access]] @@ -133,77 +132,23 @@ To exclude a cluster, use `cluster_*:logstash-*,cluster_one:-*`. Once an index pattern is configured using the {ccs} syntax, all searches and aggregations using that index pattern in {kib} take advantage of {ccs}. - -[float] -[[reload-fields]] -=== Explore and configure the data fields - -To explore and configure the data fields in your index pattern, open the main menu, then click -*Stack Management > Index Patterns*. Each field has a {ref}/mapping.html[mapping], -which indicates the type of data the field contains in {es}, -such as strings or boolean values. The field mapping also determines -how you can use the field, such as whether it can be searched or aggregated. - -When a new field is added to the index, the index pattern field list is updated -the next time the index pattern is loaded, for example, when you load the page or -move between {kib} apps. - -[role="screenshot"] -image:management/index-patterns/images/new-index-pattern.png["Create index pattern"] - -[float] -=== Format the display of common field types - -Whenever possible, {kib} uses the same field type for display as -{es}. However, some field types that {es} supports are not available -in {kib}. Using field formatters, you can manually change the field type in {kib} to display your data the way you prefer -to see it, regardless of how it is stored in {es}. - -For example, if you store -date values in {es}, you can use a {kib} field formatter to change the display to mm/dd/yyyy format. -{kib} has field formatters for -<>, -<>, -<>, -and <>. - -To customize the displayed field name provided by {es}, you can -use *Custom Label* . - -A popularity counter keeps track of the fields you use most often. -The top five most popular fields and their values are displayed in <>. - -To edit the field display, click the edit icon -(image:management/index-patterns/images/edit_icon.png[]) in the index pattern detail view. - -[role="screenshot"] -image:management/index-patterns/images/edit-field-format.png["Edit field format"] - [float] -[[default-index-pattern]] -=== Set the default index pattern +[[delete-index-pattern]] +=== Delete index patterns -The first index pattern you create is automatically designated as the default pattern, -but you can set any index pattern as the default. The default index pattern is automatically selected when you first open <> or create a visualization from scratch. +When you delete an index pattern, you are unable to recover the associated field formatters, scripted fields, source filters, +and field popularity data. Deleting an index pattern does not remove any indices or data documents from {es}. -. In *Index patterns*, click the index pattern name. -. Click the star icon (image:management/index-patterns/images/star.png[Star icon]). +WARNING: Deleting an index pattern breaks all visualizations, saved searches, and other saved objects that reference the index pattern. -[float] -[[delete-index-pattern]] -=== Delete an index pattern +. Open the main menu, then click *Stack Management > Index Patterns*. -This action removes the pattern from the list of saved objects in {kib}. -You will not be able to recover field formatters, scripted fields, source filters, -and field popularity data associated with the index pattern. Deleting an -index pattern does not remove any indices or data documents from {es}. +. Click the index pattern you want to delete. -WARNING: Deleting an index pattern breaks all visualizations, saved searches, and other saved objects that reference the pattern. - -. In *Index patterns*, click the index pattern name. -. Click the delete icon (image:management/index-patterns/images/delete.png[Delete icon]). +. Delete (image:management/index-patterns/images/delete.png[Delete icon]) the index pattern. [float] +[[reload-fields]] === What’s next -* Learn about <> and how to create data on the fly. +Learn how to <> in your index patterns. diff --git a/docs/concepts/index.asciidoc b/docs/concepts/index.asciidoc index 74e5bd4d4fb2f..cb37dceb53564 100644 --- a/docs/concepts/index.asciidoc +++ b/docs/concepts/index.asciidoc @@ -49,10 +49,9 @@ that accesses the {kib} API. {kib} uses the index pattern to show you a list of fields, such as `event.duration`. You can customize the display name and format for each field. -For example, you can tell Kibana to display `event.duration` in seconds. +For example, you can tell {kib} to display `event.duration` in seconds. {kib} has <> for strings, -dates, geopoints, -and numbers. +dates, geopoints, and numbers. [float] [[kibana-concepts-searching-your-data]] diff --git a/docs/developer/contributing/development-functional-tests.asciidoc b/docs/developer/contributing/development-functional-tests.asciidoc index f149e9de7aaba..110704a8e569a 100644 --- a/docs/developer/contributing/development-functional-tests.asciidoc +++ b/docs/developer/contributing/development-functional-tests.asciidoc @@ -140,7 +140,7 @@ export default function (/* { providerAPI } */) { ----------- **Services**::: -Services are named singleton values produced by a Service Provider. Tests and other services can retrieve service instances by asking for them by name. All functionality except the mocha API is exposed via services.\ +Services are named singleton values produced by a Service Provider. Tests and other services can retrieve service instances by asking for them by name. All functionality except the mocha API is exposed via services. **Page objects**::: Page objects are a special type of service that encapsulate behaviors common to a particular page or plugin. When you write your own plugin, you’ll likely want to add a page object (or several) that describes the common interactions your tests need to execute. diff --git a/docs/developer/getting-started/monorepo-packages.asciidoc b/docs/developer/getting-started/monorepo-packages.asciidoc index cba5b9bfadf98..1e7a95b83dd67 100644 --- a/docs/developer/getting-started/monorepo-packages.asciidoc +++ b/docs/developer/getting-started/monorepo-packages.asciidoc @@ -82,8 +82,10 @@ yarn kbn watch-bazel - @kbn/logging - @kbn/securitysolution-constants - @kbn/securitysolution-utils +- @kbn/securitysolution-es-utils - @kbn/securitysolution-io-ts-utils - @kbn/std +- @kbn/telemetry-utils - @kbn/tinymath - @kbn/utility-types - @kbn/utils diff --git a/docs/development/core/public/kibana-plugin-core-public.doclinksstart.md b/docs/development/core/public/kibana-plugin-core-public.doclinksstart.md index ac625095da2a4..78d2d8daa3d45 100644 --- a/docs/development/core/public/kibana-plugin-core-public.doclinksstart.md +++ b/docs/development/core/public/kibana-plugin-core-public.doclinksstart.md @@ -17,7 +17,5 @@ export interface DocLinksStart | --- | --- | --- | | [DOC\_LINK\_VERSION](./kibana-plugin-core-public.doclinksstart.doc_link_version.md) | string | | | [ELASTIC\_WEBSITE\_URL](./kibana-plugin-core-public.doclinksstart.elastic_website_url.md) | string | | - | [links](./kibana-plugin-core-public.doclinksstart.links.md) | {
readonly canvas: {
readonly guide: string;
};
readonly dashboard: {
readonly guide: string;
readonly drilldowns: string;
readonly drilldownsTriggerPicker: string;
readonly urlDrilldownTemplateSyntax: string;
readonly urlDrilldownVariables: string;
};
readonly discover: Record<string, string>;
readonly filebeat: {
readonly base: string;
readonly installation: string;
readonly configuration: string;
readonly elasticsearchOutput: string;
readonly elasticsearchModule: string;
readonly startup: string;
readonly exportedFields: string;
};
readonly auditbeat: {
readonly base: string;
};
readonly metricbeat: {
readonly base: string;
readonly configure: string;
readonly httpEndpoint: string;
readonly install: string;
readonly start: string;
};
readonly enterpriseSearch: {
readonly base: string;
readonly appSearchBase: string;
readonly workplaceSearchBase: string;
};
readonly heartbeat: {
readonly base: string;
};
readonly logstash: {
readonly base: string;
};
readonly functionbeat: {
readonly base: string;
};
readonly winlogbeat: {
readonly base: string;
};
readonly aggs: {
readonly composite: string;
readonly composite_missing_bucket: string;
readonly date_histogram: string;
readonly date_range: string;
readonly date_format_pattern: string;
readonly filter: string;
readonly filters: string;
readonly geohash_grid: string;
readonly histogram: string;
readonly ip_range: string;
readonly range: string;
readonly significant_terms: string;
readonly terms: string;
readonly avg: string;
readonly avg_bucket: string;
readonly max_bucket: string;
readonly min_bucket: string;
readonly sum_bucket: string;
readonly cardinality: string;
readonly count: string;
readonly cumulative_sum: string;
readonly derivative: string;
readonly geo_bounds: string;
readonly geo_centroid: string;
readonly max: string;
readonly median: string;
readonly min: string;
readonly moving_avg: string;
readonly percentile_ranks: string;
readonly serial_diff: string;
readonly std_dev: string;
readonly sum: string;
readonly top_hits: string;
};
readonly runtimeFields: {
readonly overview: string;
readonly mapping: string;
};
readonly scriptedFields: {
readonly scriptFields: string;
readonly scriptAggs: string;
readonly painless: string;
readonly painlessApi: string;
readonly painlessLangSpec: string;
readonly painlessSyntax: string;
readonly painlessWalkthrough: string;
readonly luceneExpressions: string;
};
readonly search: {
readonly sessions: string;
};
readonly indexPatterns: {
readonly introduction: string;
readonly fieldFormattersNumber: string;
readonly fieldFormattersString: string;
};
readonly addData: string;
readonly kibana: string;
readonly upgradeAssistant: string;
readonly elasticsearch: Record<string, string>;
readonly siem: {
readonly guide: string;
readonly gettingStarted: string;
};
readonly query: {
readonly eql: string;
readonly kueryQuerySyntax: string;
readonly luceneQuerySyntax: string;
readonly percolate: string;
readonly queryDsl: string;
};
readonly date: {
readonly dateMath: string;
readonly dateMathIndexNames: string;
};
readonly management: Record<string, string>;
readonly ml: Record<string, string>;
readonly transforms: Record<string, string>;
readonly visualize: Record<string, string>;
readonly apis: Readonly<{
bulkIndexAlias: string;
byteSizeUnits: string;
createAutoFollowPattern: string;
createFollower: string;
createIndex: string;
createSnapshotLifecyclePolicy: string;
createRoleMapping: string;
createRoleMappingTemplates: string;
createRollupJobsRequest: string;
createApiKey: string;
createPipeline: string;
createTransformRequest: string;
cronExpressions: string;
executeWatchActionModes: string;
indexExists: string;
openIndex: string;
putComponentTemplate: string;
painlessExecute: string;
painlessExecuteAPIContexts: string;
putComponentTemplateMetadata: string;
putSnapshotLifecyclePolicy: string;
putIndexTemplateV1: string;
putWatch: string;
simulatePipeline: string;
timeUnits: string;
updateTransform: string;
}>;
readonly observability: Record<string, string>;
readonly alerting: Record<string, string>;
readonly maps: Record<string, string>;
readonly monitoring: Record<string, string>;
readonly security: Readonly<{
apiKeyServiceSettings: string;
clusterPrivileges: string;
elasticsearchSettings: string;
elasticsearchEnableSecurity: string;
indicesPrivileges: string;
kibanaTLS: string;
kibanaPrivileges: string;
mappingRoles: string;
mappingRolesFieldRules: string;
runAsPrivilege: string;
}>;
readonly watcher: Record<string, string>;
readonly ccs: Record<string, string>;
readonly plugins: Record<string, string>;
readonly snapshotRestore: Record<string, string>;
readonly ingest: Record<string, string>;
} | | - diff --git a/docs/development/core/server/kibana-plugin-core-server.assistanceapiresponse.md b/docs/development/core/server/kibana-plugin-core-server.assistanceapiresponse.md index 4778c98493b5b..1daaf95a73d5d 100644 --- a/docs/development/core/server/kibana-plugin-core-server.assistanceapiresponse.md +++ b/docs/development/core/server/kibana-plugin-core-server.assistanceapiresponse.md @@ -6,6 +6,7 @@ > Warning: This API is now obsolete. > +> 7.16 > Signature: diff --git a/docs/development/core/server/kibana-plugin-core-server.assistantapiclientparams.md b/docs/development/core/server/kibana-plugin-core-server.assistantapiclientparams.md index 6d3f8df2fa518..1031d733fed4a 100644 --- a/docs/development/core/server/kibana-plugin-core-server.assistantapiclientparams.md +++ b/docs/development/core/server/kibana-plugin-core-server.assistantapiclientparams.md @@ -6,6 +6,7 @@ > Warning: This API is now obsolete. > +> 7.16 > Signature: diff --git a/docs/development/core/server/kibana-plugin-core-server.deprecationapiclientparams.md b/docs/development/core/server/kibana-plugin-core-server.deprecationapiclientparams.md index ed64d61e75fab..fc1748d4db907 100644 --- a/docs/development/core/server/kibana-plugin-core-server.deprecationapiclientparams.md +++ b/docs/development/core/server/kibana-plugin-core-server.deprecationapiclientparams.md @@ -6,6 +6,7 @@ > Warning: This API is now obsolete. > +> 7.16 > Signature: diff --git a/docs/development/core/server/kibana-plugin-core-server.deprecationapiresponse.md b/docs/development/core/server/kibana-plugin-core-server.deprecationapiresponse.md index 1d837d9b4705d..ce40bd7c750f0 100644 --- a/docs/development/core/server/kibana-plugin-core-server.deprecationapiresponse.md +++ b/docs/development/core/server/kibana-plugin-core-server.deprecationapiresponse.md @@ -6,6 +6,7 @@ > Warning: This API is now obsolete. > +> 7.16 > Signature: diff --git a/docs/development/core/server/kibana-plugin-core-server.deprecationinfo.md b/docs/development/core/server/kibana-plugin-core-server.deprecationinfo.md index 8eeb5ef638a82..d9d1c6c3edb41 100644 --- a/docs/development/core/server/kibana-plugin-core-server.deprecationinfo.md +++ b/docs/development/core/server/kibana-plugin-core-server.deprecationinfo.md @@ -6,6 +6,7 @@ > Warning: This API is now obsolete. > +> 7.16 > Signature: diff --git a/docs/development/core/server/kibana-plugin-core-server.ilegacyclusterclient.md b/docs/development/core/server/kibana-plugin-core-server.ilegacyclusterclient.md index b5fbb3d54b972..d1e87feba0f03 100644 --- a/docs/development/core/server/kibana-plugin-core-server.ilegacyclusterclient.md +++ b/docs/development/core/server/kibana-plugin-core-server.ilegacyclusterclient.md @@ -6,7 +6,7 @@ > Warning: This API is now obsolete. > -> Use [IClusterClient](./kibana-plugin-core-server.iclusterclient.md). +> Use [IClusterClient](./kibana-plugin-core-server.iclusterclient.md). 7.16 > Represents an Elasticsearch cluster API client created by the platform. It allows to call API on behalf of the internal Kibana user and the actual user that is derived from the request headers (via `asScoped(...)`). diff --git a/docs/development/core/server/kibana-plugin-core-server.ilegacycustomclusterclient.md b/docs/development/core/server/kibana-plugin-core-server.ilegacycustomclusterclient.md index 4da121984d084..c004ad2548802 100644 --- a/docs/development/core/server/kibana-plugin-core-server.ilegacycustomclusterclient.md +++ b/docs/development/core/server/kibana-plugin-core-server.ilegacycustomclusterclient.md @@ -6,7 +6,7 @@ > Warning: This API is now obsolete. > -> Use [ICustomClusterClient](./kibana-plugin-core-server.icustomclusterclient.md). +> Use [ICustomClusterClient](./kibana-plugin-core-server.icustomclusterclient.md). 7.16 > Represents an Elasticsearch cluster API client created by a plugin. It allows to call API on behalf of the internal Kibana user and the actual user that is derived from the request headers (via `asScoped(...)`). diff --git a/docs/development/core/server/kibana-plugin-core-server.ilegacyscopedclusterclient.md b/docs/development/core/server/kibana-plugin-core-server.ilegacyscopedclusterclient.md index 51d0b2e4882cb..8e7ecdb9f7ec2 100644 --- a/docs/development/core/server/kibana-plugin-core-server.ilegacyscopedclusterclient.md +++ b/docs/development/core/server/kibana-plugin-core-server.ilegacyscopedclusterclient.md @@ -6,7 +6,7 @@ > Warning: This API is now obsolete. > -> Use [IScopedClusterClient](./kibana-plugin-core-server.iscopedclusterclient.md). +> Use [IScopedClusterClient](./kibana-plugin-core-server.iscopedclusterclient.md). 7.16 > Serves the same purpose as "normal" `ClusterClient` but exposes additional `callAsCurrentUser` method that doesn't use credentials of the Kibana internal user (as `callAsInternalUser` does) to request Elasticsearch API, but rather passes HTTP headers extracted from the current user request to the API. diff --git a/docs/development/core/server/kibana-plugin-core-server.indexsettingsdeprecationinfo.md b/docs/development/core/server/kibana-plugin-core-server.indexsettingsdeprecationinfo.md index 706898c4ad9aa..9103f9cfc6740 100644 --- a/docs/development/core/server/kibana-plugin-core-server.indexsettingsdeprecationinfo.md +++ b/docs/development/core/server/kibana-plugin-core-server.indexsettingsdeprecationinfo.md @@ -6,6 +6,7 @@ > Warning: This API is now obsolete. > +> 7.16 > Signature: diff --git a/docs/development/core/server/kibana-plugin-core-server.legacyapicaller.md b/docs/development/core/server/kibana-plugin-core-server.legacyapicaller.md index 168209659046e..2378e61484da5 100644 --- a/docs/development/core/server/kibana-plugin-core-server.legacyapicaller.md +++ b/docs/development/core/server/kibana-plugin-core-server.legacyapicaller.md @@ -6,6 +6,7 @@ > Warning: This API is now obsolete. > +> 7.16 > Signature: diff --git a/docs/development/core/server/kibana-plugin-core-server.legacycallapioptions.md b/docs/development/core/server/kibana-plugin-core-server.legacycallapioptions.md index 40def157114ef..219180af26fd8 100644 --- a/docs/development/core/server/kibana-plugin-core-server.legacycallapioptions.md +++ b/docs/development/core/server/kibana-plugin-core-server.legacycallapioptions.md @@ -6,6 +6,7 @@ > Warning: This API is now obsolete. > +> 7.16 > The set of options that defines how API call should be made and result be processed. diff --git a/docs/development/core/server/kibana-plugin-core-server.legacyclusterclient.md b/docs/development/core/server/kibana-plugin-core-server.legacyclusterclient.md index 0872e5ba7c219..05855c31477c3 100644 --- a/docs/development/core/server/kibana-plugin-core-server.legacyclusterclient.md +++ b/docs/development/core/server/kibana-plugin-core-server.legacyclusterclient.md @@ -6,7 +6,7 @@ > Warning: This API is now obsolete. > -> Use [IClusterClient](./kibana-plugin-core-server.iclusterclient.md). +> Use [IClusterClient](./kibana-plugin-core-server.iclusterclient.md). 7.16 > Represents an Elasticsearch cluster API client created by the platform. It allows to call API on behalf of the internal Kibana user and the actual user that is derived from the request headers (via `asScoped(...)`). diff --git a/docs/development/core/server/kibana-plugin-core-server.legacyelasticsearcherror.md b/docs/development/core/server/kibana-plugin-core-server.legacyelasticsearcherror.md index 7c53356615ee9..7cf696ad8d73f 100644 --- a/docs/development/core/server/kibana-plugin-core-server.legacyelasticsearcherror.md +++ b/docs/development/core/server/kibana-plugin-core-server.legacyelasticsearcherror.md @@ -4,7 +4,7 @@ ## LegacyElasticsearchError interface -@deprecated. The new elasticsearch client doesn't wrap errors anymore. +@deprecated. The new elasticsearch client doesn't wrap errors anymore. 7.16 Signature: diff --git a/docs/development/core/server/kibana-plugin-core-server.legacyscopedclusterclient.callascurrentuser.md b/docs/development/core/server/kibana-plugin-core-server.legacyscopedclusterclient.callascurrentuser.md index 7517acc59ac80..0f2d653e41a55 100644 --- a/docs/development/core/server/kibana-plugin-core-server.legacyscopedclusterclient.callascurrentuser.md +++ b/docs/development/core/server/kibana-plugin-core-server.legacyscopedclusterclient.callascurrentuser.md @@ -6,7 +6,7 @@ > Warning: This API is now obsolete. > -> Use [IScopedClusterClient.asCurrentUser](./kibana-plugin-core-server.iscopedclusterclient.ascurrentuser.md). +> Use [IScopedClusterClient.asCurrentUser](./kibana-plugin-core-server.iscopedclusterclient.ascurrentuser.md). 7.16 > Calls specified `endpoint` with provided `clientParams` on behalf of the user initiated request to the Kibana server (via HTTP request headers). See [LegacyAPICaller](./kibana-plugin-core-server.legacyapicaller.md). diff --git a/docs/development/core/server/kibana-plugin-core-server.legacyscopedclusterclient.callasinternaluser.md b/docs/development/core/server/kibana-plugin-core-server.legacyscopedclusterclient.callasinternaluser.md index b683d3945f9ff..2c184b0fde5b3 100644 --- a/docs/development/core/server/kibana-plugin-core-server.legacyscopedclusterclient.callasinternaluser.md +++ b/docs/development/core/server/kibana-plugin-core-server.legacyscopedclusterclient.callasinternaluser.md @@ -6,7 +6,7 @@ > Warning: This API is now obsolete. > -> Use [IScopedClusterClient.asInternalUser](./kibana-plugin-core-server.iscopedclusterclient.asinternaluser.md). +> Use [IScopedClusterClient.asInternalUser](./kibana-plugin-core-server.iscopedclusterclient.asinternaluser.md). 7.16 > Calls specified `endpoint` with provided `clientParams` on behalf of the Kibana internal user. See [LegacyAPICaller](./kibana-plugin-core-server.legacyapicaller.md). diff --git a/docs/development/core/server/kibana-plugin-core-server.legacyscopedclusterclient.md b/docs/development/core/server/kibana-plugin-core-server.legacyscopedclusterclient.md index 6b6649e833a92..6678c3bc16d53 100644 --- a/docs/development/core/server/kibana-plugin-core-server.legacyscopedclusterclient.md +++ b/docs/development/core/server/kibana-plugin-core-server.legacyscopedclusterclient.md @@ -6,7 +6,7 @@ > Warning: This API is now obsolete. > -> Use [scoped cluster client](./kibana-plugin-core-server.iscopedclusterclient.md). +> Use [scoped cluster client](./kibana-plugin-core-server.iscopedclusterclient.md). 7.16 > Serves the same purpose as the normal [cluster client](./kibana-plugin-core-server.iclusterclient.md) but exposes an additional `asCurrentUser` method that doesn't use credentials of the Kibana internal user (as `asInternalUser` does) to request Elasticsearch API, but rather passes HTTP headers extracted from the current user request to the API instead. diff --git a/docs/development/core/server/kibana-plugin-core-server.md b/docs/development/core/server/kibana-plugin-core-server.md index 4df8d074ba9c8..3a9118a9c56bd 100644 --- a/docs/development/core/server/kibana-plugin-core-server.md +++ b/docs/development/core/server/kibana-plugin-core-server.md @@ -108,7 +108,7 @@ The plugin integrates with the core system via lifecycle events: `setup` | [KibanaRequestRoute](./kibana-plugin-core-server.kibanarequestroute.md) | Request specific route information exposed to a handler. | | [LegacyAPICaller](./kibana-plugin-core-server.legacyapicaller.md) | | | [LegacyCallAPIOptions](./kibana-plugin-core-server.legacycallapioptions.md) | The set of options that defines how API call should be made and result be processed. | -| [LegacyElasticsearchError](./kibana-plugin-core-server.legacyelasticsearcherror.md) | @deprecated. The new elasticsearch client doesn't wrap errors anymore. | +| [LegacyElasticsearchError](./kibana-plugin-core-server.legacyelasticsearcherror.md) | @deprecated. The new elasticsearch client doesn't wrap errors anymore. 7.16 | | [LegacyRequest](./kibana-plugin-core-server.legacyrequest.md) | | | [LoggerContextConfigInput](./kibana-plugin-core-server.loggercontextconfiginput.md) | | | [LoggingServiceSetup](./kibana-plugin-core-server.loggingservicesetup.md) | Provides APIs to plugins for customizing the plugin's logger. | diff --git a/docs/development/core/server/kibana-plugin-core-server.migration_assistance_index_action.md b/docs/development/core/server/kibana-plugin-core-server.migration_assistance_index_action.md index a924f0cea6b6b..ea0a277931eaf 100644 --- a/docs/development/core/server/kibana-plugin-core-server.migration_assistance_index_action.md +++ b/docs/development/core/server/kibana-plugin-core-server.migration_assistance_index_action.md @@ -6,6 +6,7 @@ > Warning: This API is now obsolete. > +> 7.16 > Signature: diff --git a/docs/development/core/server/kibana-plugin-core-server.migration_deprecation_level.md b/docs/development/core/server/kibana-plugin-core-server.migration_deprecation_level.md index 0fcae8c847cb4..f71e6e78a4c34 100644 --- a/docs/development/core/server/kibana-plugin-core-server.migration_deprecation_level.md +++ b/docs/development/core/server/kibana-plugin-core-server.migration_deprecation_level.md @@ -6,6 +6,7 @@ > Warning: This API is now obsolete. > +> 7.16 > Signature: diff --git a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.search.md b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.search.md index 259009c1c5668..3c99ae4c86c63 100644 --- a/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.search.md +++ b/docs/development/plugins/data/public/kibana-plugin-plugins-data-public.search.md @@ -21,7 +21,7 @@ search: { })[]; InvalidEsCalendarIntervalError: typeof InvalidEsCalendarIntervalError; InvalidEsIntervalFormatError: typeof InvalidEsIntervalFormatError; - Ipv4Address: typeof Ipv4Address; + IpAddress: typeof IpAddress; isDateHistogramBucketAggConfig: typeof isDateHistogramBucketAggConfig; isNumberType: (agg: import("../common").AggConfig) => boolean; isStringType: (agg: import("../common").AggConfig) => boolean; diff --git a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.search.md b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.search.md index 930f7710f9a00..7072f25489db2 100644 --- a/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.search.md +++ b/docs/development/plugins/data/server/kibana-plugin-plugins-data-server.search.md @@ -21,7 +21,7 @@ search: { })[]; InvalidEsCalendarIntervalError: typeof InvalidEsCalendarIntervalError; InvalidEsIntervalFormatError: typeof InvalidEsIntervalFormatError; - Ipv4Address: typeof Ipv4Address; + IpAddress: typeof IpAddress; isNumberType: (agg: import("../common").AggConfig) => boolean; isStringType: (agg: import("../common").AggConfig) => boolean; isType: (...types: string[]) => (agg: import("../common").AggConfig) => boolean; diff --git a/docs/discover/images/add-field-to-pattern.png b/docs/discover/images/add-field-to-pattern.png new file mode 100644 index 0000000000000..84dfcb0745c69 Binary files /dev/null and b/docs/discover/images/add-field-to-pattern.png differ diff --git a/docs/discover/images/hello-field.png b/docs/discover/images/hello-field.png new file mode 100644 index 0000000000000..07d97e054d7ec Binary files /dev/null and b/docs/discover/images/hello-field.png differ diff --git a/docs/management/field-formatters/color-formatter.asciidoc b/docs/management/field-formatters/color-formatter.asciidoc index d9ba5e9be1165..488fb37153799 100644 --- a/docs/management/field-formatters/color-formatter.asciidoc +++ b/docs/management/field-formatters/color-formatter.asciidoc @@ -1,10 +1,5 @@ -The `Color` field formatter enables you to specify colors with specific ranges of values for a numeric field. +The *Color* field formatter enables you to specify colors with ranges of values for a number field. -When you select the `Color` field formatter, Kibana displays the *Range*, *Font Color*, *Background Color*, and -*Example* fields. - -Click the *Add Color* button to add a range of values to associate with a particular color. You can click in the *Font -Color* and *Background Color* fields to display a color picker. You can also enter a specific hex code value in the -field. The effect of your current color choices are displayed in the *Example* field. +When you select the *Color* formatter, click *Add Color*, then specify the *Range*, *Text color*, and *Background color*. image::images/colorformatter.png[] diff --git a/docs/management/field-formatters/duration-formatter.asciidoc b/docs/management/field-formatters/duration-formatter.asciidoc index 36a73f61f6227..873a4ac94c291 100644 --- a/docs/management/field-formatters/duration-formatter.asciidoc +++ b/docs/management/field-formatters/duration-formatter.asciidoc @@ -1,4 +1,4 @@ -The `Duration` field formatter can display the numeric value of a field in the following increments: +The *Duration* field formatter displays the numeric value of a field in the following increments: * Picoseconds * Nanoseconds @@ -12,4 +12,4 @@ The `Duration` field formatter can display the numeric value of a field in the f * Months * Years -You can specify these increments with up to 20 decimal places for both input and output formats. +You can specify these increments with up to 20 decimal places for input and output formats. diff --git a/docs/management/field-formatters/string-formatter.asciidoc b/docs/management/field-formatters/string-formatter.asciidoc index ed3aa45873284..f32eee7dc8396 100644 --- a/docs/management/field-formatters/string-formatter.asciidoc +++ b/docs/management/field-formatters/string-formatter.asciidoc @@ -1,11 +1,20 @@ -The `String` field formatter can apply the following transformations to the field's contents: +The *String* field formatter enables you to apply transforms to the field. + +Supported transformations include: * Convert to lowercase + * Convert to uppercase + * Convert to title case -* Apply the short dots transformation, which replaces the content before a `.` character with the first character of -that content, as in the following example: + +* Apply the short dots transformation, which replaces the content before the `.` character with the first character of +the content. For example: [horizontal] *Original*:: *Becomes* `com.organizations.project.ClassName`:: `c.o.p.ClassName` + +* Base64 decode + +* URL param decode diff --git a/docs/management/field-formatters/url-formatter.asciidoc b/docs/management/field-formatters/url-formatter.asciidoc index 41d4f75603dc6..8b0e43c9f2496 100644 --- a/docs/management/field-formatters/url-formatter.asciidoc +++ b/docs/management/field-formatters/url-formatter.asciidoc @@ -1,33 +1,32 @@ -The `Url` field formatter can take on the following types: +You can specify the following types to the `Url` field formatter: -* The *Link* type turn the contents of the field into an URL. -* The *Image* type can be used to specify an image directory where a specified image is located. -* The *Audio* type can be used to specify an audio directory where a specified audio file is located. +* *Link* — Converts the contents of the field into an URL. You can specify the width and height of the image, while keeping the aspect ratio. +When the image is smaller than the specified paramters, the image is unable to upscale. +* *Image* — Specifies the image directory. +* *Audio* — Specify the audio directory. -For an *Image* type you can specify width and height attributes. These will be used to set the max width / max height of the image, while keeping the aspect ratio. Image will not be upscaled if it's smaller than the provided size parameters. - -You can customize either type of URL field formats with templates. A _URL template_ enables you to add specific values -to a partial URL. Use the string `{{value}}` to add the contents of the field to a fixed URL. +To customize URL field formats, use templates. An *URL template* enables you to add values +to a partial URL. To add the contents of the field to a fixed URL, use the `{{value}}` string. For example, when: * A field contains a user ID -* That field uses the `Url` field formatter +* A field uses the `Url` field formatter * The URI template is `http://company.net/profiles?user_id={­{value}­}` The resulting URL replaces `{{value}}` with the user ID from the field. The `{{value}}` template string URL-encodes the contents of the field. When a field encoded into a URL contains -non-ASCII characters, these characters are replaced with a `%` character and the appropriate hexadecimal code. For +non-ASCII characters, the characters are replaced with a `%` character and the appropriate hexadecimal code. For example, field contents `users/admin` result in the URL template adding `users%2Fadmin`. -When the formatter type is set to *Image*, the `{{value}}` template string specifies the name of an image at the +When the formatter type is *Image*, the `{{value}}` template string specifies the name of an image at the specified URI. -When the formatter type is set to *Audio*, the `{{value}}` template string specifies the name of an audio file at the specified URI. +When the formatter type is *Audio*, the `{{value}}` template string specifies the name of an audio file at the specified URI. -In order to pass unescaped values directly to the URL, use the `{{rawValue}}` string. +To pass unescaped values directly to the URL, use the `{{rawValue}}` string. -A _Label Template_ enables you to specify a text string that displays instead of the raw URL. You can use the +A *Label template* enables you to specify a text string that appears instead of the raw URL. You can use the `{{value}}` template string normally in label templates. You can also use the `{{url}}` template string to display the formatted URL. diff --git a/docs/management/images/colorformatter.png b/docs/management/images/colorformatter.png index df5dc34dd31e5..3c2cfad62d76a 100644 Binary files a/docs/management/images/colorformatter.png and b/docs/management/images/colorformatter.png differ diff --git a/docs/management/manage-index-patterns.asciidoc b/docs/management/manage-index-patterns.asciidoc new file mode 100644 index 0000000000000..94870733174ad --- /dev/null +++ b/docs/management/manage-index-patterns.asciidoc @@ -0,0 +1,264 @@ +[[managing-index-patterns]] +== Manage index pattern data fields + +To customize the data fields in your index pattern, you can add runtime fields to the existing documents, add scrited fields to compute data on the fly, and change how {kib} displays the data fields. + +[float] +[[runtime-fields]] +=== Explore your data with runtime fields + +Runtime fields are fields that you add to documents after you've ingested, and are evaluated at query time. With runtime fields, you allow for a smaller index and faster ingest time so that you can use less resources and reduce your operating costs. You can use runtime fields anywhere index patterns are used. + +When you use runtime fields, you can: + +* Define fields for a specific use without modifying the underlying schema. + +* Override the returned values from index fields. + +* Start working on your data without first understanding the structure. + +* Add fields to existing documents without reindexing your data. + +* Explore runtime field data in *Discover*. + +* Create visualizations with runtime field data using *Lens*, *Maps*, and *TSVB*. + +WARNING: Runtime fields can impact {kib} performance. When you run a query, {es} uses the fields you index first to shorten the response time. +Index the fields that you commonly search for and filter on, such as `timestamp`, then use runtime fields to limit the number of fields {es} uses to calculate values. + +For more information, refer to {ref}/runtime.html[Runtime fields]. + +[float] +[[create-runtime-fields]] +==== Create runtime fields + +Create runtime fields in your index patterns, or create runtime fields in *Discover* and *Lens*. + +. Open the main menu, then click *Stack Management > Index Patterns*. + +. Select the index pattern you want to add the runtime field to, then click *Add field*. + +. Enter a *Name* for the runtime field, then select the field *Type*. + +. Select *Set value*, then define the field value by emitting a single value using the {ref}/modules-scripting-painless.html[Painless scripting language]. ++ +The script must match the field *Type*, or the script fails. + +. Click *Create field*. +//+ +//For information on how to create runtime fields in *Discover*, refer to <>. ++ +For information on how to create runtime fields in *Lens*, refer to <>. + +[float] +[[runtime-field-examples]] +==== Runtime field examples + +Try the runtime field examples on your own using the *Sample web logs* data index pattern. + +[float] +[[simple-hello-world-example]] +==== Return a keyword value + +To return `Hello World!` value: + +[source,text] +---- +emit("Hello World!"); +---- + +[float] +[[perform-a-calculation-on-a-single-field]] +===== Perform a calculation on a single field + +Calculate kilobytes from bytes: + +[source,text] +---- +emit(doc['bytes'].value / 1024) +---- + +[float] +[[return-substring]] +===== Return a substring + +Return the string that appears after the last slash in the URL: + +[source,text] +---- +def path = doc["url.keyword"].value; +if (path != null) { + int lastSlashIndex = path.lastIndexOf('/'); + if (lastSlashIndex > 0) { + emit(path.substring(lastSlashIndex+1)); + return; + } +} +emit(""); +---- + +[float] +[[replace-nulls-with-blanks]] +===== Replace nulls with blanks + +Replace null values with none values: + +[source,text] +---- +def source = doc['referer'].value; +if (source != null) { + emit(source); + return; +} +else { + emit("None"); +} +---- + +Specify operating system condition: + +[source,text] +---- +def source = doc['machine.os.keyword'].value; +if (source != "") { + emit(source); +} +else { + emit("None"); +} +---- + +[float] +[[manage-runtime-fields]] +==== Manage runtime fields + +Edit the settings for runtime fields, or remove runtime fields from index patterns. + +. Open the main menu, then click *Stack Management > Index Patterns*. + +. Select the index pattern that contains the runtime field you want to manage, then open the runtime field edit options or delete the runtime field. + +[float] +[[scripted-fields]] +=== Add scripted fields to index patterns + +deprecated::[7.13,Use {ref}/runtime.html[runtime fields] instead of scripted fields. Runtime fields support Painless scripts and provide greater flexibility.] + +Scripted fields compute data on the fly from the data in your {es} indices. The data is shown on +the Discover tab as part of the document data, and you can use scripted fields in your visualizations. You query scripted fields with the <>, and can filter them using the filter bar. The scripted field values are computed at query time, so they aren't indexed and cannot be searched using the {kib} default +query language. + +WARNING: Computing data on the fly with scripted fields can be very resource intensive and can have a direct impact on +{kib} performance. Keep in mind that there's no built-in validation of a scripted field. If your scripts are +buggy, you'll get exceptions whenever you try to view the dynamically generated data. + +When you define a scripted field in {kib}, you have a choice of the {ref}/modules-scripting-expression.html[Lucene expressions] or the +{ref}/modules-scripting-painless.html[Painless] scripting language. + +You can reference any single value numeric field in your expressions, for example: + +---- +doc['field_name'].value +---- + +For more information on scripted fields and additional examples, refer to +https://www.elastic.co/blog/using-painless-kibana-scripted-fields[Using Painless in {kib} scripted fields] + +[float] +[[create-scripted-field]] +==== Create scripted fields + +Create and add scripted fields to your index patterns. + +. Open the main menu, then click *Stack Management > Index Patterns*. + +. Select the index pattern you want to add a scripted field to. + +. Select the *Scripted fields* tab, then click *Add scripted field*. + +. Enter a *Name* for the scripted field, then enter the *Script* you want to use to compute a value on the fly from your index data. + +. Click *Create field*. + +For more information about scripted fields in {es}, refer to {ref}/modules-scripting.html[Scripting]. + +[float] +[[update-scripted-field]] +==== Manage scripted fields + +. Open the main menu, then click *Stack Management > Index Patterns*. + +. Select the index pattern that contains the scripted field you want to manage. + +. Select the *Scripted fields* tab, then open the scripted field edit options or delete the scripted field. + +WARNING: Built-in validation is unsupported for scripted fields. When your scripts contain errors, you receive +exceptions when you view the dynamically generated data. + +[float] +[[managing-fields]] +=== Format data fields + +{kib} uses the same field types as {es}, however, some {es} field types are unsupported in {kib}. +To customize how {kib} displays data fields, use the formatting options. + +. Open the main menu, then click *Stack Management > Index Patterns*. + +. Click the index pattern that contains the field you want to change. + +. Find the field, then open the edit options (image:management/index-patterns/images/edit_icon.png[Data field edit icon]). + +. Select *Set custom label*, then enter a *Custom label* for the field. + +. Select *Set format*, then enter the *Format* for the field. + +[float] +[[string-field-formatters]] +==== String field formatters + +String fields support *String* and *Url* formatters. + +include::field-formatters/string-formatter.asciidoc[] + +include::field-formatters/url-formatter.asciidoc[] + +[float] +[[field-formatters-date]] +==== Date field formatters + +Date fields support *Date*, *String*, and *Url* formatters. + +The *Date* formatter enables you to choose the display format of date stamps using the https://momentjs.com/[moment.js] +standard format definitions. + +include::field-formatters/string-formatter.asciidoc[] + +include::field-formatters/url-formatter.asciidoc[] + +[float] +[[field-formatters-geopoint]] +==== Geographic point field formatters + +Geographic point fields support the *String* formatter. + +include::field-formatters/string-formatter.asciidoc[] + +[float] +[[field-formatters-numeric]] +==== Number field formatters + +Numeric fields support *Bytes*, *Color*, *Duration*, *Histogram*, *Number*, *Percentage*, *String*, and *Url* formatters. + +The *Bytes*, *Number*, and *Percentage* formatters enable you to choose the display formats of numbers in the field using +the <> syntax that {kib} maintains. + +The *Histogram* formatter is used only for the {ref}/histogram.html[histogram field type]. When you use the *Histogram* formatter, +you can apply the *Bytes*, *Number*, or *Percentage* format to aggregated data. + +include::field-formatters/url-formatter.asciidoc[] + +include::field-formatters/string-formatter.asciidoc[] + +include::field-formatters/duration-formatter.asciidoc[] + +include::field-formatters/color-formatter.asciidoc[] \ No newline at end of file diff --git a/docs/management/managing-fields.asciidoc b/docs/management/managing-fields.asciidoc deleted file mode 100644 index 505f6853c7906..0000000000000 --- a/docs/management/managing-fields.asciidoc +++ /dev/null @@ -1,134 +0,0 @@ -[[managing-fields]] -== Field management - -Whenever possible, -{kib} uses the same field type for display as {es}. However, a few field types -{es} supports are not available in {kib}. Use field formatters to customize how your -fields are displayed in Kibana, regardless of how they are stored in {es}. - -Kibana provides these field formatters: - -* <> -* <> -* <> -* <> - -To format a field: - -. Open the main menu, and click *Stack Management > Index Patterns*. -. Click the index pattern that contains the field you want to format. -. Find the field you want to format and click the edit icon (image:management/index-patterns/images/edit_icon.png[]). -. Enter a custom label for the field, if needed. -. Select a format and fill in the details. -+ -[role="screenshot"] -image:management/index-patterns/images/edit-field-format.png["Edit field format"] - - - -[[field-formatters-string]] -=== String field formatters - -String fields support the `String` and `Url` formatters. - -include::field-formatters/string-formatter.asciidoc[] - -include::field-formatters/url-formatter.asciidoc[] - -[[field-formatters-date]] -=== Date field formatters - -Date fields support the `Date`, `Url`, and `String` formatters. - -The `Date` formatter enables you to choose the display format of date stamps using the https://momentjs.com/[moment.js] -standard format definitions. - -include::field-formatters/string-formatter.asciidoc[] - -include::field-formatters/url-formatter.asciidoc[] - -[[field-formatters-geopoint]] -=== Geographic point field formatters - -Geographic point fields support the `String` formatter. - -include::field-formatters/string-formatter.asciidoc[] - -[[field-formatters-numeric]] -=== Numeric field formatters - -Numeric fields support the `Url`, `Bytes`, `Duration`, `Number`, `Percentage`, `Histogram`, `String`, and `Color` formatters. - -The `Bytes`, `Number`, and `Percentage` formatters enable you to choose the display formats of numbers in this field using -the <> syntax that {kib} maintains. - -The `Histogram` formatter is only used for the {ref}/histogram.html[histogram field type]. When using the `Histogram` formatter, -you can apply the `Number`, `Bytes`, or `Percentage` format to the aggregated data. - -`Number`, and `Percentage` formatters enable you to choose the display formats of numbers in this field using -the <> syntax that {kib} maintains. - -include::field-formatters/url-formatter.asciidoc[] - -include::field-formatters/string-formatter.asciidoc[] - -include::field-formatters/duration-formatter.asciidoc[] - -include::field-formatters/color-formatter.asciidoc[] - -[[scripted-fields]] -=== Scripted fields -deprecated::[7.13,Use {ref}/runtime.html[runtime fields] instead of scripted fields. Runtime fields support Painless scripts and provide greater flexibility.] - -Scripted fields compute data on the fly from the data in your {es} indices. The data is shown on -the Discover tab as part of the document data, and you can use scripted fields in your visualizations. You query scripted fields with the <>, and can filter them using the filter bar. The scripted field values are computed at query time, so they aren't indexed and cannot be searched using the {kib} default -query language. - -WARNING: Computing data on the fly with scripted fields can be very resource intensive and can have a direct impact on -{kib} performance. Keep in mind that there's no built-in validation of a scripted field. If your scripts are -buggy, you'll get exceptions whenever you try to view the dynamically generated data. - -When you define a scripted field in {kib}, you have a choice of the {ref}/modules-scripting-expression.html[Lucene expressions] or the -{ref}/modules-scripting-painless.html[Painless] scripting language. - -You can reference any single value numeric field in your expressions, for example: - ----- -doc['field_name'].value ----- - -For more information on scripted fields and additional examples, refer to -https://www.elastic.co/blog/using-painless-kibana-scripted-fields[Using Painless in {kib} scripted fields] - -[float] -[[create-scripted-field]] -=== Create a scripted field - -. Open the main menu, then click *Stack Management > Index Patterns*. -. Select the index pattern you want to add a scripted field to. -. Go to the *Scripted fields* tab for the index pattern, then click *Add scripted field*. -. Enter a name for the scripted field. -. Enter the expression that you want to use to compute a value on the fly from your index data. -. Click *Create field*. - -For more information about scripted fields in {es}, see -{ref}/modules-scripting.html[Scripting]. - -[float] -[[update-scripted-field]] -=== Update a scripted field - -. Click the *Scripted fields* tab for the index pattern. -. Click the *Edit* button for the scripted field you want to change. -. Make your changes, then click *Save field*. - -WARNING: Built-in validation is unsupported for scripted fields. If your scripts are buggy, you'll get -exceptions whenever you try to view the dynamically generated data. - -[float] -[[delete-scripted-field]] -=== Delete a scripted field - -. Click the *Scripted fields* tab for the index pattern. -. Click *Delete* for the scripted field you want to remove. -. Click *Delete* on the confirmation window. diff --git a/docs/maps/trouble-shooting.asciidoc b/docs/maps/trouble-shooting.asciidoc index 0e1ed0b9e1bec..a58e8ac8902b8 100644 --- a/docs/maps/trouble-shooting.asciidoc +++ b/docs/maps/trouble-shooting.asciidoc @@ -26,7 +26,7 @@ image::maps/images/inspector.png[] * Verify your geospatial data is correctly mapped as {ref}/geo-point.html[geo_point] or {ref}/geo-shape.html[geo_shape]. ** Run `GET myIndexPatternTitle/_field_caps?fields=myGeoFieldName` in <>, replacing `myIndexPatternTitle` and `myGeoFieldName` with your index pattern title and geospatial field name. ** Ensure response specifies `type` as `geo_point` or `geo_shape`. -* Verify your geospatial data is correctly mapped in your <>. +* Verify your geospatial data is correctly mapped in your <>. ** Open your index pattern in <>. ** Ensure your geospatial field type is `geo_point` or `geo_shape`. ** Ensure your geospatial field is searchable and aggregatable. diff --git a/docs/maps/vector-tooltips.asciidoc b/docs/maps/vector-tooltips.asciidoc index b0498c9088e4e..2dda35aa28f76 100644 --- a/docs/maps/vector-tooltips.asciidoc +++ b/docs/maps/vector-tooltips.asciidoc @@ -18,7 +18,7 @@ image::maps/images/multifeature_tooltip.png[] ==== Format tooltips You can format the attributes in a tooltip by adding <> to your -Kibana index pattern. You can use field formatters to round numbers, provide units, +index pattern. You can use field formatters to round numbers, provide units, and even display images in your tooltip. [float] diff --git a/docs/redirects.asciidoc b/docs/redirects.asciidoc index 4aedb0f516b20..a14bda2bf5a98 100644 --- a/docs/redirects.asciidoc +++ b/docs/redirects.asciidoc @@ -279,26 +279,32 @@ This content has moved. Refer to <>. [role="exclude",id="ingest-node-pipelines"] == Ingest Node Pipelines -This content has moved. See {ref}/ingest.html[Ingest pipelines]. +This content has moved. Refer to {ref}/ingest.html[Ingest pipelines]. [role="exclude",id="create-panels-with-timelion"] == Timelion -This content has moved. refer to <>. +This content has moved. Refer to <>. [role="exclude",id="space-rbac-tutorial"] == Tutorial: Use role-based access control to customize Kibana spaces -This content has moved. refer to <>. +This content has moved. Refer to <>. [role="exclude",id="search"] == Search your data -This content has moved. refer to <>. +This content has moved. Refer to <>. [role="exclude",id="discover-document-context"] == View surrounding documents -This content has moved. refer to <>. +This content has moved. Refer to <>. + +[role="exclude",id="field-formatters-string"] +== String field formatters + +This content has moved. Refer to <>. + diff --git a/docs/user/dashboard/images/lens_advanced_2_1.png b/docs/user/dashboard/images/lens_advanced_2_1.png index 5090f0d3b2841..dab32369d71a1 100644 Binary files a/docs/user/dashboard/images/lens_advanced_2_1.png and b/docs/user/dashboard/images/lens_advanced_2_1.png differ diff --git a/docs/user/dashboard/images/lens_advanced_2_1_1.png b/docs/user/dashboard/images/lens_advanced_2_1_1.png deleted file mode 100644 index f4d9ca488782e..0000000000000 Binary files a/docs/user/dashboard/images/lens_advanced_2_1_1.png and /dev/null differ diff --git a/docs/user/dashboard/images/lens_advanced_3_1.gif b/docs/user/dashboard/images/lens_advanced_2_2.gif similarity index 100% rename from docs/user/dashboard/images/lens_advanced_3_1.gif rename to docs/user/dashboard/images/lens_advanced_2_2.gif diff --git a/docs/user/dashboard/images/lens_advanced_2_2.png b/docs/user/dashboard/images/lens_advanced_2_2.png index 820bc3bd4dfa9..1d88bcd238ca3 100644 Binary files a/docs/user/dashboard/images/lens_advanced_2_2.png and b/docs/user/dashboard/images/lens_advanced_2_2.png differ diff --git a/docs/user/dashboard/images/lens_advanced_2_2_1.png b/docs/user/dashboard/images/lens_advanced_2_2_1.png index 3044f1070367d..c3fb697666b46 100644 Binary files a/docs/user/dashboard/images/lens_advanced_2_2_1.png and b/docs/user/dashboard/images/lens_advanced_2_2_1.png differ diff --git a/docs/user/dashboard/images/lens_advanced_3_3.png b/docs/user/dashboard/images/lens_advanced_2_3.png similarity index 100% rename from docs/user/dashboard/images/lens_advanced_3_3.png rename to docs/user/dashboard/images/lens_advanced_2_3.png diff --git a/docs/user/dashboard/images/lens_advanced_3_1.png b/docs/user/dashboard/images/lens_advanced_3_1.png new file mode 100644 index 0000000000000..1473b203924a3 Binary files /dev/null and b/docs/user/dashboard/images/lens_advanced_3_1.png differ diff --git a/docs/user/dashboard/images/lens_advanced_3_1_1.png b/docs/user/dashboard/images/lens_advanced_3_1_1.png deleted file mode 100644 index c3fb697666b46..0000000000000 Binary files a/docs/user/dashboard/images/lens_advanced_3_1_1.png and /dev/null differ diff --git a/docs/user/dashboard/images/lens_advanced_3_2.png b/docs/user/dashboard/images/lens_advanced_3_2.png index 20da2ed706dfd..15f2f0228a0fc 100644 Binary files a/docs/user/dashboard/images/lens_advanced_3_2.png and b/docs/user/dashboard/images/lens_advanced_3_2.png differ diff --git a/docs/user/dashboard/images/lens_advanced_4_1.png b/docs/user/dashboard/images/lens_advanced_4_1.png index 43c8db213d482..50d1affa268dd 100644 Binary files a/docs/user/dashboard/images/lens_advanced_4_1.png and b/docs/user/dashboard/images/lens_advanced_4_1.png differ diff --git a/docs/user/dashboard/images/lens_advanced_4_2.png b/docs/user/dashboard/images/lens_advanced_4_2.png deleted file mode 100644 index 4b3e98910e7b7..0000000000000 Binary files a/docs/user/dashboard/images/lens_advanced_4_2.png and /dev/null differ diff --git a/docs/user/dashboard/images/lens_advanced_5_1.png b/docs/user/dashboard/images/lens_advanced_5_1.png new file mode 100644 index 0000000000000..5090f0d3b2841 Binary files /dev/null and b/docs/user/dashboard/images/lens_advanced_5_1.png differ diff --git a/docs/user/dashboard/images/lens_advanced_5_2.png b/docs/user/dashboard/images/lens_advanced_5_2.png new file mode 100644 index 0000000000000..820bc3bd4dfa9 Binary files /dev/null and b/docs/user/dashboard/images/lens_advanced_5_2.png differ diff --git a/docs/user/dashboard/images/lens_advanced_5_2_1.png b/docs/user/dashboard/images/lens_advanced_5_2_1.png new file mode 100644 index 0000000000000..3044f1070367d Binary files /dev/null and b/docs/user/dashboard/images/lens_advanced_5_2_1.png differ diff --git a/docs/user/dashboard/images/lens_advanced_6_1.png b/docs/user/dashboard/images/lens_advanced_6_1.png new file mode 100644 index 0000000000000..5d5cefa472a13 Binary files /dev/null and b/docs/user/dashboard/images/lens_advanced_6_1.png differ diff --git a/docs/user/dashboard/images/lens_advanced_7_1.png b/docs/user/dashboard/images/lens_advanced_7_1.png new file mode 100644 index 0000000000000..3d66d5d7e4579 Binary files /dev/null and b/docs/user/dashboard/images/lens_advanced_7_1.png differ diff --git a/docs/user/dashboard/images/lens_advanced_result.png b/docs/user/dashboard/images/lens_advanced_result.png index 19963d87c8e1c..8cf087f936abd 100644 Binary files a/docs/user/dashboard/images/lens_advanced_result.png and b/docs/user/dashboard/images/lens_advanced_result.png differ diff --git a/docs/user/dashboard/images/manage-runtime-field.gif b/docs/user/dashboard/images/manage-runtime-field.gif new file mode 100644 index 0000000000000..c6ecf0caf8180 Binary files /dev/null and b/docs/user/dashboard/images/manage-runtime-field.gif differ diff --git a/docs/user/dashboard/images/runtime-field-menu.png b/docs/user/dashboard/images/runtime-field-menu.png new file mode 100644 index 0000000000000..891de38bb6833 Binary files /dev/null and b/docs/user/dashboard/images/runtime-field-menu.png differ diff --git a/docs/user/dashboard/lens-advanced.asciidoc b/docs/user/dashboard/lens-advanced.asciidoc index 6b090f6017f5d..e49db0c0d026d 100644 --- a/docs/user/dashboard/lens-advanced.asciidoc +++ b/docs/user/dashboard/lens-advanced.asciidoc @@ -1,11 +1,11 @@ [[create-a-dashboard-of-panels-with-ecommerce-data]] -== Tutorial: Create a dashboard of panels with ecommerce sales data +== Time series analysis with Lens -You collected sales data from your store, and you want to visualize and analyze the data on a dashboard. +The tutorial uses sample data from the perspective of a shop owner looking +at sales trends, but this type of dashboard works on any type of data. To create dashboard panels of the data, open the *Lens* visualization builder, then create the visualization panels that best display the data. - -When you've completed the tutorial, you'll have a dashboard that provides you with a complete overview of your ecommerce sales data. +Before using this tutorial, you should be familiar with the <>. [role="screenshot"] image::images/lens_advanced_result.png[Dashboard view] @@ -14,36 +14,52 @@ image::images/lens_advanced_result.png[Dashboard view] [[add-the-data-and-create-the-dashboard-advanced]] === Add the data and create the dashboard -To create visualizations of the data from your store, add the data set, then create the dashboard. +If you are working with your own data, you should already have an <>. +To install the sample sales data: . From the {kib} *Home* page, click *Try our sample data*. . From *Sample eCommerce orders*, click *Add data*. +Then create a new dashboard: + . Open the main menu, then click *Dashboard*. . On the *Dashboards* page, click *Create dashboard*. +. Set the <> to *Last 30 days*. + [float] [[open-and-set-up-lens-advanced]] === Open and set up Lens -Open the *Lens* editor, then make sure the correct fields appear. +*Lens* is designed to help you quickly build visualizations for your dashboard, as shown in <>, while providing support for advanced usage as well. -. From the dashboard, click *Create panel*. +Open the *Lens* editor, then make sure the correct fields appear. -. On the *New visualization* window, click *Lens*. -+ -[role="screenshot"] -image::images/lens_end_to_end_1_1.png[New visualization popover] +. From the dashboard, click *Create visualization*. -. Make sure the *kibana_sample_data_ecommerce_* index appears. +. Make sure the *kibana_sample_data_ecommerce* index appears. [discrete] -[[view-the-number-of-transactions-per-day]] -=== View the number of transactions per hour +[[custom-time-interval]] +=== View a date histogram with a custom time interval + +It is common to use the automatic date histogram interval, but sometimes you want a larger or smaller +interval. *Lens* only lets you choose the minimum time interval, not the exact time interval, for +performance reasons. The performance limit is controlled by the <> +advanced setting and the overall time range. To see hourly sales over a 30 day time period, choose +one of these options: -To determine the number of orders made every hour, create a bar chart, then add the chart to the dashboard. +* View less than 30 days at a time, then use the time picker to select each day separately. + +* Increase `histogram:maxBars` from 100 to at least 720, which the number of hours in 30 days. +This affects all visualizations and can reduce performance. + +* If approximation is okay, use the *Normalize unit* option. This can convert *Average sales per 12 hours* +into *Average sales per 12 hours (per hour)* by dividing the number of hours. + +For the sample data, approximation is okay. To use the *Normalize unit* option: . Set the <> to *Last 30 days*. @@ -75,183 +91,258 @@ image::images/lens_advanced_1_2.png[Orders per day] . Click *Save and return*. [discrete] -[[view-the-cumulative-number-of-products-sold-over-time]] -=== View the cumulative number of products sold on weekends +[[add-a-data-layer-advanced]] +=== Monitor multiple series within a date histogram -To determine the number of orders made only on Saturday and Sunday, create an area chart, then add it to the dashboard. +It is often required to monitor multiple series within a time interval. These series can be have similar configurations with few changes between one and another. +*Lens* copies a function when you drag and drop it to the *Drop a field or click to add* +field within the same group, or when you drag and drop to the *Duplicate* field on a different group. +You can also drag and drop using your keyboard. For more information, refer to <>. -. Open *Lens*. +To quickly create many copies of a percentile metric that shows distribution of price over time: -. From the *Chart Type* dropdown, select *Area*. +. From the *Chart Type* dropdown, select *Line*. + [role="screenshot"] -image::images/lens_advanced_2_1_1.png[Chart type menu with Area selected] +image::images/lens_advanced_2_1.png[Chart type menu with Line selected] -. Configure the cumulative sum of the store orders. - -.. From the *Available fields* list, drag and drop *Records* to the visualization builder. +. From the *Available fields* list, drag and drop *products.price* to the visualization builder. -.. From the editor, click *Count of Records*. +. Create the 95th percentile. -.. From *Select a function*, click *Cumulative sum*. +.. In the editor, click *Median of products.price*. -.. In the *Display name* field, enter `Cumulative orders during weekend days`, then click *Close*. +.. From *Select a function*, click *Percentile*. -. Filter the results to display the data for only Saturday and Sunday. +.. In the *Display name* field, enter `95th`, then click *Close*. -.. From the editor, click the *Drop a field or click to add* field for *Break down by*. +. To create the 90th percentile, duplicate the `95th` percentile. -.. From *Select a function*, click *Filters*. +.. Drag and drop *95th* to *Drop a field or click to add*. -.. Click *All records*. +.. Click *95th [1]*, then enter `90` in the *Percentile* field. -.. In the *KQL* field, enter `day_of_week : "Saturday" or day_of_week : "Sunday"`, then press Return. -+ -The <> displays all documents where `day_of_week` matches `Saturday` or `Sunday`. +.. In the *Display name* field enter `90th`, then click *Close*. + [role="screenshot"] -image::images/lens_advanced_2_1.png[Filter aggregation to filter weekend days] +image::images/lens_advanced_2_2.gif[Easily duplicate the items with drag and drop] -. To hide the legend, open the *Legend* menu, then click *Hide*. +.. Repeat the duplication steps to create the `50th` and `10th` percentile, naming them accordingly. + +. To change the left axis label, open the *Left Axis* menu, then enter `Percentiles for product prices` in the *Axis name* field. + [role="screenshot"] -image::images/lens_advanced_2_2_1.png[Legend menu] +image::images/lens_advanced_2_2_1.png[Left Axis menu] + -You have an area chart that shows you how many orders your store received during the weekend. +You have a line chart that shows you the price distribution of products sold over time. + [role="screenshot"] -image::images/lens_advanced_2_2.png[Line chart with cumulative sum of orders made on the weekend] +image::images/lens_advanced_2_3.png[Percentiles for product prices chart] -. Click *Save and return*. +. Add the filter for the redirect codes. [discrete] -[[add-a-data-layer-advanced]] -=== Create multiple key percentiles of product prices - -To view the price distribution of products sold over time, create a percentile chart, then add it to the dashboard. - -. Open *Lens*. - -. From the *Chart Type* dropdown, select *Line*. - -. From the *Available fields* list, drag and drop the data fields to the *Drop a field or click to add* fields in the editor. - -* Drag and drop *products.price* to the *Vertical axis* field. +[[add-a-data-layer]] +==== Multiple chart types or index patterns in one visualization -* Drag and drop *order_date* to the *Horizontal axis* field. +You can add multiple metrics to a single chart type, but if you want to overlay +multiple chart types or index patterns, use a second layer. When building layered charts, +it is important to match the data on the horizontal axis so that it uses the same +scale. To add a line chart layer on top of an existing chart: -. Create the 95th percentile. +To compare product prices with customers traffic: -.. In the editor, click *Median of products.price*. +. From the *Available fields* list, drag and drop *products.price* to the visualization builder. -.. From *Select a function*, click *Percentile*. +.. In the *KQL* field, enter `response.keyword>=500 AND response.keyword<600`. -.. In the *Display name* field, enter `95th`, then click *Close*. +.. From *Select a function*, click *Average*. -. To create the 90th percentile, duplicate the `95th` percentile. +.. In the *Display name* field, enter `Average of prices`, then click *Close*. -.. Drag and drop *95th* to *Drop a field or click to add*. +. From the *Chart Type* dropdown, select *Area*. -.. Click *95th [1]*, then enter `90` in the *Percentile* field. +. Create a new layer to overlay with custom traffic. -.. In the *Display name* field enter `90th`, then click *Close*. +. In the editor, click *+*. + [role="screenshot"] -image::images/lens_advanced_3_1.gif[Easily duplicate the items with drag and drop] - -. Create the 50th percentile. - -.. Drag and drop *90th* to *Drop a field or click to add*. +image::images/lens_advanced_3_1.png[Add new layer button] -.. Click *90th [1]*, then enter `50` in the *Percentile* field. +. From the *Available fields* list, drag and drop *customer_id* to the *Vertical Axis* of the newly created layer. -.. In the *Display name* field enter `50th`, then click *Close*. +.. In the editor, click *Unique count of customer_id*. -. Create the 10th percentile. +.. In the *Display name* field, enter `Unique customers`, then click *Close*. -.. Drag and drop *50th* to *Drop a field or click to add*. +. In the *Series color* field, enter *#D36086*, then click *Close*. -.. Click *50th [1]*, then enter `10` in the *Percentile* field. +. For *Axis side*, click *Right*, then click *Close*. -.. In the *Display name* field enter `10th`, then click *Close*. +. From the *Available fields* list, drag and drop *order_date* to the *Horizontal Axis* of the newly created layer. -. To change the left axis label, open the *Left Axis* menu, then enter `Percentiles for product prices` in the *Axis name* field. +. From the new layer editor, click the *Chart type* dropdown, then click the line chart. + [role="screenshot"] -image::images/lens_advanced_3_1_1.png[Left Axis menu] -+ -You have a line chart that shows you the price distribution of products sold over time. -+ -[role="screenshot"] -image::images/lens_advanced_3_3.png[Percentiles for product prices chart] +image::images/lens_advanced_3_2.png[Change layer type] + +The visualization is done, but the legend uses a lot of space. Change the legend position to the top of the chart. + +. From the *Legend* dropdown, select the top position. . Click *Save and return*. [discrete] -[[add-the-response-code-filters-advanced]] -=== View the moving average of inventory prices +[[percentage-stacked-area]] +=== Compare the change in percentage over time -To view and analyze the prices of shoes, accessories, and clothing in the store inventory, create a line chart. +By default, *Lens* shows *date histograms* using a stacked chart visualization, which helps understand how distinct sets of documents perform over time. Sometimes it is useful to understand how the distributions of these sets change over time. +Combine *filters* and *date histogram* functions to see the change over time in specific +sets of documents. To view this as a percentage, use a *stacked percentage* bar or area chart. -. Open *Lens*. +To see sales change of product by type over time: -. From the *Chart Type* dropdown, select *Line*. +. From the *Available fields* list, drag and drop *Records* to the visualization builder. -. From the *Available fields* list, drag and drop *products.price* to the visualization builder. +. Click *Bar vertical stacked*, then select *Area percentage*. -. In the editor, click the *Drop a field or click to add* field for *Break down by*. +For each category type that you want to break down, create a filter. + +. In the editor, click the *Drop a field or click to add* field for *Break down by*. . From *Select a function*, click *Filters*. -. Add a filter for shoes. +. Add the filter for the clothing category. .. Click *All records*. -.. In the *KQL* field, enter `category.keyword : *Shoes*`. +.. In the *KQL* field, enter `category.keyword : *Clothing`. + +.. In the *Label* field, enter `Clothing`, then press Return. + +. Add the filter for the shoes category. + +.. Click *Add a filter*. + +.. In the *KQL* field, enter `category.keyword : *Shoes`. .. In the *Label* field, enter `Shoes`, then press Return. -. Add a filter for accessories. +. Add the filter for the accessories category. .. Click *Add a filter*. -.. In the *KQL* field, enter `category.keyword : *Accessories*`. +.. In the *KQL* field, enter `category.keyword : *Accessories`. .. In the *Label* field, enter `Accessories`, then press Return. -. Add a filter for clothing. +Change the legend position to the top of the chart. -.. Click *Add a filter*. +. From the *Legend* dropdown, select the top position. -.. In the *KQL* field, enter `category.keyword : *Clothing*`. ++ +[role="screenshot"] +image::images/lens_advanced_4_1.png[Prices share by category] -.. In the *Label* field, enter `Clothing`, then press Return. + Click *Save and return*. + +[discrete] +[[view-the-cumulative-number-of-products-sold-on-weekends]] +=== View the cumulative number of products sold on weekends + +To determine the number of orders made only on Saturday and Sunday, create an area chart, then add it to the dashboard. + +. Open *Lens*. + +. From the *Chart Type* dropdown, select *Area*. + +. Configure the cumulative sum of the store orders. + +.. From the *Available fields* list, drag and drop *Records* to the visualization builder. + +.. From the editor, click *Count of Records*. + +.. From *Select a function*, click *Cumulative sum*. + +.. In the *Display name* field, enter `Cumulative orders during weekend days`, then click *Close*. + +. Filter the results to display the data for only Saturday and Sunday. + +.. From the editor, click the *Drop a field or click to add* field for *Break down by*. + +.. From *Select a function*, click *Filters*. + +.. Click *All records*. + +.. In the *KQL* field, enter `day_of_week : "Saturday" or day_of_week : "Sunday"`, then press Return. ++ +The <> displays all documents where `day_of_week` matches `Saturday` or `Sunday`. ++ +[role="screenshot"] +image::images/lens_advanced_5_1.png[Filter aggregation to filter weekend days] + +. To hide the legend, open the *Legend* menu, then click *Hide*. ++ +[role="screenshot"] +image::images/lens_advanced_5_2_1.png[Legend menu] ++ +You have an area chart that shows you how many orders your store received during the weekend. -. Click *Close* +. Click *Bar vertical stacked*, then select *Area*. + [role="screenshot"] -image::images/lens_advanced_4_1.png[Median prices chart for different categories] +image::images/lens_advanced_5_2.png[Line chart with cumulative sum of orders made on the weekend] + +. Click *Save and return*. [discrete] -[[add-the-moving-average]] -==== Add the moving average +[[view-customers-over-time-by-continents]] +=== View table of customers by category over time + +Tables are an alternative type of visualization for time series, useful when you want to read the actual values. +You can build a date histogram table, and group the customer count metric by category, like the continent registered in their profile. + +In *Lens* you can split the metric in a table leveraging the *Columns* field, where each data value from the aggregation is used as column of the table and the relative metric value is shown. + +To build a date histogram table: + +. Open *Lens*. + +. From the *Chart Type* dropdown, select *Table*. -To focus on the general trends rather than on the peaks in the data, add the moving average, then add the visualization to the dashboard. +.. From the *Available fields* list, drag and drop *customer_id* to the *Metrics* field of the editor. -. In the editor, click the *Median of products.price*. +.. From the editor, click *Unique count of customer_id*. -. From *Select a function*, click *Moving average*. +.. In the *Display name* field, enter `Customers`, then click *Close*. -. In the *Window size* field, enter `7`, then click *Close*. +.. From the *Available fields* list, drag and drop *order_date* to the *Rows* field of the editor. + +.. From the editor *Rows*, click the *order_date* field just dropped. + +. Select *Customize time interval*. + +. Change the *Minimum interval* to `1 days`, then click *Close*. + +.. In the *Display name* field, enter `Sale`, then click *Close*. + +To split the customers count by continent: + +. From the *Available fields* list, drag and drop *geoip.continent_name* to the *Columns* field of the editor. + [role="screenshot"] -image::images/lens_advanced_4_2.png[Moving average prices chart for different categories] +image::images/lens_advanced_6_1.png[Table with daily customers by continent configuration] . Click *Save and return*. [discrete] === Save the dashboard +By default the dashboard attempts to match the palette across panels, but in this case there's no need for that, so it can be disabled. + +[role="screenshot"] +image::images/lens_advanced_7_1.png[Disable palette sync in dashboard] + Now that you have a complete overview of your ecommerce sales data, save the dashboard. . In the toolbar, click *Save*. diff --git a/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_1_1.png b/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_1_1.png deleted file mode 100644 index 1c752972791af..0000000000000 Binary files a/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_1_1.png and /dev/null differ diff --git a/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_1_2.png b/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_1_2.png index de719c852dfde..ea58057433a19 100644 Binary files a/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_1_2.png and b/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_1_2.png differ diff --git a/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_1_2_1.png b/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_1_2_1.png index 9408cd736b27e..fa404d50b4dd1 100644 Binary files a/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_1_2_1.png and b/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_1_2_1.png differ diff --git a/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_1_3.png b/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_1_3.png index 4302c95eb8a25..b069acae43163 100644 Binary files a/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_1_3.png and b/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_1_3.png differ diff --git a/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_2_1.png b/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_2_1.png deleted file mode 100644 index c08a865e52137..0000000000000 Binary files a/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_2_1.png and /dev/null differ diff --git a/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_2_1_1.png b/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_2_1_1.png index 48712ec5ef224..e996b58520d41 100644 Binary files a/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_2_1_1.png and b/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_2_1_1.png differ diff --git a/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_2_1_2.png b/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_2_1_2.png new file mode 100644 index 0000000000000..a524dd1e456f6 Binary files /dev/null and b/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_2_1_2.png differ diff --git a/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_4_3.png b/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_4_3.png index b70910fdd2500..2cd75d2797a98 100644 Binary files a/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_4_3.png and b/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_4_3.png differ diff --git a/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_6_2.png b/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_6_2.png deleted file mode 100644 index e95b3d9a97b8d..0000000000000 Binary files a/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_6_2.png and /dev/null differ diff --git a/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_7_1.png b/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_7_1.png deleted file mode 100644 index e63f7e77f1e9f..0000000000000 Binary files a/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_7_1.png and /dev/null differ diff --git a/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_dashboard.png b/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_dashboard.png index 9bce65401f4c9..9e9a5ed3d758e 100644 Binary files a/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_dashboard.png and b/docs/user/dashboard/lens-end-to-end/images/lens_end_to_end_dashboard.png differ diff --git a/docs/user/dashboard/lens.asciidoc b/docs/user/dashboard/lens.asciidoc index 58476bcae87df..94c9db1462760 100644 --- a/docs/user/dashboard/lens.asciidoc +++ b/docs/user/dashboard/lens.asciidoc @@ -75,6 +75,31 @@ Drag and drop the fields on to the visualization builder, then [role="screenshot"] image::images/lens_value_labels_xychart_toggle.png[Lens Bar chart value labels menu] +[float] +[[add-fields-in-lens]] +===== Add fields + +Add and define fields to the index pattern that you want to visualize using the {ref}/modules-scripting-painless.html[Painless scripting language]. +The fields that you are add are saved to the index pattern and appear in all visualizations, saved searches, and saved objects that use the index pattern. + +. Click *...*, then select *Add field to index pattern*. ++ +[role="screenshot"] +image:images/runtime-field-menu.png[Dropdown menu located next to index pattern field with items for adding and managing fields, width=50%] + +. Enter a *Name* for the field, then select the field *Type*. + +. Select *Set value*, then define the field value by emitting a single value using the {ref}/modules-scripting-painless.html[Painless scripting language]. + +. Click *Save*. ++ +To manage the field, click the field, then click *Edit index pattern field* or *Remove index pattern field*. ++ +[role="screenshot"] +image:images/manage-runtime-field.gif[Field menu to edit or remove field from index pattern, width=50%] + +For more information about adding fields to index patterns and Painless scripting language examples, refer to <>. + [float] [[drag-and-drop-keyboard-navigation]] ===== Create visualization panels with keyboard navigation diff --git a/docs/user/dashboard/tsvb.asciidoc b/docs/user/dashboard/tsvb.asciidoc index ff1c16c14d467..5c4ce8e365e86 100644 --- a/docs/user/dashboard/tsvb.asciidoc +++ b/docs/user/dashboard/tsvb.asciidoc @@ -2,7 +2,8 @@ === TSVB *TSVB* enables you to visualize the data from multiple data series, supports <>, multiple visualization types, custom functions, and some math. To use *TSVB*, your data must have a date field. +most {es} metric aggregations>>, multiple visualization types, custom functions, and some math. +To create *TSVB* visualization panels, your data must have a time field. [role="screenshot"] image::visualize/images/tsvb-screenshot.png[TSVB overview] @@ -17,15 +18,19 @@ Open *TSVB*, then make sure the required settings are configured. . On the *New visualization* window, click *TSVB*. -. In *TSVB*, click *Panel options*, then make sure the following settings are configured: +. In *TSVB*, click *Panel options*, then specify the required *Data* settings. -* *Index pattern* -* *Time field* -* *Interval* +.. From the *Index pattern* dropdown, select the index pattern you want to visualize. ++ +To visualize an {es} index, open the *Index pattern select mode* menu, deselect *Use only {kib} index patterns*, then enter the {es} index. + +.. From the *Time field* dropdown, select the field you want to visualize, then enter the field *Interval*. -. Select a *Drop last bucket* option. It is dropped by default because the time filter intersects the time range of the last bucket, but can be enabled to see the partial data. +.. Select a *Drop last bucket* option. ++ +By default, *TSVB* drops the last bucket because the time filter intersects the time range of the last bucket. To view the partial data, select *No*. -. In the *Panel filter* field, specify any <> to select specific documents. +.. In the *Panel filter* field, enter <> to view specific documents. [float] [[configure-the-data-series]] diff --git a/docs/user/dashboard/tutorial-create-a-dashboard-of-lens-panels.asciidoc b/docs/user/dashboard/tutorial-create-a-dashboard-of-lens-panels.asciidoc index 22483b2801848..1375aa8d5934e 100644 --- a/docs/user/dashboard/tutorial-create-a-dashboard-of-lens-panels.asciidoc +++ b/docs/user/dashboard/tutorial-create-a-dashboard-of-lens-panels.asciidoc @@ -1,10 +1,10 @@ [[create-a-dashboard-of-panels-with-web-server-data]] -== Tutorial: Create a dashboard of panels with web server data +== Build your first dashboard -You collected data from your web server, and you want to visualize and analyze the data on a dashboard. To create dashboard panels of the data, open the *Lens* visualization builder, then -create the visualization panels that best display the data. - -When you've completed the tutorial, you'll have a dashboard that provides you with a complete overview of your web server data. +Learn the most common ways to build a dashboard from your own data. +The tutorial will use sample data from the perspective of an analyst looking +at website logs, but this type of dashboard works on any type of data. +Before using this tutorial, you should be familiar with the <>. [role="screenshot"] image::images/lens_end_to_end_dashboard.png[Final dashboard vis] @@ -13,46 +13,78 @@ image::images/lens_end_to_end_dashboard.png[Final dashboard vis] [[add-the-data-and-create-the-dashboard]] === Add the data and create the dashboard -To create visualizations of the data from the web server, add the data set, then create the dashboard. +Install the sample web logs data that you'll use to create your dashboard. -. From the {kib} *Home* page, click *Try our sample data*. +. On the {kib} *Home* page, click *Try our sample data*. . From *Sample web logs*, click *Add data*. -. Open the main menu, click *Dashboard*. +Then create a new dashboard: + +. Open the main menu, then click *Dashboard*. . Click *Create dashboard*. +. Set the <> to *Last 90 days*. + [float] [[open-and-set-up-lens]] -=== Open and set up Lens +=== Open Lens and get familiar with the data -With *Lens*, you identify the data fields you want to visualize, drag and drop the fields, then watch as -*Lens* uses heuristics to apply the fields and create a visualization for you. +. On the dashboard, click *Create visualization*. -. From the dashboard, click *Create panel*. - -. On the *New visualization* window, click *Lens*. +. Make sure the *kibana_sample_data_logs* index appears. You might need to select +a different index pattern from the dropdown: + [role="screenshot"] -image::images/lens_end_to_end_1_1.png[New visualization popover] +image::images/lens_end_to_end_1_2.png[Lens index pattern selector, width=50%] + +This tutorial uses `Records`, timestamp`, `bytes`, `clientip`, and `referer.keyword`. +To see the most frequent values of a field, click the field name to view a summary. + +The main elements of *Lens* are named: -. Make sure the *kibana_sample_data_logs* index appears. +Workspace panel:: Displays your visualization. You can drag and drop onto this area. +Dimensions:: Each dimension is a function with extra options. Dimensions are grouped +for each visualization type, for example the *Vertical axis* is a group that allows +multiple dimensions. Each dimension starts empty with the label *Drop a field or click to add*. +Functions:: There are two main types of functions: *buckets* and *metrics*, which are +equivalent to what {es} provides. [discrete] [[view-the-number-of-website-visitors]] -=== View the number of website visitors +=== Create your first visualization -To determine how many users have visited your website within the last 90 days, create a metric visualization, then add it to the dashboard. +Every time you build a visualization in *Lens*, you need to: -. Set the <> to *Last 90 days*. +* *Choose your visualization.* Do you know the type of visualization you'd like to use? +If you do, select the type before dragging any fields. If you don't, you can change the +visualization type after configuring your functions. + +* *Choose your field.* Do you know the dimension group you want to use the field in? If you do, +drag and drop the field from the field list to your chosen dimension and Lens will pick a function for you. +If you don't, drag and drop the field onto the workspace panel. Skip this step if you are +using the *Filters* function. + +* *Edit and delete.* To change the function or styling options, click the dimension to open +the configuration panel. To delete a specific dimension, close the configuration panel and click +the delete button. To reset the entire visualization, click *Reset layer*. + +To put this into practice, pick a field you want to analyze, such as `clientip`. If you want +to analyze only this field, you can use *Metric* to show a big number. +The only number function that you can use with `clientip` is *Unique count*. +*Unique count*, also known as cardinality, approximates the number of unique values +of the `clientip` field. -. From the *Chart Type* dropdown, select *Metric*. +. To select the visualization type, click *Bar vertical stacked* to open the chart type dropdown, then select *Metric*. + [role="screenshot"] -image::images/lens_end_to_end_1_2_1.png[Chart Type dropdown with Metric selected] +image::images/lens_end_to_end_1_2_1.png[Chart Type dropdown with Metric selected, width=50%] -. From the *Available fields* list, drag and drop *clientip* to the visualization builder. +. From the *Available fields* list, drag and drop `clientip` to the workspace panel. +Lens selects *Unique count* because it is the only numeric function +that works for IP addresses. You can also drag and drop `clientip` onto +the empty dimension for the same result. + [role="screenshot"] image::images/lens_end_to_end_1_3.png[Changed type and dropped clientip field] @@ -68,294 +100,220 @@ image::images/lens_end_to_end_1_4.png[Flyout config open] . Click *Save and return*. -[discrete] -[[view-the-distribution-of-visitors-by-operating-system]] -=== View the distribution of visitors by operating system - -To determine the operating systems you should continue to support, and the importance of mobile traffic from iOS devices, -create a donut chart that displays the top operating systems that your visitors used to access your website within the last 90 days. +. Customize the newly added panel: -. Open *Lens*, then set the <> to *Last 90 days*. +.. Drag the bottom corner of the panel until the metric takes up one quarter of the screen +width. The row for the metric will have 4 items on it later. -. From the *Chart Type* dropdown, select *Donut*. +.. The metric visualization has its own label, so you do not need to add a panel title. -. From the *Available fields* list, drag and drop the data fields to the *Drop a field or click to add* fields in the editor. +. Click *Save* on the dashboard menu -.. Drag and drop *clientip* to the *Size by* field. +. In the *Title* field, enter `Logs dashboard`. -.. Drag and drop *machine.os.keyword* to the *Slice by* field. -+ -[role="screenshot"] -image::images/lens_end_to_end_2_1_1.png[Donut chart with clientip and machine.os.keyword fields] - -. Change the color palette. - -.. In the editor, click *Top values of machine.os.keyword*. - -.. From the *Color palette* dropdown, select *Compatibility*. - -.. Click *Close*. -+ -[role="screenshot"] -image::images/lens_end_to_end_2_1.png[Donut chart with open config panel] +. Select *Store time with dashboard* box, then click *Save*. -. Click *Save and return*. +. After the dashboard refreshes, click *Edit* again. [discrete] [[mixed-multiaxis]] -=== View the average of bytes transfer per day +=== View a metric over time -To prevent potential server failures, and optimize the cost of website maintenance, create an area chart that displays the average of bytes transfer. To compare -the data to the number of visitors to your website, add a line chart layer. +*Lens* has two shortcuts that simplify viewing metrics over time. +If you drag and drop a numeric field to the workspace panel, *Lens* adds the default +time field from the index pattern. If the *Date histogram* function is being used, +quickly replace the time field by dragging and dropping on the workspace panel. -. Open *Lens*. +To visualize the `bytes` field over time without choosing a visualization type or function: -. From the *Available fields* list, drag and drop *bytes* to the visualization builder. +. From the *Available fields* list, drag and drop `bytes` onto the workspace panel to have *Lens* automatically +create a chart. *Lens* creates a bar chart with two dimensions, *timestamp* and *Median of bytes*. + +. *Lens* automatically chooses a date interval. To zoom in on the data you want to view, +click and drag your cursor across the bars. -. To zoom in on the data you want to view, click and drag your cursor across the bars. -+ [role="screenshot"] image::images/lens_end_to_end_3_1_1.gif[Zoom in on the data] -. Change the *timestamp* interval. - -.. In the editor, click *timestamp*. +To emphasize the change in *Median of bytes* over time, use a line chart. +To change the visualization type, use one of the following ways: -.. Select *Customize time interval*. +* From the *Suggestions*, click the line chart. +* Click *Bar vertical stacked*, then select *Line*. +* Click the chart type icon above *Horizontal axis*, then click the line icon. -.. Change the *Minimum interval* to `1 days`, then click *Close*. -+ -[role="screenshot"] -image::images/lens_end_to_end_3_1.png[Customize time interval] +Most users use the automatic time interval. You can increase and decrease +the minimum interval that *Lens* uses, but you cannot decrease the interval +below the {kib} advanced settings. To set the minimum time interval: -. From the *Chart Type* dropdown, select *Area*. +. In the editor, click *timestamp*. -[discrete] -[[add-a-data-layer]] -==== Add the line chart layer +. Click *How it works* to learn about the *Lens* minimum interval -To compare the average of bytes transfer to the number of users that visit your website, add a line chart layer. +. Select *Customize time interval*. -. In the editor, click *+*. +. Increase the *Minimum interval* to `1 days`, then click *Close*. + [role="screenshot"] -image::images/lens_end_to_end_3_2.png[Add new layer button] +image::images/lens_end_to_end_3_1.png[Customize time interval] -. From the new layer editor, click the *Chart type* dropdown, then click the line chart. +To save space on the dashboard, so to save space, hide the vertical and horizontal +axis labels. + +. Open the *Left axis* menu, then deselect *Show*. + [role="screenshot"] -image::images/lens_end_to_end_3_3.png[Change layer type] -+ -The chart type for the visualization changes to *Mixed XY*. - -. From the *Available fields* list, drag and drop the data fields to the *Drop a field or click to add* fields in the editor. - -.. Drag and drop *timestamp* to the *Horizontal axis* field. - -.. Drag and drop *clientip* to the *Vertical axis* field. - -. Change the *timestamp* interval. +image::images/lens_end_to_end_4_3.png[Turn off axis name] -.. In the editor, click *timestamp* in the line chart layer. +. Open the *Bottom axis* menu, then deselect *Show*. -.. Select *Customize time interval*. +. Click *Save and return* -.. Change the *Minimum interval* to `1 days`, then click *Close*. +. On the dashboard, move the panel so that it is in the same row as the *Metric* visualization panel. The two should +take up half the screen width. -. Change the *Unique count of clientip* label and color. +. Add a panel title to explain the panel, which is necessary because you removed the axis labels. -.. In the editor, click *Unique count of clientip*. +.. Open the panel menu and choose *Edit panel title*. -.. In the *Display name* field, enter `Unique visitors` in the line chart layer. +.. In the *Title* field, enter `Median of bytes`, then click *Save*. -.. In the *Series color* field, enter *#CA8EAE*, then click *Close*. +. In the toolbar, click *Save*. [discrete] -[[configure-the-multiaxis-chart]] -==== Configure the y-axes - -There is a significant difference between the *timestamp per day* and *Unique visitors* data, which makes the *Unique visitors* data difficult to read. To improve the readability, -display the *Unique visitors* data along a second y-axis, then change the formatting. When functions contain multiple formats, separate axes are created by default. +[[view-the-distribution-of-visitors-by-operating-system]] +=== View the top values of a field -. In the editor, click *Unique visitors* in the line chart layer. +The *Top values* function ranks the unique values of a field by another dimension. +The values are the most frequent when ranked by a *Count* dimension. +The values are the largest when ranked by a *Sum* dimension. -. For *Axis side*, click *Right*, then click *Close*. +When you drag and drop a text or IP address field onto the workspace panel, +*Lens* adds a *Top values* function ranked by *Count of records* to show the most frequent values. -[float] -[[change-the-visualization-type]] -==== Change the visualization type +For this tutorial, you have picked a field and function, but not a visualization type. +You want to see the most frequent values of `request.keyword` on your website, ranked by the unique visitors. +This means that you want to use *Top values of request.keyword* ranked by *Unique count of clientip*, instead of +being ranked by *Count of records*. -. In the editor, click *Average of bytes* in the area chart layer. +. From the *Available fields* list, drag and drop `clientip` onto the *Vertical axis*. +*Lens* chooses the function for you when you drop onto a dimension, which is *Unique count* here. +Do not drop the field into the main workspace because `clientip` will be added to the wrong axis. -. From the *Value format* dropdown, select *Bytes (1024)*, then click *Close*. +. Drag and drop `request.keyword` to the main workspace. *Lens* adds *Top values of request.keyword* +to the *Horizontal axis*. + [role="screenshot"] -image::images/lens_end_to_end_3_4.png[Multiaxis chart] - -[discrete] -[[lens-legend-position]] -==== Change the legend position +image::images/lens_end_to_end_2_1_1.png[Vertical bar chart with top values of request.keyword by most unique visitors] -The visualization is done, but the legend uses a lot of space. Change the legend position to the top of the chart. +This chart is hard to read because the `request.keyword` field contains long text. You could try +using one of the *Suggestions*, but the suggestions also have issues with long text. Instead, switch +to the *Table* visualization. -. From the *Legend* dropdown, select the top position. +Click *Bar vertical stacked*, then select *Table*. + [role="screenshot"] -image::images/lens_end_to_end_3_5.png[legend position] - -. Click *Save and return*. - -[discrete] -[[percentage-stacked-area]] -=== View the health of your website +image::images/lens_end_to_end_2_1_2.png[Table with top values of request.keyword by most unique visitors] -To detect unusual traffic, bad website links, and server errors, create a percentage stacked area chart that displays the associated response codes. +Next, customize the table. -. Open *Lens*. +. Click the *Top values of request.keyword* dimension. -. From the *Available fields* list, drag and drop the data fields to the *Drop a field or click to add* fields in the editor. +.. Increase the *Number of values*. The maximum allowed value is 1000. -.. Drag and drop *Records* to the *Vertical axis* field. +.. In the *Display name* field, enter `Page URL`, then click *Close*. -.. Drag and drop *@timestamp* to the *Horizontal axis* field. +. Click *Save and return*. -. From the *Chart Type* dropdown, select *Percentage bar*. +. Move the table panel so that it has its own row, but do not change the size. -. To remove the vertical axis label, click *Left axis*, then deselect *Show*. -+ -[role="screenshot"] -image::images/lens_end_to_end_4_3.png[Turn off axis name] +NOTE: You do not need a panel title because the table columns are clearly labeled. [discrete] -[[add-the-response-code-filters]] -==== Add the response code filters - -For each response code that you want to display, create a filter. - -. In the editor, click the *Drop a field or click to add* field for *Break down by*. - -. From *Select a function*, click *Filters*. - -. Add the filter for the successful response codes. +[[custom-ranges]] +=== Compare a subset of documents to all documents -.. Click *All records*. +To compare a field on subset of documents to all documents, you need to select two or more sets of documents that add up to 100%. +For this example, we are comparing documents where the `bytes` field is under 10 Kb to documents where `bytes` is over 10 Kb, +which are two sets that do not overlap. -.. In the *KQL* field, enter `response.keyword>=200 AND response.keyword<300`. +Use *Intervals* to select documents based on the number range of a field. Use *Filters* when your criteria +is not numeric, or when your query needs multiple clauses. -.. In the *Label* field, enter `2XX`, then press Return. -+ -[role="screenshot"] -image::images/lens_end_to_end_4_1.png[First filter in filters aggregation] +Use a proportion chart to display the values as a percentage of the sum of all values. Lens has 5 types of proportion charts: +pie, donut, treemap, percentage bar and percentage area. -. Add the filter for the redirect codes. +To determine if your users transfer more `bytes` from small files versus large files, +configure dimensions with *Intervals* and *Sum*, then switch to a pie chart to display as a percentage: -.. Click *Add a filter*. +. From the *Available fields* list, drag and drop `bytes` to *Vertical axis* in the editor. -.. In the *KQL* field, enter `response.keyword>=300 AND response.keyword<400`. +. Click *Median of bytes*, select *Sum*, then click *Close*. -.. In the *Label* field, enter `3XX`, then press Return. +. From the *Available fields* list, drag and drop `bytes` to *Break down by* in the editor, then specify the file size ranges. -. Add the filter for the client error codes. +.. In the editor, click *bytes*. -.. Click *Add a filter*. +.. Click *Create custom ranges*, enter the following, then press Return: -.. In the *KQL* field, enter `response.keyword>=400 AND response.keyword<500`. +* *Ranges* — `0` -> `10240` -.. In the *Label* field, enter `4XX`, then press Return. +* *Label* — `Below 10KB` -. Add the filter for the server error codes. +.. Click *Add range*, enter the following, then press Return: -.. Click *Add a filter*. +* *Ranges* — `10240` -> `+∞` -.. In the *KQL* field, enter `response.keyword>=500 AND response.keyword<600`. +* *Label* — `Above 10KB` ++ +[role="screenshot"] +image::images/lens_end_to_end_6_1.png[Custom ranges configuration] -.. In the *Label* field, enter `5XX`, then press Return. +.. From the *Value format* dropdown, select *Bytes (1024)*, then click *Close*. -. To change the color palette, select *Status* from the *Color palette* dropdown, then click *Close*. +. From the *Chart Type* dropdown, select *Pie*. . Click *Save and return*. [discrete] [[histogram]] -=== View the traffic for your website by the hour - -To find the best time to shut down your website for maintenance, create a histogram that displays the traffic for your website by the hour. - -. Open *Lens*. - -. From the *Available fields* list, drag and drop *bytes* to *Vertical axis* in the editor, then configure the options. +=== View a the distribution of a number field -.. Click *Average of bytes*. +Knowing the distribution of a number helps to find patterns. For example, you could +look at the website traffic per hour to find the best time to do routine maintenance. +Use *Intervals* to see an evenly spaced distribution of a number field. -.. From *Select a function*, click *Sum*. +. From the *Available fields* list, drag and drop `bytes` to *Vertical axis* in the editor. -.. In the *Display name* field, enter `Transferred bytes`. +. Click *Median of bytes*, then select *Sum*. -.. From the *Value format* dropdown, select `Bytes (1024)`, then click *Close*. +. In the *Display name* field, enter `Transferred bytes`. -. From the *Available fields* list, drag and drop *hour_of_day* to *Horizontal axis* in the editor, then configure the options. +. From the *Value format* dropdown, select `Bytes (1024)`, then click *Close*. -.. Click *hour_of_day*. +. From the *Available fields* list, drag and drop *hour_of_day* to *Horizontal axis* in the editor. -.. Click and slide the *Intervals granularity* slider until the horizontal axis displays hourly intervals. +. Click *hour_of_day*, and then slide the *Intervals granularity* slider until the horizontal axis displays hourly intervals. + [role="screenshot"] image::images/lens_end_to_end_5_2.png[Create custom ranges] . Click *Save and return*. -[discrete] -[[custom-ranges]] -=== View the percent of small versus large transferred files - -To determine if your users transfer more small files versus large files, create a pie chart that displays the percentage of each size. - -. Open *Lens*. - -. From the *Available fields* list, drag and drop *bytes* to *Vertical axis* in the editor, then configure the options. - -.. Click *Average of bytes*. - -.. From *Select a function*, click *Sum*, then click *Close*. - -. From the *Available fields* list, drag and drop *bytes* to *Break down by* in the editor, then specify the file size ranges. - -.. Click *bytes*. - -.. Click *Create custom ranges*, enter the following, then press Return: - -* *Ranges* — `0` -> `10240` - -* *Label* — `Below 10KB` - -.. Click *Add range*, enter the following, then press Return: - -* *Ranges* — `10240` -> `+∞` - -* *Label* — `Above 10KB` -+ -[role="screenshot"] -image::images/lens_end_to_end_6_1.png[Custom ranges configuration] - -.. From the *Value format* dropdown, select *Bytes (1024)*, then click *Close*. +. Decrease the panel size, then drag and drop it to the first row next to the `Median of bytes` panel. There +should be four panels in a row. -. From the *Chart Type* dropdown, select *Pie*. -+ -[role="screenshot"] -image::images/lens_end_to_end_6_2.png[Files size distribution] - -. Click *Save and return*. +. You do not need a panel title because the axis labels are self-explanatory. [discrete] [[treemap]] -=== View the top sources of website traffic - -To determine how users find out about your website and where your users are located, create a treemap that displays the percentage of users that -enter your website from specific social media websites, and the top countries where users are located. +=== Create a multi-level chart -. Open *Lens*. +*Lens* lets you use multiple functions in the data table and proportion charts. For example, +to create a chart that breaks down the traffic sources and user geography, use *Filters* and +*Top values*. -. From the *Chart Type* dropdown, select *Treemap*. +. Click *Bar vertical stacked*, then select *Treemap*. . From the *Available fields* list, drag and drop *Records* to the *Size by* field in the editor. @@ -377,21 +335,15 @@ enter your website from specific social media websites, and the top countries wh .. Click *Add a filter*, enter the following, then press Return: -* *KQL* — `NOT referer : *twitter* OR NOT referer: *facebook.com*` +* *KQL* — `NOT referer : *twitter.com* OR NOT referer: *facebook.com*` * *Label* — `Other` .. Click *Close*. -[discrete] -[[add-the-countries]] -==== Add the geographic data - -To determine the top countries where users are located, add the geographic data. +Add the next break down by geography: -Compare the top sources of website traffic data to the top three countries. - -. From the *Available fields* list, drag and drop *geo.src* to the visualization builder. +. From the *Available fields* list, drag and drop *geo.src* to the main workspace. . To change the *Group by* order, click and drag *Top values of geo.src* so that it appears first in the editor. + @@ -409,6 +361,12 @@ image::images/lens_end_to_end_7_3.png[Group other values as Other] . Click *Save and return*. +. Arrange the panel so that it is in the same row as the table. + +.. Click the gear icon and choose *Edit panel title*. + +.. Enter "Page views by location and referer" as the panel title, then click *Save*. + [discrete] === Save the dashboard @@ -417,3 +375,5 @@ Now that you have a complete overview of your web server data, save the dashboar . In the toolbar, click *Save*. . On the *Save dashboard* window, enter `Web server data`, then click *Save*. + +. If this was not the first time you saved the dashboard, click *Switch to view mode* diff --git a/docs/user/discover.asciidoc b/docs/user/discover.asciidoc index 6e3a7f697073d..ea413747a2aad 100644 --- a/docs/user/discover.asciidoc +++ b/docs/user/discover.asciidoc @@ -6,7 +6,7 @@ **_Gain insight to your data._** *Discover* enables you to quickly search and filter your data, get information -about structure of the fields, and visualize your data with *Lens* and *Maps*. +about the structure of the fields, and visualize your data with *Lens* and *Maps*. You can customize and save your searches and place them on a dashboard. ++++ @@ -110,6 +110,43 @@ image:images/document-table.png[Document table with fields for manufacturer, geo . To rearrange the table columns, hover the mouse over a column header, and then use the move and sort controls. +[float] +[[add-field-in-discover]] +=== Add a field + +What happens if you forgot to define an important value as a separate field? Or, what if you +want to combine two fields and treat them as one? +You can add a field to your index pattern from inside of **Discover**, +and then use that field for analysis and visualizations, +the same way you do with other fields. + +. Click the ellipsis icon (...), and then click *Add field to index pattern*. ++ +[role="screenshot"] +image:images/add-field-to-pattern.png[Dropdown menu located next to index pattern field with item for adding a field to an index pattern, width=50%] + +. In the *Create field* form, enter `hello` for the name. + +. Turn on *Set value*. + +. Use the Painless scripting language to define the field: ++ +```ts +emit("Hello World!"); +``` + +. Click *Save*. + +. In the fields list, search for the *hello* field, and then click it. ++ +You'll see the top values for the field. The pop-up also includes actions for filtering, +editing, and deleting the field. ++ +[role="screenshot"] +image:images/hello-field.png[Top values for the hello field, width=50%] + +For more information on adding fields and Painless scripting language examples, refer to <>. + [float] [[search-in-discover]] @@ -186,7 +223,8 @@ You can bookmark this document and share the link. === Save your search for later use Save your search so you can repeat it later, generate a CSV report, or use it in visualizations, dashboards, and Canvas workpads. -Saving a search saves the query and the filters. +Saving a search saves the query text, filters, +and current view of *Discover*—the columns selected in the document table, the sort order, and the index pattern. . In the toolbar, click **Save**. diff --git a/docs/user/management.asciidoc b/docs/user/management.asciidoc index 83e18734f65d4..397ab1717183b 100644 --- a/docs/user/management.asciidoc +++ b/docs/user/management.asciidoc @@ -131,8 +131,8 @@ Kerberos, PKI, OIDC, and SAML. [cols="50, 50"] |=== -a| <> -|Create and manage the index patterns that retrieve your data from {es}. +a| <> +|Manage the data fields in the index patterns that retrieve your data from {es}. | <> | Copy, edit, delete, import, and export your saved objects. @@ -186,10 +186,10 @@ include::{kib-repo-dir}/management/managing-beats.asciidoc[] include::{kib-repo-dir}/management/action-types.asciidoc[] -include::{kib-repo-dir}/management/managing-fields.asciidoc[] - include::{kib-repo-dir}/management/managing-licenses.asciidoc[] +include::{kib-repo-dir}/management/manage-index-patterns.asciidoc[] + include::{kib-repo-dir}/management/numeral.asciidoc[] include::{kib-repo-dir}/management/rollups/create_and_manage_rollups.asciidoc[] diff --git a/docs/user/reporting/reporting-troubleshooting.asciidoc b/docs/user/reporting/reporting-troubleshooting.asciidoc index c43e9210dd7c8..4305b39653f8d 100644 --- a/docs/user/reporting/reporting-troubleshooting.asciidoc +++ b/docs/user/reporting/reporting-troubleshooting.asciidoc @@ -16,6 +16,7 @@ Having trouble? Here are solutions to common problems you might encounter while * <> * <> * <> +* <> [float] [[reporting-diagnostics]] @@ -163,3 +164,12 @@ In this case, try increasing the memory for the {kib} instance to 2GB. === ARM systems Chromium is not compatible with ARM RHEL/CentOS. + +[float] +[[reporting-troubleshooting-maps-ems]] +=== Unable to connect to Elastic Maps Service + +https://www.elastic.co/elastic-maps-service[{ems} ({ems-init})] is a service that hosts +tile layers and vector shapes of administrative boundaries. +If a report contains a map with a missing basemap layer or administrative boundary, the {kib} server does not have access to {ems-init}. +See <> for information on how to connect your {kib} server to {ems-init}. diff --git a/examples/search_examples/common/index.ts b/examples/search_examples/common/index.ts index cc47c0f575973..c61de9d3c6dee 100644 --- a/examples/search_examples/common/index.ts +++ b/examples/search_examples/common/index.ts @@ -6,17 +6,7 @@ * Side Public License, v 1. */ -import { IEsSearchResponse, IEsSearchRequest } from '../../../src/plugins/data/common'; - export const PLUGIN_ID = 'searchExamples'; export const PLUGIN_NAME = 'Search Examples'; -export interface IMyStrategyRequest extends IEsSearchRequest { - get_cool: boolean; -} -export interface IMyStrategyResponse extends IEsSearchResponse { - cool: string; - executed_at: number; -} - export const SERVER_SEARCH_ROUTE_PATH = '/api/examples/search'; diff --git a/examples/search_examples/common/types.ts b/examples/search_examples/common/types.ts new file mode 100644 index 0000000000000..8bb38ea0b2d0d --- /dev/null +++ b/examples/search_examples/common/types.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { + IEsSearchRequest, + IEsSearchResponse, + IKibanaSearchRequest, + IKibanaSearchResponse, +} from '../../../src/plugins/data/common'; + +export interface IMyStrategyRequest extends IEsSearchRequest { + get_cool: boolean; +} +export interface IMyStrategyResponse extends IEsSearchResponse { + cool: string; + executed_at: number; +} + +export type FibonacciRequest = IKibanaSearchRequest<{ n: number }>; + +export type FibonacciResponse = IKibanaSearchResponse<{ values: number[] }>; diff --git a/examples/search_examples/public/search/app.tsx b/examples/search_examples/public/search/app.tsx index c9ede2ff2b45f..06f9426b4965c 100644 --- a/examples/search_examples/public/search/app.tsx +++ b/examples/search_examples/public/search/app.tsx @@ -26,27 +26,27 @@ import { EuiCode, EuiComboBox, EuiFormLabel, + EuiFieldNumber, + EuiProgress, EuiTabbedContent, + EuiTabbedContentTab, } from '@elastic/eui'; import { CoreStart } from '../../../../src/core/public'; import { mountReactNode } from '../../../../src/core/public/utils'; import { NavigationPublicPluginStart } from '../../../../src/plugins/navigation/public'; -import { - PLUGIN_ID, - PLUGIN_NAME, - IMyStrategyResponse, - SERVER_SEARCH_ROUTE_PATH, -} from '../../common'; +import { PLUGIN_ID, PLUGIN_NAME, SERVER_SEARCH_ROUTE_PATH } from '../../common'; import { DataPublicPluginStart, + IKibanaSearchResponse, IndexPattern, IndexPatternField, isCompleteResponse, isErrorResponse, } from '../../../../src/plugins/data/public'; +import { IMyStrategyResponse } from '../../common/types'; interface SearchExamplesAppDeps { notifications: CoreStart['notifications']; @@ -88,7 +88,10 @@ export const SearchExamplesApp = ({ }: SearchExamplesAppDeps) => { const { IndexPatternSelect } = data.ui; const [getCool, setGetCool] = useState(false); + const [fibonacciN, setFibonacciN] = useState(10); const [timeTook, setTimeTook] = useState(); + const [total, setTotal] = useState(100); + const [loaded, setLoaded] = useState(0); const [indexPattern, setIndexPattern] = useState(); const [fields, setFields] = useState(); const [selectedFields, setSelectedFields] = useState([]); @@ -99,7 +102,15 @@ export const SearchExamplesApp = ({ IndexPatternField | null | undefined >(); const [request, setRequest] = useState>({}); - const [response, setResponse] = useState>({}); + const [rawResponse, setRawResponse] = useState>({}); + const [selectedTab, setSelectedTab] = useState(0); + + function setResponse(response: IKibanaSearchResponse) { + setRawResponse(response.rawResponse); + setLoaded(response.loaded!); + setTotal(response.total!); + setTimeTook(response.rawResponse.took); + } // Fetch the default index pattern using the `data.indexPatterns` service, as the component is mounted. useEffect(() => { @@ -152,8 +163,7 @@ export const SearchExamplesApp = ({ .subscribe({ next: (res) => { if (isCompleteResponse(res)) { - setResponse(res.rawResponse); - setTimeTook(res.rawResponse.took); + setResponse(res); const avgResult: number | undefined = res.rawResponse.aggregations ? // @ts-expect-error @elastic/elasticsearch no way to declare a type for aggregation in the search response res.rawResponse.aggregations[1].value @@ -234,7 +244,7 @@ export const SearchExamplesApp = ({ setRequest(searchSource.getSearchRequestBody()); const { rawResponse: res } = await searchSource.fetch$().toPromise(); - setResponse(res); + setRawResponse(res); const message = Searched {res.hits.total} documents.; notifications.toasts.addSuccess( @@ -247,7 +257,7 @@ export const SearchExamplesApp = ({ } ); } catch (e) { - setResponse(e.body); + setRawResponse(e.body); notifications.toasts.addWarning(`An error has occurred: ${e.message}`); } }; @@ -260,6 +270,41 @@ export const SearchExamplesApp = ({ doAsyncSearch('myStrategy'); }; + const onPartialResultsClickHandler = () => { + setSelectedTab(1); + const req = { + params: { + n: fibonacciN, + }, + }; + + // Submit the search request using the `data.search` service. + setRequest(req.params); + const searchSubscription$ = data.search + .search(req, { + strategy: 'fibonacciStrategy', + }) + .subscribe({ + next: (res) => { + setResponse(res); + if (isCompleteResponse(res)) { + notifications.toasts.addSuccess({ + title: 'Query result', + text: 'Query finished', + }); + searchSubscription$.unsubscribe(); + } else if (isErrorResponse(res)) { + // TODO: Make response error status clearer + notifications.toasts.addWarning('An error has occurred'); + searchSubscription$.unsubscribe(); + } + }, + error: () => { + notifications.toasts.addDanger('Failed to run search'); + }, + }); + }; + const onClientSideSessionCacheClickHandler = () => { doAsyncSearch('myStrategy', data.search.session.getSessionId()); }; @@ -284,7 +329,7 @@ export const SearchExamplesApp = ({ doSearchSourceSearch(withOtherBucket); }; - const reqTabs = [ + const reqTabs: EuiTabbedContentTab[] = [ { id: 'request', name: Request, @@ -318,6 +363,7 @@ export const SearchExamplesApp = ({ values={{ time: timeTook ?? 'Unknown' }} /> + - {JSON.stringify(response, null, 2)} + {JSON.stringify(rawResponse, null, 2)} ), @@ -484,6 +530,37 @@ export const SearchExamplesApp = ({ + +

Handling partial results

+ + + The observable returned from data.search provides partial results + when the response is not yet complete. These can be handled to update a chart or + simply a progress bar: + + + <EuiProgress value={response.loaded} max={response.total} + /> + + Below is an example showing a custom search strategy that emits partial Fibonacci + sequences up to the length provided, updates the response with each partial result, + and updates a progress bar (see the Response tab). + setFibonacciN(parseInt(event.target.value, 10))} + /> + + Request Fibonacci sequence + + +

Writing a custom search strategy

 @@ -567,8 +644,13 @@ export const SearchExamplesApp = ({ + - + setSelectedTab(reqTabs.indexOf(tab))} + /> diff --git a/examples/search_examples/server/fibonacci_strategy.ts b/examples/search_examples/server/fibonacci_strategy.ts new file mode 100644 index 0000000000000..a37438aba7055 --- /dev/null +++ b/examples/search_examples/server/fibonacci_strategy.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import uuid from 'uuid'; +import { ISearchStrategy } from '../../../src/plugins/data/server'; +import { FibonacciRequest, FibonacciResponse } from '../common/types'; + +export const fibonacciStrategyProvider = (): ISearchStrategy< + FibonacciRequest, + FibonacciResponse +> => { + const responseMap = new Map(); + return ({ + search: (request: FibonacciRequest) => { + const id = request.id ?? uuid(); + const [sequence, total, started] = responseMap.get(id) ?? [ + [], + request.params?.n ?? 0, + Date.now(), + ]; + if (sequence.length < 2) { + if (total > 0) sequence.push(sequence.length); + } else { + const [a, b] = sequence.slice(-2); + sequence.push(a + b); + } + const loaded = sequence.length; + responseMap.set(id, [sequence, total, started]); + if (loaded >= total) { + responseMap.delete(id); + } + + const isRunning = loaded < total; + const isPartial = isRunning; + const took = Date.now() - started; + const values = sequence.slice(0, loaded); + + // Usually we'd do something like "of()" but for some reason it breaks in tests with the error + // "You provided an invalid object where a stream was expected." which is why we have to cast + // down below as well + return [{ id, loaded, total, isRunning, isPartial, rawResponse: { took, values } }]; + }, + cancel: async (id: string) => { + responseMap.delete(id); + }, + } as unknown) as ISearchStrategy; +}; diff --git a/examples/search_examples/server/my_strategy.ts b/examples/search_examples/server/my_strategy.ts index 0a64788960091..db8cd903f23d6 100644 --- a/examples/search_examples/server/my_strategy.ts +++ b/examples/search_examples/server/my_strategy.ts @@ -8,7 +8,7 @@ import { map } from 'rxjs/operators'; import { ISearchStrategy, PluginStart } from '../../../src/plugins/data/server'; -import { IMyStrategyResponse, IMyStrategyRequest } from '../common'; +import { IMyStrategyRequest, IMyStrategyResponse } from '../common/types'; export const mySearchStrategyProvider = ( data: PluginStart diff --git a/examples/search_examples/server/plugin.ts b/examples/search_examples/server/plugin.ts index 84f082d890bb0..984d3201220eb 100644 --- a/examples/search_examples/server/plugin.ts +++ b/examples/search_examples/server/plugin.ts @@ -24,6 +24,7 @@ import { } from './types'; import { mySearchStrategyProvider } from './my_strategy'; import { registerRoutes } from './routes'; +import { fibonacciStrategyProvider } from './fibonacci_strategy'; export class SearchExamplesPlugin implements @@ -48,7 +49,9 @@ export class SearchExamplesPlugin core.getStartServices().then(([_, depsStart]) => { const myStrategy = mySearchStrategyProvider(depsStart.data); + const fibonacciStrategy = fibonacciStrategyProvider(); deps.data.search.registerSearchStrategy('myStrategy', myStrategy); + deps.data.search.registerSearchStrategy('fibonacciStrategy', fibonacciStrategy); registerRoutes(router); }); diff --git a/package.json b/package.json index 81ffe913f0192..d46617f2a6f2a 100644 --- a/package.json +++ b/package.json @@ -136,8 +136,9 @@ "@kbn/logging": "link:bazel-bin/packages/kbn-logging/npm_module", "@kbn/monaco": "link:packages/kbn-monaco", "@kbn/securitysolution-constants": "link:bazel-bin/packages/kbn-securitysolution-constants/npm_module", - "@kbn/securitysolution-utils": "link:bazel-bin/packages/kbn-securitysolution-utils/npm_module", + "@kbn/securitysolution-es-utils": "link:bazel-bin/packages/kbn-securitysolution-es-utils/npm_module", "@kbn/securitysolution-io-ts-utils": "link:bazel-bin/packages/kbn-securitysolution-io-ts-utils/npm_module", + "@kbn/securitysolution-utils": "link:bazel-bin/packages/kbn-securitysolution-utils/npm_module", "@kbn/server-http-tools": "link:packages/kbn-server-http-tools", "@kbn/server-route-repository": "link:packages/kbn-server-route-repository", "@kbn/std": "link:bazel-bin/packages/kbn-std/npm_module", @@ -264,6 +265,7 @@ "json-stringify-safe": "5.0.1", "jsonwebtoken": "^8.5.1", "jsts": "^1.6.2", + "@kbn/rule-data-utils": "link:packages/kbn-rule-data-utils", "kea": "^2.3.0", "leaflet": "1.5.1", "leaflet-draw": "0.4.14", @@ -431,7 +433,7 @@ "@babel/traverse": "^7.12.12", "@babel/types": "^7.12.12", "@bazel/ibazel": "^0.15.10", - "@bazel/typescript": "^3.4.2", + "@bazel/typescript": "^3.5.0", "@cypress/snapshot": "^2.1.7", "@cypress/webpack-preprocessor": "^5.6.0", "@elastic/apm-rum": "^5.6.1", @@ -453,11 +455,11 @@ "@kbn/eslint-plugin-eslint": "link:bazel-bin/packages/kbn-eslint-plugin-eslint/npm_module", "@kbn/expect": "link:bazel-bin/packages/kbn-expect/npm_module", "@kbn/optimizer": "link:packages/kbn-optimizer", - "@kbn/plugin-generator": "link:packages/kbn-plugin-generator", + "@kbn/plugin-generator": "link:bazel-bin/packages/kbn-plugin-generator/npm_module", "@kbn/plugin-helpers": "link:packages/kbn-plugin-helpers", "@kbn/pm": "link:packages/kbn-pm", "@kbn/storybook": "link:packages/kbn-storybook", - "@kbn/telemetry-tools": "link:packages/kbn-telemetry-tools", + "@kbn/telemetry-tools": "link:bazel-bin/packages/kbn-telemetry-tools/npm_module", "@kbn/test": "link:packages/kbn-test", "@kbn/test-subj-selector": "link:packages/kbn-test-subj-selector", "@loaders.gl/polyfills": "^2.3.5", diff --git a/packages/BUILD.bazel b/packages/BUILD.bazel index 8b483f54344e2..2ae04e02cffd2 100644 --- a/packages/BUILD.bazel +++ b/packages/BUILD.bazel @@ -22,10 +22,13 @@ filegroup( "//packages/kbn-expect:build", "//packages/kbn-legacy-logging:build", "//packages/kbn-logging:build", + "//packages/kbn-plugin-generator:build", "//packages/kbn-securitysolution-constants:build", "//packages/kbn-securitysolution-io-ts-utils:build", "//packages/kbn-securitysolution-utils:build", + "//packages/kbn-securitysolution-es-utils:build", "//packages/kbn-std:build", + "//packages/kbn-telemetry-tools:build", "//packages/kbn-tinymath:build", "//packages/kbn-utility-types:build", "//packages/kbn-utils:build", diff --git a/packages/kbn-crypto/BUILD.bazel b/packages/kbn-crypto/BUILD.bazel index 8f55f0e0f06a7..14e292c056db6 100644 --- a/packages/kbn-crypto/BUILD.bazel +++ b/packages/kbn-crypto/BUILD.bazel @@ -2,7 +2,7 @@ load("@npm//@bazel/typescript:index.bzl", "ts_config", "ts_project") load("@build_bazel_rules_nodejs//:index.bzl", "js_library", "pkg_npm") -PKG_BASE_NAME = "kbn-cypto" +PKG_BASE_NAME = "kbn-crypto" PKG_REQUIRE_NAME = "@kbn/crypto" SOURCE_FILES = glob( diff --git a/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts b/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts index fbef255cd9ee5..51d4f28d20f2e 100644 --- a/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts +++ b/packages/kbn-es-archiver/src/lib/indices/kibana_index.ts @@ -165,6 +165,7 @@ export async function createDefaultSpace({ { index, id: 'space:default', + refresh: 'wait_for', body: { type: 'space', updated_at: new Date().toISOString(), diff --git a/packages/kbn-es/README.md b/packages/kbn-es/README.md index 4d4c2aa94db07..80850c9e6a09c 100644 --- a/packages/kbn-es/README.md +++ b/packages/kbn-es/README.md @@ -7,6 +7,8 @@ If running elasticsearch from source, elasticsearch needs to be cloned to a sibl To run, go to the Kibana root and run `node scripts/es --help` to get the latest command line options. +The script attempts to preserve the existing interfaces used by Elasticsearch CLI. This includes passing through options with the `-E` argument and the `ES_JAVA_OPTS` environment variable for Java options. + ### Examples Run a snapshot install with a trial license diff --git a/packages/kbn-es/src/cluster.js b/packages/kbn-es/src/cluster.js index c55e5d3513c44..ad9ecb059031c 100644 --- a/packages/kbn-es/src/cluster.js +++ b/packages/kbn-es/src/cluster.js @@ -236,6 +236,7 @@ exports.Cluster = class Cluster { * @param {String} installPath * @param {Object} options * @property {string|Array} options.esArgs + * @property {string} options.esJavaOpts * @return {undefined} */ _exec(installPath, options = {}) { @@ -268,14 +269,17 @@ exports.Cluster = class Cluster { this._log.debug('%s %s', ES_BIN, args.join(' ')); - options.esEnvVars = options.esEnvVars || {}; + let esJavaOpts = `${options.esJavaOpts || ''} ${process.env.ES_JAVA_OPTS || ''}`; // ES now automatically sets heap size to 50% of the machine's available memory // so we need to set it to a smaller size for local dev and CI // especially because we currently run many instances of ES on the same machine during CI - options.esEnvVars.ES_JAVA_OPTS = - (options.esEnvVars.ES_JAVA_OPTS ? `${options.esEnvVars.ES_JAVA_OPTS} ` : '') + - '-Xms1g -Xmx1g'; + // inital and max must be the same, so we only need to check the max + if (!esJavaOpts.includes('Xmx')) { + esJavaOpts += ' -Xms1g -Xmx1g'; + } + + this._log.debug('ES_JAVA_OPTS: %s', esJavaOpts.trim()); this._process = execa(ES_BIN, args, { cwd: installPath, @@ -283,7 +287,7 @@ exports.Cluster = class Cluster { ...(installPath ? { ES_TMPDIR: path.resolve(installPath, 'ES_TMPDIR') } : {}), ...process.env, JAVA_HOME: '', // By default, we want to always unset JAVA_HOME so that the bundled JDK will be used - ...(options.esEnvVars || {}), + ES_JAVA_OPTS: esJavaOpts.trim(), }, stdio: ['ignore', 'pipe', 'pipe'], }); diff --git a/packages/kbn-es/src/integration_tests/cluster.test.js b/packages/kbn-es/src/integration_tests/cluster.test.js index 6b4025840283f..34220b08d2120 100644 --- a/packages/kbn-es/src/integration_tests/cluster.test.js +++ b/packages/kbn-es/src/integration_tests/cluster.test.js @@ -71,11 +71,17 @@ function mockEsBin({ exitCode, start }) { ); } +const initialEnv = { ...process.env }; + beforeEach(() => { jest.resetAllMocks(); extractConfigFiles.mockImplementation((config) => config); }); +afterEach(() => { + process.env = { ...initialEnv }; +}); + describe('#installSource()', () => { it('awaits installSource() promise and returns { installPath }', async () => { let resolveInstallSource; @@ -355,6 +361,25 @@ describe('#run()', () => { ] `); }); + + it('sets default Java heap', async () => { + mockEsBin({ start: true }); + + const cluster = new Cluster({ log }); + await cluster.run(); + + expect(execa.mock.calls[0][2].env.ES_JAVA_OPTS).toEqual('-Xms1g -Xmx1g'); + }); + + it('allows Java heap to be overwritten', async () => { + mockEsBin({ start: true }); + process.env.ES_JAVA_OPTS = '-Xms5g -Xmx5g'; + + const cluster = new Cluster({ log }); + await cluster.run(); + + expect(execa.mock.calls[0][2].env.ES_JAVA_OPTS).toEqual('-Xms5g -Xmx5g'); + }); }); describe('#stop()', () => { diff --git a/packages/kbn-optimizer/limits.yml b/packages/kbn-optimizer/limits.yml index 08e90ed829d4a..448b5ad650da5 100644 --- a/packages/kbn-optimizer/limits.yml +++ b/packages/kbn-optimizer/limits.yml @@ -61,7 +61,6 @@ pageLoadAssetSize: remoteClusters: 51327 reporting: 183418 rollup: 97204 - ruleRegistry: 100000 savedObjects: 108518 savedObjectsManagement: 101836 savedObjectsTagging: 59482 diff --git a/packages/kbn-plugin-generator/BUILD.bazel b/packages/kbn-plugin-generator/BUILD.bazel new file mode 100644 index 0000000000000..e22d41076db00 --- /dev/null +++ b/packages/kbn-plugin-generator/BUILD.bazel @@ -0,0 +1,106 @@ +load("@npm//@bazel/typescript:index.bzl", "ts_config", "ts_project") +load("@build_bazel_rules_nodejs//:index.bzl", "js_library", "pkg_npm") + +PKG_BASE_NAME = "kbn-plugin-generator" +PKG_REQUIRE_NAME = "@kbn/plugin-generator" + +SOURCE_FILES = glob( + [ + "src/**/*.ts", + ], + exclude = [ + "**/integration_tests/**/*", + ], +) + +SRCS = SOURCE_FILES + +filegroup( + name = "srcs", + srcs = SRCS, +) + +filegroup( + name = "template", + srcs = glob( + [ + "template/**/*", + ], + ), +) + +NPM_MODULE_EXTRA_FILES = [ + "package.json", + "README.md", + ":template", +] + +SRC_DEPS = [ + "//packages/kbn-utils", + "//packages/kbn-dev-utils", + "@npm//del", + "@npm//ejs", + "@npm//execa", + "@npm//globby", + "@npm//inquirer", + "@npm//minimatch", + "@npm//prettier", + "@npm//vinyl-fs", +] + +TYPES_DEPS = [ + "@npm//@types/ejs", + "@npm//@types/inquirer", + "@npm//@types/jest", + "@npm//@types/minimatch", + "@npm//@types/node", + "@npm//@types/prettier", + "@npm//@types/vinyl-fs", +] + +DEPS = SRC_DEPS + TYPES_DEPS + +ts_config( + name = "tsconfig", + src = "tsconfig.json", + deps = [ + "//:tsconfig.base.json", + ], +) + +ts_project( + name = "tsc", + args = ['--pretty'], + srcs = SRCS, + deps = DEPS, + declaration = True, + declaration_map = True, + incremental = True, + out_dir = "target", + source_map = True, + root_dir = "src", + tsconfig = ":tsconfig", +) + +js_library( + name = PKG_BASE_NAME, + srcs = NPM_MODULE_EXTRA_FILES, + deps = [":tsc"] + DEPS, + package_name = PKG_REQUIRE_NAME, + visibility = ["//visibility:public"], +) + +pkg_npm( + name = "npm_module", + deps = [ + ":%s" % PKG_BASE_NAME, + ] +) + +filegroup( + name = "build", + srcs = [ + ":npm_module", + ], + visibility = ["//visibility:public"], +) diff --git a/packages/kbn-plugin-generator/package.json b/packages/kbn-plugin-generator/package.json index 583085430d915..298373afd2f24 100644 --- a/packages/kbn-plugin-generator/package.json +++ b/packages/kbn-plugin-generator/package.json @@ -4,9 +4,5 @@ "private": true, "license": "SSPL-1.0 OR Elastic License 2.0", "main": "target/index.js", - "types": "target/index.d.ts", - "scripts": { - "kbn:bootstrap": "node scripts/build", - "kbn:watch": "node scripts/build --watch" - } + "types": "target/index.d.ts" } \ No newline at end of file diff --git a/packages/kbn-plugin-generator/scripts/build.js b/packages/kbn-plugin-generator/scripts/build.js deleted file mode 100644 index e17f564bc482c..0000000000000 --- a/packages/kbn-plugin-generator/scripts/build.js +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -const Path = require('path'); - -const { run } = require('@kbn/dev-utils'); -const del = require('del'); -const execa = require('execa'); - -run( - async ({ flags }) => { - await del(Path.resolve(__dirname, '../target')); - - await execa(require.resolve('typescript/bin/tsc'), flags.watch ? ['--watch'] : [], { - cwd: Path.resolve(__dirname, '..'), - stdio: 'inherit', - }); - }, - { - flags: { - boolean: ['watch'], - help: ` - --watch Watch files and rebuild on changes - `, - }, - } -); diff --git a/packages/kbn-plugin-generator/tsconfig.json b/packages/kbn-plugin-generator/tsconfig.json index 5e885527a7608..9165fd21ebea0 100644 --- a/packages/kbn-plugin-generator/tsconfig.json +++ b/packages/kbn-plugin-generator/tsconfig.json @@ -1,7 +1,7 @@ { "extends": "../../tsconfig.base.json", "compilerOptions": { - "incremental": false, + "incremental": true, "outDir": "target", "target": "ES2019", "declaration": true, diff --git a/packages/kbn-telemetry-tools/babel.config.js b/packages/kbn-rule-data-utils/jest.config.js similarity index 79% rename from packages/kbn-telemetry-tools/babel.config.js rename to packages/kbn-rule-data-utils/jest.config.js index 68c67e549b74b..26cb39fe8b55a 100644 --- a/packages/kbn-telemetry-tools/babel.config.js +++ b/packages/kbn-rule-data-utils/jest.config.js @@ -7,6 +7,7 @@ */ module.exports = { - presets: ['@kbn/babel-preset/node_preset'], - ignore: ['**/*.test.ts', '**/__fixture__/**'], + preset: '@kbn/test', + rootDir: '../..', + roots: ['/packages/kbn-rule-data-utils'], }; diff --git a/packages/kbn-rule-data-utils/package.json b/packages/kbn-rule-data-utils/package.json new file mode 100644 index 0000000000000..6f0b8439ec891 --- /dev/null +++ b/packages/kbn-rule-data-utils/package.json @@ -0,0 +1,13 @@ +{ + "name": "@kbn/rule-data-utils", + "main": "./target/index.js", + "types": "./target/index.d.ts", + "version": "1.0.0", + "license": "SSPL-1.0 OR Elastic License 2.0", + "private": true, + "scripts": { + "build": "../../node_modules/.bin/tsc", + "kbn:bootstrap": "yarn build", + "kbn:watch": "yarn build --watch" + } +} diff --git a/packages/kbn-rule-data-utils/src/index.ts b/packages/kbn-rule-data-utils/src/index.ts new file mode 100644 index 0000000000000..93a2538c7aa2c --- /dev/null +++ b/packages/kbn-rule-data-utils/src/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export * from './technical_field_names'; diff --git a/packages/kbn-rule-data-utils/src/technical_field_names.ts b/packages/kbn-rule-data-utils/src/technical_field_names.ts new file mode 100644 index 0000000000000..31779c9f08e81 --- /dev/null +++ b/packages/kbn-rule-data-utils/src/technical_field_names.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { ValuesType } from 'utility-types'; + +const ALERT_NAMESPACE = 'kibana.rac.alert'; + +const TIMESTAMP = '@timestamp' as const; +const EVENT_KIND = 'event.kind' as const; +const EVENT_ACTION = 'event.action' as const; +const RULE_UUID = 'rule.uuid' as const; +const RULE_ID = 'rule.id' as const; +const RULE_NAME = 'rule.name' as const; +const RULE_CATEGORY = 'rule.category' as const; +const TAGS = 'tags' as const; +const PRODUCER = `${ALERT_NAMESPACE}.producer` as const; +const ALERT_ID = `${ALERT_NAMESPACE}.id` as const; +const ALERT_UUID = `${ALERT_NAMESPACE}.uuid` as const; +const ALERT_START = `${ALERT_NAMESPACE}.start` as const; +const ALERT_END = `${ALERT_NAMESPACE}.end` as const; +const ALERT_DURATION = `${ALERT_NAMESPACE}.duration.us` as const; +const ALERT_SEVERITY_LEVEL = `${ALERT_NAMESPACE}.severity.level` as const; +const ALERT_SEVERITY_VALUE = `${ALERT_NAMESPACE}.severity.value` as const; +const ALERT_STATUS = `${ALERT_NAMESPACE}.status` as const; +const ALERT_EVALUATION_THRESHOLD = `${ALERT_NAMESPACE}.evaluation.threshold` as const; +const ALERT_EVALUATION_VALUE = `${ALERT_NAMESPACE}.evaluation.value` as const; + +const fields = { + TIMESTAMP, + EVENT_KIND, + EVENT_ACTION, + RULE_UUID, + RULE_ID, + RULE_NAME, + RULE_CATEGORY, + TAGS, + PRODUCER, + ALERT_ID, + ALERT_UUID, + ALERT_START, + ALERT_END, + ALERT_DURATION, + ALERT_SEVERITY_LEVEL, + ALERT_SEVERITY_VALUE, + ALERT_STATUS, + ALERT_EVALUATION_THRESHOLD, + ALERT_EVALUATION_VALUE, +}; + +export { + TIMESTAMP, + EVENT_KIND, + EVENT_ACTION, + RULE_UUID, + RULE_ID, + RULE_NAME, + RULE_CATEGORY, + TAGS, + PRODUCER, + ALERT_ID, + ALERT_UUID, + ALERT_START, + ALERT_END, + ALERT_DURATION, + ALERT_SEVERITY_LEVEL, + ALERT_SEVERITY_VALUE, + ALERT_STATUS, + ALERT_EVALUATION_THRESHOLD, + ALERT_EVALUATION_VALUE, +}; + +export type TechnicalRuleDataFieldName = ValuesType; diff --git a/packages/kbn-rule-data-utils/tsconfig.json b/packages/kbn-rule-data-utils/tsconfig.json new file mode 100644 index 0000000000000..4b1262d11f3af --- /dev/null +++ b/packages/kbn-rule-data-utils/tsconfig.json @@ -0,0 +1,19 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "incremental": false, + "outDir": "./target", + "stripInternal": false, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "sourceRoot": "../../../../packages/kbn-rule-data-utils/src", + "types": [ + "jest", + "node" + ] + }, + "include": [ + "./src/**/*.ts" + ] +} diff --git a/packages/kbn-securitysolution-es-utils/BUILD.bazel b/packages/kbn-securitysolution-es-utils/BUILD.bazel new file mode 100644 index 0000000000000..0cc27358c5da2 --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/BUILD.bazel @@ -0,0 +1,86 @@ +load("@npm//@bazel/typescript:index.bzl", "ts_config", "ts_project") +load("@build_bazel_rules_nodejs//:index.bzl", "js_library", "pkg_npm") + +PKG_BASE_NAME = "kbn-securitysolution-es-utils" + +PKG_REQUIRE_NAME = "@kbn/securitysolution-es-utils" + +SOURCE_FILES = glob( + [ + "src/**/*.ts", + ], + exclude = [ + "**/*.test.*", + "**/*.mock.*", + ], +) + +SRCS = SOURCE_FILES + +filegroup( + name = "srcs", + srcs = SRCS, +) + +NPM_MODULE_EXTRA_FILES = [ + "package.json", + "README.md", +] + +SRC_DEPS = [ + "@npm//@elastic/elasticsearch", + "@npm//@hapi/hapi", + "@npm//tslib", +] + +TYPES_DEPS = [ + "@npm//@types/jest", + "@npm//@types/node", +] + +DEPS = SRC_DEPS + TYPES_DEPS + +ts_config( + name = "tsconfig", + src = "tsconfig.json", + deps = [ + "//:tsconfig.base.json", + ], +) + +ts_project( + name = "tsc", + srcs = SRCS, + args = ["--pretty"], + declaration = True, + declaration_map = True, + incremental = True, + out_dir = "target", + root_dir = "src", + source_map = True, + tsconfig = ":tsconfig", + deps = DEPS, +) + +js_library( + name = PKG_BASE_NAME, + package_name = PKG_REQUIRE_NAME, + srcs = NPM_MODULE_EXTRA_FILES, + visibility = ["//visibility:public"], + deps = [":tsc"] + DEPS, +) + +pkg_npm( + name = "npm_module", + deps = [ + ":%s" % PKG_BASE_NAME, + ], +) + +filegroup( + name = "build", + srcs = [ + ":npm_module", + ], + visibility = ["//visibility:public"], +) diff --git a/packages/kbn-securitysolution-es-utils/README.md b/packages/kbn-securitysolution-es-utils/README.md new file mode 100644 index 0000000000000..b99aa095c84f4 --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/README.md @@ -0,0 +1,6 @@ +# kbn-securitysolution-es-utils + +This is the shared security solution elastic search utilities among plugins. This was originally created +to remove the dependencies between security_solution and other projects such as lists. This should only be +used within server side code and not client side code since it is all elastic search utilities and packages. + diff --git a/packages/kbn-securitysolution-es-utils/jest.config.js b/packages/kbn-securitysolution-es-utils/jest.config.js new file mode 100644 index 0000000000000..6b86ec6e2da52 --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/jest.config.js @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../..', + roots: ['/packages/kbn-securitysolution-es-utils'], +}; diff --git a/packages/kbn-securitysolution-es-utils/package.json b/packages/kbn-securitysolution-es-utils/package.json new file mode 100644 index 0000000000000..7d0c0993c6c32 --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/package.json @@ -0,0 +1,9 @@ +{ + "name": "@kbn/securitysolution-es-utils", + "version": "1.0.0", + "description": "security solution elastic search utilities to use across plugins such lists, security_solution, cases, etc...", + "license": "SSPL-1.0 OR Elastic License 2.0", + "main": "./target/index.js", + "types": "./target/index.d.ts", + "private": true +} diff --git a/packages/kbn-securitysolution-es-utils/src/bad_request_error/index.ts b/packages/kbn-securitysolution-es-utils/src/bad_request_error/index.ts new file mode 100644 index 0000000000000..525f6cfa5c9ff --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/bad_request_error/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export class BadRequestError extends Error {} diff --git a/packages/kbn-securitysolution-es-utils/src/create_boostrap_index/index.ts b/packages/kbn-securitysolution-es-utils/src/create_boostrap_index/index.ts new file mode 100644 index 0000000000000..9671d35dc554e --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/create_boostrap_index/index.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { ElasticsearchClient } from '../elasticsearch_client'; + +// See the reference(s) below on explanations about why -000001 was chosen and +// why the is_write_index is true as well as the bootstrapping step which is needed. +// Ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/applying-policy-to-template.html +export const createBootstrapIndex = async ( + esClient: ElasticsearchClient, + index: string +): Promise => { + return ( + await esClient.transport.request({ + path: `/${index}-000001`, + method: 'PUT', + body: { + aliases: { + [index]: { + is_write_index: true, + }, + }, + }, + }) + ).body; +}; diff --git a/packages/kbn-securitysolution-es-utils/src/delete_all_index/index.ts b/packages/kbn-securitysolution-es-utils/src/delete_all_index/index.ts new file mode 100644 index 0000000000000..4df4724aaf2b5 --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/delete_all_index/index.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { ElasticsearchClient } from '../elasticsearch_client'; + +export const deleteAllIndex = async ( + esClient: ElasticsearchClient, + pattern: string, + maxAttempts = 5 +): Promise => { + for (let attempt = 1; ; attempt++) { + if (attempt > maxAttempts) { + throw new Error( + `Failed to delete indexes with pattern [${pattern}] after ${maxAttempts} attempts` + ); + } + + // resolve pattern to concrete index names + const { body: resp } = await esClient.indices.getAlias( + { + index: pattern, + }, + { ignore: [404] } + ); + + // @ts-expect-error status doesn't exist on response + if (resp.status === 404) { + return true; + } + + const indices = Object.keys(resp) as string[]; + + // if no indexes exits then we're done with this pattern + if (!indices.length) { + return true; + } + + // delete the concrete indexes we found and try again until this pattern resolves to no indexes + await esClient.indices.delete({ + index: indices, + ignore_unavailable: true, + }); + } +}; diff --git a/packages/kbn-securitysolution-es-utils/src/delete_policy/index.ts b/packages/kbn-securitysolution-es-utils/src/delete_policy/index.ts new file mode 100644 index 0000000000000..34c1d2e5da45f --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/delete_policy/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { ElasticsearchClient } from '../elasticsearch_client'; + +export const deletePolicy = async ( + esClient: ElasticsearchClient, + policy: string +): Promise => { + return ( + await esClient.transport.request({ + path: `/_ilm/policy/${policy}`, + method: 'DELETE', + }) + ).body; +}; diff --git a/packages/kbn-securitysolution-es-utils/src/delete_template/index.ts b/packages/kbn-securitysolution-es-utils/src/delete_template/index.ts new file mode 100644 index 0000000000000..2e7a71af9f772 --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/delete_template/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { ElasticsearchClient } from '../elasticsearch_client'; + +export const deleteTemplate = async ( + esClient: ElasticsearchClient, + name: string +): Promise => { + return ( + await esClient.indices.deleteTemplate({ + name, + }) + ).body; +}; diff --git a/packages/kbn-securitysolution-es-utils/src/elasticsearch_client/index.ts b/packages/kbn-securitysolution-es-utils/src/elasticsearch_client/index.ts new file mode 100644 index 0000000000000..0c2252bdc1f03 --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/elasticsearch_client/index.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +// Copied from src/core/server/elasticsearch/client/types.ts +// as these types aren't part of any package yet. Once they are, remove this completely + +import type { KibanaClient } from '@elastic/elasticsearch/api/kibana'; +import type { + ApiResponse, + TransportRequestOptions, + TransportRequestParams, + TransportRequestPromise, +} from '@elastic/elasticsearch/lib/Transport'; + +/** + * Client used to query the elasticsearch cluster. + * @deprecated At some point use the one from src/core/server/elasticsearch/client/types.ts when it is made into a package. If it never is, then keep using this one. + * @public + */ +export type ElasticsearchClient = Omit< + KibanaClient, + 'connectionPool' | 'transport' | 'serializer' | 'extend' | 'child' | 'close' +> & { + transport: { + request( + params: TransportRequestParams, + options?: TransportRequestOptions + ): TransportRequestPromise; + }; +}; diff --git a/packages/kbn-securitysolution-es-utils/src/get_index_exists/index.ts b/packages/kbn-securitysolution-es-utils/src/get_index_exists/index.ts new file mode 100644 index 0000000000000..b7d12cab3f48c --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/get_index_exists/index.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { ElasticsearchClient } from '../elasticsearch_client'; + +export const getIndexExists = async ( + esClient: ElasticsearchClient, + index: string +): Promise => { + try { + const { body: response } = await esClient.search({ + index, + size: 0, + allow_no_indices: true, + body: { + terminate_after: 1, + }, + }); + return response._shards.total > 0; + } catch (err) { + if (err.body != null && err.body.status === 404) { + return false; + } else { + throw err.body ? err.body : err; + } + } +}; diff --git a/packages/kbn-securitysolution-es-utils/src/get_policy_exists/index.ts b/packages/kbn-securitysolution-es-utils/src/get_policy_exists/index.ts new file mode 100644 index 0000000000000..cefd47dbe9d07 --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/get_policy_exists/index.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { ElasticsearchClient } from '../elasticsearch_client'; + +export const getPolicyExists = async ( + esClient: ElasticsearchClient, + policy: string +): Promise => { + try { + await esClient.transport.request({ + path: `/_ilm/policy/${policy}`, + method: 'GET', + }); + // Return true that there exists a policy which is not 404 or some error + // Since there is not a policy exists API, this is how we create one by calling + // into the API to get it if it exists or rely on it to throw a 404 + return true; + } catch (err) { + if (err.statusCode === 404) { + return false; + } else { + throw err; + } + } +}; diff --git a/packages/kbn-securitysolution-es-utils/src/get_template_exists/index.ts b/packages/kbn-securitysolution-es-utils/src/get_template_exists/index.ts new file mode 100644 index 0000000000000..c56c5b968d45c --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/get_template_exists/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { ElasticsearchClient } from '../elasticsearch_client'; + +export const getTemplateExists = async ( + esClient: ElasticsearchClient, + template: string +): Promise => { + return ( + await esClient.indices.existsTemplate({ + name: template, + }) + ).body; +}; diff --git a/packages/kbn-securitysolution-es-utils/src/index.ts b/packages/kbn-securitysolution-es-utils/src/index.ts new file mode 100644 index 0000000000000..657a63eef15cd --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/index.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export * from './bad_request_error'; +export * from './create_boostrap_index'; +export * from './delete_all_index'; +export * from './delete_policy'; +export * from './delete_template'; +export * from './elasticsearch_client'; +export * from './get_index_exists'; +export * from './get_policy_exists'; +export * from './get_template_exists'; +export * from './read_privileges'; +export * from './set_policy'; +export * from './set_template'; +export * from './transform_error'; diff --git a/packages/kbn-securitysolution-es-utils/src/read_privileges/index.ts b/packages/kbn-securitysolution-es-utils/src/read_privileges/index.ts new file mode 100644 index 0000000000000..8b11387a1d020 --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/read_privileges/index.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/** + * Copied from src/core/server/elasticsearch/legacy/api_types.ts including its deprecation mentioned below + * TODO: Remove this and refactor the readPrivileges to utilize any newer client side ways rather than all this deprecated legacy stuff + */ +export interface LegacyCallAPIOptions { + /** + * Indicates whether `401 Unauthorized` errors returned from the Elasticsearch API + * should be wrapped into `Boom` error instances with properly set `WWW-Authenticate` + * header that could have been returned by the API itself. If API didn't specify that + * then `Basic realm="Authorization Required"` is used as `WWW-Authenticate`. + */ + wrap401Errors?: boolean; + /** + * A signal object that allows you to abort the request via an AbortController object. + */ + signal?: AbortSignal; +} + +type CallWithRequest, V> = ( + endpoint: string, + params: T, + options?: LegacyCallAPIOptions +) => Promise; + +export const readPrivileges = async ( + callWithRequest: CallWithRequest<{}, unknown>, + index: string +): Promise => { + return callWithRequest('transport.request', { + path: '/_security/user/_has_privileges', + method: 'POST', + body: { + cluster: [ + 'all', + 'create_snapshot', + 'manage', + 'manage_api_key', + 'manage_ccr', + 'manage_transform', + 'manage_ilm', + 'manage_index_templates', + 'manage_ingest_pipelines', + 'manage_ml', + 'manage_own_api_key', + 'manage_pipeline', + 'manage_rollup', + 'manage_saml', + 'manage_security', + 'manage_token', + 'manage_watcher', + 'monitor', + 'monitor_transform', + 'monitor_ml', + 'monitor_rollup', + 'monitor_watcher', + 'read_ccr', + 'read_ilm', + 'transport_client', + ], + index: [ + { + names: [index], + privileges: [ + 'all', + 'create', + 'create_doc', + 'create_index', + 'delete', + 'delete_index', + 'index', + 'manage', + 'maintenance', + 'manage_follow_index', + 'manage_ilm', + 'manage_leader_index', + 'monitor', + 'read', + 'read_cross_cluster', + 'view_index_metadata', + 'write', + ], + }, + ], + }, + }); +}; diff --git a/packages/kbn-securitysolution-es-utils/src/set_policy/index.ts b/packages/kbn-securitysolution-es-utils/src/set_policy/index.ts new file mode 100644 index 0000000000000..dc45ca3e1c089 --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/set_policy/index.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { ElasticsearchClient } from '../elasticsearch_client'; + +export const setPolicy = async ( + esClient: ElasticsearchClient, + policy: string, + body: Record +): Promise => { + return ( + await esClient.transport.request({ + path: `/_ilm/policy/${policy}`, + method: 'PUT', + body, + }) + ).body; +}; diff --git a/packages/kbn-securitysolution-es-utils/src/set_template/index.ts b/packages/kbn-securitysolution-es-utils/src/set_template/index.ts new file mode 100644 index 0000000000000..89aaa44f29e0d --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/set_template/index.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { ElasticsearchClient } from '../elasticsearch_client'; + +export const setTemplate = async ( + esClient: ElasticsearchClient, + name: string, + body: Record +): Promise => { + return ( + await esClient.indices.putTemplate({ + name, + body, + }) + ).body; +}; diff --git a/packages/kbn-securitysolution-es-utils/src/transform_error/index.test.ts b/packages/kbn-securitysolution-es-utils/src/transform_error/index.test.ts new file mode 100644 index 0000000000000..e0f520f1ebfd4 --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/transform_error/index.test.ts @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Boom from '@hapi/boom'; +import { transformError } from '.'; +import { BadRequestError } from '../bad_request_error'; +import { errors } from '@elastic/elasticsearch'; + +describe('transformError', () => { + test('returns transformed output error from boom object with a 500 and payload of internal server error', () => { + const boom = new Boom.Boom('some boom message'); + const transformed = transformError(boom); + expect(transformed).toEqual({ + message: 'An internal server error occurred', + statusCode: 500, + }); + }); + + test('returns transformed output if it is some non boom object that has a statusCode', () => { + const error: Error & { statusCode?: number } = { + statusCode: 403, + name: 'some name', + message: 'some message', + }; + const transformed = transformError(error); + expect(transformed).toEqual({ + message: 'some message', + statusCode: 403, + }); + }); + + test('returns a transformed message with the message set and statusCode', () => { + const error: Error & { statusCode?: number } = { + statusCode: 403, + name: 'some name', + message: 'some message', + }; + const transformed = transformError(error); + expect(transformed).toEqual({ + message: 'some message', + statusCode: 403, + }); + }); + + test('transforms best it can if it is some non boom object but it does not have a status Code.', () => { + const error: Error = { + name: 'some name', + message: 'some message', + }; + const transformed = transformError(error); + expect(transformed).toEqual({ + message: 'some message', + statusCode: 500, + }); + }); + + test('it detects a BadRequestError and returns a status code of 400 from that particular error type', () => { + const error: BadRequestError = new BadRequestError('I have a type error'); + const transformed = transformError(error); + expect(transformed).toEqual({ + message: 'I have a type error', + statusCode: 400, + }); + }); + + test('it detects a BadRequestError and returns a Boom status of 400', () => { + const error: BadRequestError = new BadRequestError('I have a type error'); + const transformed = transformError(error); + expect(transformed).toEqual({ + message: 'I have a type error', + statusCode: 400, + }); + }); + + it('transforms a ResponseError returned by the elasticsearch client', () => { + const error: errors.ResponseError = { + name: 'ResponseError', + message: 'illegal_argument_exception', + headers: {}, + body: { + error: { + type: 'illegal_argument_exception', + reason: 'detailed explanation', + }, + }, + meta: ({} as unknown) as errors.ResponseError['meta'], + statusCode: 400, + }; + const transformed = transformError(error); + + expect(transformed).toEqual({ + message: 'illegal_argument_exception: detailed explanation', + statusCode: 400, + }); + }); +}); diff --git a/packages/kbn-securitysolution-es-utils/src/transform_error/index.ts b/packages/kbn-securitysolution-es-utils/src/transform_error/index.ts new file mode 100644 index 0000000000000..b532dc5d1b6d0 --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/transform_error/index.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import Boom from '@hapi/boom'; +import { errors } from '@elastic/elasticsearch'; +import { BadRequestError } from '../bad_request_error'; + +export interface OutputError { + message: string; + statusCode: number; +} + +export const transformError = (err: Error & Partial): OutputError => { + if (Boom.isBoom(err)) { + return { + message: err.output.payload.message, + statusCode: err.output.statusCode, + }; + } else { + if (err.statusCode != null) { + if (err.body != null && err.body.error != null) { + return { + statusCode: err.statusCode, + message: `${err.body.error.type}: ${err.body.error.reason}`, + }; + } else { + return { + statusCode: err.statusCode, + message: err.message, + }; + } + } else if (err instanceof BadRequestError) { + // allows us to throw request validation errors in the absence of Boom + return { + message: err.message, + statusCode: 400, + }; + } else { + // natively return the err and allow the regular framework + // to deal with the error when it is a non Boom + return { + message: err.message != null ? err.message : '(unknown error message)', + statusCode: 500, + }; + } + } +}; diff --git a/packages/kbn-securitysolution-es-utils/tsconfig.json b/packages/kbn-securitysolution-es-utils/tsconfig.json new file mode 100644 index 0000000000000..be8848d781cae --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/tsconfig.json @@ -0,0 +1,19 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "declaration": true, + "declarationMap": true, + "incremental": true, + "outDir": "target", + "rootDir": "src", + "sourceMap": true, + "sourceRoot": "../../../../packages/kbn-securitysolution-es-utils/src", + "types": [ + "jest", + "node" + ] + }, + "include": [ + "src/**/*" + ] +} diff --git a/packages/kbn-securitysolution-io-ts-utils/src/non_empty_string_array/index.test.ts b/packages/kbn-securitysolution-io-ts-utils/src/non_empty_string_array/index.test.ts index f56fa7faed2a4..9fec36f46dd27 100644 --- a/packages/kbn-securitysolution-io-ts-utils/src/non_empty_string_array/index.test.ts +++ b/packages/kbn-securitysolution-io-ts-utils/src/non_empty_string_array/index.test.ts @@ -8,7 +8,7 @@ import { pipe } from 'fp-ts/lib/pipeable'; import { left } from 'fp-ts/lib/Either'; -import { foldLeftRight, getPaths } from '@kbn/securitysolution-io-ts-utils'; +import { foldLeftRight, getPaths } from '../test_utils'; import { NonEmptyStringArray } from '.'; describe('non_empty_string_array', () => { diff --git a/packages/kbn-securitysolution-utils/BUILD.bazel b/packages/kbn-securitysolution-utils/BUILD.bazel index 33c25a3091c24..42897e93593b6 100644 --- a/packages/kbn-securitysolution-utils/BUILD.bazel +++ b/packages/kbn-securitysolution-utils/BUILD.bazel @@ -35,7 +35,7 @@ SRC_DEPS = [ TYPES_DEPS = [ "@npm//@types/jest", "@npm//@types/node", - "@npm//@types/uuid" + "@npm//@types/uuid" ] DEPS = SRC_DEPS + TYPES_DEPS @@ -83,4 +83,4 @@ filegroup( ":npm_module", ], visibility = ["//visibility:public"], -) \ No newline at end of file +) diff --git a/packages/kbn-telemetry-tools/BUILD.bazel b/packages/kbn-telemetry-tools/BUILD.bazel new file mode 100644 index 0000000000000..4d1b4f21117c4 --- /dev/null +++ b/packages/kbn-telemetry-tools/BUILD.bazel @@ -0,0 +1,97 @@ +load("@npm//@bazel/typescript:index.bzl", "ts_config", "ts_project") +load("@build_bazel_rules_nodejs//:index.bzl", "js_library", "pkg_npm") + +PKG_BASE_NAME = "kbn-telemetry-tools" +PKG_REQUIRE_NAME = "@kbn/telemetry-tools" + +SOURCE_FILES = glob( + [ + "src/**/*.ts", + ], + exclude = [ + "**/*.test.*", + "**/__fixture__/**", + "**/__snapshots__/**", + ] +) + +SRCS = SOURCE_FILES + +filegroup( + name = "srcs", + srcs = SRCS, +) + +NPM_MODULE_EXTRA_FILES = [ + "package.json", + "README.md", + "GUIDELINE.md", +] + +SRC_DEPS = [ + "//packages/kbn-dev-utils", + "//packages/kbn-utility-types", + "@npm//glob", + "@npm//jest-styled-components", + "@npm//listr", + "@npm//normalize-path", + "@npm//tslib", +] + +TYPES_DEPS = [ + "@npm//@types/flot", + "@npm//@types/glob", + "@npm//@types/jest", + "@npm//@types/listr", + "@npm//@types/lodash", + "@npm//@types/node", + "@npm//@types/normalize-path", + "@npm//@types/testing-library__jest-dom", +] + +DEPS = SRC_DEPS + TYPES_DEPS + +ts_config( + name = "tsconfig", + src = "tsconfig.json", + deps = [ + "//:tsconfig.base.json", + ], +) + +ts_project( + name = "tsc", + args = ['--pretty'], + srcs = SRCS, + deps = DEPS, + declaration = True, + declaration_map = True, + incremental = True, + out_dir = "target", + source_map = True, + root_dir = "src", + tsconfig = ":tsconfig", +) + +js_library( + name = PKG_BASE_NAME, + srcs = NPM_MODULE_EXTRA_FILES, + deps = [":tsc"] + DEPS, + package_name = PKG_REQUIRE_NAME, + visibility = ["//visibility:public"], +) + +pkg_npm( + name = "npm_module", + deps = [ + ":%s" % PKG_BASE_NAME, + ] +) + +filegroup( + name = "build", + srcs = [ + ":npm_module", + ], + visibility = ["//visibility:public"], +) diff --git a/packages/kbn-telemetry-tools/package.json b/packages/kbn-telemetry-tools/package.json index 31fac5c043832..cd3fd3aa966c7 100644 --- a/packages/kbn-telemetry-tools/package.json +++ b/packages/kbn-telemetry-tools/package.json @@ -7,10 +7,5 @@ "private": true, "kibana": { "devOnly": true - }, - "scripts": { - "build": "../../node_modules/.bin/babel src --out-dir target --delete-dir-on-start --extensions .ts --source-maps=inline", - "kbn:bootstrap": "yarn build", - "kbn:watch": "yarn build --watch" } } \ No newline at end of file diff --git a/packages/kbn-telemetry-tools/tsconfig.json b/packages/kbn-telemetry-tools/tsconfig.json index 419af1d02f83b..926ebff17f639 100644 --- a/packages/kbn-telemetry-tools/tsconfig.json +++ b/packages/kbn-telemetry-tools/tsconfig.json @@ -1,10 +1,11 @@ { "extends": "../../tsconfig.base.json", "compilerOptions": { - "incremental": false, + "incremental": true, "outDir": "./target", "declaration": true, "declarationMap": true, + "rootDir": "src", "sourceMap": true, "sourceRoot": "../../../../packages/kbn-telemetry-tools/src", "isolatedModules": true diff --git a/packages/kbn-test/src/es/test_es_cluster.ts b/packages/kbn-test/src/es/test_es_cluster.ts index e802613fbaedb..658fc9382d616 100644 --- a/packages/kbn-test/src/es/test_es_cluster.ts +++ b/packages/kbn-test/src/es/test_es_cluster.ts @@ -36,7 +36,7 @@ interface TestClusterFactoryOptions { * */ dataArchive?: string; esArgs?: string[]; - esEnvVars?: Record; + esJavaOpts?: string; clusterName?: string; log: ToolingLog; ssl?: boolean; @@ -52,7 +52,7 @@ export function createTestEsCluster(options: TestClusterFactoryOptions) { esFrom = esTestConfig.getBuildFrom(), dataArchive, esArgs: customEsArgs = [], - esEnvVars, + esJavaOpts, clusterName: customClusterName = 'es-test-cluster', ssl, } = options; @@ -107,7 +107,7 @@ export function createTestEsCluster(options: TestClusterFactoryOptions) { await cluster.start(installPath, { password: config.password, esArgs, - esEnvVars, + esJavaOpts, }); } diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts index d82b7b83e8f15..65573fdbd6647 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts @@ -172,7 +172,7 @@ export const schema = Joi.object() license: Joi.string().default('basic'), from: Joi.string().default('snapshot'), serverArgs: Joi.array(), - serverEnvVars: Joi.object(), + esJavaOpts: Joi.string(), dataArchive: Joi.string(), ssl: Joi.boolean().default(false), }) diff --git a/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts b/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts index 83368783da389..7ba9a3c1c4733 100644 --- a/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts +++ b/packages/kbn-test/src/functional_tests/lib/run_elasticsearch.ts @@ -29,7 +29,7 @@ export async function runElasticsearch({ const ssl = config.get('esTestCluster.ssl'); const license = config.get('esTestCluster.license'); const esArgs = config.get('esTestCluster.serverArgs'); - const esEnvVars = config.get('esTestCluster.serverEnvVars'); + const esJavaOpts = config.get('esTestCluster.esJavaOpts'); const isSecurityEnabled = esArgs.includes('xpack.security.enabled=true'); const cluster = createTestEsCluster({ @@ -43,7 +43,7 @@ export async function runElasticsearch({ esFrom: esFrom || config.get('esTestCluster.from'), dataArchive: config.get('esTestCluster.dataArchive'), esArgs, - esEnvVars, + esJavaOpts, ssl, }); diff --git a/rfcs/images/0018_agent_manager.png b/rfcs/images/0018_agent_manager.png new file mode 100644 index 0000000000000..92a135386e258 Binary files /dev/null and b/rfcs/images/0018_agent_manager.png differ diff --git a/rfcs/images/0018_buildkite_build.png b/rfcs/images/0018_buildkite_build.png new file mode 100644 index 0000000000000..1b04fc85561c8 Binary files /dev/null and b/rfcs/images/0018_buildkite_build.png differ diff --git a/rfcs/images/0018_buildkite_deps.png b/rfcs/images/0018_buildkite_deps.png new file mode 100644 index 0000000000000..f4b0376d6e26f Binary files /dev/null and b/rfcs/images/0018_buildkite_deps.png differ diff --git a/rfcs/images/0018_buildkite_uptime.png b/rfcs/images/0018_buildkite_uptime.png new file mode 100644 index 0000000000000..f850df50b0f1b Binary files /dev/null and b/rfcs/images/0018_buildkite_uptime.png differ diff --git a/rfcs/images/0018_jenkins_pipeline_steps.png b/rfcs/images/0018_jenkins_pipeline_steps.png new file mode 100644 index 0000000000000..51ad8fde8aae0 Binary files /dev/null and b/rfcs/images/0018_jenkins_pipeline_steps.png differ diff --git a/rfcs/images/timeslider/toolbar.png b/rfcs/images/timeslider/toolbar.png new file mode 100644 index 0000000000000..3dae2af0d77f7 Binary files /dev/null and b/rfcs/images/timeslider/toolbar.png differ diff --git a/rfcs/images/timeslider/v1.png b/rfcs/images/timeslider/v1.png new file mode 100644 index 0000000000000..4766fa24c816a Binary files /dev/null and b/rfcs/images/timeslider/v1.png differ diff --git a/rfcs/images/timeslider/v2.png b/rfcs/images/timeslider/v2.png new file mode 100644 index 0000000000000..e4a77e241dc83 Binary files /dev/null and b/rfcs/images/timeslider/v2.png differ diff --git a/rfcs/text/0018_buildkite.md b/rfcs/text/0018_buildkite.md new file mode 100644 index 0000000000000..560540ae3af0b --- /dev/null +++ b/rfcs/text/0018_buildkite.md @@ -0,0 +1,923 @@ +- Start Date: 2021-03-29 +- RFC PR: [#95070](https://github.com/elastic/kibana/pull/95070) +- Kibana Issue: [#94630](https://github.com/elastic/kibana/issues/94630) + +--- + +- [Summary](#summary) +- [Motivation](#motivation) + - [Required and Desired Capabilities](#required-and-desired-capabilities) + - [Required](#required) + - [Scalable](#scalable) + - [Stable](#stable) + - [Surfaces information intuitively](#surfaces-information-intuitively) + - [Pipelines](#pipelines) + - [Advanced Pipeline logic](#advanced-pipeline-logic) + - [Cloud-friendly pricing model](#cloud-friendly-pricing-model) + - [Public access](#public-access) + - [Secrets handling](#secrets-handling) + - [Support or Documentation](#support-or-documentation) + - [Scheduled Builds](#scheduled-builds) + - [Container support](#container-support) + - [Desired](#desired) + - [Customization](#customization) + - [Core functionality is first-party](#core-functionality-is-first-party) + - [First-class support for test results](#first-class-support-for-test-results) + - [GitHub Integration](#github-integration) +- [Buildkite - Detailed design](#buildkite---detailed-design) + - [Overview](#overview) + - [Required and Desired Capabilities](#required-and-desired-capabilities-1) + - [Required](#required-1) + - [Scalable](#scalable-1) + - [Stable](#stable-1) + - [Surfaces information intuitively](#surfaces-information-intuitively-1) + - [Pipelines](#pipelines-1) + - [Advanced Pipeline logic](#advanced-pipeline-logic-1) + - [Cloud-friendly pricing model](#cloud-friendly-pricing-model-1) + - [Public access](#public-access-1) + - [Secrets handling](#secrets-handling-1) + - [Support or Documentation](#support-or-documentation-1) + - [Scheduled Builds](#scheduled-builds-1) + - [Container support](#container-support-1) + - [Desired](#desired-1) + - [Customization](#customization-1) + - [Core functionality is first-party](#core-functionality-is-first-party-1) + - [First-class support for test results](#first-class-support-for-test-results-1) + - [GitHub Integration](#github-integration-1) + - [What we will build and manage](#what-we-will-build-and-manage) + - [Elastic Buildkite Agent Manager](#elastic-buildkite-agent-manager) + - [Overview](#overview-1) + - [Design](#design) + - [Protection against creating too many instances](#protection-against-creating-too-many-instances) + - [Configuration](#configuration) + - [Build / Deploy](#build--deploy) + - [Elastic Buildkite PR Bot](#elastic-buildkite-pr-bot) + - [Overview](#overview-2) + - [Configuration](#configuration-1) + - [Build / Deploy](#build--deploy-1) + - [Infrastructure](#infrastructure) + - [Monitoring / Alerting](#monitoring--alerting) + - [Agent Image management](#agent-image-management) + - [Buildkite org-level settings management](#buildkite-org-level-settings-management) + - [IT Security Processes](#it-security-processes) +- [Drawbacks](#drawbacks) +- [Alternatives](#alternatives) + - [Jenkins](#jenkins) + - [Required](#required-2) + - [Scalable](#scalable-2) + - [Stable](#stable-2) + - [Updates](#updates) + - [Surfaces information intuitively](#surfaces-information-intuitively-2) + - [Pipelines](#pipelines-2) + - [Advanced Pipeline logic](#advanced-pipeline-logic-2) + - [Cloud-friendly pricing model](#cloud-friendly-pricing-model-2) + - [Public access](#public-access-2) + - [Secrets handling](#secrets-handling-2) + - [Support or Documentation](#support-or-documentation-2) + - [Scheduled Builds](#scheduled-builds-2) + - [Container support](#container-support-2) + - [Desired](#desired-2) + - [Customization](#customization-2) + - [Core functionality is first-party](#core-functionality-is-first-party-2) + - [First-class support for test results](#first-class-support-for-test-results-2) + - [GitHub Integration](#github-integration-2) + - [Other solutions](#other-solutions) + - [CircleCI](#circleci) + - [GitHub Actions](#github-actions) +- [Adoption strategy](#adoption-strategy) +- [How we teach this](#how-we-teach-this) + +# Summary + +Implement a CI system for Kibana teams that is highly scalable and stable, surfaces information in an intuitive way, and supports pipelines that are easy to understand and change. + +This table provides an overview of the conclusions made throughout the rest of this document. A lot of this is subjective, but we've tried to take an honest look at each system and feature, based on a large amount of research on and/or experience with each system, our requirements, and our preferences as a team. Your team would likely come to different conclusions based on your preferences and requirements. + +| | Jenkins | Buildkite | GitHub Actions | CircleCI | TeamCity | +| ------------------------------------ | ------- | --------- | -------------- | -------- | -------- | +| Scalable | No | Yes | No | Yes | No | +| Stable | No | Yes | No | Yes | Partial | +| Surfaces information intuitively | No | Yes | No | Yes | Yes | +| Pipelines | Yes | Yes | Yes | Yes | Partial | +| Advanced Pipeline logic | Yes | Yes | Partial | Partial | No | +| Cloud-friendly pricing model | Yes | Yes | Yes | No | No | +| Public access | Yes | Yes | Yes | Partial | Yes | +| Secrets handling | Yes | Partial | Yes | Partial | Partial | +| Support or Documentation | No | Yes | Yes | Partial | Yes | +| Scheduled Builds | Yes | Yes | Yes | Yes | Yes | +| Container support | Partial | Yes | Yes | Yes | Partial | +| | | | | | | +| Customization | No | Yes | No | No | No | +| Core functionality is first-party | No | Yes | Mostly | Yes | Mostly | +| First-class support for test results | Buggy | No | No | Yes | Yes | +| GitHub Integration | Yes | Limited | Yes | Yes | Yes | + +# Motivation + +We have lived with the scalability and stability problems of our current Jenkins infrastructure for several years. We have spent a significant amount of time designing around problems, and are limited in how we can design our pipelines. Since the company-wide effort to move to a new system has been cancelled for the foreseeable future, we are faced with either re-engineering the way we use Jenkins, or exploring other solutions and potentially managing one ourselves. + +This RFC is focused on the option of using a system other than Jenkins, and managing it ourselves (to the extent that it must be managed). If the RFC is rejected, the alternative will be to instead invest significantly into Jenkins to further stabilize and scale our usage of it. + +## Required and Desired Capabilities + +### Required + +#### Scalable + +- Able to run 100s of pipelines and 1000s of individual steps in parallel without issues. +- If scaling agents/hosts is self-managed, dynamically scaling up and down based on usage should be supported and reasonably easy to do. + +#### Stable + +- Every minute of downtime can affect 100s of developers. +- The Kibana Operations team can't have an on-call rotation, so we need to minimize our responsibilities around stability/uptime. +- For systems provided as a service, they should not have frequent outages. This is a bit hard to define. 1-2 hours of downtime, twice a month, during peak working hours, is extremely disruptive. 10 minutes of downtime once or twice a week can also be very disruptive, as builds might need to be re-triggered, etc. +- For self-hosted solutions, they should be reasonably easy to keep online and have a solution for high-availability. At a minimum, most upgrades should not require waiting for all currently running jobs to finish before deploying. +- Failures are ideally handled gracefully. For example, agents may continue running tasks correctly, once the primary service becomes available again. + +#### Surfaces information intuitively + +- Developers should be able to easily understand what happened during their builds, and find information related to failures. +- User interfaces should be functional and easy to use. +- Overview and details about failures and execution time are particularly important. + +#### Pipelines + +- Pipelines should be defined as code. +- Pipelines should be reasonably easy to understand and change. Kibana team members should be able to follow a simple guide and create new pipelines on their own. +- Changes to pipelines should generally be able to be tested in Pull Requests before being merged. + +#### Advanced Pipeline logic + +With such a large codebase and CI pipeline, we often have complex requirements around when and how certain tasks should run, and we want the ability to handle this built into the system we use. It can be very difficult and require complex solutions for fairly simple use cases when the system does not support advanced pipeline logic out of the box. + +For example, the flaky test suite runner that we currently have in Jenkins is fairly simple: run a given task (which might have a dependency) `N` number of times on `M` agents. This is very difficult to model in a system like TeamCity, which does not have dynamic dependencies. + +- Retries + - Automatic (e.g. run a test suite twice to account for flakiness) and manual (user-initiated) + - Full (e.g. a whole pipeline) and partial (e.g. a single step) +- Dynamic pipelines + - Conditional dependencies/steps + - Based on user input + - Based on external events/data (e.g. PR label) + - Based on source code or changes (e.g. only run this for .md changes) +- Metadata and Artifacts re-usable between tasks + - Metadata could be a docker image tag for a specific task, built from a previous step + +#### Cloud-friendly pricing model + +If the given system has a cost, the pricing model should be cloud-friendly and/or usage-based. + +A per-agent or per-build model based on peak usage in a month is not a good model, because our peak build times are generally short-lived (e.g. around feature freeze). + +A model based on build-minutes can also be bad, if it encourages running things in parallel on bigger machines to keep costs down. For example, running two tasks on a single 2-CPU machine with our own orchestration should not be cheaper than running two tasks on two 1-CPU machines using the system's built-in orchestration. + +#### Public access + +Kibana is a publicly-available repository with contributors from outside Elastic. CI information needs to be available publicly in some form. + +#### Secrets handling + +Good, first-class support for handling secrets is a must-have for any CI system. This support can take many forms. + +- Secrets should not need to be stored in plaintext, in a repo nor on the server. +- For systems provided as a service, it is ideal if secrets are kept mostly/entirely on our infrastructure. +- There should be protections against accidentally leaking secrets to the console. +- There should be programmatic ways to manage secrets. +- Secrets are, by nature, harder to handle. However, the easier the system makes it, the more likely people are to follow best practices. + +#### Support or Documentation + +For paid systems, both self-hosted and as a service, good support is important. If a problem specific to Elastic is causing us downtime, we expect quick and efficient support. Again, 100s of developers are potentially affected by downtime. + +For open source solutions, good documentation is especially important. If much of the operational knowledge of a system can only be gained by working with the system and/or reading the source code, it will be harder to solve problems quickly. + +#### Scheduled Builds + +We have certain pipelines (ES Snapshots) that run once daily, and `master` CI currently only runs once an hour. We need the ability to configure scheduled builds. + +#### Container support + +We have the desire to use containers to create fast, clean environments for CI stages that can also be used locally. We think that we can utilize [modern layer-caching options](https://github.com/moby/buildkit#cache), both local and remote, to optimize bootstrapping various CI stages, doing retries, etc. + +For self-hosted options, containers will allow us to utilize longer-running instances (with cached layers, git repos, etc) without worrying about polluting the build environment between builds. + +If we use containers for CI stages, when a test fails, developers can pull the image and reproduce the failure in the same environment that was used in CI. + +So, we need a solution that at least allows us to build and run our own containers. The more features that exist for managing this, the easier it will be. + +### Desired + +#### Customization + +We have very large CI pipelines which generate a lot of information (bundle sizes, performance numbers, etc). Being able to attach this information to builds, so that it lives with the builds in the CI system, is highly desirable. The alternative is building custom reports and UIs outside of the system. + +#### Core functionality is first-party + +Most core functionality that we depend on should be created and maintained by the organization maintaining the CI software. It's important for bugs to be addressed quickly, for security issues to be resolved, and for functionality to be tested before a new release of the system. In this way, there is a large amount of risk associated with relying on third-party solutions for too much core functionality. + +#### First-class support for test results + +One of the primary reasons we run CI is to run tests and make sure they pass. There are currently around 65,000 tests (unit, integration, and functional) that run in CI. Being able to see summaries, histories, and details of test execution directly on build pages is extremely useful. Flaky test identification is also very useful, as we deal with flaky tests on a daily basis. + +For example, being able to easily see that a build passed but included 5,000 tests fewer than the previous build can make something like a pipeline misconfiguration more obvious. Being able to click on a failed test and see other recent builds where the same test failed can help identify what kind of failure it is and how important it is to resolve it quickly (e.g is it failing in 75% of builds or 5% of builds?). + +For any system that doesn't have this kind of support, we will need to maintain our own solution, customize build pages to include this (if the system allows), or both. + +#### GitHub Integration + +- Ability to trigger jobs based on webhooks +- Integrate GitHub-specific information into UI, e.g. a build for a PR should link back to the PR +- Ability to set commit statuses based on job status +- Fine-grained permission handling for pull request triggering + +# Buildkite - Detailed design + +For the alternative system in this RFC, we are recommending Buildkite. The UI, API, and documentation have been a joy to work with, they provide most of our desired features and functionality, the team is responsive and knowledgeable, and the pricing model does not encourage bad practices to lower cost. + +## Overview + +[Buildkite](https://buildkite.com/home) is a CI system where the user manages and hosts their own agents, and Buildkite manages and hosts everything else (core services, APIs, UI). + +The [Buildkite features](https://buildkite.com/features) page is a great overview of the functionality offered. + +For some public instances of Buildkite in action, see: + +- [Bazel](https://buildkite.com/bazel) +- [Rails](https://buildkite.com/rails) +- [Chef](https://buildkite.com/chef-oss) + +## Required and Desired Capabilities + +How does Buildkite stack up against our required and desired capabilities? + +### Required + +#### Scalable + +Buildkite claims to support up to 10,000 connected agents "without breaking a sweat." + +We were able to connect 2,200 running agents and run a [single job with 1,800 parallel build steps](https://buildkite.com/elastic/kibana-custom/builds/8). The job ran with only about 15 seconds of total overhead (the rest of the time, the repo was being cloned, or the actual tasks were executing). We would likely never define a single job this large, but not only did it execute without any problems, the UI handles it very well. + +2,200 agents was the maximum that we were able to test because of quotas on our GCP account that could not easily be increased. + +We also created a job with 5 parallel steps, and triggered 300 parallel builds at once. The jobs executed and finished quickly, across ~1500 agents, with no issues and very little overhead. Interestingly, it seems that we were able to see the effects of our test in Buildkite's status page graphs (see below), but, from a user perspective, we were unable to notice any issues. + +![Status Graphs](../images/0018_buildkite_uptime.png) + +#### Stable + +So far, we have witnessed no stability issues in our testing. + +If Buildkite's status pages are accurate, they seem to be extremely stable, and respond quickly to issues. + +- [Buildkite Status](https://www.buildkitestatus.com/) +- [Historical Uptime](https://www.buildkitestatus.com/uptime) +- [Incident History](https://www.buildkitestatus.com/history) + +For agents, stability and availability will depend primarily on the infrastructure that we build and the availability of the cloud provider (GCP, primarily) running our agents. Since [we control our agents](#elastic-buildkite-agent-manager), we will be able to run agents across multiple zones, and possibly regions, in GCP for increased availability. + +They have a [99.95% uptime SLA](https://buildkite.com/enterprise) for Enterprise customers. + +#### Surfaces information intuitively + +The Buildkite UI is very easy to use, and works as expected. Here is some of the information surfaced for each build: + +- The overall status of the job, as well as which steps succeeded and failed. +- Logs for each individual step +- The timeline for each individual step, including how long it took Buildkite to schedule/handle the job on their end +- Artifacts uploaded by each step +- The entire agent/job configuration at the time the step executed, expressed as environment variables + +![Example Build](../images/0018_buildkite_build.png) + +Note that dependencies between steps are mostly not shown in the UI. See screenshot below for an example. There are several layers of dependencies between all of the steps in this pipeline. The only one that is shown is the final step (`Post All`), which executes after all steps beforehand are finished. There are some other strategies to help organize the steps (such as the new grouping functionality) if we need. + +![Dependencies](../images/0018_buildkite_deps.png) + +Buildkite has rich build page customization via "annotations" which will let us surface custom information. See the [customization section](#customization-1). + +#### Pipelines + +- [Buildkite pipelines](https://buildkite.com/docs/pipelines) must be defined as code. Even if you configure them through the UI, you still have to do so using yaml. +- This is subjective, but the yaml syntax for pipelines is friendly and straightforward. We feel that it will be easy for teams to create and modify pipelines with minimal instructions. +- If your pipeline is configured to use yaml stored in your repo for its definition, branches and PRs will use the version in their source by default. This means that PRs that change the pipeline can be tested as part of the PR CI. +- Top-level pipeline configurations, i.e. basically a pointer to a repo that has the real pipeline yaml in it, can be configured via the UI, API, or terraform. + +#### Advanced Pipeline logic + +Buildkite supports very advanced pipeline logic, and has support for generating dynamic pipeline definitions at runtime. + +- [Conditionals](https://buildkite.com/docs/pipelines/conditionals) +- [Dependencies](https://buildkite.com/docs/pipelines/dependencies) with lots of options, including being optional/conditional +- [Retries](https://buildkite.com/docs/pipelines/command-step#retry-attributes), both automatic and manual, including configuring retry conditions by different exit codes +- [Dynamic pipelines](https://buildkite.com/docs/pipelines/defining-steps#dynamic-pipelines) - pipelines can be generated by running a script at runtime +- [Metadata](https://buildkite.com/docs/pipelines/build-meta-data) can be set in one step, and read in other steps +- [Artifacts](https://buildkite.com/docs/pipelines/artifacts) can be uploaded from and downloaded in steps, and are visible in the UI +- [Parallelism and Concurrency](https://buildkite.com/docs/tutorials/parallel-builds) settings + +Here's an example of a dynamically-generated pipeline based on user input that runs a job `RUN_COUNT` times (from user input), across up to a maximum of 25 agents at once: + +```yaml +# pipeline.yml + +steps: + - input: 'Test Suite Runner' + fields: + - select: 'Test Suite' + key: 'test-suite' + required: true + options: + - label: 'Default CI Group 1' + value: 'default:cigroup:1' + - label: 'Default CI Group 2' + value: 'default:cigroup:2' + - text: 'Number of Runs' + key: 'run-count' + required: true + default: 75 + - wait + - command: .buildkite/scripts/flaky-test-suite-runner.sh | buildkite-agent pipeline upload + label: ':pipeline: Upload' +``` + +```bash +#!/usr/bin/env bash + +# flaky-test-suite-runner.sh + +set -euo pipefail + +TEST_SUITE="$(buildkite-agent meta-data get 'test-suite')" +export TEST_SUITE + +RUN_COUNT="$(buildkite-agent meta-data get 'run-count')" +export RUN_COUNT + +UUID="$(cat /proc/sys/kernel/random/uuid)" +export UUID + +cat << EOF +steps: + - command: | + echo 'Bootstrap' + label: Bootstrap + agents: + queue: bootstrap + key: bootstrap + - command: | + echo 'Build Default Distro' + label: Build Default Distro + agents: + queue: bootstrap + key: default-build + depends_on: bootstrap + - command: 'echo "Running $TEST_SUITE"; sleep 10;' + label: 'Run $TEST_SUITE' + agents: + queue: ci-group + parallelism: $RUN_COUNT + concurrency: 25 + concurrency_group: '$UUID' + depends_on: default-build +EOF +``` + +#### Cloud-friendly pricing model + +Buildkite is priced using a per-user model, where a user is effectively an Elastic employee triggering builds for Kibana via PR, merging code, or through the Buildkite UI. That means that the cost essentially grows with our company size. Most importantly, we don't need to make CI pipeline design decisions based on the Buildkite pricing model. + +However, since we manage our own agents, we will still pay for our compute usage, and will need to consider that cost when designing our pipelines. + +#### Public access + +Buildkite has read-only public access, configurable for each pipeline. An organization can contain a mix of both public and private pipelines. + +There are not fine-grained settings for this, and all information in the build is publicly accessible. + +#### Secrets handling + +[Managing Pipeline Secrets](https://buildkite.com/docs/pipelines/secrets) + +Because agents run on customers' infrastructure, secrets can stay completely in the customer's environment. For this reason, Buildkite doesn't provide a real mechanism for storing secrets, and instead provide recommendations for accessing secrets in pipelines in secure ways. + +There are two recommended methods for handling secrets: using a third-party secrets service like Vault or GCP's Secret Manager, or baking them into agent images and only letting certain jobs access them. Since Elastic already uses Vault, we could utilize Vault the same way we do in Jenkins today. + +Also, a new experimental feature, [redacted environment variables](https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables) can automatically redact the values of environment variables that match some configurable suffixes if they are accidentally written to the console. This would only redact environment variables that were set prior to execution of a build step, e.g. during the `environment` or `pre-command` hooks, and not variables that were created during execution, e.g. by accessing Vault in the middle of a build step. + +#### Support or Documentation + +[Buildkite's documentation](https://buildkite.com/docs/pipelines) is extensive and well-written, as mentioned earlier. + +Besides this, [Enterprise](https://buildkite.com/enterprise) customers get 24/7 emergency help, prioritized support, a dedicated chat channel, and guaranteed response times. They will also consult on best practices, etc. + +#### Scheduled Builds + +[Buildkite has scheduled build](https://buildkite.com/docs/pipelines/scheduled-builds) support with a cron-like syntax. Schedules are defined separately from the pipeline yaml, and can be managed via the UI, API, or terraform. + +#### Container support + +Since we will manage our own agents with Buildkite, we have full control over the container management tools we install and use. In particular, this means that we can easily use modern container tooling, such as Docker with Buildkit, and we can pre-cache layers or other data in our agent images. + +[Buildkite maintains](https://buildkite.com/docs/tutorials/docker-containerized-builds) two officially-supported plugins for making it easier to create pipelines using containers: [one for Docker](https://github.com/buildkite-plugins/docker-buildkite-plugin) and [one for Docker Compose](https://github.com/buildkite-plugins/docker-compose-buildkite-plugin). + +The Docker plugin is essentially a wrapper around `docker run` that makes it easier to define steps that run in containers, while setting various flags. It also provides some logging, and provides mechanisms for automatically propagating environment variables or mounting the workspace into the container. + +A simple, working example for running Jest tests using a container is below. The `Dockerfile` contains all dependencies for CI, and runs `yarn kbn bootstrap` so that it contains a full environment, ready to run tasks. + +```yaml +steps: + - command: | + export DOCKER_BUILDKIT=1 && \ + docker build -t gcr.io/elastic-kibana-184716/buildkite/ci/base:$BUILDKITE_COMMIT -f .ci/Dockerfile . --progress plain && \ + docker push gcr.io/elastic-kibana-184716/buildkite/ci/base:$BUILDKITE_COMMIT + - wait + - command: node scripts/jest --ci --verbose --maxWorkers=6 + label: 'Jest' + artifact_paths: target/junit/**/*.xml + plugins: + - docker#v3.8.0: + image: 'gcr.io/elastic-kibana-184716/buildkite/ci/base:$BUILDKITE_COMMIT' + propagate-environment: true + mount-checkout: false + parallelism: 2 + timeout_in_minutes: 120 +``` + +### Desired + +#### Customization + +We have very large CI pipelines which generate a lot of information (bundle sizes, performance numbers, etc). Being able to attach this information to builds, so that it lives with the builds in the CI system, is highly desirable. The alternative is building custom reports and UIs outside of the system. + +[Annotations](https://buildkite.com/docs/agent/v3/cli-annotate) provide a way to add rich, well-formatted, custom information to build pages using CommonMark Markdown. There are several built-in CSS classes for formatting and several visual styles. Images, emojis, and links can be embedded as well. Just for some examples: Metrics such as bundle sizes, links to the distro builds for that build, and screenshots for test failures could all be embedded directly into the build pages. + +The structure of logs can also be easily customized by adding [collapsible groups](https://buildkite.com/docs/pipelines/managing-log-output#collapsing-output) for log messages. + +#### Core functionality is first-party + +There's a large number of [plugins for Buildkite](https://buildkite.com/plugins), but, so far, there are only two plugins we've been considering using (one for Docker and one for test results), and they're both maintained by Buildkite. All other functionality we've assessed that we need is either built directly into Buildkite, or [we are building it](#what-we-will-build-and-manage). + +#### First-class support for test results + +Buildkite doesn't really have any built-in support specifically for handling test results. Test result reports (e.g. JUnit) can be uploaded as artifacts, and test results can be rendered on the build page using annotations. They have [a plugin](https://github.com/buildkite-plugins/junit-annotate-buildkite-plugin) for automatically annotating builds with test results from JUnit reports in a simple fashion. We would likely want to build our own annotation for this. + +This does mean that Buildkite lacks test-related features of other CI systems: tracking tests over time across build, flagging flaky tests, etc. We would likely need to ingest test results into Elasticsearch and build out Kibana dashboards/visualizations for this, or similar. + +#### GitHub Integration + +Buildkite's [GitHub Integration](https://buildkite.com/docs/integrations/github) can trigger builds based on GitHub webhooks (e.g. on commit/push for branches and PRs), and update commit statuses. Buildkite also adds basic information to build pages, such as links to commits on GitHub and links to PRs. This should cover what we need for tracked branch builds. + +However, for Pull Requests, because we have a lot of requirements around when builds should run and who can run them, we will need to [build a solution](#elastic-buildkite-pr-bot) for handling PRs ourselves. The work for this is already close to complete. + +## What we will build and manage + +### Elastic Buildkite Agent Manager + +#### Overview + +Currently, with Buildkite, the agent lifecycle is managed entirely by customers. Customers can run "static" workers that are online all of the time, or dynamically scale their agents up and down as needed. + +For AWS, Buildkite maintains an auto-scaling solution called [Elastic CI Stack for AWS](https://github.com/buildkite/elastic-ci-stack-for-aws). + +Since, we primarily need support for GCP, we built our own agent manager. It's not 100% complete, but has been working very well during our testing/evaluation of Buildkite, and can handle 1000s of agents. + +[Elastic Buildkite Agent Manager](https://github.com/brianseeders/buildkite-agent-manager) + +Features: + +- Handles many different agent configurations with one instance +- Configures long-running agents, one-time use agents, and agents that will terminate after being idle for a configured amount of time +- Configures both minimum and maximum agent limits - i.e. can ensure a certain number of agents are always online, even if no jobs currently require them +- Supports overprovisioning agents by a percentage or a fixed number +- Supports many GCE settings: zone, image/image family, machine type, disk type and size, tags, metadata, custom startup scripts +- Agent configuration is stored in a separate repo and read at runtime +- Agents are gracefully replaced (e.g. after they finish their current job) if they are running using an out-of-date agent configuration that can affect the underlying GCE instance +- Detect and remove orphaned GCP instances +- Handles 1000s of agents (tested with 2200 before we hit GCP quotas) +- Does instance creation/deletion in large, parallel batches so that demand spikes are handled quickly + +Also planned: + +- Balance creating agents across numerous GCP zones for higher availability +- Automatically gracefully replace agents if disk usage gets too high +- Scaling idle timeouts: e.g. the first agent for a configuration might have an idle timeout of 1 hour, but the 200th might be 5 minutes + +#### Design + +The agent manager is primarily concerned with ensuring that, given an agent configuration, the number of online agents for that configuration is **greater than or equal to** the desired number. Buildkite then determines how to use the agents: which jobs they should execute and when they should go offline (due to being idle, done with jobs, etc). Even when stopping agents due to having an outdated configuration, Buildkite still determines the actual time that the agent should disconnect. + +The current version of the agent manager only handles GCP-based agents, but support for other platforms could be added as well, such as AWS or Kubernetes. There's likely more complexity in managing all of the various agent images than in maintaining support in the agent manager. + +It is also designed to itself be stateless, so that it is easy to deploy and reason about. State is effectively stored in GCP and Buildkite. + +![High-Level Design](../images/0018_agent_manager.png) + +The high-level design for the agent manager is pretty straightforward. There are three primary stages during execution: + +1. Gather Current State + 1. Data and agent configuration is gathered from various sources/APIs in parallel +2. Create Plan + 1. Given the current state across the various services, a plan is created based on agent configurations, current Buildkite job queue sizes, and current GCE instances. + 2. Instances need to be created when there aren't enough online/in-progress agents of a particular configuration to satisfy the needs of its matching queue. + 3. Agents need to be stopped when the agents have been online for too long (based on their configuration) or when their configuration is out-of-date. This is a soft stop, they will terminate after finishing their current job. + 4. Instances need to be deleted if they have been stopped (which happens when their agent stops), or when they have been online past their hard stop time (based on configuration). +3. Execute Plan + 1. The different types of actions in the plan are executed in parallel. Instance creating and deleting is done in batches to handle spikes quickly. + +An error at any step, e.g. when checking current state of GCP instances, will cause the rest of the run to abort. + +Because the service gathers data about the total current state and creates a plan based on that state each run, it's reasonably resistant to errors and it's self-healing. + +##### Protection against creating too many instances + +Creating too many instances in GCP could be costly, so it is worth mentioning here. Since the agent manager itself is stateless, and only looks at the current, external state when determining an execution plan, there is the possibility of creating too many instances. + +There are two primary mechanisms to protect against this: + +One is usage of GCP quotas. Maintaining reasonable GCP quotas will ensure that we don't create too many instances in a situation where something goes catastrophically wrong during operation. It's an extra failsafe. + +The other is built into the agent manager. The agent manager checks both the number of connected agents in Buildkite for a given configuration, as well as the number of instances currently running and being created in GCP. It uses whichever number is greater as the current number of instances. + +This is a simple failsafe, but means that a large number of unnecessary instances should only be able to be created in a pretty specific scenario (keep in mind that errors will abort the current agent manager run): + +- The GCP APIs (both read and create) are returning success codes +- The GCP API for listing instances is returning partial/missing/erroneous data, with a success code +- GCP instances are successfully being created +- Created GCP instances are unable to connect to Buildkite, or Buildkite Agents API is returning partial/missing/erroneous data + +All of these things would need to be true at the same time for a large number of instances to be created. In the unlikely event that that were to happen, the GCP quotas would still be in-place. + +#### Configuration + +Here's an example configuration, which would likely reside in the `master` branch of the kibana repository. + +```js +{ + gcp: { + // Configurations at this level are defaults for all configurations defined under `agents` + project: 'elastic-kibana-184716', + zone: 'us-central1-b', + serviceAccount: 'elastic-buildkite-agent@elastic-kibana-184716.iam.gserviceaccount.com', + agents: [ + { + queue: 'default', + name: 'kibana-buildkite', + overprovision: 0, // percentage or flat number + minimumAgents: 1, + maximumAgents: 500, + gracefulStopAfterSecs: 60 * 60 * 6, + hardStopAfterSecs: 60 * 60 * 9, + idleTimeoutSecs: 60 * 60, + exitAfterOneJob: false, + imageFamily: 'kibana-bk-dev-agents', + machineType: 'n2-standard-1', + diskType: 'pd-ssd', + diskSizeGb: 75 + }, + { + // ... + }, + } +} +``` + +#### Build / Deploy + +Currently, the agent manager is built and deployed using [Google Cloud Build](https://cloud.google.com/build). It is deployed to and hosted using [GKE Auto-Pilot](https://cloud.google.com/blog/products/containers-kubernetes/introducing-gke-autopilot) (Kubernetes). GKE was used, rather than Cloud Run, primarily because the agent manager runs continuously (with a 30sec pause between executions) whereas Cloud Run is for services that respond to HTTP requests. + +It uses [Google Secret Manager](https://cloud.google.com/secret-manager) for storing/retrieving tokens for accessing Buildkite. It uses a GCP service account and [Workload Identity](https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity) to manage GCP resources. + +### Elastic Buildkite PR Bot + +#### Overview + +For TeamCity, we built a bot that was going to handle webhooks from GitHub and trigger builds for PRs based on configuration, user permissions, etc. Since we will not be moving to TeamCity, we've repurposed this bot for Buildkite, since Buildkite does not support all of our requirements around triggering builds for PRs out-of-the-box. The bot supports everything we currently use in Jenkins, and has some additional features as well. + +[Elastic Buildkite PR Bot](https://github.com/elastic/buildkite-pr-bot) + +Features supported by the bot: + +- Triggering builds on commit / when the PR is opened +- Triggering builds on comment +- Permissions for who can trigger builds based on: Elastic org membership, write and/or admin access to the repo, or user present in an allowed list +- Limit builds to PRs targeting a specific branch +- Custom regex for trigger comment, e.g. "buildkite test this" +- Triggering builds based on labels +- Setting labels, comment body, and other PR info as env vars on triggered build +- Skip triggering build if a customizable label is present +- Option to set commit status on trigger +- Capture custom arguments from comment text using capture groups and forward them to the triggered build + +#### Configuration + +The configuration is stored in a `json` file (default: `.ci/pull-requests.json`) in the repo for which pull requests will be monitored. Multiple branches in the repo can store different configurations, or one configuration (e.g. in `master`) can cover the entire repo. + +Example configuration: + +```json +{ + "jobs": [ + { + "repoOwner": "elastic", + "repoName": "kibana", + "pipelineSlug": "kibana", + + "enabled": true, + "target_branch": "master", + "allow_org_users": true, + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": ["renovate[bot]"], + "set_commit_status": true, + "commit_status_context": "kibana-buildkite", + "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^retest$" + } + ] +} +``` + +Github Webhooks must also be configured to send events to the deployed bot. + +#### Build / Deploy + +Currently, the bot is built and deployed using [Google Cloud Build](https://cloud.google.com/build). It is deployed to and hosted on [Google Cloud Run](https://cloud.google.com/run). It uses [Google Secret Manager](https://cloud.google.com/secret-manager) for storing/retrieving tokens for accessing GitHub and Buildkite. + +[Build/deploy configuration](https://github.com/elastic/buildkite-pr-bot/blob/main/cloudbuild.yaml) + +### Infrastructure + +We will need to maintain our infrastructure related to Buildkite, primarily ephemeral agents. To start, it will mean supporting infrastructure in GCP, but could later mean AWS as well. + +- Separate GCP project for CI resources +- Hosting for bots/services we maintain, such as the Agent Manager (GKE Auto-Pilot) and GitHub PR bot (Cloud Run) +- Google Storage Buckets for CI artifacts +- Networking (security, we may also need Cloud NAT) +- IAM and Security +- Agent images + +We are already using Terraform to manage most resources related to Buildkite, and will continue to do so. + +### Monitoring / Alerting + +We will need to set up and maintain monitoring and alerting for our GCP infrastructure, as well as Buildkite metrics. + +Some examples: + +GCP + +- Number of instances by type +- Age of instances +- Resource Quotas + +Buildkite + +- Agent queues +- Job wait times +- Build status + +### Agent Image management + +We will need to maintain images used to create GCP instances for our Buildkite agents. These images would need to be built on a regular basis (daily, or possibly more often). + +We could likely maintain a single linux-based image to cover all of our current CI needs. However, in the future, if we need to maintain many images across different operating systems and architectures, this is likely to become the most complex part of the CI system that we would need to maintain. Every operating system and architecture we need to support adds another group of required images, with unique dependencies and configuration automation. + +Another thing to note: Just because we need to run something on a specific OS or architecture, it doesn't necessarily mean we need to maintain an agent image for it. For example, we might use something like Vagrant to create a separate VM, using the default, cloud-provided images, that we run something on (e.g. for testing system packages), rather than running it on the same machine as the agent. In this case, we would potentially only be managing a small number of images, or even a single image. + +Also, we always have the option of running a small number of jobs using Jenkins, if we need to do so to target additional OSes and architectures. + +For our testing, we have a single GCP image, [built using Packer](https://github.com/elastic/kibana/tree/kb-bk/.buildkite/agents/packer), with the Buildkite agent installed and all of our dependencies. + +Summary of Responsibilities + +- An automated process for creating new images, at least daily, running automated smoke tests against them, and promoting them +- Delete old images when creating new ones +- Ability to roll back images easily and/or pin specific image versions +- Manage dependencies, failures, updates, etc across all supported OSes and architectures, on a regular basis + +### Buildkite org-level settings management + +There are a few settings outside of pipelines that we will need to manage. + +- Top-level pipelines and their settings +- Pipeline schedules / scheduled jobs +- Public visibility of pipelines +- Teams and Permissions +- Single Sign On settings + +Most of the content for our pipelines will be stored in repositories as YAML. However, a job still must exist in Buildkite that points to that repo and that YAML. For managing those top-level configurations, an official [Terraform provider](https://registry.terraform.io/providers/buildkite/buildkite/latest/docs/resources/pipeline) exists, which we will likely take advantage of. + +Pipeline schedules can also be managed using the Terraform provider. + +Teams can also be managed using Terraform, but it's unlikely we will need to use Teams. + +For everything else, we will likely start off using UI and build automation (or contribute to the Terraform provider) where we see fit. Most of the other settings are easy to configure, and unlikely to change. + +### IT Security Processes + +There will likely be numerous IT Security processes we will need to follow, since we will be managing infrastructure. This could include regular audits, specific software and configurations that must be baked into our agents, documentation procedures, or other conditions that we will need to satisfy. There is risk here, as the processes and workload are currently unknown to us. + +# Drawbacks + +The biggest drawback to doing this is that we will be duplicating a large amount of work and providing/maintaining a service that is already provided to us by another team at Elastic. Jenkins is already provided to us, and there is automation for creating Jenkins worker images and managing worker instances in both AWS and GCP, and IT Security policies are already being handled for all of this. It is hard to predict what the extra workload will be for the Kibana Operations team if we move our CI processes to Buildkite, but we know we will have to maintain all of the things listed under [What we will build and manage](#what-we-will-build-and-manage). + +Some other drawbacks: + +- CI Pipelines and other jobs built in Jenkins will need to be re-built, which includes building support for things like CI Stats, Slack notifications, GitHub PR comments, etc. +- Developers will need to learn a new system. +- The service is an additional cost to the company. +- There is a lot of Jenkins knowledge throughout the company, but likely little Buildkite knowledge. + +# Alternatives + +## Jenkins + +We are not happy with the experience provided by our instance of Jenkins and our current pipelines. If we stick with Jenkins, we will need to invest a likely significant amount of time in improving the experience and making our pipelines scale given the limitations we face. + +### Required + +#### Scalable + +Our current Jenkins instance only allows for 300-400 connected agents, before effectively going offline. We have struggled with this issue for several years, and completely redesigned our pipelines around this limitation. The resulting design, which involves running 20+ tasks in parallel on single, large machines, and managing all of the concurrency ourselves, is complicated and problematic. + +Other teams at Elastic, especially over the last few months, have been experiencing this same limitation with their Jenkins instances as well. The team that manages Jenkins at Elastic is well aware of this issue, and is actively investigating. It is currently unknown whether or not it is a solvable problem (without sharding) or a limitation of Jenkins. + +#### Stable + +Firstly, Jenkins was not designed for high availability. If the primary/controller goes offline, CI is offline. + +The two biggest sources of stability issues for us are currently related to scaling (see above) and updates. + +##### Updates + +The typical update process for Jenkins looks like this: + +- Put Jenkins into shutdown mode, which stops any new builds from starting +- Wait for all currently-running jobs to finish +- Shutdown Jenkins +- Do the update +- Start Jenkins + +For us, shutdown mode also means that `gobld` stops creating new agents for our jobs. This means that many running jobs will never finish executing while shutdown mode is active. + +So, for us, the typical update process is: + +- Put Jenkins into shutdown mode, which stops any new builds from starting, and many from finishing +- Hard kill all of our currently running jobs +- Shutdown Jenkins +- Do the update +- Start Jenkins +- A human manually restarts CI for all PRs that were running before the update + +This is pretty disruptive for us, as developers have to wait several hours longer before merging or seeing the status of their PRs, plus there is manual work that must be done to restart CI. If we stay with Jenkins, we'll need to fix this process, and likely build some automation for it. + +#### Surfaces information intuitively + +Our pipelines are very complex, mainly because of the issues mentioned above related to designing around scaling issues, and none of the UIs in Jenkins work well for us. + +The [Stage View](https://kibana-ci.elastic.co/job/elastic+kibana+pipeline-pull-request) only works for very simple pipelines. Even if we were able to re-design our pipelines to populate this page better, there are just too many stages to display in this manner. + +[Blue Ocean](https://kibana-ci.elastic.co/blue/organizations/jenkins/elastic%2Bkibana%2Bpipeline-pull-request/activity), which is intended to be the modern UI for Jenkins, doesn't work at all for our pipelines. We have nested parallel stages in our pipelines, which [are not supported](https://issues.jenkins.io/browse/JENKINS-54010). + +[Pipeline Steps](https://kibana-ci.elastic.co/job/elastic+kibana+pipeline-pull-request/) (Choose a build -> Pipeline Steps) shows information fairly accurately (sometimes logs/errors are not attached to any steps, and do not show), but is very difficult to read. There are entire pages of largely irrelevant information (Setting environment variables, starting a `try` block, etc), which is difficult to read through, especially developers who don't interact with Jenkins every day. + +![Pipeline Steps](../images/0018_jenkins_pipeline_steps.png) + +We push a lot of information to GitHub and Slack, and have even built custom UIs, to try to minimize how much people need to interact directly with Jenkins. In particular, when things go wrong, it is very difficult to investigate using the Jenkins UI. + +#### Pipelines + +Jenkins supports pipeline-as-code through [Pipelines](https://www.jenkins.io/doc/book/pipeline), which we currently use. + +Pros: + +- Overall pretty powerful, pipelines execute Groovy code at runtime, so pipelines can do a lot and can be pretty complex, if you're willing to write the code +- Pipeline changes can be tested in PRs +- Shared Libraries allow shared code to be used across pipelines easily + +Cons: + +- The sandbox is pretty difficult to work with. There's a [hard-coded list](https://github.com/jenkinsci/script-security-plugin/tree/e99ba9cffb0502868b05d19ef5cd205ca7e0e5bd/src/main/resources/org/jenkinsci/plugins/scriptsecurity/sandbox/whitelists) of allowed methods for pipelines. Other methods must be approved separately, or put in a separate shared repository that runs trusted code. +- Pipeline code is serialized by Jenkins, and the serialization process leads to a lot of issues that are difficult to debug and reason about. See [JENKINS-44924](https://issues.jenkins.io/browse/JENKINS-44924) - `List.sort()` doesn't work and silently returns `-1` instead of a list +- Reasonably complex pipelines are difficult to view in the UI ([see above](#surfaces-information-intuitively-2)) +- Using Pipelines to manage certain configurations (such as Build Parameters) requires running an outdated job once and letting it fail to update it +- Jobs that reference a pipeline have to be managed separately. Only third-party tools exist for managing these jobs as code (JJB and Job DSL). +- Very difficult to test code without running it live in Jenkins + +#### Advanced Pipeline logic + +See above section. Jenkins supports very advanced pipeline logic using scripted pipelines and Groovy. + +#### Cloud-friendly pricing model + +Given that Jenkins is open-source, we pay only for infrastructure and people to manage it. + +#### Public access + +- Fine-grained authorization settings +- Anonymous user access +- Per-job authorization, so some jobs can be private + +#### Secrets handling + +- Supports [Credentials](https://www.jenkins.io/doc/book/using/using-credentials/), which are stored encrypted on disk and have authorization settings + - Credentials are difficult to manage in an automated way +- Pipeline support for accessing credentials +- Credentials masked in log output +- Support for masking custom values in log output + +#### Support or Documentation + +Documentation for Jenkins is notoriously fragmented. All major functionality is provided in plugins, and documentation is spread out across the Jenkins Handbook, the CloudBees website, JIRA issues, wikis, GitHub repos, JavaDoc pages. Many plugins have poor documentation, and source code often has to be read to understand how to configure something. + +CloudBees offers paid support, but we're not familiar with it at this time. + +#### Scheduled Builds + +Jenkins supports scheduled builds via a Cron-like syntax, and can spread scheduled jobs out. For example, if many jobs are scheduled to run every day at midnight, a syntax is available that will automatically spread the triggered jobs evenly out across the midnight hour. + +#### Container support + +Jenkins has support for using Docker to [run containers for specific stages in a Pipeline](https://www.jenkins.io/doc/book/pipeline/docker/). It is effectively a wrapper around `docker run`. There are few conveniences, and figuring out how to do things like mount the workspace into the container is left up to the user. There are also gotchas that are not well-documented, such as the fact that the user running inside the container will be automatically changed using `-u`, which can cause issues. + +Though we have control over the agents running our jobs at Elastic, and thus all of the container-related tooling, it is not currently easy for the Operations team to manage our container tooling. We are mostly dependent on another team to do this for us. + +### Desired + +#### Customization + +The only way to customize information added to build pages is through custom plugins. [Creating and maintaining plugins for Jenkins](https://www.jenkins.io/doc/developer/plugin-development/) is a fairly significant investment, and we do not currently have a good way to manage plugins for Jenkins instances at Elastic. It's a pretty involved process that, at the moment, has to be done by another team. + +Given that, we feel we would be able to build a higher-quality experience in less time by creating custom applications separate from Jenkins, which we have actually [done in the past](https://ci.kibana.dev/es-snapshots). + +#### Core functionality is first-party + +Jenkins is very modular, and almost all Jenkins functionality is provided by plugins. + +It's difficult to understand which plugins are required to support which base features. For example, Pipelines support is provided by a group of many plugins, and many of them have outdated names ([Pipeline: Nodes and Processes](https://github.com/jenkinsci/workflow-durable-task-step-plugin) is actually a plugin called `workflow-durable-task-step-plugin`). + +Many plugins are maintained by CloudBees employees, but it can be very difficult to determine which ones are, without knowing the names of CloudBees employees. All Jenkins community/third-party plugins reside under the `jenkinsci` organization in GitHub, which makes finding "official" ones difficult. Given the open source nature of the Jenkins ecosystem and the way that development is handled by Cloudbees, it might be incorrect to say that any plugins outside of the Cloudbees plugins (for the Cloudbees Jenkins distribution) are "first-party". + +#### First-class support for test results + +It's a bit buggy at times (for example, if you run the same test multiple times, you have to load pages in a specific order to see the correct results in the UI), but Jenkins does have support for ingesting and displaying test results, including graphs that show changes over time. We use this feature to ingest test results from JUnit files produced by unit tests, integration tests, and end-to-end/functional tests. + +#### GitHub Integration + +Jenkins has rich support for GitHub spread across many different plugins. It can trigger builds in response to webhook payloads, automatically create jobs for repositories in an organization, has support for self-hosted GitHub, and has many settings for triggering pull requests. + +It's worth mentioning, however, that we've had and continue to have many issues with these integrations. For example, the GitHub Pull Request Builder plugin, which currently provides PR triggering for us and other teams, has been the source of several issues at Elastic. It's had performance issues, triggers builds erroneously, and has been mostly unmaintained for several years. + +## Other solutions + +### CircleCI + +CircleCI is a mature, widely-used option that is scalable and fulfills a lot of our requirements. We felt that we could create a good CI experience with this solution, but it had several disadvantages for us compared to Buildkite: + +- The pricing model for self-hosted runners felt punishing for breaking CI into smaller tasks +- Public access to build pages is gated behind a login, and gives CircleCI access to your private repos by default +- There are no customization options for adding information to build pages +- Options for advanced pipeline logic are limited compared to other solutions + +### GitHub Actions + +GitHub Actions is an interesting option, but it didn't pass our initial consideration round for one main reason: scalability. + +To ensure we're able to run the number of parallel tasks that we need to run, we'll have to use self-hosted runners. Self-hosted runners aren't subject to concurrency limits. However, managing auto-scaling runners seems to be pretty complex at the moment, and GitHub doesn't seem to have any official guidance on how to do it. + +Also, even with self-hosted runners, there is a 1,000 API request per hour hard limit, though it does not specify which APIs. Assuming even that 1 parallel step in a job is one API request, given the large number of small tasks that we'd like to split our CI into, we will likely hit this limit pretty quickly. + +# Adoption strategy + +We have already done a lot of the required legwork to begin building and running pipelines in Buildkite, including getting approval from various business groups inside Elastic. After all business groups have signed off, and a deal has been signed with Buildkite, we can begin adopting Buildkite. A rough plan outline is below. It's not meant to be a full migration plan. + +- Build minimal supporting services, automation, and pipelines to migrate a low-risk job from Jenkins to Buildkite (e.g. "Baseline" CI for tracked branches) + - The following will need to exist (some of which has already been built) + - New GCP project for infrastructure, with current implementations migrated + - Agent Manager + - Agent image build/promote + - Slack notifications for failures (possibly utilize Buildkite's built-in solution) + - The Buildkite pipeline and supporting code + - Run the job in parallel with Jenkins until we have confidence that it's working well + - Turn off the Jenkins version +- Build, test, migrate the next low-risk pipelines: ES Snapshot and/or Flaky Test Suite Runner +- Build, test, migrate tracked branch pipelines +- Build, test, migrate PR pipelines + - Will additionally need PR comment support + - PR pipelines are the most disruptive if there are problems, so we should have a high level of confidence before migrating + +# How we teach this + +The primary way that developers interact with Jenkins/CI today is through pull requests. Since we push a lot of information to pull requests via comments, developers mostly only need to interact with Jenkins when something goes wrong. + +The Buildkite UI is simple and intuitive enough that, even without documentation, there would likely be a pretty small learning curve to navigating the build page UI that will be linked from PR comments. That's not to say we're not going to provide documentation, we just think it would be easy even without it! + +We would also like to provide simple documentation that will guide developers through setting up new pipelines without our help. Getting a new job up and running with our current Jenkins setup is a bit complicated for someone who hasn't done it before, and there isn't good documentation for it. We'd like to change that if we move to Buildkite. + +To teach and inform, we will likely do some subset of these things: + +- Documentation around new CI pipelines in Buildkite +- Documentation on how to handle PR failures using Buildkite +- Documentation on the new infrastructure, supporting services, etc. +- Zoom sessions with walkthrough and Q&A +- E-mail announcement with links to documentation +- Temporarily add an extra message to PR comments, stating the change and adding links to relevant documentation diff --git a/rfcs/text/0018_timeslider.md b/rfcs/text/0018_timeslider.md new file mode 100644 index 0000000000000..aa8e7263260d2 --- /dev/null +++ b/rfcs/text/0018_timeslider.md @@ -0,0 +1,217 @@ +- Start Date: 2020-04-26 +- RFC PR: (leave this empty) +- Kibana Issue: (leave this empty) + +--- +- [1. Summary](#1-summary) +- [2. Detailed design](#2-detailed-design) +- [3. Unresolved questions](#3-unresolved-questions) + +# 1. Summary + +A timeslider is a UI component that allows users to intuitively navigate through time-based data. + +This RFC proposes adding a timeslider control to the Maps application. + +It proposes a two phased roll-out in the Maps application. The [design proposal](#2-detailed-design) focuses on the first phase. + +Since the timeslider UI is relevant for other Kibana apps, the implementation should be portable. We propose to implement the control as a React-component +without implicit dependencies on Kibana-state or Maps-state. + +The RFC details the integration of this control in Maps. It includes specific consideration to how timeslider affects data-refresh in the context of Maps. + +This RFC also outlines a possible integration of this Timeslider-React component with an Embeddable, and the introduction of a new piece of embeddable-state `Timeslice`. + +This RFC does not address how this component should _behave_ in apps other than the Maps-app. + +# 2. Detailed design + +Below outlines: +- the two delivery phases intended for Kibana Maps +- outline of the Timeslider UI component implementation (phase 1) +- outline of the integration in Maps of the Timeslider UI component (phase 1) + +## 2.1 Design phases overview + + + +### 2.1.1 Time-range selection and stepped navigation + +A first phase includes arbitrary time-range selection and stepped navigation. + +![Timeslider version 1](../images/timeslider/v1.png) + +This is the focus of this RFC. + +Check [https://github.com/elastic/kibana/pull/96791](https://github.com/elastic/kibana/pull/96791) for a POC implementation. + +### 2.2.2 Data distribution preview with histogram and playback + +A second phase adds a date histogram showing the preview of the data. + +![Timeslider version 2](../images/timeslider/v2.png) + +Details on this phase 2 are beyond the scope of this RFC. + +## 2.2 The timeslider UI React-component (phase 1) + +This focuses on Phase 1. Phase 2, with date histogram preview and auto-playback is out of scope for now. + +### 2.2.1 Interface of the React-component + +The core timeslider-UI is a React-component. + +The component has no implicit dependencies on any Kibana-state or Maps-store state. + +Its interface is fully defined by its `props`-contract. + +``` +export type Timeslice = { + from: number; // epoch timestamp + to: number; // epoch timestamp +}; + +export interface TimesliderProps { + onTimesliceChanged: (timeslice: Timeslice) => void; + timerange: TimeRange; // TimeRange The start and end time of the entire time-range. TimeRange is defined in `data/common` + timeslice?: Timeslice; // The currently displayed timeslice. Needs to be set after onTimesliceChange to be reflected back in UI. If ommitted, the control selects the first timeslice. +} +``` + +`timeslice` is clamped to the bounds of `timeRange`. + +Any change to `timeslice`, either by dragging the handles of the timeslider, or pressing the back or forward buttons, calls the `onTimesliceChanged` handler. + +Since the initial use is inside Maps, the initial location of this React-component is inside the Maps plugin, `x-pack/plugins/maps/public/timeslider`. + +Nonetheless, this UI-component should be easily "cut-and-pastable" to another location. + +### 2.2.2 Internals + +The timeslider automatically subdivides the timerange with equal breaks that are heuristically determined. + +It assigns about 6-10 breaks within the `timerange`, snaps the "ticks" to a natural "pretty date" using `calcAutoIntervalNear` from `data/common`. + +For example; +- a `timerange` of 8.5 days, it would assign a 8 day-steps, plus some padding on either end, depending on the entire `timerange`. +- a `timerange` of 6.9 years would snap to year-long step, plus some padding on either end, depending on the entire `timerange`. + +The slider itself is a ``. + +### 2.2.2 End-user behavior + +- the user can manipulate the `timeslice`-double ranged slider to any arbitrary range within `timerange`. +- the user can press the forward and back buttons for a stepped navigation. The range of the current time-slice is preserved when there is room for the `timeslice` within the `timerange`. + - when the user has _not modified the width_ of the `timeslice`, using the buttons means stepping through the pre-determined ticks (e.g. by year, by day, ...) + - when the user has _already modified the width_ of the `timeslice`, it means stepping through the `timerange`, with a stride of the width of the `timeslice`. +- the `timeslice` "snaps" to the beginning or end (depending on direction) of the `timerange`. In practice, this means the `timeslice` will collaps or reduce in width. + +## 2.3 Maps integration of the timeslider React component + +This control will be integrated in the Maps-UI. + +Maps is Redux-based, so `timeslice` selection and activation/de-activation all propagates to the Redux-store. + +#### 2.3.1 Position in the UI + +The timeslider control is enabled/disabled by the timeslider on/off toggle-button in the main toolbar. + +![Timeslider version 1](../images/timeslider/toolbar.png) + + +#### 2.3.2 Layer interactions + + +Enabling the Timeslider will automatically retrigger refresh of _all_ time-based layers to the currently selected `timeslice`. + +The Layer-TOC will indicate which layer is currently "time-filtered" by the timeslider. + +On a layer-per-layer basis, users will be able to explicitly opt-out if they should be governed by the timerange or not. This is an existing toggle in Maps already. +This is relevant for having users add contextual layers that should _not_ depend on the time. + + +#### 2.3.3 Omitting timeslider on a dashboard + +Maps will not display the timeslider-activation button on Maps that are embedded in a Dashboard. + +We believe that a Timeslider-embeddable would be a better vehicle to bring timeslider-functionality do Dashboards. See also the [last section](#3-unresolved-questions). + +#### 2.3.3 Data-fetch considerations + +--- +**NOTE** + +The below section is very Maps-specific, although similar challenges would be present in other applications as well. + +Some of these considerations will not generalize to all of Kibana. + +The main ways that Maps distinguishes in data-fetch from other use-cases: + - the majority of the data-fetching for layers in Maps depends on the scale and extent. Ie. different data is requested based on the current zoom-level and current-extent of the Map. So for example, even if two views share the same time, query and filter-state, if their extent and/or scale is different, their requests to ES will be different. + - for some layer-types, Maps will fetch individual documents, rather than the result of an aggregation. + +--- + +Data-fetch for timeslider should be responsive and smooth. A user dragging the slider should have an immediate visual result on the map. + +In addition, we expect the timeslider will be used a lot for "comparisons". For example, imagine a user stepping back&forth between timeslices. + +For this reason, apps using a timeslider (such as Maps) ideally: +- pre-fetch data when possible +- cache data when possible + +For Maps specifically, when introducing timeslider, layers will therefore need to implement time-based data fetch based on _two_ pieces of state +- the entire `timerange` (aka. the global Kibana timerange) +- the selected `timeslice` (aka. the `timeslice` chosen by the user using the UI-component) + +##### 2.3.3.1 Pre-fetching individual documents and masking of data + +ES-document layers (which display individual documents) can prefetch all documents within the entire `timerange`, when the total number of docs is below some threshold. In the context of Maps, this threshold is the default index-search-size of the index. + +Maps can then just mask data on the map based on some filter-expression. The evaluation of this filter-expression is done by mapbox-gl is fast because it occurs on the GPU. There is immediate visual feedback to the user as they manipulate the timeslider, because it does not require a roundtrip to update the data. + +##### 2.3.3.2 Caching of aggregated searches + +Aggregated data can be cached on the client, so toggling between timeslices can avoid a round-trip data-fetch. +The main technique here is for layers to use `.mvt`-data format to request data. Tiled-based data can be cached client-side + +We do _not_ propose _pre-fetching_ of aggregated data in this initial phase of the Maps timeslider effort. There is a couple reasons: +- Based on the intended user-interactions for timeslider, because a user can flexibly select a `timeslice` of arbitrary widths, it would be really hard to determine how to determine which timeslices to aggregate up front. +- Maps already strains the maximum bucket sizes it can retrieve from Elasticsearch. Cluster/grid-layers often push up to 10k or more buckets, and terms-aggregations for choropleth maps also is going up to 10k buckets. Prefetching this for timeslices (e.g. say x10 timeslices) would easily exceed the default bucket limit sizes of Elasticsearch. + + +##### 2.3.3.3 Decouple data-fetch from UI-effort + +Apart from refactoring the data-fetch for layers to now use two pieces of time-based state, the implementation will decouple any data-fetch considerations from the actual timeslider-UI work. + +The idea is that dial in data-fetch optimizations can be dialed into Maps in a parallel work-thread, not necessarily dependent on any changes or additions to the UI. +Any optimizations would not only affect timeslider users, but support all interaction patterns that require smooth data-updates (e.g. panning back&forth to two locations, toggling back&forth between two filters, ...) + +The main effort to support efficient data-fetch in a maps-context is to use `.mvt` as the default data format ([https://github.com/elastic/kibana/issues/79868](https://github.com/elastic/kibana/issues/79868)). This is a stack-wide effort in collaboration with the Elasticsearch team ([https://github.com/elastic/elasticsearch/issues/58696](https://github.com/elastic/elasticsearch/issues/58696), which will add `.mvt` as a core response format to Elasticsearch. + +Growing the use of `mvt`([https://docs.mapbox.com/vector-tiles/reference/](https://docs.mapbox.com/vector-tiles/reference/)) in Maps will help with both pre-fetching and client-side caching: +- `mvt` is a binary format which allows more data to be packed inside, as compared to Json. Multiple tiles are patched together, so this introduces a form of parallelization as well. Due to growing the amount of data inside a single tile, and due to the parallelization, Maps has a pathway to increase the number of features that can be time-filtered. +- Because vector tiles have fixed extents and scales (defined by a `{x}/{y}/{scale}`-tuple), this type of data-fetching allows tiles to be cached on the client. This cache can be the implicit browser disc-cache, or the transient in-mem cache of mapbox-gl. Using mvt thus provides a pathway for fast toggling back&forth between timeslices, without round-trips to fetch data. + + +##### 2.3.3.4 timeslider and async search + +It is unclear on what the practical uses for async-search would be in the context of a timeslider-control in Maps. + +Timeslider is a highly interactive control that require immediate visual feedback. We also do not intend to activate timeslider in Maps on a Dashboard (see above). + +End-users who need to view a dashboard with a long-running background search will need to manipulate the _global Kibana time picker_ to select the time-range, and will not be able to use the timeslider to do so. + +# 3. Unresolved questions + +## Making Timeslider a Kibana Embeddable + +This below is a forward looking section. It is a proposal of how the Timeslider-UI can be exposed as an Embeddable, when that time should come. + +We expect a few steps: +- This would require the extraction of the timeslider React-component out of Maps into a separate plugin. As outlined above, this migration should be fairly straightforward, a "cut and paste". +- It would require the creation of a `TimesliderEmbeddable` which wraps this UI-component. +- It would also require the introduction of a new piece of embeddable-state, `Timeslice`, which can be controlled by the `TimesliderEmbeddable`. +We believe it is important to keep `timeslice` and `timerange` separate, as individual apps and other embeddables will have different mechanism to efficiently fetch data and respond to changes in `timeslice` and/or `timerange`. + +Having timeslider as a core Embeddable likely provides a better pathway to integrate timeslider-functionality in Dashboards or apps other than Maps. + diff --git a/src/core/public/doc_links/doc_links_service.ts b/src/core/public/doc_links/doc_links_service.ts index 2a202a426c16c..b2eec43cc5ad7 100644 --- a/src/core/public/doc_links/doc_links_service.ts +++ b/src/core/public/doc_links/doc_links_service.ts @@ -347,7 +347,7 @@ export class DocLinksService { snapshotRestoreRepos: `${PLUGIN_DOCS}repository.html`, }, snapshotRestore: { - guide: `${ELASTICSEARCH_DOCS}snapshot-restore.html`, + guide: `${KIBANA_DOCS}snapshot-repositories.html`, changeIndexSettings: `${ELASTICSEARCH_DOCS}snapshots-restore-snapshot.html#change-index-settings-during-restore`, createSnapshot: `${ELASTICSEARCH_DOCS}snapshots-take-snapshot.html`, registerSharedFileSystem: `${ELASTICSEARCH_DOCS}snapshots-register-repository.html#snapshots-filesystem-repository`, diff --git a/src/core/server/elasticsearch/legacy/api_types.ts b/src/core/server/elasticsearch/legacy/api_types.ts index a3b442549942f..e4ff4816527b4 100644 --- a/src/core/server/elasticsearch/legacy/api_types.ts +++ b/src/core/server/elasticsearch/legacy/api_types.ts @@ -140,6 +140,7 @@ import { * * @public * @deprecated + * @removeBy 7.16 */ export interface LegacyCallAPIOptions { /** @@ -157,6 +158,7 @@ export interface LegacyCallAPIOptions { /** * @deprecated + * @removeBy 7.16 * @public * */ export interface LegacyAPICaller { @@ -312,6 +314,7 @@ export interface LegacyAPICaller { /** * @deprecated + * @removeBy 7.16 * @public * */ export interface AssistantAPIClientParams extends GenericParams { @@ -321,17 +324,20 @@ export interface AssistantAPIClientParams extends GenericParams { /** * @deprecated + * @removeBy 7.16 * @public * */ export type MIGRATION_ASSISTANCE_INDEX_ACTION = 'upgrade' | 'reindex'; /** * @deprecated + * @removeBy 7.16 * @public * */ export type MIGRATION_DEPRECATION_LEVEL = 'none' | 'info' | 'warning' | 'critical'; /** * @deprecated + * @removeBy 7.16 * @public * */ export interface AssistanceAPIResponse { @@ -344,6 +350,7 @@ export interface AssistanceAPIResponse { /** * @deprecated + * @removeBy 7.16 * @public * */ export interface DeprecationAPIClientParams extends GenericParams { @@ -353,6 +360,7 @@ export interface DeprecationAPIClientParams extends GenericParams { /** * @deprecated + * @removeBy 7.16 * @public * */ export interface DeprecationInfo { @@ -364,6 +372,7 @@ export interface DeprecationInfo { /** * @deprecated + * @removeBy 7.16 * @public * */ export interface IndexSettingsDeprecationInfo { @@ -372,6 +381,7 @@ export interface IndexSettingsDeprecationInfo { /** * @deprecated + * @removeBy 7.16 * @public * */ export interface DeprecationAPIResponse { diff --git a/src/core/server/elasticsearch/legacy/cluster_client.ts b/src/core/server/elasticsearch/legacy/cluster_client.ts index e4b72abe0bc1c..bdb2ca4d01b3c 100644 --- a/src/core/server/elasticsearch/legacy/cluster_client.ts +++ b/src/core/server/elasticsearch/legacy/cluster_client.ts @@ -78,6 +78,7 @@ const callAPI = async ( * See {@link LegacyClusterClient}. * * @deprecated Use {@link IClusterClient}. + * @removeBy 7.16 * @public */ export type ILegacyClusterClient = Pick; @@ -89,6 +90,7 @@ export type ILegacyClusterClient = Pick; /** * @deprecated + * @removeBy 7.16 * Use {@link ElasticsearchServiceStart.legacy | ElasticsearchServiceStart.legacy.createClient} instead. * * Create application specific Elasticsearch cluster API client with customized config. See {@link ILegacyClusterClient}. @@ -60,6 +61,7 @@ export interface ElasticsearchServiceSetup { ) => ILegacyCustomClusterClient; /** + * @removeBy 7.16 * @deprecated * Use {@link ElasticsearchServiceStart.legacy | ElasticsearchServiceStart.legacy.client} instead. * @@ -131,6 +133,9 @@ export interface ElasticsearchServiceStart { /** * Create application specific Elasticsearch cluster API client with customized config. See {@link ILegacyClusterClient}. * + * @deprecated + * @removeBy 7.16 + * * @param type Unique identifier of the client * @param clientConfig A config consists of Elasticsearch JS client options and * valid sub-set of Elasticsearch service config. @@ -153,6 +158,9 @@ export interface ElasticsearchServiceStart { /** * A pre-configured {@link ILegacyClusterClient | legacy Elasticsearch client}. * + * @deprecated + * @removeBy 7.16 + * * @example * ```js * const client = core.elasticsearch.legacy.client; diff --git a/src/core/server/saved_objects/migrationsv2/actions/index.ts b/src/core/server/saved_objects/migrationsv2/actions/index.ts index d0623de51e4c3..c2e0476960c3b 100644 --- a/src/core/server/saved_objects/migrationsv2/actions/index.ts +++ b/src/core/server/saved_objects/migrationsv2/actions/index.ts @@ -456,6 +456,7 @@ export const openPit = ( export interface ReadWithPit { outdatedDocuments: SavedObjectsRawDoc[]; readonly lastHitSortValue: number[] | undefined; + readonly totalHits: number | undefined; } /* @@ -481,13 +482,20 @@ export const readWithPit = ( pit: { id: pitId, keep_alive: pitKeepAlive }, size: batchSize, search_after: searchAfter, - // Improve performance by not calculating the total number of hits - // matching the query. - track_total_hits: false, + /** + * We want to know how many documents we need to process so we can log the progress. + * But we also want to increase the performance of these requests, + * so we ask ES to report the total count only on the first request (when searchAfter does not exist) + */ + track_total_hits: typeof searchAfter === 'undefined', query, }, }) .then((response) => { + const totalHits = + typeof response.body.hits.total === 'number' + ? response.body.hits.total // This format is to be removed in 8.0 + : response.body.hits.total?.value; const hits = response.body.hits.hits; if (hits.length > 0) { @@ -495,12 +503,14 @@ export const readWithPit = ( // @ts-expect-error @elastic/elasticsearch _source is optional outdatedDocuments: hits as SavedObjectsRawDoc[], lastHitSortValue: hits[hits.length - 1].sort as number[], + totalHits, }); } return Either.right({ outdatedDocuments: [], lastHitSortValue: undefined, + totalHits, }); }) .catch(catchRetryableEsClientErrors); diff --git a/src/core/server/saved_objects/migrationsv2/model.test.ts b/src/core/server/saved_objects/migrationsv2/model.test.ts index bdaedba9c9ea3..adeb78e568af3 100644 --- a/src/core/server/saved_objects/migrationsv2/model.test.ts +++ b/src/core/server/saved_objects/migrationsv2/model.test.ts @@ -45,6 +45,7 @@ import { createInitialState, model } from './model'; import { ResponseType } from './next'; import { SavedObjectsMigrationConfigType } from '../saved_objects_config'; import { TransformErrorObjects, TransformSavedObjectDocumentError } from '../migrations/core'; +import { createInitialProgress } from './progress'; describe('migrations v2 model', () => { const baseState: BaseState = { @@ -768,6 +769,8 @@ describe('migrations v2 model', () => { expect(newState.controlState).toBe('REINDEX_SOURCE_TO_TEMP_READ'); expect(newState.sourceIndexPitId).toBe('pit_id'); expect(newState.lastHitSortValue).toBe(undefined); + expect(newState.progress.processed).toBe(undefined); + expect(newState.progress.total).toBe(undefined); }); }); @@ -783,6 +786,7 @@ describe('migrations v2 model', () => { lastHitSortValue: undefined, corruptDocumentIds: [], transformErrors: [], + progress: createInitialProgress(), }; it('REINDEX_SOURCE_TO_TEMP_READ -> REINDEX_SOURCE_TO_TEMP_INDEX if the index has outdated documents to reindex', () => { @@ -791,21 +795,34 @@ describe('migrations v2 model', () => { const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_READ'> = Either.right({ outdatedDocuments, lastHitSortValue, + totalHits: 1, }); const newState = model(state, res) as ReindexSourceToTempIndex; expect(newState.controlState).toBe('REINDEX_SOURCE_TO_TEMP_INDEX'); expect(newState.outdatedDocuments).toBe(outdatedDocuments); expect(newState.lastHitSortValue).toBe(lastHitSortValue); + expect(newState.progress.processed).toBe(undefined); + expect(newState.progress.total).toBe(1); + expect(newState.logs).toMatchInlineSnapshot(` + Array [ + Object { + "level": "info", + "message": "Starting to process 1 documents.", + }, + ] + `); }); it('REINDEX_SOURCE_TO_TEMP_READ -> REINDEX_SOURCE_TO_TEMP_CLOSE_PIT if no outdated documents to reindex', () => { const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_READ'> = Either.right({ outdatedDocuments: [], lastHitSortValue: undefined, + totalHits: undefined, }); const newState = model(state, res) as ReindexSourceToTempClosePit; expect(newState.controlState).toBe('REINDEX_SOURCE_TO_TEMP_CLOSE_PIT'); expect(newState.sourceIndexPitId).toBe('pit_id'); + expect(newState.logs).toStrictEqual([]); // No logs because no hits }); it('REINDEX_SOURCE_TO_TEMP_READ -> FATAL if no outdated documents to reindex and transform failures seen with previous outdated documents', () => { @@ -817,12 +834,14 @@ describe('migrations v2 model', () => { const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_READ'> = Either.right({ outdatedDocuments: [], lastHitSortValue: undefined, + totalHits: undefined, }); const newState = model(testState, res) as FatalState; expect(newState.controlState).toBe('FATAL'); expect(newState.reason).toMatchInlineSnapshot( `"Migrations failed. Reason: Corrupt saved object documents: a:b. To allow migrations to proceed, please delete these documents."` ); + expect(newState.logs).toStrictEqual([]); // No logs because no hits }); }); @@ -857,6 +876,7 @@ describe('migrations v2 model', () => { lastHitSortValue: undefined, corruptDocumentIds: [], transformErrors: [], + progress: { processed: undefined, total: 1 }, }; const processedDocs = [ { @@ -869,8 +889,28 @@ describe('migrations v2 model', () => { const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_INDEX'> = Either.right({ processedDocs, }); - const newState = model(state, res); + const newState = model(state, res) as ReindexSourceToTempIndexBulk; + expect(newState.controlState).toEqual('REINDEX_SOURCE_TO_TEMP_INDEX_BULK'); + expect(newState.progress.processed).toBe(0); // Result of `(undefined ?? 0) + corruptDocumentsId.length` + }); + + it('increments the progress.processed counter', () => { + const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_INDEX'> = Either.right({ + processedDocs, + }); + + const testState = { + ...state, + outdatedDocuments: [{ _id: '1', _source: { type: 'vis' } }], + progress: { + processed: 1, + total: 1, + }, + }; + + const newState = model(testState, res) as ReindexSourceToTempIndexBulk; expect(newState.controlState).toEqual('REINDEX_SOURCE_TO_TEMP_INDEX_BULK'); + expect(newState.progress.processed).toBe(2); }); it('REINDEX_SOURCE_TO_TEMP_INDEX -> REINDEX_SOURCE_TO_TEMP_READ if action succeeded but we have carried through previous failures', () => { @@ -886,6 +926,7 @@ describe('migrations v2 model', () => { expect(newState.controlState).toEqual('REINDEX_SOURCE_TO_TEMP_READ'); expect(newState.corruptDocumentIds.length).toEqual(1); expect(newState.transformErrors.length).toEqual(0); + expect(newState.progress.processed).toBe(0); }); it('REINDEX_SOURCE_TO_TEMP_INDEX -> REINDEX_SOURCE_TO_TEMP_READ when response is left documents_transform_failed', () => { @@ -918,6 +959,7 @@ describe('migrations v2 model', () => { sourceIndexPitId: 'pit_id', targetIndex: '.kibana_7.11.0_001', lastHitSortValue: undefined, + progress: createInitialProgress(), }; test('REINDEX_SOURCE_TO_TEMP_INDEX_BULK -> REINDEX_SOURCE_TO_TEMP_READ if action succeeded', () => { const res: ResponseType<'REINDEX_SOURCE_TO_TEMP_INDEX_BULK'> = Either.right( @@ -1018,6 +1060,7 @@ describe('migrations v2 model', () => { hasTransformedDocs: false, corruptDocumentIds: [], transformErrors: [], + progress: createInitialProgress(), }; it('OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_TRANSFORM if found documents to transform', () => { @@ -1026,21 +1069,65 @@ describe('migrations v2 model', () => { const res: ResponseType<'OUTDATED_DOCUMENTS_SEARCH_READ'> = Either.right({ outdatedDocuments, lastHitSortValue, + totalHits: 10, }); const newState = model(state, res) as OutdatedDocumentsTransform; expect(newState.controlState).toBe('OUTDATED_DOCUMENTS_TRANSFORM'); expect(newState.outdatedDocuments).toBe(outdatedDocuments); expect(newState.lastHitSortValue).toBe(lastHitSortValue); + expect(newState.progress.processed).toBe(undefined); + expect(newState.progress.total).toBe(10); + expect(newState.logs).toMatchInlineSnapshot(` + Array [ + Object { + "level": "info", + "message": "Starting to process 10 documents.", + }, + ] + `); + }); + + it('keeps the previous progress.total if not obtained in the result', () => { + const outdatedDocuments = [{ _id: '1', _source: { type: 'vis' } }]; + const lastHitSortValue = [123456]; + const res: ResponseType<'OUTDATED_DOCUMENTS_SEARCH_READ'> = Either.right({ + outdatedDocuments, + lastHitSortValue, + totalHits: undefined, + }); + const testState = { + ...state, + progress: { + processed: 5, + total: 10, + }, + }; + const newState = model(testState, res) as OutdatedDocumentsTransform; + expect(newState.controlState).toBe('OUTDATED_DOCUMENTS_TRANSFORM'); + expect(newState.outdatedDocuments).toBe(outdatedDocuments); + expect(newState.lastHitSortValue).toBe(lastHitSortValue); + expect(newState.progress.processed).toBe(5); + expect(newState.progress.total).toBe(10); + expect(newState.logs).toMatchInlineSnapshot(` + Array [ + Object { + "level": "info", + "message": "Processed 5 documents out of 10.", + }, + ] + `); }); it('OUTDATED_DOCUMENTS_SEARCH_READ -> OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT if no outdated documents to transform', () => { const res: ResponseType<'OUTDATED_DOCUMENTS_SEARCH_READ'> = Either.right({ outdatedDocuments: [], lastHitSortValue: undefined, + totalHits: undefined, }); const newState = model(state, res) as OutdatedDocumentsSearchClosePit; expect(newState.controlState).toBe('OUTDATED_DOCUMENTS_SEARCH_CLOSE_PIT'); expect(newState.pitId).toBe('pit_id'); + expect(newState.logs).toStrictEqual([]); // No logs because no hits }); it('OUTDATED_DOCUMENTS_SEARCH_READ -> FATAL if no outdated documents to transform and we have failed document migrations', () => { @@ -1060,6 +1147,7 @@ describe('migrations v2 model', () => { const res: ResponseType<'OUTDATED_DOCUMENTS_SEARCH_READ'> = Either.right({ outdatedDocuments: [], lastHitSortValue: undefined, + totalHits: undefined, }); const transformErrorsState: OutdatedDocumentsSearchRead = { ...state, @@ -1072,6 +1160,7 @@ describe('migrations v2 model', () => { expect(newState.reason.includes('Corrupt saved object documents: ')).toBe(true); expect(newState.reason.includes('Transformation errors: ')).toBe(true); expect(newState.reason.includes('randomvis: 7.12.0')).toBe(true); + expect(newState.logs).toStrictEqual([]); // No logs because no hits }); }); @@ -1138,6 +1227,7 @@ describe('migrations v2 model', () => { pitId: 'pit_id', lastHitSortValue: [3, 4], hasTransformedDocs: false, + progress: createInitialProgress(), }; describe('OUTDATED_DOCUMENTS_TRANSFORM if action succeeds', () => { const processedDocs = [ @@ -1156,6 +1246,7 @@ describe('migrations v2 model', () => { expect(newState.transformedDocs).toEqual(processedDocs); expect(newState.retryCount).toEqual(0); expect(newState.retryDelay).toEqual(0); + expect(newState.progress.processed).toBe(outdatedDocuments.length); }); test('OUTDATED_DOCUMENTS_TRANSFORM -> OUTDATED_DOCUMENTS_SEARCH_READ if there are are existing documents that failed transformation', () => { const outdatedDocumentsTransformStateWithFailedDocuments: OutdatedDocumentsTransform = { @@ -1172,6 +1263,7 @@ describe('migrations v2 model', () => { expect(newState.corruptDocumentIds).toEqual(corruptDocumentIds); expect(newState.retryCount).toEqual(0); expect(newState.retryDelay).toEqual(0); + expect(newState.progress.processed).toBe(outdatedDocuments.length); }); test('OUTDATED_DOCUMENTS_TRANSFORM -> OUTDATED_DOCUMENTS_SEARCH_READ if there are are existing documents that failed transformation because of transform errors', () => { const outdatedDocumentsTransformStateWithFailedDocuments: OutdatedDocumentsTransform = { @@ -1189,6 +1281,7 @@ describe('migrations v2 model', () => { expect(newState.transformErrors.length).toEqual(1); expect(newState.retryCount).toEqual(0); expect(newState.retryDelay).toEqual(0); + expect(newState.progress.processed).toBe(outdatedDocuments.length); }); }); describe('OUTDATED_DOCUMENTS_TRANSFORM if action fails', () => { @@ -1204,6 +1297,7 @@ describe('migrations v2 model', () => { ) as OutdatedDocumentsSearchRead; expect(newState.controlState).toEqual('OUTDATED_DOCUMENTS_SEARCH_READ'); expect(newState.corruptDocumentIds).toEqual(corruptDocumentIds); + expect(newState.progress.processed).toBe(outdatedDocuments.length); }); test('OUTDATED_DOCUMENTS_TRANSFORM -> OUTDATED_DOCUMENTS_SEARCH_READ combines newly failed documents with those already on state if documents failed the transform', () => { const newFailedTransformDocumentIds = ['b:other', 'c:__']; @@ -1226,6 +1320,7 @@ describe('migrations v2 model', () => { ...corruptDocumentIds, ...newFailedTransformDocumentIds, ]); + expect(newState.progress.processed).toBe(outdatedDocuments.length); }); }); }); @@ -1246,6 +1341,7 @@ describe('migrations v2 model', () => { pitId: 'pit_id', lastHitSortValue: [3, 4], hasTransformedDocs: false, + progress: createInitialProgress(), }; test('TRANSFORMED_DOCUMENTS_BULK_INDEX should throw a throwBadResponse error if action failed', () => { const res: ResponseType<'TRANSFORMED_DOCUMENTS_BULK_INDEX'> = Either.left({ diff --git a/src/core/server/saved_objects/migrationsv2/model.ts b/src/core/server/saved_objects/migrationsv2/model.ts index cf9d6aec6b5b0..3ef3cb4f83b6f 100644 --- a/src/core/server/saved_objects/migrationsv2/model.ts +++ b/src/core/server/saved_objects/migrationsv2/model.ts @@ -18,6 +18,12 @@ import { SavedObjectsMigrationVersion } from '../types'; import { disableUnknownTypeMappingFields } from '../migrations/core/migration_context'; import { excludeUnusedTypesQuery, TransformErrorObjects } from '../migrations/core'; import { SavedObjectsMigrationConfigType } from '../saved_objects_config'; +import { + createInitialProgress, + incrementProcessedProgress, + logProgress, + setProgressTotal, +} from './progress'; /** * A helper function/type for ensuring that all control state's are handled. @@ -509,6 +515,7 @@ export const model = (currentState: State, resW: ResponseType): // placeholders to collect document transform problems corruptDocumentIds: [], transformErrors: [], + progress: createInitialProgress(), }; } else { throwBadResponse(stateP, res); @@ -517,12 +524,16 @@ export const model = (currentState: State, resW: ResponseType): // we carry through any failures we've seen with transforming documents on state const res = resW as ExcludeRetryableEsError>; if (Either.isRight(res)) { + const progress = setProgressTotal(stateP.progress, res.right.totalHits); + const logs = logProgress(stateP.logs, progress); if (res.right.outdatedDocuments.length > 0) { return { ...stateP, controlState: 'REINDEX_SOURCE_TO_TEMP_INDEX', outdatedDocuments: res.right.outdatedDocuments, lastHitSortValue: res.right.lastHitSortValue, + progress, + logs, }; } else { // we don't have any more outdated documents and need to either fail or move on to updating the target mappings. @@ -542,6 +553,7 @@ export const model = (currentState: State, resW: ResponseType): return { ...stateP, controlState: 'REINDEX_SOURCE_TO_TEMP_CLOSE_PIT', + logs, }; } } @@ -566,12 +578,18 @@ export const model = (currentState: State, resW: ResponseType): // collecting issues along the way rather than failing // REINDEX_SOURCE_TO_TEMP_INDEX handles the document transforms const res = resW as ExcludeRetryableEsError>; + + // Increment the processed documents, no matter what the results are. + // Otherwise the progress might look off when there are errors. + const progress = incrementProcessedProgress(stateP.progress, stateP.outdatedDocuments.length); + if (Either.isRight(res)) { if (stateP.corruptDocumentIds.length === 0 && stateP.transformErrors.length === 0) { return { ...stateP, controlState: 'REINDEX_SOURCE_TO_TEMP_INDEX_BULK', // handles the actual bulk indexing into temp index transformedDocs: [...res.right.processedDocs], + progress, }; } else { // we don't have any transform issues with the current batch of outdated docs but @@ -581,6 +599,7 @@ export const model = (currentState: State, resW: ResponseType): return { ...stateP, controlState: 'REINDEX_SOURCE_TO_TEMP_READ', + progress, }; } } else { @@ -592,6 +611,7 @@ export const model = (currentState: State, resW: ResponseType): controlState: 'REINDEX_SOURCE_TO_TEMP_READ', corruptDocumentIds: [...stateP.corruptDocumentIds, ...left.corruptDocumentIds], transformErrors: [...stateP.transformErrors, ...left.transformErrors], + progress, }; } else { // should never happen @@ -676,6 +696,7 @@ export const model = (currentState: State, resW: ResponseType): controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ', pitId: res.right.pitId, lastHitSortValue: undefined, + progress: createInitialProgress(), hasTransformedDocs: false, corruptDocumentIds: [], transformErrors: [], @@ -687,11 +708,16 @@ export const model = (currentState: State, resW: ResponseType): const res = resW as ExcludeRetryableEsError>; if (Either.isRight(res)) { if (res.right.outdatedDocuments.length > 0) { + const progress = setProgressTotal(stateP.progress, res.right.totalHits); + const logs = logProgress(stateP.logs, progress); + return { ...stateP, controlState: 'OUTDATED_DOCUMENTS_TRANSFORM', outdatedDocuments: res.right.outdatedDocuments, lastHitSortValue: res.right.lastHitSortValue, + progress, + logs, }; } else { // we don't have any more outdated documents and need to either fail or move on to updating the target mappings. @@ -720,6 +746,11 @@ export const model = (currentState: State, resW: ResponseType): } } else if (stateP.controlState === 'OUTDATED_DOCUMENTS_TRANSFORM') { const res = resW as ExcludeRetryableEsError>; + + // Increment the processed documents, no matter what the results are. + // Otherwise the progress might look off when there are errors. + const progress = incrementProcessedProgress(stateP.progress, stateP.outdatedDocuments.length); + if (Either.isRight(res)) { // we haven't seen corrupt documents or any transformation errors thus far in the migration // index the migrated docs @@ -729,6 +760,7 @@ export const model = (currentState: State, resW: ResponseType): controlState: 'TRANSFORMED_DOCUMENTS_BULK_INDEX', transformedDocs: [...res.right.processedDocs], hasTransformedDocs: true, + progress, }; } else { // We have seen corrupt documents and/or transformation errors @@ -736,6 +768,7 @@ export const model = (currentState: State, resW: ResponseType): return { ...stateP, controlState: 'OUTDATED_DOCUMENTS_SEARCH_READ', + progress, }; } } else { @@ -747,6 +780,7 @@ export const model = (currentState: State, resW: ResponseType): corruptDocumentIds: [...stateP.corruptDocumentIds, ...res.left.corruptDocumentIds], transformErrors: [...stateP.transformErrors, ...res.left.transformErrors], hasTransformedDocs: false, + progress, }; } else { throwBadResponse(stateP, res as never); diff --git a/src/core/server/saved_objects/migrationsv2/progress.test.ts b/src/core/server/saved_objects/migrationsv2/progress.test.ts new file mode 100644 index 0000000000000..a0d89c2c63300 --- /dev/null +++ b/src/core/server/saved_objects/migrationsv2/progress.test.ts @@ -0,0 +1,140 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { MigrationLog } from './types'; +import { + createInitialProgress, + incrementProcessedProgress, + logProgress, + setProgressTotal, +} from './progress'; + +describe('createInitialProgress', () => { + test('create initial progress', () => { + expect(createInitialProgress()).toStrictEqual({ + processed: undefined, + total: undefined, + }); + }); +}); + +describe('setProgressTotal', () => { + const previousProgress = { + processed: undefined, + total: 10, + }; + test('should keep the previous total if not provided', () => { + expect(setProgressTotal(previousProgress)).toStrictEqual(previousProgress); + }); + + test('should keep the previous total is undefined', () => { + expect(setProgressTotal(previousProgress, undefined)).toStrictEqual(previousProgress); + }); + + test('should overwrite if the previous total is provided', () => { + expect(setProgressTotal(previousProgress, 20)).toStrictEqual({ + processed: undefined, + total: 20, + }); + }); +}); + +describe('logProgress', () => { + const previousLogs: MigrationLog[] = []; + + test('should not log anything if there is no total', () => { + const progress = { + processed: undefined, + total: undefined, + }; + expect(logProgress(previousLogs, progress)).toStrictEqual([]); + }); + + test('should not log anything if total is 0', () => { + const progress = { + processed: undefined, + total: 0, + }; + expect(logProgress(previousLogs, progress)).toStrictEqual([]); + }); + + test('should log the "Starting..." log', () => { + const progress = { + processed: undefined, + total: 10, + }; + expect(logProgress(previousLogs, progress)).toStrictEqual([ + { + level: 'info', + message: 'Starting to process 10 documents.', + }, + ]); + }); + + test('should log the "Processed..." log', () => { + const progress = { + processed: 5, + total: 10, + }; + expect(logProgress(previousLogs, progress)).toStrictEqual([ + { + level: 'info', + message: 'Processed 5 documents out of 10.', + }, + ]); + }); +}); + +describe('incrementProcessedProgress', () => { + const previousProgress = { + processed: undefined, + total: 10, + }; + test('should not increment if the incrementValue is not defined', () => { + expect(incrementProcessedProgress(previousProgress)).toStrictEqual({ + processed: 0, + total: 10, + }); + }); + + test('should not increment if the incrementValue is undefined', () => { + expect(incrementProcessedProgress(previousProgress, undefined)).toStrictEqual({ + processed: 0, + total: 10, + }); + }); + + test('should not increment if the incrementValue is not defined (with some processed values)', () => { + const testPreviousProgress = { + ...previousProgress, + processed: 1, + }; + expect(incrementProcessedProgress(testPreviousProgress, undefined)).toStrictEqual({ + processed: 1, + total: 10, + }); + }); + + test('should increment if the incrementValue is defined', () => { + expect(incrementProcessedProgress(previousProgress, 5)).toStrictEqual({ + processed: 5, + total: 10, + }); + }); + + test('should increment if the incrementValue is defined (with some processed values)', () => { + const testPreviousProgress = { + ...previousProgress, + processed: 5, + }; + expect(incrementProcessedProgress(testPreviousProgress, 5)).toStrictEqual({ + processed: 10, + total: 10, + }); + }); +}); diff --git a/src/core/server/saved_objects/migrationsv2/progress.ts b/src/core/server/saved_objects/migrationsv2/progress.ts new file mode 100644 index 0000000000000..d626cd6528902 --- /dev/null +++ b/src/core/server/saved_objects/migrationsv2/progress.ts @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { MigrationLog, Progress } from './types'; + +/** + * Returns an initial state of the progress object (everything undefined) + */ +export function createInitialProgress(): Progress { + return { + processed: undefined, + total: undefined, + }; +} + +/** + * Overwrites the total of the progress if anything provided + * @param previousProgress + * @param total + */ +export function setProgressTotal( + previousProgress: Progress, + total = previousProgress.total +): Progress { + return { + ...previousProgress, + total, + }; +} + +/** + * Returns a new list of MigrationLogs with the info entry about the progress + * @param previousLogs + * @param progress + */ +export function logProgress(previousLogs: MigrationLog[], progress: Progress): MigrationLog[] { + const logs = [...previousLogs]; + + if (progress.total) { + if (typeof progress.processed === 'undefined') { + logs.push({ + level: 'info', + message: `Starting to process ${progress.total} documents.`, + }); + } else { + logs.push({ + level: 'info', + message: `Processed ${progress.processed} documents out of ${progress.total}.`, + }); + } + } + + return logs; +} + +/** + * Increments the processed count and returns a new Progress + * @param previousProgress Previous state of the progress + * @param incrementProcessedBy Amount to increase the processed count by + */ +export function incrementProcessedProgress( + previousProgress: Progress, + incrementProcessedBy = 0 +): Progress { + return { + ...previousProgress, + processed: (previousProgress.processed ?? 0) + incrementProcessedBy, + }; +} diff --git a/src/core/server/saved_objects/migrationsv2/types.ts b/src/core/server/saved_objects/migrationsv2/types.ts index f5800a3cd9570..e3e52212d56cb 100644 --- a/src/core/server/saved_objects/migrationsv2/types.ts +++ b/src/core/server/saved_objects/migrationsv2/types.ts @@ -26,6 +26,11 @@ export interface MigrationLog { message: string; } +export interface Progress { + processed: number | undefined; + total: number | undefined; +} + export interface BaseState extends ControlState { /** The first part of the index name such as `.kibana` or `.kibana_task_manager` */ readonly indexPrefix: string; @@ -183,6 +188,7 @@ export interface ReindexSourceToTempRead extends PostInitState { readonly lastHitSortValue: number[] | undefined; readonly corruptDocumentIds: string[]; readonly transformErrors: TransformErrorObjects[]; + readonly progress: Progress; } export interface ReindexSourceToTempClosePit extends PostInitState { @@ -197,6 +203,7 @@ export interface ReindexSourceToTempIndex extends PostInitState { readonly lastHitSortValue: number[] | undefined; readonly corruptDocumentIds: string[]; readonly transformErrors: TransformErrorObjects[]; + readonly progress: Progress; } export interface ReindexSourceToTempIndexBulk extends PostInitState { @@ -204,6 +211,7 @@ export interface ReindexSourceToTempIndexBulk extends PostInitState { readonly transformedDocs: SavedObjectsRawDoc[]; readonly sourceIndexPitId: string; readonly lastHitSortValue: number[] | undefined; + readonly progress: Progress; } export type SetTempWriteBlock = PostInitState & { @@ -252,6 +260,7 @@ export interface OutdatedDocumentsSearchRead extends PostInitState { readonly hasTransformedDocs: boolean; readonly corruptDocumentIds: string[]; readonly transformErrors: TransformErrorObjects[]; + readonly progress: Progress; } export interface OutdatedDocumentsSearchClosePit extends PostInitState { @@ -276,6 +285,7 @@ export interface OutdatedDocumentsTransform extends PostInitState { readonly hasTransformedDocs: boolean; readonly corruptDocumentIds: string[]; readonly transformErrors: TransformErrorObjects[]; + readonly progress: Progress; } export interface TransformedDocumentsBulkIndex extends PostInitState { /** @@ -286,6 +296,7 @@ export interface TransformedDocumentsBulkIndex extends PostInitState { readonly lastHitSortValue: number[] | undefined; readonly hasTransformedDocs: boolean; readonly pitId: string; + readonly progress: Progress; } export interface MarkVersionIndexReady extends PostInitState { diff --git a/src/plugins/dashboard/public/application/dashboard_router.tsx b/src/plugins/dashboard/public/application/dashboard_router.tsx index ed68afc5e97b1..9050aee5ce38c 100644 --- a/src/plugins/dashboard/public/application/dashboard_router.tsx +++ b/src/plugins/dashboard/public/application/dashboard_router.tsx @@ -84,7 +84,8 @@ export async function mountApp({ } = pluginsStart; const spacesApi = pluginsStart.spacesOss?.isSpacesAvailable ? pluginsStart.spacesOss : undefined; - const activeSpaceId = spacesApi && (await spacesApi.activeSpace$.pipe(first()).toPromise())?.id; + const activeSpaceId = + spacesApi && (await spacesApi.getActiveSpace$().pipe(first()).toPromise())?.id; let globalEmbedSettings: DashboardEmbedSettings | undefined; const dashboardServices: DashboardAppServices = { diff --git a/src/plugins/data/common/es_query/filters/range_filter.test.ts b/src/plugins/data/common/es_query/filters/range_filter.test.ts index c7ff84f61d0fc..bb7ecc09ebc34 100644 --- a/src/plugins/data/common/es_query/filters/range_filter.test.ts +++ b/src/plugins/data/common/es_query/filters/range_filter.test.ts @@ -64,6 +64,29 @@ describe('Range filter builder', () => { }); }); + it('should convert strings to numbers if the field is scripted and type number', () => { + const field = getField('script number'); + + expect(buildRangeFilter(field, { gte: '1', lte: '3' }, indexPattern)).toEqual({ + meta: { + field: 'script number', + index: 'id', + params: {}, + }, + script: { + script: { + lang: 'expression', + source: '(' + field!.script + ')>=gte && (' + field!.script + ')<=lte', + params: { + value: '>=1 <=3', + gte: 1, + lte: 3, + }, + }, + }, + }); + }); + it('should wrap painless scripts in comparator lambdas', () => { const field = getField('script date'); const expected = diff --git a/src/plugins/data/common/es_query/filters/range_filter.ts b/src/plugins/data/common/es_query/filters/range_filter.ts index ba9055e26dbda..fb8426655583e 100644 --- a/src/plugins/data/common/es_query/filters/range_filter.ts +++ b/src/plugins/data/common/es_query/filters/range_filter.ts @@ -138,7 +138,10 @@ export const buildRangeFilter = ( }; export const getRangeScript = (field: IFieldType, params: RangeFilterParams) => { - const knownParams = pickBy(params, (val, key: any) => key in operators); + const knownParams = mapValues( + pickBy(params, (val, key: any) => key in operators), + (value) => (field.type === 'number' && typeof value === 'string' ? parseFloat(value) : value) + ); let script = map( knownParams, (val: any, key: string) => '(' + field.script + ')' + get(operators, key) + key diff --git a/src/plugins/data/common/search/aggs/buckets/ip_range.ts b/src/plugins/data/common/search/aggs/buckets/ip_range.ts index c99676d71957a..5bcd614d9debf 100644 --- a/src/plugins/data/common/search/aggs/buckets/ip_range.ts +++ b/src/plugins/data/common/search/aggs/buckets/ip_range.ts @@ -18,7 +18,7 @@ import { KBN_FIELD_TYPES } from '../../../../common'; import { BaseAggParams } from '../types'; const ipRangeTitle = i18n.translate('data.search.aggs.buckets.ipRangeTitle', { - defaultMessage: 'IPv4 Range', + defaultMessage: 'IP Range', }); export enum IP_RANGE_TYPES { diff --git a/src/plugins/data/common/search/aggs/buckets/lib/cidr_mask.test.ts b/src/plugins/data/common/search/aggs/buckets/lib/cidr_mask.test.ts index 04363ee792289..b0466b78529e7 100644 --- a/src/plugins/data/common/search/aggs/buckets/lib/cidr_mask.test.ts +++ b/src/plugins/data/common/search/aggs/buckets/lib/cidr_mask.test.ts @@ -9,56 +9,47 @@ import { CidrMask } from './cidr_mask'; describe('CidrMask', () => { - test('should throw errors with invalid CIDR masks', () => { - expect( - () => - // @ts-ignore - new CidrMask() - ).toThrowError(); - - expect(() => new CidrMask('')).toThrowError(); - expect(() => new CidrMask('hello, world')).toThrowError(); - expect(() => new CidrMask('0.0.0.0')).toThrowError(); - expect(() => new CidrMask('0.0.0.0/0')).toThrowError(); - expect(() => new CidrMask('0.0.0.0/33')).toThrowError(); - expect(() => new CidrMask('256.0.0.0/32')).toThrowError(); - expect(() => new CidrMask('0.0.0.0/32/32')).toThrowError(); - expect(() => new CidrMask('1.2.3/1')).toThrowError(); - expect(() => new CidrMask('0.0.0.0/123d')).toThrowError(); + describe('constructor', () => { + it.each` + mask + ${''} + ${'hello, world'} + ${'0.0.0.0'} + ${'0.0.0.0/33'} + ${'256.0.0.0/32'} + ${'0.0.0.0/32/32'} + ${'0.0.0.0/123d'} + ${'::1'} + ${'::1/129'} + ${'fffff::/128'} + ${'ffff::/128/128'} + `('should throw an error on $mask', ({ mask }) => { + expect(() => new CidrMask(mask)).toThrowError(); + }); }); - test('should correctly grab IP address and prefix length', () => { - let mask = new CidrMask('0.0.0.0/1'); - expect(mask.initialAddress.toString()).toBe('0.0.0.0'); - expect(mask.prefixLength).toBe(1); - - mask = new CidrMask('128.0.0.1/31'); - expect(mask.initialAddress.toString()).toBe('128.0.0.1'); - expect(mask.prefixLength).toBe(31); + describe('toString', () => { + it.each` + mask | expected + ${'192.168.1.1/24'} | ${'192.168.1.1/24'} + ${'192.168.257/32'} | ${'192.168.1.1/32'} + ${'ffff:0:0:0:0:0:0:0/128'} | ${'ffff::/128'} + `('should format $mask as $expected', ({ mask, expected }) => { + expect(new CidrMask(mask).toString()).toBe(expected); + }); }); - test('should calculate a range of IP addresses', () => { - let mask = new CidrMask('0.0.0.0/1'); - let range = mask.getRange(); - expect(range.from.toString()).toBe('0.0.0.0'); - expect(range.to.toString()).toBe('127.255.255.255'); - - mask = new CidrMask('1.2.3.4/2'); - range = mask.getRange(); - expect(range.from.toString()).toBe('0.0.0.0'); - expect(range.to.toString()).toBe('63.255.255.255'); - - mask = new CidrMask('67.129.65.201/27'); - range = mask.getRange(); - expect(range.from.toString()).toBe('67.129.65.192'); - expect(range.to.toString()).toBe('67.129.65.223'); - }); - - test('toString()', () => { - let mask = new CidrMask('.../1'); - expect(mask.toString()).toBe('0.0.0.0/1'); - - mask = new CidrMask('128.0.0.1/31'); - expect(mask.toString()).toBe('128.0.0.1/31'); + describe('getRange', () => { + it.each` + mask | from | to + ${'0.0.0.0/0'} | ${'0.0.0.0'} | ${'255.255.255.255'} + ${'0.0.0.0/1'} | ${'0.0.0.0'} | ${'127.255.255.255'} + ${'1.2.3.4/2'} | ${'0.0.0.0'} | ${'63.255.255.255'} + ${'67.129.65.201/27'} | ${'67.129.65.192'} | ${'67.129.65.223'} + ${'::/1'} | ${'::'} | ${'7fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff'} + ${'8000::/1'} | ${'8000::'} | ${'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff'} + `('should return $from-$to for $mask', ({ mask, from, to }) => { + expect(new CidrMask(mask).getRange()).toEqual({ from, to }); + }); }); }); diff --git a/src/plugins/data/common/search/aggs/buckets/lib/cidr_mask.ts b/src/plugins/data/common/search/aggs/buckets/lib/cidr_mask.ts index 1a1c9b8157838..93ccbffaeb89d 100644 --- a/src/plugins/data/common/search/aggs/buckets/lib/cidr_mask.ts +++ b/src/plugins/data/common/search/aggs/buckets/lib/cidr_mask.ts @@ -6,42 +6,60 @@ * Side Public License, v 1. */ -import { Ipv4Address } from '../../utils'; - -const NUM_BITS = 32; - -function throwError(mask: string) { - throw Error('Invalid CIDR mask: ' + mask); -} +import ipaddr from 'ipaddr.js'; +import { IpAddress } from '../../utils'; export class CidrMask { - public readonly initialAddress: Ipv4Address; - public readonly prefixLength: number; + private address: number[]; + private netmask: number; - constructor(mask: string) { - const splits = mask.split('/'); - if (splits.length !== 2) { - throwError(mask); - } - this.initialAddress = new Ipv4Address(splits[0]); - this.prefixLength = Number(splits[1]); - if (isNaN(this.prefixLength) || this.prefixLength < 1 || this.prefixLength > NUM_BITS) { - throwError(mask); + constructor(cidr: string) { + try { + const [address, netmask] = ipaddr.parseCIDR(cidr); + + this.address = address.toByteArray(); + this.netmask = netmask; + } catch { + throw Error('Invalid CIDR mask: ' + cidr); } } - public getRange() { - const variableBits = NUM_BITS - this.prefixLength; - // eslint-disable-next-line no-bitwise - const fromAddress = ((this.initialAddress.valueOf() >> variableBits) << variableBits) >>> 0; // >>> 0 coerces to unsigned - const numAddresses = Math.pow(2, variableBits); + private getBroadcastAddress() { + /* eslint-disable no-bitwise */ + const netmask = (1n << BigInt(this.address.length * 8 - this.netmask)) - 1n; + const broadcast = this.address.map((byte, index) => { + const offset = BigInt(this.address.length - index - 1) * 8n; + const mask = Number((netmask >> offset) & 255n); + + return byte | mask; + }); + /* eslint-enable no-bitwise */ + + return new IpAddress(broadcast).toString(); + } + + private getNetworkAddress() { + /* eslint-disable no-bitwise */ + const netmask = (1n << BigInt(this.address.length * 8 - this.netmask)) - 1n; + const network = this.address.map((byte, index) => { + const offset = BigInt(this.address.length - index - 1) * 8n; + const mask = Number((netmask >> offset) & 255n) ^ 255; + + return byte & mask; + }); + /* eslint-enable no-bitwise */ + + return new IpAddress(network).toString(); + } + + getRange() { return { - from: new Ipv4Address(fromAddress).toString(), - to: new Ipv4Address(fromAddress + numAddresses - 1).toString(), + from: this.getNetworkAddress(), + to: this.getBroadcastAddress(), }; } - public toString() { - return this.initialAddress.toString() + '/' + this.prefixLength; + toString() { + return `${new IpAddress(this.address)}/${this.netmask}`; } } diff --git a/src/plugins/data/common/search/aggs/buckets/lib/time_buckets/calc_es_interval.ts b/src/plugins/data/common/search/aggs/buckets/lib/time_buckets/calc_es_interval.ts index 72d59af037d90..4b3ec90bb2cc2 100644 --- a/src/plugins/data/common/search/aggs/buckets/lib/time_buckets/calc_es_interval.ts +++ b/src/plugins/data/common/search/aggs/buckets/lib/time_buckets/calc_es_interval.ts @@ -12,7 +12,7 @@ import dateMath, { Unit } from '@elastic/datemath'; import { parseEsInterval } from '../../../utils'; const unitsDesc = dateMath.unitsDesc; -const largeMax = unitsDesc.indexOf('M'); +const largeMax = unitsDesc.indexOf('w'); export interface EsInterval { expression: string; diff --git a/src/plugins/data/common/search/aggs/buckets/lib/time_buckets/time_buckets.test.ts b/src/plugins/data/common/search/aggs/buckets/lib/time_buckets/time_buckets.test.ts index 6fbaddb09b226..fd3f861dce07e 100644 --- a/src/plugins/data/common/search/aggs/buckets/lib/time_buckets/time_buckets.test.ts +++ b/src/plugins/data/common/search/aggs/buckets/lib/time_buckets/time_buckets.test.ts @@ -10,6 +10,7 @@ import moment from 'moment'; import { TimeBuckets, TimeBucketsConfig } from './time_buckets'; import { autoInterval } from '../../_interval_options'; +import { InvalidEsCalendarIntervalError } from '../../../utils'; describe('TimeBuckets', () => { const timeBucketConfig: TimeBucketsConfig = { @@ -137,4 +138,14 @@ describe('TimeBuckets', () => { const format = timeBuckets.getScaledDateFormat(); expect(format).toEqual('HH:mm'); }); + + test('allows days but throws error on weeks', () => { + const timeBuckets = new TimeBuckets(timeBucketConfig); + timeBuckets.setInterval('14d'); + const interval = timeBuckets.getInterval(false); + expect(interval.esUnit).toEqual('d'); + + timeBuckets.setInterval('2w'); + expect(() => timeBuckets.getInterval(false)).toThrow(InvalidEsCalendarIntervalError); + }); }); diff --git a/src/plugins/data/common/search/aggs/utils/index.ts b/src/plugins/data/common/search/aggs/utils/index.ts index c92653e843233..f1625070b6f75 100644 --- a/src/plugins/data/common/search/aggs/utils/index.ts +++ b/src/plugins/data/common/search/aggs/utils/index.ts @@ -11,7 +11,7 @@ export { getNumberHistogramIntervalByDatatableColumn } from './get_number_histog export { getDateHistogramMetaDataByDatatableColumn } from './get_date_histogram_meta'; export * from './date_interval_utils'; export * from './get_format_with_aggs'; -export * from './ipv4_address'; +export * from './ip_address'; export * from './prop_filter'; export * from './to_angular_json'; export * from './infer_time_zone'; diff --git a/src/plugins/data/common/search/aggs/utils/ip_address.test.ts b/src/plugins/data/common/search/aggs/utils/ip_address.test.ts new file mode 100644 index 0000000000000..966408cf6fe27 --- /dev/null +++ b/src/plugins/data/common/search/aggs/utils/ip_address.test.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { IpAddress } from './ip_address'; + +describe('IpAddress', () => { + describe('constructor', () => { + it.each` + address + ${''} + ${'hello, world'} + ${'256.0.0.0'} + ${'-1.0.0.0'} + ${Number.MAX_SAFE_INTEGER} + ${'fffff::'} + ${'ffff:0:0:0:0:0:0:0:0'} + `('should throw an error on $address', ({ address }) => { + expect(() => new IpAddress(address)).toThrowError(); + }); + + it.each` + address | expected + ${'192.168.257'} | ${'192.168.1.1'} + ${2116932386} | ${'126.45.211.34'} + ${'126.45.211.34'} | ${'126.45.211.34'} + ${[126, 45, 211, 34]} | ${'126.45.211.34'} + ${'ffff:0:0:0:0:0:0:0'} | ${'ffff::'} + ${'ffff::'} | ${'ffff::'} + ${[0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]} | ${'ffff::'} + `('should parse $address', ({ address, expected }) => { + expect(new IpAddress(address).toString()).toBe(expected); + }); + }); + + describe('valueOf', () => { + it.each` + address | expected + ${'0.0.0.0'} | ${'0'} + ${'0.0.0.1'} | ${'1'} + ${'126.45.211.34'} | ${'2116932386'} + ${'ffff::'} | ${'340277174624079928635746076935438991360'} + `( + 'should return $expected as a decimal representation of $address', + ({ address, expected }) => { + expect(new IpAddress(address).valueOf().toString()).toBe(expected); + } + ); + }); + + describe('toString()', () => { + it.each` + address | expected + ${'0.000.00000.1'} | ${'0.0.0.1'} + ${'192.168.257'} | ${'192.168.1.1'} + ${'ffff:0:0:0:0:0:0:0'} | ${'ffff::'} + ${'0:0:0:0:0:0:0:ffff'} | ${'::ffff'} + ${'f:0:0:0:0:0:0:f'} | ${'f::f'} + `('should serialize $address as $expected', ({ address, expected }) => { + expect(new IpAddress(address).toString()).toBe(expected); + }); + }); +}); diff --git a/src/plugins/data/common/search/aggs/utils/ip_address.ts b/src/plugins/data/common/search/aggs/utils/ip_address.ts new file mode 100644 index 0000000000000..2fffbc468046f --- /dev/null +++ b/src/plugins/data/common/search/aggs/utils/ip_address.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import ipaddr, { IPv4, IPv6 } from 'ipaddr.js'; + +function isIPv6(value: IPv4 | IPv6): value is IPv6 { + return value.kind() === 'ipv6'; +} + +export class IpAddress { + private value: IPv4 | IPv6; + + constructor(ipAddress: string | number | number[]) { + try { + this.value = Array.isArray(ipAddress) + ? ipaddr.fromByteArray(ipAddress) + : ipaddr.parse(`${ipAddress}`); + } catch { + throw Error('Invalid IP address: ' + ipAddress); + } + } + + toString() { + if (isIPv6(this.value)) { + return this.value.toRFC5952String(); + } + + return this.value.toString(); + } + + valueOf(): number | bigint { + const value = this.value + .toByteArray() + .reduce((result, octet) => result * 256n + BigInt(octet), 0n); + + if (value > Number.MAX_SAFE_INTEGER) { + return value; + } + + return Number(value); + } +} diff --git a/src/plugins/data/common/search/aggs/utils/ipv4_address.test.ts b/src/plugins/data/common/search/aggs/utils/ipv4_address.test.ts deleted file mode 100644 index 4be406f54390f..0000000000000 --- a/src/plugins/data/common/search/aggs/utils/ipv4_address.test.ts +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import expect from '@kbn/expect'; -import { Ipv4Address } from './ipv4_address'; - -describe('Ipv4Address', () => { - it('should throw errors with invalid IP addresses', () => { - // @ts-ignore - expect(() => new Ipv4Address()).to.throwError(); - - expect(() => new Ipv4Address('')).to.throwError(); - - expect(() => new Ipv4Address('hello, world')).to.throwError(); - - expect(() => new Ipv4Address('0.0.0')).to.throwError(); - - expect(() => new Ipv4Address('256.0.0.0')).to.throwError(); - - expect(() => new Ipv4Address('-1.0.0.0')).to.throwError(); - - expect(() => new Ipv4Address(Number.MAX_SAFE_INTEGER)).to.throwError(); - }); - - it('should allow creation with an integer or string', () => { - expect(new Ipv4Address(2116932386).toString()).to.be( - new Ipv4Address('126.45.211.34').toString() - ); - }); - - it('should correctly calculate the decimal representation of an IP address', () => { - let ipAddress = new Ipv4Address('0.0.0.0'); - expect(ipAddress.valueOf()).to.be(0); - - ipAddress = new Ipv4Address('0.0.0.1'); - expect(ipAddress.valueOf()).to.be(1); - - ipAddress = new Ipv4Address('126.45.211.34'); - expect(ipAddress.valueOf()).to.be(2116932386); - }); - - it('toString()', () => { - let ipAddress = new Ipv4Address('0.000.00000.1'); - expect(ipAddress.toString()).to.be('0.0.0.1'); - - ipAddress = new Ipv4Address('123.123.123.123'); - expect(ipAddress.toString()).to.be('123.123.123.123'); - }); -}); diff --git a/src/plugins/data/common/search/aggs/utils/ipv4_address.ts b/src/plugins/data/common/search/aggs/utils/ipv4_address.ts deleted file mode 100644 index aa6588349c9db..0000000000000 --- a/src/plugins/data/common/search/aggs/utils/ipv4_address.ts +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -const NUM_BYTES = 4; -const BYTE_SIZE = 256; - -function throwError(ipAddress: string | number) { - throw Error('Invalid IPv4 address: ' + ipAddress); -} - -function isIntegerInRange(integer: number, min: number, max: number) { - return ( - !isNaN(integer as number) && integer >= min && integer < max && (integer as number) % 1 === 0 - ); -} - -export class Ipv4Address { - private value: number; - - constructor(ipAddress: string | number) { - if (typeof ipAddress === 'string') { - this.value = 0; - - const bytes = ipAddress.split('.'); - if (bytes.length !== NUM_BYTES) { - throwError(ipAddress); - } - - for (let i = 0; i < bytes.length; i++) { - const byte = Number(bytes[i]); - if (!isIntegerInRange(byte, 0, BYTE_SIZE)) { - throwError(ipAddress); - } - this.value += Math.pow(BYTE_SIZE, NUM_BYTES - 1 - i) * byte; - } - } else { - this.value = ipAddress; - } - - if (!isIntegerInRange(this.value, 0, Math.pow(BYTE_SIZE, NUM_BYTES))) { - throwError(ipAddress); - } - } - - public toString() { - let value = this.value; - const bytes = []; - for (let i = 0; i < NUM_BYTES; i++) { - bytes.unshift(value % 256); - value = Math.floor(value / 256); - } - return bytes.join('.'); - } - - public valueOf() { - return this.value; - } -} diff --git a/src/plugins/data/public/actions/filters/create_filters_from_value_click.test.ts b/src/plugins/data/public/actions/filters/create_filters_from_value_click.test.ts index 30667d59eff84..14ae24a2a5626 100644 --- a/src/plugins/data/public/actions/filters/create_filters_from_value_click.test.ts +++ b/src/plugins/data/public/actions/filters/create_filters_from_value_click.test.ts @@ -105,4 +105,11 @@ describe('createFiltersFromValueClick', () => { expect(rangeFilter.range.bytes.lt).toEqual(2078); } }); + + test('handles non-unique filters', async () => { + const [point] = dataPoints; + const filters = await createFiltersFromValueClickAction({ data: [point, point] }); + + expect(filters.length).toEqual(1); + }); }); diff --git a/src/plugins/data/public/actions/filters/create_filters_from_value_click.ts b/src/plugins/data/public/actions/filters/create_filters_from_value_click.ts index fb8240a0cdc27..13cb9ba419bf9 100644 --- a/src/plugins/data/public/actions/filters/create_filters_from_value_click.ts +++ b/src/plugins/data/public/actions/filters/create_filters_from_value_click.ts @@ -6,6 +6,7 @@ * Side Public License, v 1. */ +import _ from 'lodash'; import { Datatable } from '../../../../../plugins/expressions/public'; import { esFilters, Filter } from '../../../public'; import { getIndexPatterns, getSearchService } from '../../../public/services'; @@ -140,5 +141,7 @@ export const createFiltersFromValueClickAction = async ({ }) ); - return esFilters.mapAndFlattenFilters(filters); + return _.uniqWith(esFilters.mapAndFlattenFilters(filters), (a, b) => + esFilters.compareFilters(a, b, esFilters.COMPARE_ALL_OPTIONS) + ); }; diff --git a/src/plugins/data/public/index.ts b/src/plugins/data/public/index.ts index f2a61e94a07d9..ba873952c9841 100644 --- a/src/plugins/data/public/index.ts +++ b/src/plugins/data/public/index.ts @@ -305,7 +305,7 @@ import { dateHistogramInterval, InvalidEsCalendarIntervalError, InvalidEsIntervalFormatError, - Ipv4Address, + IpAddress, isValidEsInterval, isValidInterval, parseEsInterval, @@ -411,7 +411,7 @@ export const search = { intervalOptions, InvalidEsCalendarIntervalError, InvalidEsIntervalFormatError, - Ipv4Address, + IpAddress, isDateHistogramBucketAggConfig, // TODO: remove in build_pipeline refactor isNumberType, isStringType, diff --git a/src/plugins/data/public/public.api.md b/src/plugins/data/public/public.api.md index bb1684c658362..ad71b6dfc9024 100644 --- a/src/plugins/data/public/public.api.md +++ b/src/plugins/data/public/public.api.md @@ -2302,7 +2302,7 @@ export const search: { })[]; InvalidEsCalendarIntervalError: typeof InvalidEsCalendarIntervalError; InvalidEsIntervalFormatError: typeof InvalidEsIntervalFormatError; - Ipv4Address: typeof Ipv4Address; + IpAddress: typeof IpAddress; isDateHistogramBucketAggConfig: typeof isDateHistogramBucketAggConfig; isNumberType: (agg: import("../common").AggConfig) => boolean; isStringType: (agg: import("../common").AggConfig) => boolean; @@ -2740,7 +2740,7 @@ export interface WaitUntilNextSessionCompletesOptions { // src/plugins/data/public/index.ts:410:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts // src/plugins/data/public/index.ts:419:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts // src/plugins/data/public/index.ts:420:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts -// src/plugins/data/public/index.ts:421:1 - (ae-forgotten-export) The symbol "Ipv4Address" needs to be exported by the entry point index.d.ts +// src/plugins/data/public/index.ts:421:1 - (ae-forgotten-export) The symbol "IpAddress" needs to be exported by the entry point index.d.ts // src/plugins/data/public/index.ts:422:1 - (ae-forgotten-export) The symbol "isDateHistogramBucketAggConfig" needs to be exported by the entry point index.d.ts // src/plugins/data/public/index.ts:426:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts // src/plugins/data/public/index.ts:427:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts diff --git a/src/plugins/data/public/query/filter_manager/lib/generate_filter.test.ts b/src/plugins/data/public/query/filter_manager/lib/generate_filter.test.ts index 1a29a57ff058a..0daaf804e7b40 100644 --- a/src/plugins/data/public/query/filter_manager/lib/generate_filter.test.ts +++ b/src/plugins/data/public/query/filter_manager/lib/generate_filter.test.ts @@ -174,4 +174,22 @@ describe('Generate filters', () => { [FIELD.name]: ANOTHER_PHRASE, }); }); + + it('should use only distinct values', () => { + const ANOTHER_PHRASE = 'another-value'; + const filters = generateFilters( + mockFilterManager, + FIELD, + [PHRASE_VALUE, ANOTHER_PHRASE, PHRASE_VALUE, ANOTHER_PHRASE], + '', + INDEX_NAME + ); + expect(filters).toHaveLength(2); + expect((filters[0] as PhraseFilter).query.match_phrase).toEqual({ + [FIELD.name]: PHRASE_VALUE, + }); + expect((filters[1] as PhraseFilter).query.match_phrase).toEqual({ + [FIELD.name]: ANOTHER_PHRASE, + }); + }); }); diff --git a/src/plugins/data/public/query/filter_manager/lib/generate_filters.ts b/src/plugins/data/public/query/filter_manager/lib/generate_filters.ts index 9a52a02edcd4e..0a4998a159523 100644 --- a/src/plugins/data/public/query/filter_manager/lib/generate_filters.ts +++ b/src/plugins/data/public/query/filter_manager/lib/generate_filters.ts @@ -71,7 +71,7 @@ export function generateFilters( operation: string, index: string ): Filter[] { - values = Array.isArray(values) ? values : [values]; + values = Array.isArray(values) ? _.uniq(values) : [values]; const fieldObj = (_.isObject(field) ? field : { diff --git a/src/plugins/data/public/ui/filter_bar/filter_editor/lib/filter_editor_utils.ts b/src/plugins/data/public/ui/filter_bar/filter_editor/lib/filter_editor_utils.ts index d6e5b57f5878b..e15f667128c4f 100644 --- a/src/plugins/data/public/ui/filter_bar/filter_editor/lib/filter_editor_utils.ts +++ b/src/plugins/data/public/ui/filter_bar/filter_editor/lib/filter_editor_utils.ts @@ -12,7 +12,7 @@ import { isFilterable, IIndexPattern, IFieldType, - Ipv4Address, + IpAddress, Filter, FieldFilter, } from '../../../../../common'; @@ -44,7 +44,7 @@ export function validateParams(params: any, type: string) { return Boolean(typeof params === 'string' && moment && moment.isValid()); case 'ip': try { - return Boolean(new Ipv4Address(params)); + return Boolean(new IpAddress(params)); } catch (e) { return false; } diff --git a/src/plugins/data/public/ui/search_bar/search_bar.tsx b/src/plugins/data/public/ui/search_bar/search_bar.tsx index 9c84b9b73d5e5..1608b68ed01b2 100644 --- a/src/plugins/data/public/ui/search_bar/search_bar.tsx +++ b/src/plugins/data/public/ui/search_bar/search_bar.tsx @@ -10,7 +10,6 @@ import { compact } from 'lodash'; import { InjectedIntl, injectI18n } from '@kbn/i18n/react'; import classNames from 'classnames'; import React, { Component } from 'react'; -import ResizeObserver from 'resize-observer-polyfill'; import { get, isEqual } from 'lodash'; import { EuiIconProps } from '@elastic/eui'; @@ -100,8 +99,6 @@ class SearchBarUI extends Component { private services = this.props.kibana.services; private savedQueryService = this.services.data.query.savedQueries; - public filterBarRef: Element | null = null; - public filterBarWrapperRef: Element | null = null; public static getDerivedStateFromProps(nextProps: SearchBarProps, prevState: State) { if (isEqual(prevState.currentProps, nextProps)) { @@ -212,19 +209,6 @@ class SearchBarUI extends Component { ); } - public setFilterBarHeight = () => { - requestAnimationFrame(() => { - const height = - this.filterBarRef && this.state.isFiltersVisible ? this.filterBarRef.clientHeight : 0; - if (this.filterBarWrapperRef) { - this.filterBarWrapperRef.setAttribute('style', `height: ${height}px`); - } - }); - }; - - // member-ordering rules conflict with use-before-declaration rules - public ro = new ResizeObserver(this.setFilterBarHeight); - public onSave = async (savedQueryMeta: SavedQueryMeta, saveAsNew = false) => { if (!this.state.query) return; @@ -352,20 +336,6 @@ class SearchBarUI extends Component { } }; - public componentDidMount() { - if (this.filterBarRef) { - this.setFilterBarHeight(); - this.ro.observe(this.filterBarRef); - } - } - - public componentDidUpdate() { - if (this.filterBarRef) { - this.setFilterBarHeight(); - this.ro.unobserve(this.filterBarRef); - } - } - public render() { const savedQueryManagement = this.state.query && this.props.onClearSavedQuery && ( { 'globalFilterGroup__wrapper-isVisible': this.state.isFiltersVisible, }); filterBar = ( -
{ - this.filterBarWrapperRef = node; - }} - className={filterGroupClasses} - > -
{ - this.filterBarRef = node; - }} - > - -
+
+
); } diff --git a/src/plugins/data/server/index.ts b/src/plugins/data/server/index.ts index c4e132e33fc3b..e782fc0d4ffea 100644 --- a/src/plugins/data/server/index.ts +++ b/src/plugins/data/server/index.ts @@ -170,7 +170,7 @@ import { dateHistogramInterval, InvalidEsCalendarIntervalError, InvalidEsIntervalFormatError, - Ipv4Address, + IpAddress, isValidEsInterval, isValidInterval, parseEsInterval, @@ -247,7 +247,7 @@ export const search = { intervalOptions, InvalidEsCalendarIntervalError, InvalidEsIntervalFormatError, - Ipv4Address, + IpAddress, isNumberType, isStringType, isType, diff --git a/src/plugins/data/server/search/strategies/ese_search/ese_search_strategy.ts b/src/plugins/data/server/search/strategies/ese_search/ese_search_strategy.ts index e7f5005e7e837..cec4b9a2dbf9f 100644 --- a/src/plugins/data/server/search/strategies/ese_search/ese_search_strategy.ts +++ b/src/plugins/data/server/search/strategies/ese_search/ese_search_strategy.ts @@ -39,7 +39,7 @@ export const enhancedEsSearchStrategyProvider = ( legacyConfig$: Observable, logger: Logger, usage?: SearchUsage -): ISearchStrategy => { +): ISearchStrategy => { async function cancelAsyncSearch(id: string, esClient: IScopedClusterClient) { try { await esClient.asCurrentUser.asyncSearch.delete({ id }); diff --git a/src/plugins/data/server/server.api.md b/src/plugins/data/server/server.api.md index 8ef10451398a6..185f096e34d86 100644 --- a/src/plugins/data/server/server.api.md +++ b/src/plugins/data/server/server.api.md @@ -1341,7 +1341,7 @@ export const search: { })[]; InvalidEsCalendarIntervalError: typeof InvalidEsCalendarIntervalError; InvalidEsIntervalFormatError: typeof InvalidEsIntervalFormatError; - Ipv4Address: typeof Ipv4Address; + IpAddress: typeof IpAddress; isNumberType: (agg: import("../common").AggConfig) => boolean; isStringType: (agg: import("../common").AggConfig) => boolean; isType: (...types: string[]) => (agg: import("../common").AggConfig) => boolean; @@ -1544,7 +1544,7 @@ export function usageProvider(core: CoreSetup_2): SearchUsage; // src/plugins/data/server/index.ts:246:1 - (ae-forgotten-export) The symbol "dateHistogramInterval" needs to be exported by the entry point index.d.ts // src/plugins/data/server/index.ts:255:1 - (ae-forgotten-export) The symbol "InvalidEsCalendarIntervalError" needs to be exported by the entry point index.d.ts // src/plugins/data/server/index.ts:256:1 - (ae-forgotten-export) The symbol "InvalidEsIntervalFormatError" needs to be exported by the entry point index.d.ts -// src/plugins/data/server/index.ts:257:1 - (ae-forgotten-export) The symbol "Ipv4Address" needs to be exported by the entry point index.d.ts +// src/plugins/data/server/index.ts:257:1 - (ae-forgotten-export) The symbol "IpAddress" needs to be exported by the entry point index.d.ts // src/plugins/data/server/index.ts:261:1 - (ae-forgotten-export) The symbol "isValidEsInterval" needs to be exported by the entry point index.d.ts // src/plugins/data/server/index.ts:262:1 - (ae-forgotten-export) The symbol "isValidInterval" needs to be exported by the entry point index.d.ts // src/plugins/data/server/index.ts:266:1 - (ae-forgotten-export) The symbol "propFilter" needs to be exported by the entry point index.d.ts diff --git a/src/plugins/discover/public/application/components/discover_grid/discover_grid.test.tsx b/src/plugins/discover/public/application/components/discover_grid/discover_grid.test.tsx index 8037022085f02..b2be40c008200 100644 --- a/src/plugins/discover/public/application/components/discover_grid/discover_grid.test.tsx +++ b/src/plugins/discover/public/application/components/discover_grid/discover_grid.test.tsx @@ -124,6 +124,33 @@ describe('DiscoverGrid', () => { expect(getDisplayedDocNr(component)).toBe(5); }); + test('showing selected documents, underlying data changes, all documents are displayed, selection is gone', async () => { + await toggleDocSelection(component, esHits[0]); + await toggleDocSelection(component, esHits[1]); + expect(getSelectedDocNr(component)).toBe(2); + findTestSubject(component, 'dscGridSelectionBtn').simulate('click'); + findTestSubject(component, 'dscGridShowSelectedDocuments').simulate('click'); + expect(getDisplayedDocNr(component)).toBe(2); + component.setProps({ + rows: [ + { + _index: 'i', + _id: '6', + _score: 1, + _type: '_doc', + _source: { + date: '2020-20-02T12:12:12.128', + name: 'test6', + extension: 'doc', + bytes: 50, + }, + }, + ], + }); + expect(getDisplayedDocNr(component)).toBe(1); + expect(getSelectedDocNr(component)).toBe(0); + }); + test('showing only selected documents and remove filter deselecting each doc manually', async () => { await toggleDocSelection(component, esHits[0]); findTestSubject(component, 'dscGridSelectionBtn').simulate('click'); diff --git a/src/plugins/discover/public/application/components/discover_grid/discover_grid.tsx b/src/plugins/discover/public/application/components/discover_grid/discover_grid.tsx index be38f166fa1c0..f969eb32f3791 100644 --- a/src/plugins/discover/public/application/components/discover_grid/discover_grid.tsx +++ b/src/plugins/discover/public/application/components/discover_grid/discover_grid.tsx @@ -164,17 +164,33 @@ export const DiscoverGrid = ({ const [isFilterActive, setIsFilterActive] = useState(false); const displayedColumns = getDisplayedColumns(columns, indexPattern); const defaultColumns = displayedColumns.includes('_source'); + const usedSelectedDocs = useMemo(() => { + if (!selectedDocs.length || !rows?.length) { + return []; + } + const idMap = rows.reduce((map, row) => map.set(getDocId(row), true), new Map()); + // filter out selected docs that are no longer part of the current data + const result = selectedDocs.filter((docId) => idMap.get(docId)); + if (result.length === 0 && isFilterActive) { + setIsFilterActive(false); + } + return result; + }, [selectedDocs, rows, isFilterActive]); + const displayedRows = useMemo(() => { if (!rows) { return []; } - if (!isFilterActive || selectedDocs.length === 0) { + if (!isFilterActive || usedSelectedDocs.length === 0) { + return rows; + } + const rowsFiltered = rows.filter((row) => usedSelectedDocs.includes(getDocId(row))); + if (!rowsFiltered.length) { + // in case the selected docs are no longer part of the sample of 500, show all docs return rows; } - return rows.filter((row) => { - return selectedDocs.includes(getDocId(row)); - }); - }, [rows, selectedDocs, isFilterActive]); + return rowsFiltered; + }, [rows, usedSelectedDocs, isFilterActive]); /** * Pagination @@ -258,16 +274,16 @@ export const DiscoverGrid = ({ const additionalControls = useMemo( () => - selectedDocs.length ? ( + usedSelectedDocs.length ? ( ) : null, - [selectedDocs, isFilterActive, rows, setIsFilterActive] + [usedSelectedDocs, isFilterActive, rows, setIsFilterActive] ); if (!rowCount) { @@ -291,7 +307,7 @@ export const DiscoverGrid = ({ onFilter, indexPattern, isDarkMode: services.uiSettings.get('theme:darkMode'), - selectedDocs, + selectedDocs: usedSelectedDocs, setSelectedDocs: (newSelectedDocs) => { setSelectedDocs(newSelectedDocs); if (isFilterActive && newSelectedDocs.length === 0) { diff --git a/src/plugins/discover/public/application/components/top_nav/get_top_nav_links.ts b/src/plugins/discover/public/application/components/top_nav/get_top_nav_links.ts index 9a12cb51eac0c..38bc831225c68 100644 --- a/src/plugins/discover/public/application/components/top_nav/get_top_nav_links.ts +++ b/src/plugins/discover/public/application/components/top_nav/get_top_nav_links.ts @@ -7,6 +7,7 @@ */ import { i18n } from '@kbn/i18n'; +import moment from 'moment'; import { showOpenSearchPanel } from './show_open_search_panel'; import { getSharingData, showPublicUrlSwitch } from '../../helpers/get_sharing_data'; import { unhashUrl } from '../../../../../kibana_utils/public'; @@ -122,7 +123,13 @@ export const getTopNavLinks = ({ objectType: 'search', sharingData: { ...sharingData, - title: savedSearch.title, + // CSV reports can be generated without a saved search so we provide a fallback title + title: + savedSearch.title || + i18n.translate('discover.localMenu.fallbackReportTitle', { + defaultMessage: 'Discover search [{date}]', + values: { date: moment().toISOString(true) }, + }), }, isDirty: !savedSearch.id || state.isAppStateDirty(), showPublicUrlSwitch, diff --git a/src/plugins/expressions/common/execution/execution.test.ts b/src/plugins/expressions/common/execution/execution.test.ts index 343ea9ef7f03c..69687f75f3098 100644 --- a/src/plugins/expressions/common/execution/execution.test.ts +++ b/src/plugins/expressions/common/execution/execution.test.ts @@ -362,14 +362,21 @@ describe('Execution', () => { }); test('result is undefined until execution completes', async () => { + jest.useFakeTimers(); const execution = createExecution('sleep 10'); expect(execution.state.get().result).toBe(undefined); execution.start(null).subscribe(jest.fn()); expect(execution.state.get().result).toBe(undefined); - await new Promise((r) => setTimeout(r, 1)); + + jest.advanceTimersByTime(1); + await new Promise(process.nextTick); expect(execution.state.get().result).toBe(undefined); - await new Promise((r) => setTimeout(r, 11)); + + jest.advanceTimersByTime(10); + await new Promise(process.nextTick); expect(execution.state.get().result).toBe(null); + + jest.useRealTimers(); }); test('handles functions returning observables', () => { diff --git a/src/plugins/kibana_react/public/code_editor/code_editor.tsx b/src/plugins/kibana_react/public/code_editor/code_editor.tsx index 51344e2d28ab6..55e10e7861e51 100644 --- a/src/plugins/kibana_react/public/code_editor/code_editor.tsx +++ b/src/plugins/kibana_react/public/code_editor/code_editor.tsx @@ -34,14 +34,14 @@ export interface Props { value: string; /** Function invoked when text in editor is changed */ - onChange: (value: string) => void; + onChange: (value: string, event: monaco.editor.IModelContentChangedEvent) => void; /** * Options for the Monaco Code Editor * Documentation of options can be found here: - * https://microsoft.github.io/monaco-editor/api/interfaces/monaco.editor.ieditorconstructionoptions.html + * https://microsoft.github.io/monaco-editor/api/interfaces/monaco.editor.istandaloneeditorconstructionoptions.html */ - options?: monaco.editor.IEditorConstructionOptions; + options?: monaco.editor.IStandaloneEditorConstructionOptions; /** * Suggestion provider for autocompletion diff --git a/src/plugins/kibana_react/public/code_editor/index.tsx b/src/plugins/kibana_react/public/code_editor/index.tsx index 635e84b1d8c20..2440974c3b1d1 100644 --- a/src/plugins/kibana_react/public/code_editor/index.tsx +++ b/src/plugins/kibana_react/public/code_editor/index.tsx @@ -16,7 +16,7 @@ import { import darkTheme from '@elastic/eui/dist/eui_theme_dark.json'; import lightTheme from '@elastic/eui/dist/eui_theme_light.json'; import { useUiSetting } from '../ui_settings'; -import type { Props } from './code_editor'; +import { Props } from './code_editor'; const LazyBaseEditor = React.lazy(() => import('./code_editor')); @@ -26,6 +26,8 @@ const Fallback = () => ( ); +export type CodeEditorProps = Props; + /** * Renders a Monaco code editor with EUI color theme. * diff --git a/src/plugins/spaces_oss/public/api.mock.ts b/src/plugins/spaces_oss/public/api.mock.ts index 84b22459a96e2..9ad7599b5ae61 100644 --- a/src/plugins/spaces_oss/public/api.mock.ts +++ b/src/plugins/spaces_oss/public/api.mock.ts @@ -11,7 +11,7 @@ import { of } from 'rxjs'; import type { SpacesApi, SpacesApiUi, SpacesApiUiComponent } from './api'; const createApiMock = (): jest.Mocked => ({ - activeSpace$: of(), + getActiveSpace$: jest.fn().mockReturnValue(of()), getActiveSpace: jest.fn(), ui: createApiUiMock(), }); diff --git a/src/plugins/spaces_oss/public/api.ts b/src/plugins/spaces_oss/public/api.ts index bd452c2fca00e..e460d9a43ef6b 100644 --- a/src/plugins/spaces_oss/public/api.ts +++ b/src/plugins/spaces_oss/public/api.ts @@ -19,7 +19,7 @@ export interface SpacesApi { * Observable representing the currently active space. * The details of the space can change without a full page reload (such as display name, color, etc.) */ - readonly activeSpace$: Observable; + getActiveSpace$(): Observable; /** * Retrieve the currently active space. diff --git a/src/plugins/vis_default_editor/public/components/controls/components/from_to_list.tsx b/src/plugins/vis_default_editor/public/components/controls/components/from_to_list.tsx index a6e0ee0d2aec4..1c721cde37b2b 100644 --- a/src/plugins/vis_default_editor/public/components/controls/components/from_to_list.tsx +++ b/src/plugins/vis_default_editor/public/components/controls/components/from_to_list.tsx @@ -38,7 +38,7 @@ const defaultConfig = { from: { value: '0.0.0.0', model: '0.0.0.0', isInvalid: false }, to: { value: '255.255.255.255', model: '255.255.255.255', isInvalid: false }, }, - validateClass: search.aggs.Ipv4Address, + validateClass: search.aggs.IpAddress, getModelValue: (item: FromToObject = {}) => ({ from: { value: item.from || EMPTY_STRING, diff --git a/src/plugins/vis_type_timeseries/common/enums/metric_types.ts b/src/plugins/vis_type_timeseries/common/enums/metric_types.ts index 17352f0f9da25..8e2bc8f346eb6 100644 --- a/src/plugins/vis_type_timeseries/common/enums/metric_types.ts +++ b/src/plugins/vis_type_timeseries/common/enums/metric_types.ts @@ -27,6 +27,7 @@ export enum METRIC_TYPES { // We should probably use BUCKET_TYPES from data plugin in future. export enum BUCKET_TYPES { TERMS = 'terms', + FILTERS = 'filters', } export const EXTENDED_STATS_TYPES = [ diff --git a/src/plugins/vis_type_timeseries/public/application/components/lib/convert_series_to_datatable.test.ts b/src/plugins/vis_type_timeseries/public/application/components/lib/convert_series_to_datatable.test.ts index df0874fdd73ec..ff7b2f497fd70 100644 --- a/src/plugins/vis_type_timeseries/public/application/components/lib/convert_series_to_datatable.test.ts +++ b/src/plugins/vis_type_timeseries/public/application/components/lib/convert_series_to_datatable.test.ts @@ -15,6 +15,18 @@ jest.mock('../../../services', () => { getDataStart: jest.fn(() => { return { indexPatterns: jest.fn(), + query: { + timefilter: { + timefilter: { + getTime: jest.fn(() => { + return { + from: '2021-04-30T16:42:24.502Z', + to: '2021-05-05T14:42:24.502Z', + }; + }), + }, + }, + }, }; }), }; @@ -25,9 +37,12 @@ describe('convert series to datatables', () => { beforeEach(() => { const fieldMap: Record = { - test1: { name: 'test1', spec: { type: 'date' } } as IndexPatternField, - test2: { name: 'test2' } as IndexPatternField, - test3: { name: 'test3', spec: { type: 'boolean' } } as IndexPatternField, + test1: { name: 'test1', spec: { type: 'date', name: 'test1' } } as IndexPatternField, + test2: { + name: 'test2', + spec: { type: 'number', name: 'Average of test2' }, + } as IndexPatternField, + test3: { name: 'test3', spec: { type: 'boolean', name: 'test3' } } as IndexPatternField, }; const getFieldByName = (name: string): IndexPatternField | undefined => fieldMap[name]; @@ -41,8 +56,8 @@ describe('convert series to datatables', () => { describe('addMetaColumns()', () => { test('adds the correct meta to a date column', () => { - const columns = [{ id: 0, name: 'test1', isSplit: false }]; - const columnsWithMeta = addMetaToColumns(columns, indexPattern, 'count'); + const columns = [{ id: 0, name: 'test1', isMetric: true, type: 'date_histogram' }]; + const columnsWithMeta = addMetaToColumns(columns, indexPattern); expect(columnsWithMeta).toEqual([ { id: '0', @@ -54,6 +69,13 @@ describe('convert series to datatables', () => { enabled: true, indexPatternId: 'index1', type: 'date_histogram', + schema: 'metric', + params: { + timeRange: { + from: '2021-04-30T16:42:24.502Z', + to: '2021-05-05T14:42:24.502Z', + }, + }, }, type: 'date', }, @@ -63,8 +85,8 @@ describe('convert series to datatables', () => { }); test('adds the correct meta to a non date column', () => { - const columns = [{ id: 1, name: 'Average of test2', isSplit: false }]; - const columnsWithMeta = addMetaToColumns(columns, indexPattern, 'avg'); + const columns = [{ id: 1, name: 'test2', isMetric: true, type: 'avg' }]; + const columnsWithMeta = addMetaToColumns(columns, indexPattern); expect(columnsWithMeta).toEqual([ { id: '1', @@ -76,17 +98,21 @@ describe('convert series to datatables', () => { enabled: true, indexPatternId: 'index1', type: 'avg', + schema: 'metric', + params: { + field: 'Average of test2', + }, }, type: 'number', }, - name: 'Average of test2', + name: 'test2', }, ]); }); test('adds the correct meta for a split column', () => { - const columns = [{ id: 2, name: 'test3', isSplit: true }]; - const columnsWithMeta = addMetaToColumns(columns, indexPattern, 'avg'); + const columns = [{ id: 2, name: 'test3', isMetric: false, type: 'terms' }]; + const columnsWithMeta = addMetaToColumns(columns, indexPattern); expect(columnsWithMeta).toEqual([ { id: '2', @@ -98,6 +124,10 @@ describe('convert series to datatables', () => { enabled: true, indexPatternId: 'index1', type: 'terms', + schema: 'group', + params: { + field: 'test3', + }, }, type: 'boolean', }, diff --git a/src/plugins/vis_type_timeseries/public/application/components/lib/convert_series_to_datatable.ts b/src/plugins/vis_type_timeseries/public/application/components/lib/convert_series_to_datatable.ts index 35506f9e62081..19a1910afbe2f 100644 --- a/src/plugins/vis_type_timeseries/public/application/components/lib/convert_series_to_datatable.ts +++ b/src/plugins/vis_type_timeseries/public/application/components/lib/convert_series_to_datatable.ts @@ -6,51 +6,74 @@ * Side Public License, v 1. */ import { IndexPattern } from 'src/plugins/data/public'; -import { - Datatable, - DatatableRow, - DatatableColumn, - DatatableColumnType, -} from 'src/plugins/expressions/public'; +import { DatatableRow, DatatableColumn, DatatableColumnType } from 'src/plugins/expressions/public'; +import { Query } from 'src/plugins/data/common'; import { TimeseriesVisParams } from '../../../types'; import type { PanelData } from '../../../../common/types'; +import { BUCKET_TYPES } from '../../../../common/enums'; import { fetchIndexPattern } from '../../../../common/index_patterns_utils'; import { getDataStart } from '../../../services'; import { X_ACCESSOR_INDEX } from '../../visualizations/constants'; +import type { TSVBTables } from './types'; -interface TSVBTables { - [key: string]: Datatable; +interface FilterParams { + filter?: Query; + label?: string; + field?: string; } interface TSVBColumns { id: number; name: string; - isSplit: boolean; + isMetric: boolean; + type: string; + params?: FilterParams[]; } export const addMetaToColumns = ( columns: TSVBColumns[], - indexPattern: IndexPattern, - metricsType: string + indexPattern: IndexPattern ): DatatableColumn[] => { return columns.map((column) => { const field = indexPattern.getFieldByName(column.name); const type = (field?.spec.type as DatatableColumnType) || 'number'; + + let params: unknown = { + field: field?.spec.name, + }; + if (column.type === BUCKET_TYPES.FILTERS && column.params) { + const filters = column.params.map((col) => ({ + input: col.filter, + label: col.label, + })); + params = { + filters, + }; + } else if (column.type === 'date_histogram') { + const { query } = getDataStart(); + const timeRange = query.timefilter.timefilter.getTime(); + params = { + timeRange, + }; + } + const cleanedColumn = { id: column.id.toString(), name: column.name, meta: { type, - field: column.name, + field: field?.spec.name, index: indexPattern.title, source: 'esaggs', sourceParams: { enabled: true, indexPatternId: indexPattern?.id, - type: type === 'date' ? 'date_histogram' : column.isSplit ? 'terms' : metricsType, + type: column.type, + schema: column.isMetric ? 'metric' : 'group', + params, }, }, - }; + } as DatatableColumn; return cleanedColumn; }); }; @@ -73,30 +96,58 @@ export const convertSeriesToDataTable = async ( usedIndexPattern = indexPattern; } } - const isGroupedByTerms = layer.split_mode === 'terms'; + const isGroupedByTerms = layer.split_mode === BUCKET_TYPES.TERMS; + const isGroupedByFilters = layer.split_mode === BUCKET_TYPES.FILTERS; const seriesPerLayer = series.filter((s) => s.seriesId === layer.id); let id = X_ACCESSOR_INDEX; const columns: TSVBColumns[] = [ - { id, name: usedIndexPattern.timeFieldName || '', isSplit: false }, + { + id, + name: usedIndexPattern.timeFieldName || '', + isMetric: false, + type: 'date_histogram', + }, ]; + if (seriesPerLayer.length) { id++; - columns.push({ id, name: seriesPerLayer[0].splitByLabel, isSplit: false }); - // Adds an extra column, if the layer is split by terms aggregation + const metrics = layer.metrics; + columns.push({ + id, + name: metrics[metrics.length - 1].field || seriesPerLayer[0].splitByLabel, + isMetric: true, + type: metrics[metrics.length - 1].type, + }); + + // Adds an extra column, if the layer is split by terms or filters aggregation if (isGroupedByTerms) { id++; - columns.push({ id, name: layer.terms_field || '', isSplit: true }); + columns.push({ + id, + name: layer.terms_field || '', + isMetric: false, + type: BUCKET_TYPES.TERMS, + }); + } else if (isGroupedByFilters) { + id++; + columns.push({ + id, + name: BUCKET_TYPES.FILTERS, + isMetric: false, + params: layer?.split_filters as FilterParams[], + type: BUCKET_TYPES.FILTERS, + }); } } - const columnsWithMeta = addMetaToColumns(columns, usedIndexPattern, layer.metrics[0].type); - + const columnsWithMeta = addMetaToColumns(columns, usedIndexPattern); + const filtersColumn = columns.find((col) => col.type === BUCKET_TYPES.FILTERS); let rows: DatatableRow[] = []; for (let j = 0; j < seriesPerLayer.length; j++) { const data = seriesPerLayer[j].data.map((rowData) => { const row: DatatableRow = [rowData[0], rowData[1]]; // If the layer is split by terms aggregation, the data array should also contain the split value. - if (isGroupedByTerms) { + if (isGroupedByTerms || filtersColumn) { row.push(seriesPerLayer[j].label); } return row; diff --git a/src/plugins/vis_type_timeseries/public/application/components/lib/get_click_filter_data.test.ts b/src/plugins/vis_type_timeseries/public/application/components/lib/get_click_filter_data.test.ts new file mode 100644 index 0000000000000..2b9a1f930c15a --- /dev/null +++ b/src/plugins/vis_type_timeseries/public/application/components/lib/get_click_filter_data.test.ts @@ -0,0 +1,207 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { XYChartSeriesIdentifier, GeometryValue } from '@elastic/charts'; +import { getClickFilterData } from './get_click_filter_data'; +import type { TSVBTables } from './types'; +import { TimeseriesVisParams } from '../../../types'; + +describe('getClickFilterData', () => { + test('gets the correct data for a group by everything timeseries chart', () => { + const points = [ + [ + { + x: 1620032400000, + y: 72, + accessor: 'y1', + datum: [1620032400000, 72], + }, + { + key: + 'groupId{yaxis_50589930-ad98-11eb-b6f6-59011d7388b7_main_group}spec{61ca57f1-469d-11e7-af02-69e470af7417}yAccessor{1}splitAccessors{}', + specId: '61ca57f1-469d-11e7-af02-69e470af7417', + yAccessor: 1, + }, + ], + ] as Array<[GeometryValue, XYChartSeriesIdentifier]>; + const tables = { + '61ca57f1-469d-11e7-af02-69e470af7417': { + type: 'datatable', + rows: [ + [1620010800000, 8], + [1620021600000, 33], + [1620032400000, 72], + [1620043200000, 66], + [1620054000000, 36], + [1620064800000, 11], + ], + columns: [ + { + id: '0', + name: 'timestamp', + meta: { + type: 'date', + field: 'timestamp', + index: 'kibana_sample_data_logs', + source: 'esaggs', + sourceParams: { + enabled: true, + indexPatternId: '90943e30-9a47-11e8-b64d-95841ca0b247', + type: 'date_histogram', + schema: 'group', + params: { + field: 'timestamp', + }, + }, + }, + }, + { + id: '1', + name: 'Count', + meta: { + type: 'number', + index: 'kibana_sample_data_logs', + source: 'esaggs', + sourceParams: { + enabled: true, + indexPatternId: '90943e30-9a47-11e8-b64d-95841ca0b247', + type: 'count', + schema: 'metric', + params: {}, + }, + }, + }, + ], + }, + } as TSVBTables; + const model = { + series: [ + { + id: '61ca57f1-469d-11e7-af02-69e470af7417', + split_mode: 'everything', + }, + ], + } as TimeseriesVisParams; + const data = getClickFilterData(points, tables, model); + expect(data[0].column).toEqual(0); + expect(data[0].row).toEqual(2); + expect(data[0].value).toEqual(points[0][0].x); + }); + + test('gets the correct data for a group by terms timeseries chart', () => { + const points = [ + [ + { + x: 1619481600000, + y: 3, + accessor: 'y1', + datum: [1619481600000, 3], + }, + { + key: + 'groupId{yaxis_6e0353a0-ad9b-11eb-b112-89cce8e43380_main_group}spec{61ca57f1-469d-11e7-af02-69e470af7417:1}yAccessor{1}splitAccessors{}', + specId: '61ca57f1-469d-11e7-af02-69e470af7417:1', + }, + ], + ] as Array<[GeometryValue, XYChartSeriesIdentifier]>; + const tables = { + '61ca57f1-469d-11e7-af02-69e470af7417': { + type: 'datatable', + rows: [ + [1619449200000, 31, 0], + [1619460000000, 36, 0], + [1619470800000, 35, 0], + [1619481600000, 40, 0], + [1619492400000, 38, 0], + [1619503200000, 30, 0], + [1620172800000, 0, 0], + [1619449200000, 4, 1], + [1619460000000, 4, 1], + [1619470800000, 3, 1], + [1619481600000, 3, 1], + [1619492400000, 2, 1], + [1619503200000, 3, 1], + ], + columns: [ + { + id: '0', + name: 'timestamp', + meta: { + type: 'date', + field: 'timestamp', + index: 'kibana_sample_data_flights', + source: 'esaggs', + sourceParams: { + enabled: true, + indexPatternId: 'd3d7af60-4c81-11e8-b3d7-01146121b73d', + type: 'date_histogram', + schema: 'group', + params: { + field: 'timestamp', + }, + }, + }, + }, + { + id: '1', + name: 'Count', + meta: { + type: 'number', + index: 'kibana_sample_data_flights', + source: 'esaggs', + sourceParams: { + enabled: true, + indexPatternId: 'd3d7af60-4c81-11e8-b3d7-01146121b73d', + type: 'count', + schema: 'metric', + params: {}, + }, + }, + }, + { + id: '2', + name: 'Cancelled', + meta: { + type: 'boolean', + field: 'Cancelled', + index: 'kibana_sample_data_flights', + source: 'esaggs', + sourceParams: { + enabled: true, + indexPatternId: 'd3d7af60-4c81-11e8-b3d7-01146121b73d', + type: 'terms', + schema: 'group', + params: { + field: 'Cancelled', + }, + }, + }, + }, + ], + }, + } as TSVBTables; + const model = { + series: [ + { + id: '61ca57f1-469d-11e7-af02-69e470af7417', + split_mode: 'terms', + }, + ], + } as TimeseriesVisParams; + const data = getClickFilterData(points, tables, model); + expect(data.length).toEqual(2); + expect(data[0].column).toEqual(0); + expect(data[0].row).toEqual(10); + expect(data[0].value).toEqual(points[0][0].x); + + expect(data[1].column).toEqual(2); + expect(data[1].row).toEqual(10); + // expect(data).toEqual([]); + const splitValue = points[0][1].specId.split(':'); + expect(data[1].value).toEqual(parseInt(splitValue[1], 10)); + }); +}); diff --git a/src/plugins/vis_type_timeseries/public/application/components/lib/get_click_filter_data.ts b/src/plugins/vis_type_timeseries/public/application/components/lib/get_click_filter_data.ts new file mode 100644 index 0000000000000..a6e35aa6e1032 --- /dev/null +++ b/src/plugins/vis_type_timeseries/public/application/components/lib/get_click_filter_data.ts @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { XYChartSeriesIdentifier, GeometryValue } from '@elastic/charts'; +import { ValueClickContext } from 'src/plugins/embeddable/public'; +import { X_ACCESSOR_INDEX } from '../../visualizations/constants'; +import { BUCKET_TYPES } from '../../../../common/enums'; +import { TimeseriesVisParams } from '../../../types'; +import type { TSVBTables } from './types'; + +export const getClickFilterData = ( + points: Array<[GeometryValue, XYChartSeriesIdentifier]>, + tables: TSVBTables, + model: TimeseriesVisParams +) => { + const data: ValueClickContext['data']['data'] = []; + points.forEach((point) => { + const [geometry] = point; + const { specId } = point[1]; + // specId for a split series has the format + // 61ca57f1-469d-11e7-af02-69e470af7417:Men's Accessories, : + const [layerId, splitLabel] = specId.split(':'); + const table = tables[layerId]; + + const layer = model.series.filter(({ id }) => id === layerId); + let label = splitLabel; + // compute label for filters split mode + if (splitLabel && layer.length && layer[0].split_mode === BUCKET_TYPES.FILTERS) { + const filter = layer[0]?.split_filters?.filter(({ id }) => id === splitLabel); + label = filter?.[0].label || (filter?.[0].filter?.query as string); + } + const index = table.rows.findIndex((row) => { + const condition = + geometry.x === row[X_ACCESSOR_INDEX] && geometry.y === row[X_ACCESSOR_INDEX + 1]; + return splitLabel ? condition && row[X_ACCESSOR_INDEX + 2].toString() === label : condition; + }); + if (index < 0) return; + + // Filter out the metric column + const bucketCols = table.columns.filter((col) => col.meta.sourceParams?.schema === 'group'); + + const newData = bucketCols.map(({ id }) => ({ + table, + column: parseInt(id, 10), + row: index, + value: table.rows[index][id] ?? null, + })); + if (newData.length) { + data.push(...newData); + } + }); + return data; +}; diff --git a/src/plugins/vis_type_timeseries/public/application/components/lib/types.ts b/src/plugins/vis_type_timeseries/public/application/components/lib/types.ts new file mode 100644 index 0000000000000..fee6670b49b9c --- /dev/null +++ b/src/plugins/vis_type_timeseries/public/application/components/lib/types.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { Datatable } from 'src/plugins/expressions/public'; + +export interface TSVBTables { + [key: string]: Datatable; +} diff --git a/src/plugins/vis_type_timeseries/public/application/components/timeseries_visualization.tsx b/src/plugins/vis_type_timeseries/public/application/components/timeseries_visualization.tsx index 9683089094054..5391bf319ee57 100644 --- a/src/plugins/vis_type_timeseries/public/application/components/timeseries_visualization.tsx +++ b/src/plugins/vis_type_timeseries/public/application/components/timeseries_visualization.tsx @@ -12,7 +12,7 @@ import React, { useCallback, useEffect } from 'react'; import { get } from 'lodash'; import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; - +import { XYChartSeriesIdentifier, GeometryValue } from '@elastic/charts'; import { IUiSettingsClient } from 'src/core/public'; import { IInterpreterRenderHandlers } from 'src/plugins/expressions'; import { PersistedState } from 'src/plugins/visualizations/public'; @@ -27,6 +27,7 @@ import { fetchIndexPattern } from '../../../common/index_patterns_utils'; import { TimeseriesVisParams } from '../../types'; import { getDataStart } from '../../services'; import { convertSeriesToDataTable } from './lib/convert_series_to_datatable'; +import { getClickFilterData } from './lib/get_click_filter_data'; import { X_ACCESSOR_INDEX } from '../visualizations/constants'; import { LastValueModeIndicator } from './last_value_mode_indicator'; import { getInterval } from './lib/get_interval'; @@ -102,6 +103,37 @@ function TimeseriesVisualization({ [handlers, model] ); + const handleFilterClick = useCallback( + async (series: PanelData[], points: Array<[GeometryValue, XYChartSeriesIdentifier]>) => { + const indexPatternValue = model.index_pattern || ''; + const { indexPatterns } = getDataStart(); + const { indexPattern } = await fetchIndexPattern(indexPatternValue, indexPatterns); + + // it should work only if index pattern is found + if (!indexPattern) return; + + const tables = indexPattern + ? await convertSeriesToDataTable(model, series, indexPattern) + : null; + + if (!tables) return; + + const data = getClickFilterData(points, tables, model); + + const event = { + name: 'filterBucket', + data: { + data, + negate: false, + timeFieldName: indexPattern.timeFieldName, + }, + }; + + handlers.event(event); + }, + [handlers, model] + ); + const handleUiState = useCallback( (field: string, value: { column: string; order: string }) => { uiState.set(field, value); @@ -156,6 +188,7 @@ function TimeseriesVisualization({ visData={visData} uiState={uiState} onBrush={onBrush} + onFilterClick={handleFilterClick} onUiState={handleUiState} syncColors={syncColors} palettesService={palettesService} diff --git a/src/plugins/vis_type_timeseries/public/application/components/vis_editor.tsx b/src/plugins/vis_type_timeseries/public/application/components/vis_editor.tsx index de3163c44f95b..99f2564328402 100644 --- a/src/plugins/vis_type_timeseries/public/application/components/vis_editor.tsx +++ b/src/plugins/vis_type_timeseries/public/application/components/vis_editor.tsx @@ -8,6 +8,7 @@ import React, { Component } from 'react'; import * as Rx from 'rxjs'; +import uuid from 'uuid/v4'; import { share } from 'rxjs/operators'; import { isEqual, isEmpty, debounce } from 'lodash'; import { EventEmitter } from 'events'; @@ -79,6 +80,9 @@ export class VisEditor extends Component Promise; + onFilterClick: ( + series: PanelData[], + points: Array<[GeometryValue, XYChartSeriesIdentifier]> + ) => Promise; onUiState: ( field: string, value: { diff --git a/src/plugins/vis_type_timeseries/public/application/components/vis_types/markdown/vis.js b/src/plugins/vis_type_timeseries/public/application/components/vis_types/markdown/vis.js index 6510ad24615a6..ef6b30be30a30 100644 --- a/src/plugins/vis_type_timeseries/public/application/components/vis_types/markdown/vis.js +++ b/src/plugins/vis_type_timeseries/public/application/components/vis_types/markdown/vis.js @@ -91,6 +91,7 @@ MarkdownVisualization.propTypes = { className: PropTypes.string, model: PropTypes.object, onBrush: PropTypes.func, + onFilterClick: PropTypes.func, onChange: PropTypes.func, visData: PropTypes.object, getConfig: PropTypes.func, diff --git a/src/plugins/vis_type_timeseries/public/application/components/vis_types/metric/vis.js b/src/plugins/vis_type_timeseries/public/application/components/vis_types/metric/vis.js index 2b58145a4928d..3029bba04b450 100644 --- a/src/plugins/vis_type_timeseries/public/application/components/vis_types/metric/vis.js +++ b/src/plugins/vis_type_timeseries/public/application/components/vis_types/metric/vis.js @@ -79,6 +79,7 @@ MetricVisualization.propTypes = { additionalLabel: PropTypes.string, model: PropTypes.object, onBrush: PropTypes.func, + onFilterClick: PropTypes.func, onChange: PropTypes.func, visData: PropTypes.object, getConfig: PropTypes.func, diff --git a/src/plugins/vis_type_timeseries/public/application/components/vis_types/timeseries/vis.js b/src/plugins/vis_type_timeseries/public/application/components/vis_types/timeseries/vis.js index 29560c4bd9368..8e59e8e1bb628 100644 --- a/src/plugins/vis_type_timeseries/public/application/components/vis_types/timeseries/vis.js +++ b/src/plugins/vis_type_timeseries/public/application/components/vis_types/timeseries/vis.js @@ -26,6 +26,7 @@ class TimeseriesVisualization extends Component { static propTypes = { model: PropTypes.object, onBrush: PropTypes.func, + onFilterClick: PropTypes.func, visData: PropTypes.object, getConfig: PropTypes.func, }; @@ -136,7 +137,7 @@ class TimeseriesVisualization extends Component { }; render() { - const { model, visData, onBrush, syncColors, palettesService } = this.props; + const { model, visData, onBrush, onFilterClick, syncColors, palettesService } = this.props; const series = get(visData, `${model.id}.series`, []); const interval = getInterval(visData, model); const yAxisIdGenerator = htmlIdGenerator('yaxis'); @@ -230,6 +231,7 @@ class TimeseriesVisualization extends Component { series={series} yAxis={yAxis} onBrush={onBrush} + onFilterClick={onFilterClick} backgroundColor={model.background_color} showGrid={Boolean(model.show_grid)} legend={Boolean(model.show_legend)} diff --git a/src/plugins/vis_type_timeseries/public/application/components/vis_types/top_n/vis.js b/src/plugins/vis_type_timeseries/public/application/components/vis_types/top_n/vis.js index 8150d2e4d6e1b..41e6236cbc39b 100644 --- a/src/plugins/vis_type_timeseries/public/application/components/vis_types/top_n/vis.js +++ b/src/plugins/vis_type_timeseries/public/application/components/vis_types/top_n/vis.js @@ -96,6 +96,7 @@ TopNVisualization.propTypes = { className: PropTypes.string, model: PropTypes.object, onBrush: PropTypes.func, + onFilterClick: PropTypes.func, onChange: PropTypes.func, visData: PropTypes.object, getConfig: PropTypes.func, diff --git a/src/plugins/vis_type_timeseries/public/application/components/vis_with_splits.js b/src/plugins/vis_type_timeseries/public/application/components/vis_with_splits.js index 7dc6a26185e2b..4b933bc81d882 100644 --- a/src/plugins/vis_type_timeseries/public/application/components/vis_with_splits.js +++ b/src/plugins/vis_type_timeseries/public/application/components/vis_with_splits.js @@ -109,6 +109,7 @@ export function visWithSplits(WrappedComponent) { model={model} visData={newVisData} onBrush={props.onBrush} + onFilterClick={props.onFilterClick} additionalLabel={additionalLabel || emptyLabel} backgroundColor={props.backgroundColor} getConfig={props.getConfig} diff --git a/src/plugins/vis_type_timeseries/public/application/visualizations/views/timeseries/index.js b/src/plugins/vis_type_timeseries/public/application/visualizations/views/timeseries/index.js index 44c1742e144be..a4d834ea8d217 100644 --- a/src/plugins/vis_type_timeseries/public/application/visualizations/views/timeseries/index.js +++ b/src/plugins/vis_type_timeseries/public/application/visualizations/views/timeseries/index.js @@ -61,6 +61,7 @@ export const TimeSeries = ({ series, yAxis, onBrush, + onFilterClick, xAxisFormatter, annotations, syncColors, @@ -118,6 +119,10 @@ export const TimeSeries = ({ onBrush(min, max, series); }; + const handleElementClick = (points) => { + onFilterClick(series, points); + }; + const getSeriesColor = useCallback( (seriesName, seriesGroupId, seriesId) => { const seriesById = series.filter((s) => s.seriesId === seriesGroupId); @@ -142,6 +147,7 @@ export const TimeSeries = ({ showLegendExtra={true} legendPosition={legendPosition} onBrushEnd={onBrushEndListener} + onElementClick={(args) => handleElementClick(args)} animateData={false} onPointerUpdate={handleCursorUpdate} theme={[ diff --git a/src/plugins/vis_type_timeseries/public/metrics_type.ts b/src/plugins/vis_type_timeseries/public/metrics_type.ts index 66128b2abe856..3cb4faaacf25b 100644 --- a/src/plugins/vis_type_timeseries/public/metrics_type.ts +++ b/src/plugins/vis_type_timeseries/public/metrics_type.ts @@ -7,7 +7,7 @@ */ import { i18n } from '@kbn/i18n'; - +import uuid from 'uuid/v4'; import { TSVB_EDITOR_NAME } from './application/editor_controller'; import { PANEL_TYPES, TOOLTIP_MODES } from '../common/enums'; import { isStringTypeIndexPattern } from '../common/index_patterns_utils'; @@ -25,11 +25,11 @@ export const metricsVisDefinition = { group: VisGroups.PROMOTED, visConfig: { defaults: { - id: '61ca57f0-469d-11e7-af02-69e470af7417', + id: uuid(), type: PANEL_TYPES.TIMESERIES, series: [ { - id: '61ca57f1-469d-11e7-af02-69e470af7417', + id: uuid(), color: '#68BC00', split_mode: 'everything', palette: { @@ -38,7 +38,7 @@ export const metricsVisDefinition = { }, metrics: [ { - id: '61ca57f2-469d-11e7-af02-69e470af7417', + id: uuid(), type: 'count', }, ], @@ -75,7 +75,7 @@ export const metricsVisDefinition = { }, toExpressionAst, getSupportedTriggers: () => { - return [VIS_EVENT_TO_TRIGGER.brush]; + return [VIS_EVENT_TO_TRIGGER.filter, VIS_EVENT_TO_TRIGGER.brush]; }, inspectorAdapters: {}, getUsedIndexPattern: async (params: VisParams) => { diff --git a/src/plugins/vis_type_timeseries/public/plugin.ts b/src/plugins/vis_type_timeseries/public/plugin.ts index 1c1212add3d8c..479c3f47435cc 100644 --- a/src/plugins/vis_type_timeseries/public/plugin.ts +++ b/src/plugins/vis_type_timeseries/public/plugin.ts @@ -15,7 +15,6 @@ import { EditorController, TSVB_EDITOR_NAME } from './application/editor_control import { createMetricsFn } from './metrics_fn'; import { metricsVisDefinition } from './metrics_type'; import { - setSavedObjectsClient, setUISettings, setI18n, setFieldFormats, @@ -65,7 +64,6 @@ export class MetricsPlugin implements Plugin { } public start(core: CoreStart, { data }: MetricsPluginStartDependencies) { - setSavedObjectsClient(core.savedObjects); setI18n(core.i18n); setFieldFormats(data.fieldFormats); setDataStart(data); diff --git a/src/plugins/vis_type_timeseries/public/services.ts b/src/plugins/vis_type_timeseries/public/services.ts index 17c4a6c7de153..22f99f95691de 100644 --- a/src/plugins/vis_type_timeseries/public/services.ts +++ b/src/plugins/vis_type_timeseries/public/services.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -import { I18nStart, SavedObjectsStart, IUiSettingsClient, CoreStart } from 'src/core/public'; +import { I18nStart, IUiSettingsClient, CoreStart } from 'src/core/public'; import { createGetterSetter } from '../../kibana_utils/public'; import { ChartsPluginSetup } from '../../charts/public'; import { DataPublicPluginStart } from '../../data/public'; @@ -17,10 +17,6 @@ export const [getFieldFormats, setFieldFormats] = createGetterSetter< DataPublicPluginStart['fieldFormats'] >('FieldFormats'); -export const [getSavedObjectsClient, setSavedObjectsClient] = createGetterSetter( - 'SavedObjectsClient' -); - export const [getCoreStart, setCoreStart] = createGetterSetter('CoreStart'); export const [getDataStart, setDataStart] = createGetterSetter('DataStart'); diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/date_histogram.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/date_histogram.js index 29cf3f274dc24..f82f332df19fd 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/date_histogram.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/date_histogram.js @@ -68,6 +68,7 @@ export function dateHistogram( bucketSize, seriesId: series.id, index: panel.use_kibana_indexes ? seriesIndex.indexPattern?.id : undefined, + panelId: panel.id, }); return next(doc); diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/date_histogram.test.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/date_histogram.test.js index a1fd242dc150e..741eb93267f4c 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/date_histogram.test.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/date_histogram.test.js @@ -32,6 +32,7 @@ describe('dateHistogram(req, panel, series)', () => { index_pattern: '*', time_field: '@timestamp', interval: '10s', + id: 'panelId', }; series = { id: 'test' }; config = { @@ -89,6 +90,7 @@ describe('dateHistogram(req, panel, series)', () => { intervalString: '10s', timeField: '@timestamp', seriesId: 'test', + panelId: 'panelId', }, }, }, @@ -130,6 +132,7 @@ describe('dateHistogram(req, panel, series)', () => { intervalString: '10s', timeField: '@timestamp', seriesId: 'test', + panelId: 'panelId', }, }, }, @@ -174,6 +177,7 @@ describe('dateHistogram(req, panel, series)', () => { intervalString: '20s', timeField: 'timestamp', seriesId: 'test', + panelId: 'panelId', }, }, }, @@ -230,6 +234,7 @@ describe('dateHistogram(req, panel, series)', () => { seriesId: 'test', bucketSize: 10, intervalString: '10s', + panelId: 'panelId', }, }, }, diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/normalize_query.test.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/normalize_query.test.js index 0194e1ce3e5d9..a3a9022a67191 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/normalize_query.test.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/series/normalize_query.test.js @@ -11,6 +11,7 @@ import { normalizeQuery } from './normalize_query'; describe('normalizeQuery', () => { const req = 'req'; const seriesId = '61ca57f1-469d-11e7-af02-69e470af7417'; + const panelId = '39d49073-a924-426b-aa32-35acb40a9bb7'; let next; let panel; @@ -48,6 +49,7 @@ describe('normalizeQuery', () => { intervalString: '10s', bucketSize: 10, seriesId: [seriesId], + panelId, }, }, }, @@ -78,6 +80,7 @@ describe('normalizeQuery', () => { intervalString: '10s', bucketSize: 10, seriesId: [seriesId], + panelId: panelId, }); }); diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/table/date_histogram.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/table/date_histogram.js index f0989cf0fa08b..3e883abc9e5e0 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/table/date_histogram.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/table/date_histogram.js @@ -24,6 +24,7 @@ export function dateHistogram(req, panel, esQueryConfig, seriesIndex, capabiliti const meta = { timeField, index: panel.use_kibana_indexes ? seriesIndex.indexPattern?.id : undefined, + panelId: panel.id, }; const getDateHistogramForLastBucketMode = () => { diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/table/normalize_query.test.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/table/normalize_query.test.js index e2a7e7b17b548..71381f32599e3 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/table/normalize_query.test.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/request_processors/table/normalize_query.test.js @@ -11,6 +11,7 @@ import { normalizeQuery } from './normalize_query'; describe('normalizeQuery', () => { const req = 'req'; const seriesId = '61ca57f1-469d-11e7-af02-69e470af7417'; + const panelId = '39d49073-a924-426b-aa32-35acb40a9bb7'; let next; let panel; @@ -56,6 +57,7 @@ describe('normalizeQuery', () => { timeField: 'order_date', intervalString: '10s', bucketSize: 10, + panelId, }, }, }, @@ -87,6 +89,7 @@ describe('normalizeQuery', () => { timeField: 'order_date', intervalString: '10s', bucketSize: 10, + panelId, }); }); diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/series/build_request_body.test.ts b/src/plugins/vis_type_timeseries/server/lib/vis_data/series/build_request_body.test.ts index 89d87da5f3d7e..5b865d451003a 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/series/build_request_body.test.ts +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/series/build_request_body.test.ts @@ -157,6 +157,7 @@ describe('buildRequestBody(req)', () => { intervalString: '10s', seriesId: 'c9b5f9c0-e403-11e6-be91-6f7688e9fac7', timeField: '@timestamp', + panelId: 'c9b5d2b0-e403-11e6-be91-6f7688e9fac7', }, }, }, diff --git a/src/plugins/vis_type_vega/public/vega_view/vega_map_view/view.test.ts b/src/plugins/vis_type_vega/public/vega_view/vega_map_view/view.test.ts index d5a67f2eac064..4fd19aa45e69e 100644 --- a/src/plugins/vis_type_vega/public/vega_view/vega_map_view/view.test.ts +++ b/src/plugins/vis_type_vega/public/vega_view/vega_map_view/view.test.ts @@ -28,9 +28,12 @@ import { setUISettings, } from '../../services'; import { initVegaLayer, initTmsRasterLayer } from './layers'; -import { Map, NavigationControl, Style } from 'mapbox-gl'; -jest.mock('mapbox-gl', () => ({ +// @ts-expect-error +import mapboxgl from 'mapbox-gl/dist/mapbox-gl-csp'; + +jest.mock('mapbox-gl/dist/mapbox-gl-csp', () => ({ + setRTLTextPlugin: jest.fn(), Map: jest.fn().mockImplementation(() => ({ getLayer: () => '', removeLayer: jest.fn(), @@ -75,9 +78,10 @@ describe('vega_map_view/view', () => { setUISettings(coreStart.uiSettings); const getTmsService = jest.fn().mockReturnValue(({ - getVectorStyleSheet: (): Style => ({ + getVectorStyleSheet: () => ({ version: 8, sources: {}, + // @ts-expect-error layers: [], }), getMaxZoom: async () => 20, @@ -144,7 +148,7 @@ describe('vega_map_view/view', () => { await vegaMapView.init(); const { longitude, latitude, scrollWheelZoom } = vegaMapView._parser.mapConfig; - expect(Map).toHaveBeenCalledWith({ + expect(mapboxgl.Map).toHaveBeenCalledWith({ style: { version: 8, sources: {}, @@ -170,7 +174,7 @@ describe('vega_map_view/view', () => { await vegaMapView.init(); const { longitude, latitude, scrollWheelZoom } = vegaMapView._parser.mapConfig; - expect(Map).toHaveBeenCalledWith({ + expect(mapboxgl.Map).toHaveBeenCalledWith({ style: { version: 8, sources: {}, @@ -195,7 +199,7 @@ describe('vega_map_view/view', () => { await vegaMapView.init(); - expect(NavigationControl).toHaveBeenCalled(); + expect(mapboxgl.NavigationControl).toHaveBeenCalled(); }); }); }); diff --git a/src/plugins/vis_type_vega/public/vega_view/vega_map_view/view.ts b/src/plugins/vis_type_vega/public/vega_view/vega_map_view/view.ts index 453e9596a2a4c..e899057819a19 100644 --- a/src/plugins/vis_type_vega/public/vega_view/vega_map_view/view.ts +++ b/src/plugins/vis_type_vega/public/vega_view/vega_map_view/view.ts @@ -7,9 +7,11 @@ */ import { i18n } from '@kbn/i18n'; -import { Map, Style, NavigationControl, MapboxOptions } from 'mapbox-gl'; +import type { Map, Style, MapboxOptions } from 'mapbox-gl'; import { View, parse } from 'vega'; +// @ts-expect-error +import mapboxgl from 'mapbox-gl/dist/mapbox-gl-csp'; import { initTmsRasterLayer, initVegaLayer } from './layers'; import { VegaBaseView } from '../vega_base_view'; import { getMapServiceSettings } from '../../services'; @@ -22,11 +24,17 @@ import { userConfiguredLayerId, vegaLayerId, } from './constants'; - import { validateZoomSettings, injectMapPropsIntoSpec } from './utils'; - import './vega_map_view.scss'; +// @ts-expect-error +import mbRtlPlugin from '!!file-loader!@mapbox/mapbox-gl-rtl-text/mapbox-gl-rtl-text.min.js'; +// @ts-expect-error +import mbWorkerUrl from '!!file-loader!mapbox-gl/dist/mapbox-gl-csp-worker'; + +mapboxgl.workerUrl = mbWorkerUrl; +mapboxgl.setRTLTextPlugin(mbRtlPlugin); + async function updateVegaView(mapBoxInstance: Map, vegaView: View) { const mapCanvas = mapBoxInstance.getCanvas(); const { lat, lng } = mapBoxInstance.getCenter(); @@ -115,7 +123,7 @@ export class VegaMapView extends VegaBaseView { // In some cases, Vega may be initialized twice, e.g. after awaiting... if (!this._$container) return; - const mapBoxInstance = new Map({ + const mapBoxInstance = new mapboxgl.Map({ style, customAttribution, container: this._$container.get(0), @@ -142,7 +150,7 @@ export class VegaMapView extends VegaBaseView { private initControls(mapBoxInstance: Map) { if (this.shouldShowZoomControl) { - mapBoxInstance.addControl(new NavigationControl({ showCompass: false }), 'top-left'); + mapBoxInstance.addControl(new mapboxgl.NavigationControl({ showCompass: false }), 'top-left'); } // disable map rotation using right click + drag diff --git a/src/plugins/visualizations/public/embeddable/visualize_embeddable.ts b/src/plugins/visualizations/public/embeddable/visualize_embeddable.ts index 3bb52eb15758a..f5bf6b59aa0ae 100644 --- a/src/plugins/visualizations/public/embeddable/visualize_embeddable.ts +++ b/src/plugins/visualizations/public/embeddable/visualize_embeddable.ts @@ -337,6 +337,14 @@ export class VisualizeEmbeddable data: { timeFieldName: this.vis.data.indexPattern?.timeFieldName!, ...event.data }, }; } + // do not trigger the filter click event if the filter bar is not visible + if ( + triggerId === VIS_EVENT_TO_TRIGGER.filter && + !this.input.id && + !this.vis.type.options.showFilterBar + ) { + return; + } getUiActions().getTrigger(triggerId).exec(context); } diff --git a/test/accessibility/apps/dashboard_panel.ts b/test/accessibility/apps/dashboard_panel.ts index 77b6cf2dbb6da..2a6c290172a9e 100644 --- a/test/accessibility/apps/dashboard_panel.ts +++ b/test/accessibility/apps/dashboard_panel.ts @@ -63,6 +63,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('dashboard panel full screen', async () => { const header = await dashboardPanelActions.getPanelHeading('[Flights] Airline Carrier'); await dashboardPanelActions.toggleContextMenu(header); + await dashboardPanelActions.clickContextMenuMoreItem(); + await testSubjects.click('embeddablePanelAction-togglePanel'); await a11y.testAppSnapshot(); }); diff --git a/test/api_integration/apis/home/sample_data.ts b/test/api_integration/apis/home/sample_data.ts index 99327901ec8c3..c681ad325e56f 100644 --- a/test/api_integration/apis/home/sample_data.ts +++ b/test/api_integration/apis/home/sample_data.ts @@ -43,7 +43,7 @@ export default function ({ getService }: FtrProviderContext) { expect(resp.body).to.eql({ elasticsearchIndicesCreated: { kibana_sample_data_flights: 13059 }, - kibanaSavedObjectsLoaded: 20, + kibanaSavedObjectsLoaded: 23, }); }); diff --git a/test/api_integration/apis/search/search.ts b/test/api_integration/apis/search/search.ts index bc092dd3889bb..7ef82cd5467ab 100644 --- a/test/api_integration/apis/search/search.ts +++ b/test/api_integration/apis/search/search.ts @@ -99,26 +99,6 @@ export default function ({ getService }: FtrProviderContext) { expect(resp.body.message).to.contain('banana not found'); }); - it('should return 400 when index type is provided in OSS', async () => { - const resp = await supertest - .post(`/internal/search/es`) - .send({ - indexType: 'baad', - params: { - body: { - query: { - match_all: {}, - }, - }, - }, - }) - .expect(400); - - verifyErrorResponse(resp.body, 400); - - expect(resp.body.message).to.contain('Unsupported index pattern'); - }); - it('should return 400 with illegal ES argument', async () => { const resp = await supertest .post(`/internal/search/es`) diff --git a/test/api_integration/apis/telemetry/index.js b/test/api_integration/apis/telemetry/index.js index db95bf92cd44f..5394b54062d89 100644 --- a/test/api_integration/apis/telemetry/index.js +++ b/test/api_integration/apis/telemetry/index.js @@ -8,7 +8,6 @@ export default function ({ loadTestFile }) { describe('Telemetry', () => { - loadTestFile(require.resolve('./telemetry_local')); loadTestFile(require.resolve('./opt_in')); loadTestFile(require.resolve('./telemetry_optin_notice_seen')); }); diff --git a/test/api_integration/apis/telemetry/telemetry_local.ts b/test/api_integration/apis/telemetry/telemetry_local.ts deleted file mode 100644 index f20c6a3d129dd..0000000000000 --- a/test/api_integration/apis/telemetry/telemetry_local.ts +++ /dev/null @@ -1,331 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -import expect from '@kbn/expect'; -import supertestAsPromised from 'supertest-as-promised'; -import { omit } from 'lodash'; -import { basicUiCounters } from './__fixtures__/ui_counters'; -import { basicUsageCounters } from './__fixtures__/usage_counters'; -import type { FtrProviderContext } from '../../ftr_provider_context'; -import type { SavedObject } from '../../../../src/core/server'; -import ossRootTelemetrySchema from '../../../../src/plugins/telemetry/schema/oss_root.json'; -import ossPluginsTelemetrySchema from '../../../../src/plugins/telemetry/schema/oss_plugins.json'; -import { assertTelemetryPayload, flatKeys } from './utils'; - -async function retrieveTelemetry( - supertest: supertestAsPromised.SuperTest -) { - const { body } = await supertest - .post('/api/telemetry/v2/clusters/_stats') - .set('kbn-xsrf', 'xxx') - .send({ unencrypted: true }) - .expect(200); - - expect(body.length).to.be(1); - return body[0]; -} - -export default function ({ getService }: FtrProviderContext) { - const supertest = getService('supertest'); - const es = getService('es'); - const esArchiver = getService('esArchiver'); - - describe('/api/telemetry/v2/clusters/_stats', () => { - before('make sure there are some saved objects', () => esArchiver.load('saved_objects/basic')); - after('cleanup saved objects changes', () => esArchiver.unload('saved_objects/basic')); - - before('create some telemetry-data tracked indices', async () => { - await es.indices.create({ index: 'filebeat-telemetry_tests_logs' }); - }); - - after('cleanup telemetry-data tracked indices', async () => { - await es.indices.delete({ index: 'filebeat-telemetry_tests_logs' }); - }); - - describe('validate data types', () => { - let stats: Record; - - before('pull local stats', async () => { - stats = await retrieveTelemetry(supertest); - }); - - it('should pass the schema validation', () => { - try { - assertTelemetryPayload( - { root: ossRootTelemetrySchema, plugins: ossPluginsTelemetrySchema }, - stats - ); - } catch (err) { - err.message = `The telemetry schemas in 'src/plugins/telemetry/schema/' are out-of-date, please update it as required: ${err.message}`; - throw err; - } - }); - - it('should pass ad-hoc enforced validations', () => { - expect(stats.collection).to.be('local'); - expect(stats.collectionSource).to.be('local'); - expect(stats.license).to.be(undefined); // OSS cannot get the license - expect(stats.stack_stats.kibana.count).to.be.a('number'); - expect(stats.stack_stats.kibana.indices).to.be.a('number'); - expect(stats.stack_stats.kibana.os.platforms[0].platform).to.be.a('string'); - expect(stats.stack_stats.kibana.os.platforms[0].count).to.be(1); - expect(stats.stack_stats.kibana.os.platformReleases[0].platformRelease).to.be.a('string'); - expect(stats.stack_stats.kibana.os.platformReleases[0].count).to.be(1); - expect(stats.stack_stats.kibana.plugins.telemetry.opt_in_status).to.be(false); - expect(stats.stack_stats.kibana.plugins.telemetry.usage_fetcher).to.be.a('string'); - expect(stats.stack_stats.kibana.plugins.stack_management).to.be.an('object'); - expect(stats.stack_stats.kibana.plugins.ui_metric).to.be.an('object'); - expect(stats.stack_stats.kibana.plugins.ui_counters).to.be.an('object'); - expect(stats.stack_stats.kibana.plugins.application_usage).to.be.an('object'); - expect(stats.stack_stats.kibana.plugins.kql.defaultQueryLanguage).to.be.a('string'); - expect(stats.stack_stats.kibana.plugins.localization).to.be.an('object'); - expect(stats.stack_stats.kibana.plugins.csp.strict).to.be(true); - expect(stats.stack_stats.kibana.plugins.csp.warnLegacyBrowsers).to.be(true); - expect(stats.stack_stats.kibana.plugins.csp.rulesChangedFromDefault).to.be(false); - expect(stats.stack_stats.kibana.plugins.kibana_config_usage).to.be.an('object'); - // non-default kibana configs. Configs set at 'test/api_integration/config.js'. - expect(omit(stats.stack_stats.kibana.plugins.kibana_config_usage, 'server.port')).to.eql({ - 'elasticsearch.username': '[redacted]', - 'elasticsearch.password': '[redacted]', - 'elasticsearch.hosts': '[redacted]', - 'elasticsearch.healthCheck.delay': 3600000, - 'plugins.paths': '[redacted]', - 'logging.json': false, - 'server.xsrf.disableProtection': true, - 'server.compression.referrerWhitelist': '[redacted]', - 'server.maxPayload': 1679958, - 'status.allowAnonymous': true, - 'home.disableWelcomeScreen': true, - 'data.search.aggs.shardDelay.enabled': true, - 'security.showInsecureClusterWarning': false, - 'telemetry.banner': false, - 'telemetry.url': '[redacted]', - 'telemetry.optInStatusUrl': '[redacted]', - 'telemetry.optIn': false, - 'newsfeed.service.urlRoot': '[redacted]', - 'newsfeed.service.pathTemplate': '[redacted]', - 'savedObjects.maxImportPayloadBytes': 10485760, - 'savedObjects.maxImportExportSize': 10001, - 'usageCollection.usageCounters.bufferDuration': 0, - }); - expect(stats.stack_stats.kibana.plugins.kibana_config_usage['server.port']).to.be.a( - 'number' - ); - - // Testing stack_stats.data - expect(stats.stack_stats.data).to.be.an('object'); - expect(stats.stack_stats.data).to.be.an('array'); - expect(stats.stack_stats.data[0]).to.be.an('object'); - expect(stats.stack_stats.data[0].pattern_name).to.be('filebeat'); - expect(stats.stack_stats.data[0].shipper).to.be('filebeat'); - expect(stats.stack_stats.data[0].index_count).to.be(1); - expect(stats.stack_stats.data[0].doc_count).to.be(0); - expect(stats.stack_stats.data[0].ecs_index_count).to.be(0); - expect(stats.stack_stats.data[0].size_in_bytes).to.be.a('number'); - - expect(stats.stack_stats.kibana.plugins.saved_objects_counts).to.be.an('object'); - expect(stats.stack_stats.kibana.plugins.saved_objects_counts.by_type).to.be.an('array'); - expect(stats.stack_stats.kibana.plugins.saved_objects_counts.by_type).to.eql([ - { type: 'config', count: 2 }, - { type: 'dashboard', count: 2 }, - { type: 'index-pattern', count: 2 }, - { type: 'visualization', count: 2 }, - ]); - }); - - it('should validate mandatory fields exist', () => { - const actual = flatKeys(stats); - expect(actual).to.be.an('array'); - const expected = [ - 'cluster_name', - 'cluster_stats.cluster_uuid', - 'cluster_stats.indices.analysis', - 'cluster_stats.indices.completion', - 'cluster_stats.indices.count', - 'cluster_stats.indices.docs', - 'cluster_stats.indices.fielddata', - 'cluster_stats.indices.mappings', - 'cluster_stats.indices.query_cache', - 'cluster_stats.indices.segments', - 'cluster_stats.indices.shards', - 'cluster_stats.indices.store', - 'cluster_stats.nodes.count', - 'cluster_stats.nodes.discovery_types', - 'cluster_stats.nodes.fs', - 'cluster_stats.nodes.ingest', - 'cluster_stats.nodes.jvm', - 'cluster_stats.nodes.network_types', - 'cluster_stats.nodes.os', - 'cluster_stats.nodes.packaging_types', - 'cluster_stats.nodes.plugins', - 'cluster_stats.nodes.process', - 'cluster_stats.nodes.versions', - 'cluster_stats.nodes.usage', - 'cluster_stats.status', - 'cluster_stats.timestamp', - 'cluster_uuid', - 'collection', - 'collectionSource', - 'stack_stats.kibana.count', - 'stack_stats.kibana.indices', - 'stack_stats.kibana.os', - 'stack_stats.kibana.plugins', - 'stack_stats.kibana.versions', - 'timestamp', - 'version', - ]; - - expect(expected.every((m) => actual.includes(m))).to.be.ok(); - }); - }); - - describe('UI Counters telemetry', () => { - before('Add UI Counters saved objects', () => esArchiver.load('saved_objects/ui_counters')); - after('cleanup saved objects changes', () => esArchiver.unload('saved_objects/ui_counters')); - it('returns ui counters aggregated by day', async () => { - const stats = await retrieveTelemetry(supertest); - expect(stats.stack_stats.kibana.plugins.ui_counters).to.eql(basicUiCounters); - }); - }); - - describe('Usage Counters telemetry', () => { - before('Add UI Counters saved objects', () => - esArchiver.load('saved_objects/usage_counters') - ); - after('cleanup saved objects changes', () => - esArchiver.unload('saved_objects/usage_counters') - ); - - it('returns usage counters aggregated by day', async () => { - const stats = await retrieveTelemetry(supertest); - expect(stats.stack_stats.kibana.plugins.usage_counters).to.eql(basicUsageCounters); - }); - }); - - describe('application usage limits', () => { - function createSavedObject(viewId?: string) { - return supertest - .post('/api/saved_objects/application_usage_daily') - .send({ - attributes: { - appId: 'test-app', - viewId, - minutesOnScreen: 10.33, - numberOfClicks: 10, - timestamp: new Date().toISOString(), - }, - }) - .expect(200) - .then((resp) => resp.body.id); - } - - describe('basic behaviour', () => { - let savedObjectIds: string[] = []; - before('create application usage entries', async () => { - await esArchiver.emptyKibanaIndex(); - savedObjectIds = await Promise.all([ - createSavedObject(), - createSavedObject('appView1'), - createSavedObject(), - ]); - }); - after('cleanup', async () => { - await Promise.all( - savedObjectIds.map((savedObjectId) => { - return supertest - .delete(`/api/saved_objects/application_usage_daily/${savedObjectId}`) - .expect(200); - }) - ); - }); - - it('should return application_usage data', async () => { - const stats = await retrieveTelemetry(supertest); - expect(stats.stack_stats.kibana.plugins.application_usage).to.eql({ - 'test-app': { - appId: 'test-app', - viewId: 'main', - clicks_total: 20, - clicks_7_days: 20, - clicks_30_days: 20, - clicks_90_days: 20, - minutes_on_screen_total: 20.66, - minutes_on_screen_7_days: 20.66, - minutes_on_screen_30_days: 20.66, - minutes_on_screen_90_days: 20.66, - views: [ - { - appId: 'test-app', - viewId: 'appView1', - clicks_total: 10, - clicks_7_days: 10, - clicks_30_days: 10, - clicks_90_days: 10, - minutes_on_screen_total: 10.33, - minutes_on_screen_7_days: 10.33, - minutes_on_screen_30_days: 10.33, - minutes_on_screen_90_days: 10.33, - }, - ], - }, - }); - }); - }); - - describe('10k + 1', () => { - const savedObjectIds = []; - before('create 10k + 1 entries for application usage', async () => { - await supertest - .post('/api/saved_objects/_bulk_create') - .send( - new Array(10001).fill(0).map(() => ({ - type: 'application_usage_daily', - attributes: { - appId: 'test-app', - minutesOnScreen: 1, - numberOfClicks: 1, - timestamp: new Date().toISOString(), - }, - })) - ) - .expect(200) - .then((resp) => - resp.body.saved_objects.forEach(({ id }: SavedObject) => savedObjectIds.push(id)) - ); - }); - after('clean them all', async () => { - // The SavedObjects API does not allow bulk deleting, and deleting one by one takes ages and the tests timeout - await es.deleteByQuery({ - index: '.kibana', - body: { query: { term: { type: 'application_usage_daily' } } }, - conflicts: 'proceed', - }); - }); - - it("should only use the first 10k docs for the application_usage data (they'll be rolled up in a later process)", async () => { - const stats = await retrieveTelemetry(supertest); - expect(stats.stack_stats.kibana.plugins.application_usage).to.eql({ - 'test-app': { - appId: 'test-app', - viewId: 'main', - clicks_total: 10000, - clicks_7_days: 10000, - clicks_30_days: 10000, - clicks_90_days: 10000, - minutes_on_screen_total: 10000, - minutes_on_screen_7_days: 10000, - minutes_on_screen_30_days: 10000, - minutes_on_screen_90_days: 10000, - views: [], - }, - }); - }); - }); - }); - }); -} diff --git a/test/api_integration/fixtures/es_archiver/saved_objects/basic/data.json.gz b/test/api_integration/fixtures/es_archiver/saved_objects/basic/data.json.gz index c72f46370fa12..d65e1920b2342 100644 Binary files a/test/api_integration/fixtures/es_archiver/saved_objects/basic/data.json.gz and b/test/api_integration/fixtures/es_archiver/saved_objects/basic/data.json.gz differ diff --git a/test/api_integration/fixtures/es_archiver/saved_objects/basic/mappings.json b/test/api_integration/fixtures/es_archiver/saved_objects/basic/mappings.json index e601c43431437..6dac52137ef01 100644 --- a/test/api_integration/fixtures/es_archiver/saved_objects/basic/mappings.json +++ b/test/api_integration/fixtures/es_archiver/saved_objects/basic/mappings.json @@ -1,148 +1,2613 @@ { "type": "index", "value": { - "index": ".kibana", - "settings": { - "index": { - "number_of_shards": "1", - "number_of_replicas": "1" + "aliases": { + ".kibana": { } }, + "index": ".kibana_1", "mappings": { + "_meta": { + "migrationMappingPropertyHashes": { + "action": "6e96ac5e648f57523879661ea72525b7", + "action_task_params": "a9d49f184ee89641044be0ca2950fa3a", + "alert": "05d57e6963593484582a4de341446974", + "api_key_pending_invalidation": "16f515278a295f6245149ad7c5ddedb7", + "apm-indices": "9bb9b2bf1fa636ed8619cbab5ce6a1dd", + "apm-telemetry": "3d1b76c39bfb2cc8296b024d73854724", + "app_search_telemetry": "3d1b76c39bfb2cc8296b024d73854724", + "application_usage_daily": "43b8830d5d0df85a6823d290885fc9fd", + "application_usage_totals": "3d1b76c39bfb2cc8296b024d73854724", + "application_usage_transactional": "3d1b76c39bfb2cc8296b024d73854724", + "canvas-element": "7390014e1091044523666d97247392fc", + "canvas-workpad": "b0a1706d356228dbdcb4a17e6b9eb231", + "canvas-workpad-template": "ae2673f678281e2c055d764b153e9715", + "cases": "7c28a18fbac7c2a4e79449e9802ef476", + "cases-comments": "112cefc2b6737e613a8ef033234755e6", + "cases-configure": "387c5f3a3bda7e0ae0dd4e106f914a69", + "cases-connector-mappings": "6bc7e49411d38be4969dc6aa8bd43776", + "cases-sub-case": "2dc9dbf1fc7144e2b18fffc017358ff9", + "cases-user-actions": "32277330ec6b721abe3b846cfd939a71", + "config": "c63748b75f39d0c54de12d12c1ccbc20", + "core-usage-stats": "3d1b76c39bfb2cc8296b024d73854724", + "coreMigrationVersion": "2f4316de49999235636386fe51dc06c1", + "dashboard": "40554caf09725935e2c02e02563a2d07", + "endpoint:user-artifact": "4a11183eee21e6fbad864f7a30b39ad0", + "endpoint:user-artifact-manifest": "a0d7b04ad405eed54d76e279c3727862", + "enterprise_search_telemetry": "3d1b76c39bfb2cc8296b024d73854724", + "epm-packages": "0cbbb16506734d341a96aaed65ec6413", + "epm-packages-assets": "44621b2f6052ef966da47b7c3a00f33b", + "exception-list": "baf108c9934dda844921f692a513adae", + "exception-list-agnostic": "baf108c9934dda844921f692a513adae", + "file-upload-usage-collection-telemetry": "a34fbb8e3263d105044869264860c697", + "fleet-agent-actions": "9511b565b1cc6441a42033db3d5de8e9", + "fleet-agent-events": "e20a508b6e805189356be381dbfac8db", + "fleet-agents": "59fd74f819f028f8555776db198d2562", + "fleet-enrollment-api-keys": "a69ef7ae661dab31561d6c6f052ef2a7", + "graph-workspace": "27a94b2edcb0610c6aea54a7c56d7752", + "index-pattern": "45915a1ad866812242df474eb0479052", + "infrastructure-ui-source": "3d1b76c39bfb2cc8296b024d73854724", + "ingest-agent-policies": "8df93787e2927f227dc80db8b6d309b9", + "ingest-outputs": "8854f34453a47e26f86a29f8f3b80b4e", + "ingest-package-policies": "c91ca97b1ff700f0fc64dc6b13d65a85", + "ingest_manager_settings": "22d4d1288c2687ef6a8f6e83159b4542", + "inventory-view": "3d1b76c39bfb2cc8296b024d73854724", + "kql-telemetry": "d12a98a6f19a2d273696597547e064ee", + "legacy-url-alias": "3d1b76c39bfb2cc8296b024d73854724", + "lens": "52346cfec69ff7b47d5f0c12361a2797", + "lens-ui-telemetry": "509bfa5978586998e05f9e303c07a327", + "map": "9134b47593116d7953f6adba096fc463", + "maps-telemetry": "5ef305b18111b77789afefbd36b66171", + "metrics-explorer-view": "3d1b76c39bfb2cc8296b024d73854724", + "migrationVersion": "4a1746014a75ade3a714e1db5763276f", + "ml-job": "3bb64c31915acf93fc724af137a0891b", + "ml-module": "46ef4f0d6682636f0fff9799d6a2d7ac", + "monitoring-telemetry": "2669d5ec15e82391cf58df4294ee9c68", + "namespace": "2f4316de49999235636386fe51dc06c1", + "namespaces": "2f4316de49999235636386fe51dc06c1", + "originId": "2f4316de49999235636386fe51dc06c1", + "query": "11aaeb7f5f7fa5bb43f25e18ce26e7d9", + "references": "7997cf5a56cc02bdc9c93361bde732b0", + "sample-data-telemetry": "7d3cfeb915303c9641c59681967ffeb4", + "search": "db2c00e39b36f40930a3b9fc71c823e1", + "search-session": "33157cf0119e41cd4e7a1d24266beff4", + "search-telemetry": "3d1b76c39bfb2cc8296b024d73854724", + "security-solution-signals-migration": "72761fd374ca11122ac8025a92b84fca", + "siem-detection-engine-rule-actions": "6569b288c169539db10cb262bf79de18", + "siem-detection-engine-rule-status": "ae783f41c6937db6b7a2ef5c93a9e9b0", + "siem-ui-timeline": "3e97beae13cdfc6d62bc1846119f7276", + "siem-ui-timeline-note": "8874706eedc49059d4cf0f5094559084", + "siem-ui-timeline-pinned-event": "20638091112f0e14f0e443d512301c29", + "space": "c5ca8acafa0beaa4d08d014a97b6bc6b", + "spaces-usage-stats": "3d1b76c39bfb2cc8296b024d73854724", + "tag": "83d55da58f6530f7055415717ec06474", + "telemetry": "36a616f7026dfa617d6655df850fe16d", + "timelion-sheet": "9a2a2748877c7a7b582fef201ab1d4cf", + "type": "2f4316de49999235636386fe51dc06c1", + "ui-counter": "0d409297dc5ebe1e3a1da691c6ee32e3", + "ui-metric": "0d409297dc5ebe1e3a1da691c6ee32e3", + "updated_at": "00da57df13e94e9d98437d13ace4bfe0", + "upgrade-assistant-reindex-operation": "215107c281839ea9b3ad5f6419819763", + "upgrade-assistant-telemetry": "56702cec857e0a9dacfb696655b4ff7b", + "uptime-dynamic-settings": "3d1b76c39bfb2cc8296b024d73854724", + "url": "c7f66a0df8b1b52f17c28c4adb111105", + "visualization": "f819cf6636b75c9e76ba733a0c6ef355", + "workplace_search_telemetry": "3d1b76c39bfb2cc8296b024d73854724" + } + }, "dynamic": "strict", "properties": { - "config": { - "dynamic": "true", + "action": { "properties": { - "buildNum": { + "actionTypeId": { + "type": "keyword" + }, + "config": { + "enabled": false, + "type": "object" + }, + "name": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + }, + "secrets": { + "type": "binary" + } + } + }, + "action_task_params": { + "properties": { + "actionId": { + "type": "keyword" + }, + "apiKey": { + "type": "binary" + }, + "params": { + "enabled": false, + "type": "object" + } + } + }, + "alert": { + "properties": { + "actions": { + "properties": { + "actionRef": { + "type": "keyword" + }, + "actionTypeId": { + "type": "keyword" + }, + "group": { + "type": "keyword" + }, + "params": { + "enabled": false, + "type": "object" + } + }, + "type": "nested" + }, + "alertTypeId": { + "type": "keyword" + }, + "apiKey": { + "type": "binary" + }, + "apiKeyOwner": { + "type": "keyword" + }, + "consumer": { + "type": "keyword" + }, + "createdAt": { + "type": "date" + }, + "createdBy": { + "type": "keyword" + }, + "enabled": { + "type": "boolean" + }, + "executionStatus": { + "properties": { + "error": { + "properties": { + "message": { + "type": "keyword" + }, + "reason": { + "type": "keyword" + } + } + }, + "lastExecutionDate": { + "type": "date" + }, + "status": { + "type": "keyword" + } + } + }, + "meta": { + "properties": { + "versionApiKeyLastmodified": { + "type": "keyword" + } + } + }, + "muteAll": { + "type": "boolean" + }, + "mutedInstanceIds": { + "type": "keyword" + }, + "name": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + }, + "notifyWhen": { + "type": "keyword" + }, + "params": { + "type": "flattened" + }, + "schedule": { + "properties": { + "interval": { + "type": "keyword" + } + } + }, + "scheduledTaskId": { + "type": "keyword" + }, + "tags": { + "type": "keyword" + }, + "throttle": { + "type": "keyword" + }, + "updatedAt": { + "type": "date" + }, + "updatedBy": { + "type": "keyword" + } + } + }, + "api_key_pending_invalidation": { + "properties": { + "apiKeyId": { + "type": "keyword" + }, + "createdAt": { + "type": "date" + } + } + }, + "apm-indices": { + "properties": { + "apm_oss": { + "properties": { + "errorIndices": { + "type": "keyword" + }, + "metricsIndices": { + "type": "keyword" + }, + "onboardingIndices": { + "type": "keyword" + }, + "sourcemapIndices": { + "type": "keyword" + }, + "spanIndices": { + "type": "keyword" + }, + "transactionIndices": { + "type": "keyword" + } + } + } + } + }, + "apm-telemetry": { + "dynamic": "false", + "type": "object" + }, + "app_search_telemetry": { + "dynamic": "false", + "type": "object" + }, + "application_usage_daily": { + "dynamic": "false", + "properties": { + "timestamp": { + "type": "date" + } + } + }, + "application_usage_totals": { + "dynamic": "false", + "type": "object" + }, + "application_usage_transactional": { + "dynamic": "false", + "type": "object" + }, + "canvas-element": { + "dynamic": "false", + "properties": { + "@created": { + "type": "date" + }, + "@timestamp": { + "type": "date" + }, + "content": { + "type": "text" + }, + "help": { + "type": "text" + }, + "image": { + "type": "text" + }, + "name": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "canvas-workpad": { + "dynamic": "false", + "properties": { + "@created": { + "type": "date" + }, + "@timestamp": { + "type": "date" + }, + "name": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "canvas-workpad-template": { + "dynamic": "false", + "properties": { + "help": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + }, + "name": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + }, + "tags": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + }, + "template_key": { + "type": "keyword" + } + } + }, + "cases": { + "properties": { + "closed_at": { + "type": "date" + }, + "closed_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "connector": { + "properties": { + "fields": { + "properties": { + "key": { + "type": "text" + }, + "value": { + "type": "text" + } + } + }, + "id": { + "type": "keyword" + }, + "name": { + "type": "text" + }, + "type": { + "type": "keyword" + } + } + }, + "created_at": { + "type": "date" + }, + "created_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "description": { + "type": "text" + }, + "external_service": { + "properties": { + "connector_id": { + "type": "keyword" + }, + "connector_name": { + "type": "keyword" + }, + "external_id": { + "type": "keyword" + }, + "external_title": { + "type": "text" + }, + "external_url": { + "type": "text" + }, + "pushed_at": { + "type": "date" + }, + "pushed_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + } + } + }, + "settings": { + "properties": { + "syncAlerts": { + "type": "boolean" + } + } + }, + "status": { "type": "keyword" }, - "defaultIndex": { - "type": "text", - "fields": { - "keyword": { - "type": "keyword", - "ignore_above": 256 + "tags": { + "type": "keyword" + }, + "title": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "updated_at": { + "type": "date" + }, + "updated_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + } + } + }, + "cases-comments": { + "properties": { + "alertId": { + "type": "keyword" + }, + "associationType": { + "type": "keyword" + }, + "comment": { + "type": "text" + }, + "created_at": { + "type": "date" + }, + "created_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "index": { + "type": "keyword" + }, + "pushed_at": { + "type": "date" + }, + "pushed_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "rule": { + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + }, + "type": { + "type": "keyword" + }, + "updated_at": { + "type": "date" + }, + "updated_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + } + } + }, + "cases-configure": { + "properties": { + "closure_type": { + "type": "keyword" + }, + "connector": { + "properties": { + "fields": { + "properties": { + "key": { + "type": "text" + }, + "value": { + "type": "text" + } + } + }, + "id": { + "type": "keyword" + }, + "name": { + "type": "text" + }, + "type": { + "type": "keyword" + } + } + }, + "created_at": { + "type": "date" + }, + "created_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "updated_at": { + "type": "date" + }, + "updated_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + } + } + }, + "cases-connector-mappings": { + "properties": { + "mappings": { + "properties": { + "action_type": { + "type": "keyword" + }, + "source": { + "type": "keyword" + }, + "target": { + "type": "keyword" + } + } + } + } + }, + "cases-sub-case": { + "properties": { + "closed_at": { + "type": "date" + }, + "closed_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "created_at": { + "type": "date" + }, + "created_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "status": { + "type": "keyword" + }, + "updated_at": { + "type": "date" + }, + "updated_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + } + } + }, + "cases-user-actions": { + "properties": { + "action": { + "type": "keyword" + }, + "action_at": { + "type": "date" + }, + "action_by": { + "properties": { + "email": { + "type": "keyword" + }, + "full_name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "action_field": { + "type": "keyword" + }, + "new_value": { + "type": "text" + }, + "old_value": { + "type": "text" + } + } + }, + "config": { + "dynamic": "false", + "properties": { + "buildNum": { + "type": "keyword" + } + } + }, + "core-usage-stats": { + "dynamic": "false", + "type": "object" + }, + "coreMigrationVersion": { + "type": "keyword" + }, + "dashboard": { + "properties": { + "description": { + "type": "text" + }, + "hits": { + "doc_values": false, + "index": false, + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "index": false, + "type": "text" + } + } + }, + "optionsJSON": { + "index": false, + "type": "text" + }, + "panelsJSON": { + "index": false, + "type": "text" + }, + "refreshInterval": { + "properties": { + "display": { + "doc_values": false, + "index": false, + "type": "keyword" + }, + "pause": { + "doc_values": false, + "index": false, + "type": "boolean" + }, + "section": { + "doc_values": false, + "index": false, + "type": "integer" + }, + "value": { + "doc_values": false, + "index": false, + "type": "integer" + } + } + }, + "timeFrom": { + "doc_values": false, + "index": false, + "type": "keyword" + }, + "timeRestore": { + "doc_values": false, + "index": false, + "type": "boolean" + }, + "timeTo": { + "doc_values": false, + "index": false, + "type": "keyword" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "endpoint:user-artifact": { + "properties": { + "body": { + "type": "binary" + }, + "compressionAlgorithm": { + "index": false, + "type": "keyword" + }, + "created": { + "index": false, + "type": "date" + }, + "decodedSha256": { + "index": false, + "type": "keyword" + }, + "decodedSize": { + "index": false, + "type": "long" + }, + "encodedSha256": { + "type": "keyword" + }, + "encodedSize": { + "index": false, + "type": "long" + }, + "encryptionAlgorithm": { + "index": false, + "type": "keyword" + }, + "identifier": { + "type": "keyword" + } + } + }, + "endpoint:user-artifact-manifest": { + "properties": { + "artifacts": { + "properties": { + "artifactId": { + "index": false, + "type": "keyword" + }, + "policyId": { + "index": false, + "type": "keyword" + } + }, + "type": "nested" + }, + "created": { + "index": false, + "type": "date" + }, + "schemaVersion": { + "type": "keyword" + }, + "semanticVersion": { + "index": false, + "type": "keyword" + } + } + }, + "enterprise_search_telemetry": { + "dynamic": "false", + "type": "object" + }, + "epm-packages": { + "properties": { + "es_index_patterns": { + "enabled": false, + "type": "object" + }, + "install_source": { + "type": "keyword" + }, + "install_started_at": { + "type": "date" + }, + "install_status": { + "type": "keyword" + }, + "install_version": { + "type": "keyword" + }, + "installed_es": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + }, + "type": "nested" + }, + "installed_kibana": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + }, + "type": "nested" + }, + "internal": { + "type": "boolean" + }, + "name": { + "type": "keyword" + }, + "package_assets": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + }, + "type": "nested" + }, + "removable": { + "type": "boolean" + }, + "version": { + "type": "keyword" + } + } + }, + "epm-packages-assets": { + "properties": { + "asset_path": { + "type": "keyword" + }, + "data_base64": { + "type": "binary" + }, + "data_utf8": { + "index": false, + "type": "text" + }, + "install_source": { + "type": "keyword" + }, + "media_type": { + "type": "keyword" + }, + "package_name": { + "type": "keyword" + }, + "package_version": { + "type": "keyword" + } + } + }, + "exception-list": { + "properties": { + "_tags": { + "type": "keyword" + }, + "comments": { + "properties": { + "comment": { + "type": "keyword" + }, + "created_at": { + "type": "keyword" + }, + "created_by": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "updated_at": { + "type": "keyword" + }, + "updated_by": { + "type": "keyword" + } + } + }, + "created_at": { + "type": "keyword" + }, + "created_by": { + "type": "keyword" + }, + "description": { + "type": "keyword" + }, + "entries": { + "properties": { + "entries": { + "properties": { + "field": { + "type": "keyword" + }, + "operator": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "value": { + "fields": { + "text": { + "type": "text" + } + }, + "type": "keyword" + } + } + }, + "field": { + "type": "keyword" + }, + "list": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + }, + "operator": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "value": { + "fields": { + "text": { + "type": "text" + } + }, + "type": "keyword" + } + } + }, + "immutable": { + "type": "boolean" + }, + "item_id": { + "type": "keyword" + }, + "list_id": { + "type": "keyword" + }, + "list_type": { + "type": "keyword" + }, + "meta": { + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "type": "text" + } + }, + "type": "keyword" + }, + "os_types": { + "type": "keyword" + }, + "tags": { + "fields": { + "text": { + "type": "text" + } + }, + "type": "keyword" + }, + "tie_breaker_id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "updated_by": { + "type": "keyword" + }, + "version": { + "type": "keyword" + } + } + }, + "exception-list-agnostic": { + "properties": { + "_tags": { + "type": "keyword" + }, + "comments": { + "properties": { + "comment": { + "type": "keyword" + }, + "created_at": { + "type": "keyword" + }, + "created_by": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "updated_at": { + "type": "keyword" + }, + "updated_by": { + "type": "keyword" + } + } + }, + "created_at": { + "type": "keyword" + }, + "created_by": { + "type": "keyword" + }, + "description": { + "type": "keyword" + }, + "entries": { + "properties": { + "entries": { + "properties": { + "field": { + "type": "keyword" + }, + "operator": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "value": { + "fields": { + "text": { + "type": "text" + } + }, + "type": "keyword" + } + } + }, + "field": { + "type": "keyword" + }, + "list": { + "properties": { + "id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + }, + "operator": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "value": { + "fields": { + "text": { + "type": "text" + } + }, + "type": "keyword" + } + } + }, + "immutable": { + "type": "boolean" + }, + "item_id": { + "type": "keyword" + }, + "list_id": { + "type": "keyword" + }, + "list_type": { + "type": "keyword" + }, + "meta": { + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "type": "text" + } + }, + "type": "keyword" + }, + "os_types": { + "type": "keyword" + }, + "tags": { + "fields": { + "text": { + "type": "text" + } + }, + "type": "keyword" + }, + "tie_breaker_id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "updated_by": { + "type": "keyword" + }, + "version": { + "type": "keyword" + } + } + }, + "file-upload-usage-collection-telemetry": { + "properties": { + "file_upload": { + "properties": { + "index_creation_count": { + "type": "long" + } + } + } + } + }, + "fleet-agent-actions": { + "properties": { + "ack_data": { + "type": "text" + }, + "agent_id": { + "type": "keyword" + }, + "created_at": { + "type": "date" + }, + "data": { + "type": "binary" + }, + "policy_id": { + "type": "keyword" + }, + "policy_revision": { + "type": "integer" + }, + "sent_at": { + "type": "date" + }, + "type": { + "type": "keyword" + } + } + }, + "fleet-agent-events": { + "properties": { + "action_id": { + "type": "keyword" + }, + "agent_id": { + "type": "keyword" + }, + "data": { + "type": "text" + }, + "message": { + "type": "text" + }, + "payload": { + "type": "text" + }, + "policy_id": { + "type": "keyword" + }, + "stream_id": { + "type": "keyword" + }, + "subtype": { + "type": "keyword" + }, + "timestamp": { + "type": "date" + }, + "type": { + "type": "keyword" + } + } + }, + "fleet-agents": { + "properties": { + "access_api_key_id": { + "type": "keyword" + }, + "active": { + "type": "boolean" + }, + "current_error_events": { + "index": false, + "type": "text" + }, + "default_api_key": { + "type": "binary" + }, + "default_api_key_id": { + "type": "keyword" + }, + "enrolled_at": { + "type": "date" + }, + "last_checkin": { + "type": "date" + }, + "last_checkin_status": { + "type": "keyword" + }, + "last_updated": { + "type": "date" + }, + "local_metadata": { + "type": "flattened" + }, + "packages": { + "type": "keyword" + }, + "policy_id": { + "type": "keyword" + }, + "policy_revision": { + "type": "integer" + }, + "type": { + "type": "keyword" + }, + "unenrolled_at": { + "type": "date" + }, + "unenrollment_started_at": { + "type": "date" + }, + "updated_at": { + "type": "date" + }, + "upgrade_started_at": { + "type": "date" + }, + "upgraded_at": { + "type": "date" + }, + "user_provided_metadata": { + "type": "flattened" + }, + "version": { + "type": "keyword" + } + } + }, + "fleet-enrollment-api-keys": { + "properties": { + "active": { + "type": "boolean" + }, + "api_key": { + "type": "binary" + }, + "api_key_id": { + "type": "keyword" + }, + "created_at": { + "type": "date" + }, + "expire_at": { + "type": "date" + }, + "name": { + "type": "keyword" + }, + "policy_id": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "updated_at": { + "type": "date" + } + } + }, + "graph-workspace": { + "properties": { + "description": { + "type": "text" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "legacyIndexPatternRef": { + "index": false, + "type": "text" + }, + "numLinks": { + "type": "integer" + }, + "numVertices": { + "type": "integer" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + }, + "wsState": { + "type": "text" + } + } + }, + "index-pattern": { + "dynamic": "false", + "properties": { + "title": { + "type": "text" + }, + "type": { + "type": "keyword" + } + } + }, + "infrastructure-ui-source": { + "dynamic": "false", + "type": "object" + }, + "ingest-agent-policies": { + "properties": { + "description": { + "type": "text" + }, + "is_default": { + "type": "boolean" + }, + "is_default_fleet_server": { + "type": "boolean" + }, + "is_managed": { + "type": "boolean" + }, + "monitoring_enabled": { + "index": false, + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "namespace": { + "type": "keyword" + }, + "package_policies": { + "type": "keyword" + }, + "revision": { + "type": "integer" + }, + "status": { + "type": "keyword" + }, + "updated_at": { + "type": "date" + }, + "updated_by": { + "type": "keyword" + } + } + }, + "ingest-outputs": { + "properties": { + "ca_sha256": { + "index": false, + "type": "keyword" + }, + "config": { + "type": "flattened" + }, + "config_yaml": { + "type": "text" + }, + "fleet_enroll_password": { + "type": "binary" + }, + "fleet_enroll_username": { + "type": "binary" + }, + "hosts": { + "type": "keyword" + }, + "is_default": { + "type": "boolean" + }, + "name": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + }, + "ingest-package-policies": { + "properties": { + "created_at": { + "type": "date" + }, + "created_by": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "enabled": { + "type": "boolean" + }, + "inputs": { + "enabled": false, + "properties": { + "compiled_input": { + "type": "flattened" + }, + "config": { + "type": "flattened" + }, + "enabled": { + "type": "boolean" + }, + "streams": { + "properties": { + "compiled_stream": { + "type": "flattened" + }, + "config": { + "type": "flattened" + }, + "data_stream": { + "properties": { + "dataset": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + }, + "enabled": { + "type": "boolean" + }, + "id": { + "type": "keyword" + }, + "vars": { + "type": "flattened" + } + }, + "type": "nested" + }, + "type": { + "type": "keyword" + }, + "vars": { + "type": "flattened" + } + }, + "type": "nested" + }, + "name": { + "type": "keyword" + }, + "namespace": { + "type": "keyword" + }, + "output_id": { + "type": "keyword" + }, + "package": { + "properties": { + "name": { + "type": "keyword" + }, + "title": { + "type": "keyword" + }, + "version": { + "type": "keyword" + } + } + }, + "policy_id": { + "type": "keyword" + }, + "revision": { + "type": "integer" + }, + "updated_at": { + "type": "date" + }, + "updated_by": { + "type": "keyword" + } + } + }, + "ingest_manager_settings": { + "properties": { + "has_seen_add_data_notice": { + "index": false, + "type": "boolean" + }, + "kibana_ca_sha256": { + "type": "keyword" + }, + "kibana_urls": { + "type": "keyword" + } + } + }, + "inventory-view": { + "dynamic": "false", + "type": "object" + }, + "kql-telemetry": { + "properties": { + "optInCount": { + "type": "long" + }, + "optOutCount": { + "type": "long" + } + } + }, + "legacy-url-alias": { + "dynamic": "false", + "type": "object" + }, + "lens": { + "properties": { + "description": { + "type": "text" + }, + "expression": { + "doc_values": false, + "index": false, + "type": "keyword" + }, + "state": { + "type": "flattened" + }, + "title": { + "type": "text" + }, + "visualizationType": { + "type": "keyword" + } + } + }, + "lens-ui-telemetry": { + "properties": { + "count": { + "type": "integer" + }, + "date": { + "type": "date" + }, + "name": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + }, + "map": { + "properties": { + "bounds": { + "dynamic": "false", + "type": "object" + }, + "description": { + "type": "text" + }, + "layerListJSON": { + "type": "text" + }, + "mapStateJSON": { + "type": "text" + }, + "title": { + "type": "text" + }, + "uiStateJSON": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "maps-telemetry": { + "enabled": false, + "type": "object" + }, + "metrics-explorer-view": { + "dynamic": "false", + "type": "object" + }, + "migrationVersion": { + "dynamic": "true", + "properties": { + "config": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "dashboard": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "index-pattern": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "visualization": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "ml-job": { + "properties": { + "datafeed_id": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + }, + "job_id": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + }, + "type": { + "type": "keyword" + } + } + }, + "ml-module": { + "dynamic": "false", + "properties": { + "datafeeds": { + "type": "object" + }, + "defaultIndexPattern": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + }, + "description": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + }, + "id": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + }, + "jobs": { + "type": "object" + }, + "logo": { + "type": "object" + }, + "query": { + "type": "object" + }, + "title": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + }, + "type": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "monitoring-telemetry": { + "properties": { + "reportedClusterUuids": { + "type": "keyword" + } + } + }, + "namespace": { + "type": "keyword" + }, + "namespaces": { + "type": "keyword" + }, + "originId": { + "type": "keyword" + }, + "query": { + "properties": { + "description": { + "type": "text" + }, + "filters": { + "enabled": false, + "type": "object" + }, + "query": { + "properties": { + "language": { + "type": "keyword" + }, + "query": { + "index": false, + "type": "keyword" + } + } + }, + "timefilter": { + "enabled": false, + "type": "object" + }, + "title": { + "type": "text" + } + } + }, + "references": { + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + }, + "type": "nested" + }, + "sample-data-telemetry": { + "properties": { + "installCount": { + "type": "long" + }, + "unInstallCount": { + "type": "long" + } + } + }, + "search": { + "properties": { + "columns": { + "doc_values": false, + "index": false, + "type": "keyword" + }, + "description": { + "type": "text" + }, + "grid": { + "enabled": false, + "type": "object" + }, + "hideChart": { + "doc_values": false, + "index": false, + "type": "boolean" + }, + "hits": { + "doc_values": false, + "index": false, + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "index": false, + "type": "text" + } + } + }, + "sort": { + "doc_values": false, + "index": false, + "type": "keyword" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "search-session": { + "properties": { + "appId": { + "type": "keyword" + }, + "created": { + "type": "date" + }, + "expires": { + "type": "date" + }, + "idMapping": { + "enabled": false, + "type": "object" + }, + "initialState": { + "enabled": false, + "type": "object" + }, + "name": { + "type": "keyword" + }, + "persisted": { + "type": "boolean" + }, + "realmName": { + "type": "keyword" + }, + "realmType": { + "type": "keyword" + }, + "restoreState": { + "enabled": false, + "type": "object" + }, + "sessionId": { + "type": "keyword" + }, + "status": { + "type": "keyword" + }, + "touched": { + "type": "date" + }, + "urlGeneratorId": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "search-telemetry": { + "dynamic": "false", + "type": "object" + }, + "security-solution-signals-migration": { + "properties": { + "created": { + "index": false, + "type": "date" + }, + "createdBy": { + "index": false, + "type": "text" + }, + "destinationIndex": { + "index": false, + "type": "keyword" + }, + "error": { + "index": false, + "type": "text" + }, + "sourceIndex": { + "type": "keyword" + }, + "status": { + "index": false, + "type": "keyword" + }, + "taskId": { + "index": false, + "type": "keyword" + }, + "updated": { + "index": false, + "type": "date" + }, + "updatedBy": { + "index": false, + "type": "text" + }, + "version": { + "type": "long" + } + } + }, + "server": { + "dynamic": "false", + "type": "object" + }, + "siem-detection-engine-rule-actions": { + "properties": { + "actions": { + "properties": { + "action_type_id": { + "type": "keyword" + }, + "group": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "params": { + "enabled": false, + "type": "object" + } + } + }, + "alertThrottle": { + "type": "keyword" + }, + "ruleAlertId": { + "type": "keyword" + }, + "ruleThrottle": { + "type": "keyword" + } + } + }, + "siem-detection-engine-rule-status": { + "properties": { + "alertId": { + "type": "keyword" + }, + "bulkCreateTimeDurations": { + "type": "float" + }, + "gap": { + "type": "text" + }, + "lastFailureAt": { + "type": "date" + }, + "lastFailureMessage": { + "type": "text" + }, + "lastLookBackDate": { + "type": "date" + }, + "lastSuccessAt": { + "type": "date" + }, + "lastSuccessMessage": { + "type": "text" + }, + "searchAfterTimeDurations": { + "type": "float" + }, + "status": { + "type": "keyword" + }, + "statusDate": { + "type": "date" + } + } + }, + "siem-ui-timeline": { + "properties": { + "columns": { + "properties": { + "aggregatable": { + "type": "boolean" + }, + "category": { + "type": "keyword" + }, + "columnHeaderType": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "example": { + "type": "text" + }, + "id": { + "type": "keyword" + }, + "indexes": { + "type": "keyword" + }, + "name": { + "type": "text" + }, + "placeholder": { + "type": "text" + }, + "searchable": { + "type": "boolean" + }, + "type": { + "type": "keyword" + } + } + }, + "created": { + "type": "date" + }, + "createdBy": { + "type": "text" + }, + "dataProviders": { + "properties": { + "and": { + "properties": { + "enabled": { + "type": "boolean" + }, + "excluded": { + "type": "boolean" + }, + "id": { + "type": "keyword" + }, + "kqlQuery": { + "type": "text" + }, + "name": { + "type": "text" + }, + "queryMatch": { + "properties": { + "displayField": { + "type": "text" + }, + "displayValue": { + "type": "text" + }, + "field": { + "type": "text" + }, + "operator": { + "type": "text" + }, + "value": { + "type": "text" + } + } + }, + "type": { + "type": "text" + } + } + }, + "enabled": { + "type": "boolean" + }, + "excluded": { + "type": "boolean" + }, + "id": { + "type": "keyword" + }, + "kqlQuery": { + "type": "text" + }, + "name": { + "type": "text" + }, + "queryMatch": { + "properties": { + "displayField": { + "type": "text" + }, + "displayValue": { + "type": "text" + }, + "field": { + "type": "text" + }, + "operator": { + "type": "text" + }, + "value": { + "type": "text" + } + } + }, + "type": { + "type": "text" + } + } + }, + "dateRange": { + "properties": { + "end": { + "type": "date" + }, + "start": { + "type": "date" } } - } - } - }, - "dashboard": { - "properties": { + }, "description": { "type": "text" }, - "hits": { - "type": "integer" - }, - "kibanaSavedObjectMeta": { + "eqlOptions": { "properties": { - "searchSourceJSON": { + "eventCategoryField": { + "type": "text" + }, + "query": { + "type": "text" + }, + "size": { + "type": "text" + }, + "tiebreakerField": { + "type": "text" + }, + "timestampField": { "type": "text" } } }, - "optionsJSON": { + "eventType": { + "type": "keyword" + }, + "excludedRowRendererIds": { "type": "text" }, - "panelsJSON": { + "favorite": { + "properties": { + "favoriteDate": { + "type": "date" + }, + "fullName": { + "type": "text" + }, + "keySearch": { + "type": "text" + }, + "userName": { + "type": "text" + } + } + }, + "filters": { + "properties": { + "exists": { + "type": "text" + }, + "match_all": { + "type": "text" + }, + "meta": { + "properties": { + "alias": { + "type": "text" + }, + "controlledBy": { + "type": "text" + }, + "disabled": { + "type": "boolean" + }, + "field": { + "type": "text" + }, + "formattedValue": { + "type": "text" + }, + "index": { + "type": "keyword" + }, + "key": { + "type": "keyword" + }, + "negate": { + "type": "boolean" + }, + "params": { + "type": "text" + }, + "type": { + "type": "keyword" + }, + "value": { + "type": "text" + } + } + }, + "missing": { + "type": "text" + }, + "query": { + "type": "text" + }, + "range": { + "type": "text" + }, + "script": { + "type": "text" + } + } + }, + "indexNames": { "type": "text" }, - "refreshInterval": { + "kqlMode": { + "type": "keyword" + }, + "kqlQuery": { "properties": { - "display": { + "filterQuery": { + "properties": { + "kuery": { + "properties": { + "expression": { + "type": "text" + }, + "kind": { + "type": "keyword" + } + } + }, + "serializedQuery": { + "type": "text" + } + } + } + } + }, + "savedQueryId": { + "type": "keyword" + }, + "sort": { + "dynamic": "false", + "properties": { + "columnId": { "type": "keyword" }, - "pause": { - "type": "boolean" - }, - "section": { - "type": "integer" + "columnType": { + "type": "keyword" }, - "value": { - "type": "integer" + "sortDirection": { + "type": "keyword" } } }, - "timeFrom": { + "status": { "type": "keyword" }, - "timeRestore": { - "type": "boolean" + "templateTimelineId": { + "type": "text" }, - "timeTo": { + "templateTimelineVersion": { + "type": "integer" + }, + "timelineType": { "type": "keyword" }, "title": { "type": "text" }, - "uiStateJSON": { - "type": "text" + "updated": { + "type": "date" }, - "version": { - "type": "integer" + "updatedBy": { + "type": "text" } } }, - "index-pattern": { + "siem-ui-timeline-note": { "properties": { - "fieldFormatMap": { + "created": { + "type": "date" + }, + "createdBy": { "type": "text" }, - "fields": { + "eventId": { + "type": "keyword" + }, + "note": { "type": "text" }, - "intervalName": { + "timelineId": { "type": "keyword" }, - "notExpandable": { - "type": "boolean" + "updated": { + "type": "date" + }, + "updatedBy": { + "type": "text" + } + } + }, + "siem-ui-timeline-pinned-event": { + "properties": { + "created": { + "type": "date" }, - "sourceFilters": { + "createdBy": { "type": "text" }, - "timeFieldName": { + "eventId": { "type": "keyword" }, - "title": { + "timelineId": { + "type": "keyword" + }, + "updated": { + "type": "date" + }, + "updatedBy": { "type": "text" } } }, - "search": { + "space": { "properties": { - "columns": { + "_reserved": { + "type": "boolean" + }, + "color": { "type": "keyword" }, "description": { "type": "text" }, - "hits": { - "type": "integer" + "disabledFeatures": { + "type": "keyword" }, - "kibanaSavedObjectMeta": { - "properties": { - "searchSourceJSON": { - "type": "text" - } - } + "imageUrl": { + "index": false, + "type": "text" }, - "sort": { + "initials": { "type": "keyword" }, - "title": { + "name": { + "fields": { + "keyword": { + "ignore_above": 2048, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "spaces-usage-stats": { + "dynamic": "false", + "type": "object" + }, + "tag": { + "properties": { + "color": { "type": "text" }, - "version": { - "type": "integer" + "description": { + "type": "text" + }, + "name": { + "type": "text" } } }, - "server": { + "telemetry": { "properties": { - "uuid": { + "allowChangingOptInStatus": { + "type": "boolean" + }, + "enabled": { + "type": "boolean" + }, + "lastReported": { + "type": "date" + }, + "lastVersionChecked": { + "type": "keyword" + }, + "reportFailureCount": { + "type": "integer" + }, + "reportFailureVersion": { + "type": "keyword" + }, + "sendUsageFrom": { "type": "keyword" + }, + "userHasSeenNotice": { + "type": "boolean" } } }, @@ -187,28 +2652,147 @@ } } }, - "namespace": { + "type": { "type": "keyword" }, - "references": { + "ui-counter": { "properties": { - "id": { - "type": "keyword" + "count": { + "type": "integer" + } + } + }, + "ui-metric": { + "properties": { + "count": { + "type": "integer" + } + } + }, + "updated_at": { + "type": "date" + }, + "upgrade-assistant-reindex-operation": { + "properties": { + "errorMessage": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" }, - "name": { + "indexName": { "type": "keyword" }, - "type": { - "type": "keyword" + "lastCompletedStep": { + "type": "long" + }, + "locked": { + "type": "date" + }, + "newIndexName": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "reindexOptions": { + "properties": { + "openAndClose": { + "type": "boolean" + }, + "queueSettings": { + "properties": { + "queuedAt": { + "type": "long" + }, + "startedAt": { + "type": "long" + } + } + } + } + }, + "reindexTaskId": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "reindexTaskPercComplete": { + "type": "float" + }, + "runningReindexCount": { + "type": "integer" + }, + "status": { + "type": "integer" } - }, - "type": "nested" + } }, - "type": { - "type": "keyword" + "upgrade-assistant-telemetry": { + "properties": { + "features": { + "properties": { + "deprecation_logging": { + "properties": { + "enabled": { + "null_value": true, + "type": "boolean" + } + } + } + } + }, + "ui_open": { + "properties": { + "cluster": { + "null_value": 0, + "type": "long" + }, + "indices": { + "null_value": 0, + "type": "long" + }, + "overview": { + "null_value": 0, + "type": "long" + } + } + }, + "ui_reindex": { + "properties": { + "close": { + "null_value": 0, + "type": "long" + }, + "open": { + "null_value": 0, + "type": "long" + }, + "start": { + "null_value": 0, + "type": "long" + }, + "stop": { + "null_value": 0, + "type": "long" + } + } + } + } }, - "updated_at": { - "type": "date" + "uptime-dynamic-settings": { + "dynamic": "false", + "type": "object" }, "url": { "properties": { @@ -222,13 +2806,13 @@ "type": "date" }, "url": { - "type": "text", "fields": { "keyword": { - "type": "keyword", - "ignore_above": 2048 + "ignore_above": 2048, + "type": "keyword" } - } + }, + "type": "text" } } }, @@ -240,28 +2824,43 @@ "kibanaSavedObjectMeta": { "properties": { "searchSourceJSON": { + "index": false, "type": "text" } } }, - "savedSearchId": { + "savedSearchRefName": { + "doc_values": false, + "index": false, "type": "keyword" }, "title": { "type": "text" }, "uiStateJSON": { + "index": false, "type": "text" }, "version": { "type": "integer" }, "visState": { + "index": false, "type": "text" } } + }, + "workplace_search_telemetry": { + "dynamic": "false", + "type": "object" } } + }, + "settings": { + "index": { + "number_of_replicas": "0", + "number_of_shards": "1" + } } } -} +} \ No newline at end of file diff --git a/test/common/services/deployment.ts b/test/common/services/deployment.ts index 510124ce3d1b7..65466ca966ad2 100644 --- a/test/common/services/deployment.ts +++ b/test/common/services/deployment.ts @@ -30,14 +30,6 @@ export function DeploymentProvider({ getService }: FtrProviderContext) { return getUrl.baseUrl(config.get('servers.elasticsearch')); }, - /** - * Helper to detect an OSS licensed Kibana - * Useful for functional testing in cloud environment - */ - async isOss() { - return config.get('kbnTestServer.serverArgs').indexOf('--oss') > -1; - }, - async isCloud(): Promise { const baseUrl = this.getHostPort(); const username = config.get('servers.kibana.username'); diff --git a/test/examples/expressions_explorer/expressions.ts b/test/examples/expressions_explorer/expressions.ts index 39afa177501d5..4c240653b5fdd 100644 --- a/test/examples/expressions_explorer/expressions.ts +++ b/test/examples/expressions_explorer/expressions.ts @@ -22,7 +22,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) { await retry.try(async () => { const text = await testSubjects.getVisibleText('expressionResult'); expect(text).to.be( - '{\n "type": "error",\n "error": {\n "message": "Function markdown could not be found.",\n "name": "fn not found"\n }\n}' + '{\n "type": "render",\n "as": "markdown",\n "value": {\n "content": "## expressions explorer",\n "font": {\n "type": "style",\n "spec": {\n "fontFamily": "\'Open Sans\', Helvetica, Arial, sans-serif",\n "fontWeight": "normal",\n "fontStyle": "normal",\n "textDecoration": "none",\n "textAlign": "left",\n "fontSize": "14px",\n "lineHeight": "1"\n },\n "css": "font-family:\'Open Sans\', Helvetica, Arial, sans-serif;font-weight:normal;font-style:normal;text-decoration:none;text-align:left;font-size:14px;line-height:1"\n },\n "openLinksInNewTab": false\n }\n}' ); }); }); @@ -30,7 +30,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) { it('renders expression', async () => { await retry.try(async () => { const text = await testSubjects.getVisibleText('expressionRender'); - expect(text).to.be('Function markdown could not be found.'); + expect(text).to.be('expressions explorer rendering'); }); }); diff --git a/test/functional/apps/home/_home.js b/test/functional/apps/home/_home.js index 056f3ec6f993c..24e672463964d 100644 --- a/test/functional/apps/home/_home.js +++ b/test/functional/apps/home/_home.js @@ -11,6 +11,7 @@ import expect from '@kbn/expect'; export default function ({ getService, getPageObjects }) { const browser = getService('browser'); const globalNav = getService('globalNav'); + const testSubjects = getService('testSubjects'); const PageObjects = getPageObjects(['common', 'header', 'home']); describe('Kibana takes you home', function describeIndexTests() { @@ -25,7 +26,8 @@ export default function ({ getService, getPageObjects }) { }); it('clicking on console on homepage should take you to console app', async () => { - await PageObjects.home.clickSynopsis('console'); + await PageObjects.common.navigateToUrl('home'); + await testSubjects.click('homeDevTools'); const url = await browser.getCurrentUrl(); expect(url.includes('/app/dev_tools#/console')).to.be(true); }); diff --git a/test/functional/apps/home/_newsfeed.ts b/test/functional/apps/home/_newsfeed.ts index 449aeea013341..5b8b5a22cf439 100644 --- a/test/functional/apps/home/_newsfeed.ts +++ b/test/functional/apps/home/_newsfeed.ts @@ -11,7 +11,6 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ getService, getPageObjects }: FtrProviderContext) { const globalNav = getService('globalNav'); - const deployment = getService('deployment'); const PageObjects = getPageObjects(['newsfeed']); describe('Newsfeed', () => { @@ -38,16 +37,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('shows all news from newsfeed', async () => { const objects = await PageObjects.newsfeed.getNewsfeedList(); - if (await deployment.isOss()) { - expect(objects).to.eql([ - '21 June 2019\nYou are functionally testing the newsfeed widget with fixtures!\nSee test/common/fixtures/plugins/newsfeed/newsfeed_simulation\nGeneric feed-viewer could go here', - '21 June 2019\nStaging too!\nHello world\nGeneric feed-viewer could go here', - ]); - } else { - // can't shim the API in cloud so going to check that at least something is rendered - // to test that the API was called and returned something that could be rendered - expect(objects.length).to.be.above(0); - } + // can't shim the API in cloud so going to check that at least something is rendered + // to test that the API was called and returned something that could be rendered + expect(objects.length).to.be.above(0); }); it('clicking on newsfeed icon should close opened newsfeed', async () => { diff --git a/test/functional/apps/management/_scripted_fields.js b/test/functional/apps/management/_scripted_fields.js index fdbc419c16241..c7a4c8c51bf44 100644 --- a/test/functional/apps/management/_scripted_fields.js +++ b/test/functional/apps/management/_scripted_fields.js @@ -27,7 +27,6 @@ import expect from '@kbn/expect'; export default function ({ getService, getPageObjects }) { const esArchiver = getService('esArchiver'); const kibanaServer = getService('kibanaServer'); - const deployment = getService('deployment'); const log = getService('log'); const browser = getService('browser'); const retry = getService('retry'); @@ -187,16 +186,14 @@ export default function ({ getService, getPageObjects }) { }); it('should visualize scripted field in vertical bar chart', async function () { - const isOss = await deployment.isOss(); - if (!isOss) { - await filterBar.removeAllFilters(); - await PageObjects.discover.clickFieldListItemVisualize(scriptedPainlessFieldName); - await PageObjects.header.waitUntilLoadingHasFinished(); - // verify Lens opens a visualization - expect(await testSubjects.getVisibleTextAll('lns-dimensionTrigger')).to.contain( - 'Average of ram_Pain1' - ); - } + await filterBar.removeAllFilters(); + await PageObjects.discover.clickFieldListItemVisualize(scriptedPainlessFieldName); + await PageObjects.header.waitUntilLoadingHasFinished(); + // verify Lens opens a visualization + expect(await testSubjects.getVisibleTextAll('lns-dimensionTrigger')).to.contain( + '@timestamp', + 'Median of ram_Pain1' + ); }); }); @@ -277,15 +274,12 @@ export default function ({ getService, getPageObjects }) { }); it('should visualize scripted field in vertical bar chart', async function () { - const isOss = await deployment.isOss(); - if (!isOss) { - await PageObjects.discover.clickFieldListItemVisualize(scriptedPainlessFieldName2); - await PageObjects.header.waitUntilLoadingHasFinished(); - // verify Lens opens a visualization - expect(await testSubjects.getVisibleTextAll('lns-dimensionTrigger')).to.contain( - 'Top values of painString' - ); - } + await PageObjects.discover.clickFieldListItemVisualize(scriptedPainlessFieldName2); + await PageObjects.header.waitUntilLoadingHasFinished(); + // verify Lens opens a visualization + expect(await testSubjects.getVisibleTextAll('lns-dimensionTrigger')).to.contain( + 'Top values of painString' + ); }); }); @@ -367,15 +361,12 @@ export default function ({ getService, getPageObjects }) { }); it('should visualize scripted field in vertical bar chart', async function () { - const isOss = await deployment.isOss(); - if (!isOss) { - await PageObjects.discover.clickFieldListItemVisualize(scriptedPainlessFieldName2); - await PageObjects.header.waitUntilLoadingHasFinished(); - // verify Lens opens a visualization - expect(await testSubjects.getVisibleTextAll('lns-dimensionTrigger')).to.contain( - 'Top values of painBool' - ); - } + await PageObjects.discover.clickFieldListItemVisualize(scriptedPainlessFieldName2); + await PageObjects.header.waitUntilLoadingHasFinished(); + // verify Lens opens a visualization + expect(await testSubjects.getVisibleTextAll('lns-dimensionTrigger')).to.contain( + 'Top values of painBool' + ); }); }); @@ -460,15 +451,10 @@ export default function ({ getService, getPageObjects }) { }); it('should visualize scripted field in vertical bar chart', async function () { - const isOss = await deployment.isOss(); - if (!isOss) { - await PageObjects.discover.clickFieldListItemVisualize(scriptedPainlessFieldName2); - await PageObjects.header.waitUntilLoadingHasFinished(); - // verify Lens opens a visualization - expect(await testSubjects.getVisibleTextAll('lns-dimensionTrigger')).to.contain( - 'painDate' - ); - } + await PageObjects.discover.clickFieldListItemVisualize(scriptedPainlessFieldName2); + await PageObjects.header.waitUntilLoadingHasFinished(); + // verify Lens opens a visualization + expect(await testSubjects.getVisibleTextAll('lns-dimensionTrigger')).to.contain('painDate'); }); }); }); diff --git a/test/functional/apps/visualize/_area_chart.ts b/test/functional/apps/visualize/_area_chart.ts index f6eaa2c685f5d..2bad91565de72 100644 --- a/test/functional/apps/visualize/_area_chart.ts +++ b/test/functional/apps/visualize/_area_chart.ts @@ -505,7 +505,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); it('should show error when calendar interval invalid', async () => { - await PageObjects.visEditor.setInterval('14d', { type: 'custom' }); + await PageObjects.visEditor.setInterval('2w', { type: 'custom' }); const intervalErrorMessage = await find.byCssSelector( '[data-test-subj="visEditorInterval"] + .euiFormErrorText' ); diff --git a/test/functional/apps/visualize/_chart_types.ts b/test/functional/apps/visualize/_chart_types.ts index 4ee6e3dac21b6..71bdc75d41d9c 100644 --- a/test/functional/apps/visualize/_chart_types.ts +++ b/test/functional/apps/visualize/_chart_types.ts @@ -35,6 +35,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.visualize.clickAggBasedVisualizations(); const expectedChartTypes = [ 'Area', + 'Coordinate Map', 'Data table', 'Gauge', 'Goal', @@ -43,6 +44,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 'Line', 'Metric', 'Pie', + 'Region Map', 'Tag cloud', 'Timelion', 'Vertical bar', diff --git a/test/functional/apps/visualize/_tile_map.ts b/test/functional/apps/visualize/_tile_map.ts index 3af467affa1fb..719c2c48761f9 100644 --- a/test/functional/apps/visualize/_tile_map.ts +++ b/test/functional/apps/visualize/_tile_map.ts @@ -125,26 +125,26 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { it('Fit data bounds should zoom to level 3', async function () { const expectedPrecision2DataTable = [ - ['-', 'dr4', '127', { lat: 40, lon: -76 }], - ['-', 'dr7', '92', { lat: 41, lon: -74 }], - ['-', '9q5', '91', { lat: 34, lon: -119 }], - ['-', '9qc', '89', { lat: 38, lon: -122 }], - ['-', 'drk', '87', { lat: 41, lon: -73 }], - ['-', 'dps', '82', { lat: 42, lon: -84 }], - ['-', 'dph', '82', { lat: 40, lon: -84 }], - ['-', 'dp3', '79', { lat: 41, lon: -88 }], - ['-', 'dpe', '78', { lat: 42, lon: -86 }], - ['-', 'dp8', '77', { lat: 43, lon: -90 }], - ['-', 'dp6', '74', { lat: 41, lon: -87 }], - ['-', 'djv', '74', { lat: 33, lon: -83 }], - ['-', '9qh', '74', { lat: 34, lon: -118 }], - ['-', 'dpq', '73', { lat: 41, lon: -81 }], - ['-', 'dpp', '73', { lat: 40, lon: -80 }], - ['-', '9y7', '73', { lat: 35, lon: -97 }], - ['-', '9vg', '73', { lat: 32, lon: -97 }], - ['-', 'drs', '71', { lat: 42, lon: -73 }], - ['-', '9ys', '71', { lat: 37, lon: -95 }], - ['-', '9yn', '71', { lat: 34, lon: -93 }], + ['-', 'dn', '1,429', { lat: 36, lon: -85 }], + ['-', 'dp', '1,418', { lat: 41, lon: -85 }], + ['-', '9y', '1,215', { lat: 36, lon: -96 }], + ['-', '9z', '1,099', { lat: 42, lon: -96 }], + ['-', 'dr', '1,076', { lat: 42, lon: -74 }], + ['-', 'dj', '982', { lat: 31, lon: -85 }], + ['-', '9v', '938', { lat: 31, lon: -96 }], + ['-', '9q', '722', { lat: 36, lon: -120 }], + ['-', '9w', '475', { lat: 36, lon: -107 }], + ['-', 'cb', '457', { lat: 46, lon: -96 }], + ['-', 'c2', '453', { lat: 47, lon: -120 }], + ['-', '9x', '420', { lat: 41, lon: -107 }], + ['-', 'dq', '399', { lat: 37, lon: -78 }], + ['-', '9r', '396', { lat: 41, lon: -120 }], + ['-', '9t', '274', { lat: 32, lon: -107 }], + ['-', 'c8', '271', { lat: 47, lon: -107 }], + ['-', 'dh', '214', { lat: 26, lon: -82 }], + ['-', 'b6', '207', { lat: 60, lon: -162 }], + ['-', 'bd', '206', { lat: 59, lon: -153 }], + ['-', 'b7', '167', { lat: 64, lon: -163 }], ]; await PageObjects.tileMap.clickMapFitDataBounds(); diff --git a/test/functional/apps/visualize/_tsvb_time_series.ts b/test/functional/apps/visualize/_tsvb_time_series.ts index 85d445bc34e6c..58e8cd8dd0d46 100644 --- a/test/functional/apps/visualize/_tsvb_time_series.ts +++ b/test/functional/apps/visualize/_tsvb_time_series.ts @@ -11,9 +11,18 @@ import expect from '@kbn/expect'; import { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ getPageObjects, getService }: FtrProviderContext) { - const { visualize, visualBuilder } = getPageObjects(['visualBuilder', 'visualize']); + const { visualize, visualBuilder, timeToVisualize, dashboard } = getPageObjects([ + 'visualBuilder', + 'visualize', + 'timeToVisualize', + 'dashboard', + ]); + const testSubjects = getService('testSubjects'); const retry = getService('retry'); + const filterBar = getService('filterBar'); + const elasticChart = getService('elasticChart'); const log = getService('log'); + const browser = getService('browser'); const kibanaServer = getService('kibanaServer'); describe('visual builder', function describeIndexTests() { @@ -140,6 +149,36 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { }); }); }); + + describe('Clicking on the chart', () => { + it(`should create a filter`, async () => { + await visualBuilder.setMetricsGroupByTerms('machine.os.raw'); + await visualBuilder.clickSeriesOption(); + await testSubjects.click('visualizeSaveButton'); + + await timeToVisualize.saveFromModal('My TSVB viz 1', { + addToDashboard: 'new', + saveToLibrary: false, + }); + + await dashboard.waitForRenderComplete(); + const el = await elasticChart.getCanvas(); + // click on specific coordinates + await browser + .getActions() + .move({ x: 100, y: 110, origin: el._webElement }) + .click() + .perform(); + + await retry.try(async () => { + await testSubjects.click('applyFiltersPopoverButton'); + await testSubjects.missingOrFail('applyFiltersPopoverButton'); + }); + + const hasMachineRawFilter = await filterBar.hasFilter('machine.os.raw', 'win 7'); + expect(hasMachineRawFilter).to.be(true); + }); + }); }); }); } diff --git a/test/functional/apps/visualize/index.ts b/test/functional/apps/visualize/index.ts index 747494a690c7e..4dff3eada1f24 100644 --- a/test/functional/apps/visualize/index.ts +++ b/test/functional/apps/visualize/index.ts @@ -14,8 +14,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { const log = getService('log'); const esArchiver = getService('esArchiver'); const kibanaServer = getService('kibanaServer'); - const deployment = getService('deployment'); - let isOss = true; describe('visualize app', () => { before(async () => { @@ -28,7 +26,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { defaultIndex: 'logstash-*', [UI_SETTINGS.FORMAT_BYTES_DEFAULT_PATTERN]: '0,0.[000]b', }); - isOss = await deployment.isOss(); }); // TODO: Remove when vislib is removed @@ -66,11 +63,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./_data_table')); loadTestFile(require.resolve('./_data_table_nontimeindex')); loadTestFile(require.resolve('./_data_table_notimeindex_filters')); - - // this check is not needed when the CI doesn't run anymore for the OSS - if (!isOss) { - loadTestFile(require.resolve('./_chart_types')); - } + loadTestFile(require.resolve('./_chart_types')); }); describe('', function () { @@ -98,11 +91,8 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./_linked_saved_searches')); loadTestFile(require.resolve('./_visualize_listing')); loadTestFile(require.resolve('./_add_to_dashboard.ts')); - - if (isOss) { - loadTestFile(require.resolve('./_tile_map')); - loadTestFile(require.resolve('./_region_map')); - } + loadTestFile(require.resolve('./_tile_map')); + loadTestFile(require.resolve('./_region_map')); }); describe('', function () { diff --git a/test/functional/config.js b/test/functional/config.js index 1048bd72dc575..4a6791a3bc62f 100644 --- a/test/functional/config.js +++ b/test/functional/config.js @@ -14,6 +14,7 @@ export default async function ({ readConfigFile }) { return { testFiles: [ + require.resolve('./apps/status_page'), require.resolve('./apps/bundles'), require.resolve('./apps/console'), require.resolve('./apps/context'), @@ -23,7 +24,6 @@ export default async function ({ readConfigFile }) { require.resolve('./apps/home'), require.resolve('./apps/management'), require.resolve('./apps/saved_objects_management'), - require.resolve('./apps/status_page'), require.resolve('./apps/timelion'), require.resolve('./apps/visualize'), ], @@ -36,13 +36,15 @@ export default async function ({ readConfigFile }) { ...commonConfig.get('esTestCluster'), serverArgs: ['xpack.security.enabled=false'], }, + kbnTestServer: { ...commonConfig.get('kbnTestServer'), serverArgs: [ ...commonConfig.get('kbnTestServer.serverArgs'), - '--oss', '--telemetry.optIn=false', + '--xpack.security.enabled=false', '--savedObjects.maxImportPayloadBytes=10485760', + '--xpack.maps.showMapVisualizationTypes=true', ], }, diff --git a/test/functional/page_objects/home_page.ts b/test/functional/page_objects/home_page.ts index 05f7fb7eecb3d..f03f74ef8c61d 100644 --- a/test/functional/page_objects/home_page.ts +++ b/test/functional/page_objects/home_page.ts @@ -12,9 +12,7 @@ export function HomePageProvider({ getService, getPageObjects }: FtrProviderCont const testSubjects = getService('testSubjects'); const retry = getService('retry'); const find = getService('find'); - const deployment = getService('deployment'); const PageObjects = getPageObjects(['common']); - let isOss = true; class HomePage { async clickSynopsis(title: string) { @@ -72,10 +70,7 @@ export function HomePageProvider({ getService, getPageObjects }: FtrProviderCont async launchSampleDashboard(id: string) { await this.launchSampleDataSet(id); - isOss = await deployment.isOss(); - if (!isOss) { - await find.clickByLinkText('Dashboard'); - } + await find.clickByLinkText('Dashboard'); } async launchSampleDataSet(id: string) { diff --git a/test/scripts/jenkins_build_kibana.sh b/test/scripts/jenkins_build_kibana.sh index fa0c9522ef5fb..32fb98929e21c 100755 --- a/test/scripts/jenkins_build_kibana.sh +++ b/test/scripts/jenkins_build_kibana.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash +cd "$KIBANA_DIR" source src/dev/ci_setup/setup_env.sh if [[ ! "$TASK_QUEUE_PROCESS_ID" ]]; then @@ -12,16 +13,50 @@ export KBN_NP_PLUGINS_BUILT=true echo " -> Ensuring all functional tests are in a ciGroup" node scripts/ensure_all_tests_in_ci_group; +echo " -> Ensuring all x-pack functional tests are in a ciGroup" +node x-pack/scripts/functional_tests --assert-none-excluded \ + --include-tag ciGroup1 \ + --include-tag ciGroup2 \ + --include-tag ciGroup3 \ + --include-tag ciGroup4 \ + --include-tag ciGroup5 \ + --include-tag ciGroup6 \ + --include-tag ciGroup7 \ + --include-tag ciGroup8 \ + --include-tag ciGroup9 \ + --include-tag ciGroup10 \ + --include-tag ciGroup11 \ + --include-tag ciGroup12 \ + --include-tag ciGroup13 \ + --include-tag ciGroupDocker + # Do not build kibana for code coverage run if [[ -z "$CODE_COVERAGE" ]] ; then - echo " -> building and extracting OSS Kibana distributable for use in functional tests" - node scripts/build --debug --oss + echo " -> building and extracting default Kibana distributable for use in functional tests" + node scripts/build --debug --no-oss echo " -> shipping metrics from build to ci-stats" node scripts/ship_ci_stats \ --metrics target/optimizer_bundle_metrics.json \ --metrics packages/kbn-ui-shared-deps/target/metrics.json - mkdir -p "$WORKSPACE/kibana-build-oss" - cp -pR build/oss/kibana-*-SNAPSHOT-linux-x86_64/. $WORKSPACE/kibana-build-oss/ + linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')" + installDir="$KIBANA_DIR/install/kibana" + mkdir -p "$installDir" + tar -xzf "$linuxBuild" -C "$installDir" --strip=1 + cp "$linuxBuild" "$WORKSPACE/kibana-default.tar.gz" + + mkdir -p "$WORKSPACE/kibana-build" + cp -pR install/kibana/. $WORKSPACE/kibana-build/ + + echo " -> Archive built plugins" + shopt -s globstar + tar -zcf \ + "$WORKSPACE/kibana-default-plugins.tar.gz" \ + x-pack/plugins/**/target/public \ + x-pack/test/**/target/public \ + examples/**/target/public \ + x-pack/examples/**/target/public \ + test/**/target/public + shopt -u globstar fi diff --git a/test/scripts/jenkins_build_load_testing.sh b/test/scripts/jenkins_build_load_testing.sh index a635e34bcbeda..dd9cedea9c4d4 100755 --- a/test/scripts/jenkins_build_load_testing.sh +++ b/test/scripts/jenkins_build_load_testing.sh @@ -64,8 +64,8 @@ installDir="$KIBANA_DIR/install/kibana" mkdir -p "$installDir" tar -xzf "$linuxBuild" -C "$installDir" --strip=1 -mkdir -p "$WORKSPACE/kibana-build-xpack" -cp -pR install/kibana/. $WORKSPACE/kibana-build-xpack/ +mkdir -p "$WORKSPACE/kibana-build" +cp -pR install/kibana/. $WORKSPACE/kibana-build/ echo " -> Setup env for tests" source test/scripts/jenkins_test_setup_xpack.sh diff --git a/test/scripts/jenkins_build_plugins.sh b/test/scripts/jenkins_build_plugins.sh index 962a126ab2700..131bf0121fa14 100755 --- a/test/scripts/jenkins_build_plugins.sh +++ b/test/scripts/jenkins_build_plugins.sh @@ -4,10 +4,19 @@ source src/dev/ci_setup/setup_env.sh echo " -> building kibana platform plugins" node scripts/build_kibana_platform_plugins \ - --oss \ --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \ --scan-dir "$KIBANA_DIR/test/interpreter_functional/plugins" \ --scan-dir "$KIBANA_DIR/test/common/fixtures/plugins" \ --scan-dir "$KIBANA_DIR/examples" \ - --workers 6 \ - --verbose + --scan-dir "$KIBANA_DIR/test/plugin_functional/plugins" \ + --scan-dir "$KIBANA_DIR/test/common/fixtures/plugins" \ + --scan-dir "$XPACK_DIR/test/plugin_functional/plugins" \ + --scan-dir "$XPACK_DIR/test/functional_with_es_ssl/fixtures/plugins" \ + --scan-dir "$XPACK_DIR/test/alerting_api_integration/plugins" \ + --scan-dir "$XPACK_DIR/test/plugin_api_integration/plugins" \ + --scan-dir "$XPACK_DIR/test/plugin_api_perf/plugins" \ + --scan-dir "$XPACK_DIR/test/licensing_plugin/plugins" \ + --scan-dir "$XPACK_DIR/test/usage_collection/plugins" \ + --scan-dir "$XPACK_DIR/test/security_functional/fixtures/common" \ + --scan-dir "$XPACK_DIR/examples" \ + --workers 12 diff --git a/test/scripts/jenkins_test_setup_oss.sh b/test/scripts/jenkins_test_setup_oss.sh index 53626ce89462a..29d396667c465 100755 --- a/test/scripts/jenkins_test_setup_oss.sh +++ b/test/scripts/jenkins_test_setup_oss.sh @@ -3,11 +3,11 @@ source test/scripts/jenkins_test_setup.sh if [[ -z "$CODE_COVERAGE" ]]; then - destDir="$WORKSPACE/kibana-build-oss-${TASK_QUEUE_PROCESS_ID:-$CI_PARALLEL_PROCESS_NUMBER}" + destDir="$WORKSPACE/kibana-build-${TASK_QUEUE_PROCESS_ID:-$CI_PARALLEL_PROCESS_NUMBER}" if [[ ! -d $destDir ]]; then mkdir -p $destDir - cp -pR "$WORKSPACE/kibana-build-oss/." $destDir/ + cp -pR "$WORKSPACE/kibana-build/." $destDir/ fi export KIBANA_INSTALL_DIR="$destDir" diff --git a/test/scripts/jenkins_test_setup_xpack.sh b/test/scripts/jenkins_test_setup_xpack.sh index b9227fd8ff416..31acc4f4865e2 100755 --- a/test/scripts/jenkins_test_setup_xpack.sh +++ b/test/scripts/jenkins_test_setup_xpack.sh @@ -3,11 +3,11 @@ source test/scripts/jenkins_test_setup.sh if [[ -z "$CODE_COVERAGE" ]]; then - destDir="$WORKSPACE/kibana-build-xpack-${TASK_QUEUE_PROCESS_ID:-$CI_PARALLEL_PROCESS_NUMBER}" + destDir="$WORKSPACE/kibana-build-${TASK_QUEUE_PROCESS_ID:-$CI_PARALLEL_PROCESS_NUMBER}" if [[ ! -d $destDir ]]; then mkdir -p $destDir - cp -pR "$WORKSPACE/kibana-build-xpack/." $destDir/ + cp -pR "$WORKSPACE/kibana-build/." $destDir/ fi export KIBANA_INSTALL_DIR="$(realpath $destDir)" diff --git a/test/scripts/jenkins_xpack_baseline.sh b/test/scripts/jenkins_xpack_baseline.sh index 2755a6e0a705d..93363687b39a9 100755 --- a/test/scripts/jenkins_xpack_baseline.sh +++ b/test/scripts/jenkins_xpack_baseline.sh @@ -17,8 +17,8 @@ installDir="$KIBANA_DIR/install/kibana" mkdir -p "$installDir" tar -xzf "$linuxBuild" -C "$installDir" --strip=1 -mkdir -p "$WORKSPACE/kibana-build-xpack" -cp -pR install/kibana/. $WORKSPACE/kibana-build-xpack/ +mkdir -p "$WORKSPACE/kibana-build" +cp -pR install/kibana/. $WORKSPACE/kibana-build/ cd "$KIBANA_DIR" source "test/scripts/jenkins_xpack_saved_objects_field_metrics.sh" diff --git a/test/scripts/jenkins_xpack_build_kibana.sh b/test/scripts/jenkins_xpack_build_kibana.sh deleted file mode 100755 index b5fd7492d7961..0000000000000 --- a/test/scripts/jenkins_xpack_build_kibana.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash - -cd "$KIBANA_DIR" -source src/dev/ci_setup/setup_env.sh - -if [[ ! "$TASK_QUEUE_PROCESS_ID" ]]; then - ./test/scripts/jenkins_xpack_build_plugins.sh -fi - -# doesn't persist, also set in kibanaPipeline.groovy -export KBN_NP_PLUGINS_BUILT=true - -echo " -> Ensuring all functional tests are in a ciGroup" -cd "$XPACK_DIR" -node scripts/functional_tests --assert-none-excluded \ - --include-tag ciGroup1 \ - --include-tag ciGroup2 \ - --include-tag ciGroup3 \ - --include-tag ciGroup4 \ - --include-tag ciGroup5 \ - --include-tag ciGroup6 \ - --include-tag ciGroup7 \ - --include-tag ciGroup8 \ - --include-tag ciGroup9 \ - --include-tag ciGroup10 \ - --include-tag ciGroup11 \ - --include-tag ciGroup12 \ - --include-tag ciGroup13 \ - --include-tag ciGroupDocker - -# Do not build kibana for code coverage run -if [[ -z "$CODE_COVERAGE" ]] ; then - echo " -> building and extracting default Kibana distributable for use in functional tests" - cd "$KIBANA_DIR" - node scripts/build --debug --no-oss - - echo " -> shipping metrics from build to ci-stats" - node scripts/ship_ci_stats \ - --metrics target/optimizer_bundle_metrics.json \ - --metrics packages/kbn-ui-shared-deps/target/metrics.json - - linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')" - installDir="$KIBANA_DIR/install/kibana" - mkdir -p "$installDir" - tar -xzf "$linuxBuild" -C "$installDir" --strip=1 - cp "$linuxBuild" "$WORKSPACE/kibana-default.tar.gz" - - mkdir -p "$WORKSPACE/kibana-build-xpack" - cp -pR install/kibana/. $WORKSPACE/kibana-build-xpack/ - - echo " -> Archive built plugins" - shopt -s globstar - tar -zcf \ - "$WORKSPACE/kibana-default-plugins.tar.gz" \ - x-pack/plugins/**/target/public \ - x-pack/test/**/target/public \ - examples/**/target/public \ - x-pack/examples/**/target/public \ - test/**/target/public - shopt -u globstar -fi diff --git a/test/server_integration/http/ssl_redirect/index.js b/test/server_integration/http/ssl_redirect/index.js index bcd1b6f25ea51..8abe700e26149 100644 --- a/test/server_integration/http/ssl_redirect/index.js +++ b/test/server_integration/http/ssl_redirect/index.js @@ -17,7 +17,7 @@ export default function ({ getService }) { await supertest.get('/').expect('location', url).expect(302); - await supertest.get('/').redirects(1).expect('location', '/app/home').expect(302); + await supertest.get('/').redirects(1).expect('location', '/spaces/enter').expect(302); }); }); } diff --git a/typings/elasticsearch/search.d.ts b/typings/elasticsearch/search.d.ts index c9bf3b1d8b7bc..d75f31d388176 100644 --- a/typings/elasticsearch/search.d.ts +++ b/typings/elasticsearch/search.d.ts @@ -49,7 +49,7 @@ type ValueTypeOfField = T extends Record type MaybeArray = T | T[]; -type Fields = MaybeArray; +type Fields = Exclude['body']['fields'], undefined>; type DocValueFields = MaybeArray; export type SearchHit< @@ -58,7 +58,7 @@ export type SearchHit< TDocValueFields extends DocValueFields | undefined = undefined > = Omit & (TSource extends false ? {} : { _source: TSource }) & - (TFields extends estypes.Fields + (TFields extends Fields ? { fields: Partial, unknown[]>>; } @@ -77,7 +77,7 @@ type HitsOf< > = Array< SearchHit< TOptions extends { _source: false } ? undefined : TDocument, - TOptions extends { fields: estypes.Fields } ? TOptions['fields'] : undefined, + TOptions extends { fields: Fields } ? TOptions['fields'] : undefined, TOptions extends { docvalue_fields: DocValueFields } ? TOptions['docvalue_fields'] : undefined > >; diff --git a/vars/kibanaCoverage.groovy b/vars/kibanaCoverage.groovy index 74d3a67f4a1b8..63585a9f764e3 100644 --- a/vars/kibanaCoverage.groovy +++ b/vars/kibanaCoverage.groovy @@ -194,12 +194,12 @@ def runTests() { 'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'), 'kibana-oss-agent' : workers.functional( 'kibana-oss-tests', - { kibanaPipeline.buildOss() }, + { kibanaPipeline.buildKibana() }, ossProks() ), 'kibana-xpack-agent' : workers.functional( 'kibana-xpack-tests', - { kibanaPipeline.buildXpack() }, + { kibanaPipeline.buildKibana() }, xpackProks() ), ]) diff --git a/vars/kibanaPipeline.groovy b/vars/kibanaPipeline.groovy index 6cccdfaefecba..31cd129d87749 100644 --- a/vars/kibanaPipeline.groovy +++ b/vars/kibanaPipeline.groovy @@ -303,33 +303,23 @@ def doSetup() { } } -def buildOss(maxWorkers = '') { - notifyOnError { - withEnv(["KBN_OPTIMIZER_MAX_WORKERS=${maxWorkers}"]) { - runbld("./test/scripts/jenkins_build_kibana.sh", "Build OSS/Default Kibana") - } - } -} - def getBuildArtifactBucket() { def dir = env.ghprbPullId ? "pr-${env.ghprbPullId}" : buildState.get('checkoutInfo').branch.replace("/", "__") return "gs://ci-artifacts.kibana.dev/default-build/${dir}/${buildState.get('checkoutInfo').commit}" } -def buildXpack(maxWorkers = '', uploadArtifacts = false) { +def buildKibana(maxWorkers = '') { notifyOnError { withEnv(["KBN_OPTIMIZER_MAX_WORKERS=${maxWorkers}"]) { - runbld("./test/scripts/jenkins_xpack_build_kibana.sh", "Build X-Pack Kibana") + runbld("./test/scripts/jenkins_build_kibana.sh", "Build Kibana") } - if (uploadArtifacts) { - withGcpServiceAccount.fromVaultSecret('secret/kibana-issues/dev/ci-artifacts-key', 'value') { - bash(""" - cd "${env.WORKSPACE}" - gsutil -q -m cp 'kibana-default.tar.gz' '${getBuildArtifactBucket()}/' - gsutil -q -m cp 'kibana-default-plugins.tar.gz' '${getBuildArtifactBucket()}/' - """, "Upload Default Build artifacts to GCS") - } + withGcpServiceAccount.fromVaultSecret('secret/kibana-issues/dev/ci-artifacts-key', 'value') { + bash(""" + cd "${env.WORKSPACE}" + gsutil -q -m cp 'kibana-default.tar.gz' '${getBuildArtifactBucket()}/' + gsutil -q -m cp 'kibana-default-plugins.tar.gz' '${getBuildArtifactBucket()}/' + """, "Upload Default Build artifacts to GCS") } } } @@ -443,14 +433,10 @@ def withDocker(Closure closure) { ) } -def buildOssPlugins() { +def buildPlugins() { runbld('./test/scripts/jenkins_build_plugins.sh', 'Build OSS Plugins') } -def buildXpackPlugins() { - runbld('./test/scripts/jenkins_xpack_build_plugins.sh', 'Build X-Pack Plugins') -} - def withTasks(Map params = [:], Closure closure) { catchErrors { def config = [setupWork: {}, worker: [:], parallel: 24] + params @@ -466,8 +452,7 @@ def withTasks(Map params = [:], Closure closure) { }, // There are integration tests etc that require the plugins to be built first, so let's go ahead and build them before set up the parallel workspaces - ossPlugins: { buildOssPlugins() }, - xpackPlugins: { buildXpackPlugins() }, + plugins: { buildPlugins() }, ]) config.setupWork() @@ -487,8 +472,11 @@ def allCiTasks() { tasks.check() tasks.lint() tasks.test() - tasks.functionalOss() - tasks.functionalXpack() + task { + buildKibana(16) + tasks.functionalOss() + tasks.functionalXpack() + } tasks.storybooksCi() } }, diff --git a/vars/tasks.groovy b/vars/tasks.groovy index 1d33fd1249681..e6061de2987ba 100644 --- a/vars/tasks.groovy +++ b/vars/tasks.groovy @@ -55,8 +55,8 @@ def xpackCiGroupDocker() { kibanaPipeline.downloadDefaultBuildArtifacts() kibanaPipeline.bash(""" cd '${env.WORKSPACE}' - mkdir -p kibana-build-xpack - tar -xzf kibana-default.tar.gz -C kibana-build-xpack --strip=1 + mkdir -p kibana-build + tar -xzf kibana-default.tar.gz -C kibana-build --strip=1 tar -xzf kibana-default-plugins.tar.gz -C kibana """, "Extract Default Build artifacts") kibanaPipeline.xpackCiGroupProcess('Docker', true)() @@ -75,8 +75,6 @@ def functionalOss(Map params = [:]) { ] task { - kibanaPipeline.buildOss(6) - if (config.ciGroups) { ossCiGroups() } @@ -115,8 +113,6 @@ def functionalXpack(Map params = [:]) { ] task { - kibanaPipeline.buildXpack(10, true) - if (config.ciGroups) { xpackCiGroups() xpackCiGroupDocker() diff --git a/x-pack/plugins/apm/kibana.json b/x-pack/plugins/apm/kibana.json index 28e4a7b36e740..76d544c3bc6f5 100644 --- a/x-pack/plugins/apm/kibana.json +++ b/x-pack/plugins/apm/kibana.json @@ -10,7 +10,8 @@ "triggersActionsUi", "embeddable", "infra", - "observability" + "observability", + "ruleRegistry" ], "optionalPlugins": [ "spaces", @@ -26,8 +27,13 @@ ], "server": true, "ui": true, - "configPath": ["xpack", "apm"], - "extraPublicDirs": ["public/style/variables"], + "configPath": [ + "xpack", + "apm" + ], + "extraPublicDirs": [ + "public/style/variables" + ], "requiredBundles": [ "home", "kibanaReact", diff --git a/x-pack/plugins/apm/public/application/application.test.tsx b/x-pack/plugins/apm/public/application/application.test.tsx index e6415f76c60dc..4ec654a6c0bfd 100644 --- a/x-pack/plugins/apm/public/application/application.test.tsx +++ b/x-pack/plugins/apm/public/application/application.test.tsx @@ -39,7 +39,11 @@ describe('renderApp', () => { }); it('renders the app', () => { - const { core, config, apmRuleRegistry } = mockApmPluginContextValue; + const { + core, + config, + observabilityRuleTypeRegistry, + } = mockApmPluginContextValue; const plugins = { licensing: { license$: new Observable() }, triggersActionsUi: { actionTypeRegistry: {}, alertTypeRegistry: {} }, @@ -92,7 +96,7 @@ describe('renderApp', () => { appMountParameters: params as any, pluginsStart: startDeps as any, config, - apmRuleRegistry, + observabilityRuleTypeRegistry, }); }); diff --git a/x-pack/plugins/apm/public/application/csmApp.tsx b/x-pack/plugins/apm/public/application/csmApp.tsx index 17905074cfec1..11a2777f47f6a 100644 --- a/x-pack/plugins/apm/public/application/csmApp.tsx +++ b/x-pack/plugins/apm/public/application/csmApp.tsx @@ -12,6 +12,7 @@ import React from 'react'; import ReactDOM from 'react-dom'; import { Route, Router } from 'react-router-dom'; import { DefaultTheme, ThemeProvider } from 'styled-components'; +import type { ObservabilityRuleTypeRegistry } from '../../../observability/public'; import { euiStyled } from '../../../../../src/plugins/kibana_react/common'; import { KibanaContextProvider, @@ -26,11 +27,7 @@ import { ApmPluginContext } from '../context/apm_plugin/apm_plugin_context'; import { UrlParamsProvider } from '../context/url_params_context/url_params_context'; import { useBreadcrumbs } from '../hooks/use_breadcrumbs'; import { ConfigSchema } from '../index'; -import { - ApmPluginSetupDeps, - ApmPluginStartDeps, - ApmRuleRegistry, -} from '../plugin'; +import { ApmPluginSetupDeps, ApmPluginStartDeps } from '../plugin'; import { createCallApmApi } from '../services/rest/createCallApmApi'; import { px, units } from '../style/variables'; import { createStaticIndexPattern } from '../services/rest/index_pattern'; @@ -77,14 +74,14 @@ export function CsmAppRoot({ deps, config, corePlugins: { embeddable, maps }, - apmRuleRegistry, + observabilityRuleTypeRegistry, }: { appMountParameters: AppMountParameters; core: CoreStart; deps: ApmPluginSetupDeps; config: ConfigSchema; corePlugins: ApmPluginStartDeps; - apmRuleRegistry: ApmRuleRegistry; + observabilityRuleTypeRegistry: ObservabilityRuleTypeRegistry; }) { const { history } = appMountParameters; const i18nCore = core.i18n; @@ -94,7 +91,7 @@ export function CsmAppRoot({ config, core, plugins, - apmRuleRegistry, + observabilityRuleTypeRegistry, }; return ( @@ -125,14 +122,14 @@ export const renderApp = ({ appMountParameters, config, corePlugins, - apmRuleRegistry, + observabilityRuleTypeRegistry, }: { core: CoreStart; deps: ApmPluginSetupDeps; appMountParameters: AppMountParameters; config: ConfigSchema; corePlugins: ApmPluginStartDeps; - apmRuleRegistry: ApmRuleRegistry; + observabilityRuleTypeRegistry: ObservabilityRuleTypeRegistry; }) => { const { element } = appMountParameters; @@ -151,7 +148,7 @@ export const renderApp = ({ deps={deps} config={config} corePlugins={corePlugins} - apmRuleRegistry={apmRuleRegistry} + observabilityRuleTypeRegistry={observabilityRuleTypeRegistry} />, element ); diff --git a/x-pack/plugins/apm/public/application/index.tsx b/x-pack/plugins/apm/public/application/index.tsx index acb55a02599f1..e2a0bdb6b48b1 100644 --- a/x-pack/plugins/apm/public/application/index.tsx +++ b/x-pack/plugins/apm/public/application/index.tsx @@ -13,6 +13,7 @@ import ReactDOM from 'react-dom'; import { Route, Router, Switch } from 'react-router-dom'; import 'react-vis/dist/style.css'; import { DefaultTheme, ThemeProvider } from 'styled-components'; +import type { ObservabilityRuleTypeRegistry } from '../../../observability/public'; import { euiStyled } from '../../../../../src/plugins/kibana_react/common'; import { ConfigSchema } from '../'; import { AppMountParameters, CoreStart } from '../../../../../src/core/public'; @@ -30,11 +31,7 @@ import { import { LicenseProvider } from '../context/license/license_context'; import { UrlParamsProvider } from '../context/url_params_context/url_params_context'; import { useBreadcrumbs } from '../hooks/use_breadcrumbs'; -import { - ApmPluginSetupDeps, - ApmPluginStartDeps, - ApmRuleRegistry, -} from '../plugin'; +import { ApmPluginSetupDeps, ApmPluginStartDeps } from '../plugin'; import { createCallApmApi } from '../services/rest/createCallApmApi'; import { createStaticIndexPattern } from '../services/rest/index_pattern'; import { setHelpExtension } from '../setHelpExtension'; @@ -112,14 +109,14 @@ export const renderApp = ({ appMountParameters, config, pluginsStart, - apmRuleRegistry, + observabilityRuleTypeRegistry, }: { coreStart: CoreStart; pluginsSetup: ApmPluginSetupDeps; appMountParameters: AppMountParameters; config: ConfigSchema; pluginsStart: ApmPluginStartDeps; - apmRuleRegistry: ApmRuleRegistry; + observabilityRuleTypeRegistry: ObservabilityRuleTypeRegistry; }) => { const { element } = appMountParameters; const apmPluginContextValue = { @@ -127,7 +124,7 @@ export const renderApp = ({ config, core: coreStart, plugins: pluginsSetup, - apmRuleRegistry, + observabilityRuleTypeRegistry, }; // render APM feedback link in global help menu diff --git a/x-pack/plugins/apm/public/components/alerting/register_apm_alerts.ts b/x-pack/plugins/apm/public/components/alerting/register_apm_alerts.ts index 98c8b99411bc3..7e788016baad2 100644 --- a/x-pack/plugins/apm/public/components/alerting/register_apm_alerts.ts +++ b/x-pack/plugins/apm/public/components/alerting/register_apm_alerts.ts @@ -8,9 +8,19 @@ import { i18n } from '@kbn/i18n'; import { lazy } from 'react'; import { stringify } from 'querystring'; +import { + ALERT_EVALUATION_THRESHOLD, + ALERT_EVALUATION_VALUE, + ALERT_SEVERITY_LEVEL, +} from '@kbn/rule-data-utils/target/technical_field_names'; +import type { ObservabilityRuleTypeRegistry } from '../../../../observability/public'; import { ENVIRONMENT_ALL } from '../../../common/environment_filter_values'; import { AlertType } from '../../../common/alert_types'; -import type { ApmRuleRegistry } from '../../plugin'; + +// copied from elasticsearch_fieldnames.ts to limit page load bundle size +const SERVICE_ENVIRONMENT = 'service.environment'; +const SERVICE_NAME = 'service.name'; +const TRANSACTION_TYPE = 'transaction.type'; const format = ({ pathname, @@ -22,28 +32,32 @@ const format = ({ return `${pathname}?${stringify(query)}`; }; -export function registerApmAlerts(apmRuleRegistry: ApmRuleRegistry) { - apmRuleRegistry.registerType({ +export function registerApmAlerts( + observabilityRuleTypeRegistry: ObservabilityRuleTypeRegistry +) { + observabilityRuleTypeRegistry.register({ id: AlertType.ErrorCount, description: i18n.translate('xpack.apm.alertTypes.errorCount.description', { defaultMessage: 'Alert when the number of errors in a service exceeds a defined threshold.', }), - format: ({ alert }) => { + format: ({ fields }) => { return { reason: i18n.translate('xpack.apm.alertTypes.errorCount.reason', { defaultMessage: `Error count is greater than {threshold} (current value is {measured}) for {serviceName}`, values: { - threshold: alert['kibana.observability.evaluation.threshold'], - measured: alert['kibana.observability.evaluation.value'], - serviceName: alert['service.name']!, + threshold: fields[ALERT_EVALUATION_THRESHOLD], + measured: fields[ALERT_EVALUATION_VALUE], + serviceName: String(fields[SERVICE_NAME][0]), }, }), link: format({ - pathname: `/app/apm/services/${alert['service.name']!}/errors`, + pathname: `/app/apm/services/${String( + fields[SERVICE_NAME][0] + )}/errors`, query: { - ...(alert['service.environment'] - ? { environment: alert['service.environment'] } + ...(fields[SERVICE_ENVIRONMENT]?.[0] + ? { environment: String(fields[SERVICE_ENVIRONMENT][0]) } : { environment: ENVIRONMENT_ALL.value }), }, }), @@ -71,7 +85,7 @@ export function registerApmAlerts(apmRuleRegistry: ApmRuleRegistry) { ), }); - apmRuleRegistry.registerType({ + observabilityRuleTypeRegistry.register({ id: AlertType.TransactionDuration, description: i18n.translate( 'xpack.apm.alertTypes.transactionDuration.description', @@ -80,28 +94,24 @@ export function registerApmAlerts(apmRuleRegistry: ApmRuleRegistry) { 'Alert when the latency of a specific transaction type in a service exceeds a defined threshold.', } ), - format: ({ alert, formatters: { asDuration } }) => ({ + format: ({ fields, formatters: { asDuration } }) => ({ reason: i18n.translate( 'xpack.apm.alertTypes.transactionDuration.reason', { defaultMessage: `Latency is above {threshold} (current value is {measured}) for {serviceName}`, values: { - threshold: asDuration( - alert['kibana.observability.evaluation.threshold'] - ), - measured: asDuration( - alert['kibana.observability.evaluation.value'] - ), - serviceName: alert['service.name']!, + threshold: asDuration(fields[ALERT_EVALUATION_THRESHOLD]), + measured: asDuration(fields[ALERT_EVALUATION_VALUE]), + serviceName: String(fields[SERVICE_NAME][0]), }, } ), link: format({ - pathname: `/app/apm/services/${alert['service.name']!}`, + pathname: `/app/apm/services/${fields[SERVICE_NAME][0]!}`, query: { - transactionType: alert['transaction.type']!, - ...(alert['service.environment'] - ? { environment: alert['service.environment'] } + transactionType: fields[TRANSACTION_TYPE][0]!, + ...(fields[SERVICE_ENVIRONMENT]?.[0] + ? { environment: String(fields[SERVICE_ENVIRONMENT][0]) } : { environment: ENVIRONMENT_ALL.value }), }, }), @@ -131,7 +141,7 @@ export function registerApmAlerts(apmRuleRegistry: ApmRuleRegistry) { ), }); - apmRuleRegistry.registerType({ + observabilityRuleTypeRegistry.register({ id: AlertType.TransactionErrorRate, description: i18n.translate( 'xpack.apm.alertTypes.transactionErrorRate.description', @@ -140,30 +150,24 @@ export function registerApmAlerts(apmRuleRegistry: ApmRuleRegistry) { 'Alert when the rate of transaction errors in a service exceeds a defined threshold.', } ), - format: ({ alert, formatters: { asPercent } }) => ({ + format: ({ fields, formatters: { asPercent } }) => ({ reason: i18n.translate( 'xpack.apm.alertTypes.transactionErrorRate.reason', { defaultMessage: `Transaction error rate is greater than {threshold} (current value is {measured}) for {serviceName}`, values: { - threshold: asPercent( - alert['kibana.observability.evaluation.threshold'], - 100 - ), - measured: asPercent( - alert['kibana.observability.evaluation.value'], - 100 - ), - serviceName: alert['service.name']!, + threshold: asPercent(fields[ALERT_EVALUATION_THRESHOLD], 100), + measured: asPercent(fields[ALERT_EVALUATION_VALUE], 100), + serviceName: String(fields[SERVICE_NAME][0]), }, } ), link: format({ - pathname: `/app/apm/services/${alert['service.name']!}`, + pathname: `/app/apm/services/${String(fields[SERVICE_NAME][0]!)}`, query: { - transactionType: alert['transaction.type']!, - ...(alert['service.environment'] - ? { environment: alert['service.environment'] } + transactionType: String(fields[TRANSACTION_TYPE][0]!), + ...(fields[SERVICE_ENVIRONMENT]?.[0] + ? { environment: String(fields[SERVICE_ENVIRONMENT][0]) } : { environment: ENVIRONMENT_ALL.value }), }, }), @@ -193,7 +197,7 @@ export function registerApmAlerts(apmRuleRegistry: ApmRuleRegistry) { ), }); - apmRuleRegistry.registerType({ + observabilityRuleTypeRegistry.register({ id: AlertType.TransactionDurationAnomaly, description: i18n.translate( 'xpack.apm.alertTypes.transactionDurationAnomaly.description', @@ -201,24 +205,24 @@ export function registerApmAlerts(apmRuleRegistry: ApmRuleRegistry) { defaultMessage: 'Alert when the latency of a service is abnormal.', } ), - format: ({ alert }) => ({ + format: ({ fields }) => ({ reason: i18n.translate( 'xpack.apm.alertTypes.transactionDurationAnomaly.reason', { defaultMessage: `{severityLevel} anomaly detected for {serviceName} (score was {measured})`, values: { - serviceName: alert['service.name'], - severityLevel: alert['kibana.rac.alert.severity.level'], - measured: alert['kibana.observability.evaluation.value'], + serviceName: String(fields[SERVICE_NAME][0]), + severityLevel: String(fields[ALERT_SEVERITY_LEVEL]), + measured: Number(fields[ALERT_EVALUATION_VALUE]), }, } ), link: format({ - pathname: `/app/apm/services/${alert['service.name']!}`, + pathname: `/app/apm/services/${String(fields[SERVICE_NAME][0])}`, query: { - transactionType: alert['transaction.type']!, - ...(alert['service.environment'] - ? { environment: alert['service.environment'] } + transactionType: String(fields[TRANSACTION_TYPE][0]), + ...(fields[SERVICE_ENVIRONMENT]?.[0] + ? { environment: String(fields[SERVICE_ENVIRONMENT][0]) } : { environment: ENVIRONMENT_ALL.value }), }, }), diff --git a/x-pack/plugins/apm/public/components/app/ErrorGroupDetails/Distribution/index.tsx b/x-pack/plugins/apm/public/components/app/ErrorGroupDetails/Distribution/index.tsx index 19a567a3866bd..16ac1a35666d2 100644 --- a/x-pack/plugins/apm/public/components/app/ErrorGroupDetails/Distribution/index.tsx +++ b/x-pack/plugins/apm/public/components/app/ErrorGroupDetails/Distribution/index.tsx @@ -19,6 +19,7 @@ import { import { EuiTitle } from '@elastic/eui'; import d3 from 'd3'; import React from 'react'; +import { RULE_ID } from '@kbn/rule-data-utils/target/technical_field_names'; import { useApmServiceContext } from '../../../../context/apm_service/use_apm_service_context'; import { APIReturnType } from '../../../../services/rest/createCallApmApi'; import { asRelativeDateTimeRange } from '../../../../../common/utils/formatters'; @@ -115,7 +116,7 @@ export function ErrorDistribution({ distribution, title }: Props) { /> {getAlertAnnotations({ alerts: alerts?.filter( - (alert) => alert['rule.id'] === AlertType.ErrorCount + (alert) => alert[RULE_ID]?.[0] === AlertType.ErrorCount ), theme, })} diff --git a/x-pack/plugins/apm/public/components/app/Home/__snapshots__/Home.test.tsx.snap b/x-pack/plugins/apm/public/components/app/Home/__snapshots__/Home.test.tsx.snap index b1bcf561bed84..f13cce3fd9b40 100644 --- a/x-pack/plugins/apm/public/components/app/Home/__snapshots__/Home.test.tsx.snap +++ b/x-pack/plugins/apm/public/components/app/Home/__snapshots__/Home.test.tsx.snap @@ -4,10 +4,6 @@ exports[`Home component should render services 1`] = ` alert['kibana.rac.alert.id']! + const collapsedAlerts = uniqBy(alerts, (alert) => alert[ALERT_ID]![0]!).map( + (alert) => { + return parseTechnicalFields(alert); + } ); return ( {collapsedAlerts.map((alert) => { - const ruleType = apmRuleRegistry.getTypeByRuleId(alert['rule.id']!); + const formatter = observabilityRuleTypeRegistry.getFormatter( + alert[RULE_ID]! + ); const formatted = { link: undefined, - reason: alert['rule.name'], - ...(ruleType?.format?.({ - alert, + reason: alert[RULE_NAME], + ...(formatter?.({ + fields: alert, formatters: { asDuration, asPercent }, }) ?? {}), }; @@ -55,7 +65,7 @@ export function AlertDetails({ alerts }: AlertDetailProps) { : undefined; return ( - + {parsedLink ? ( @@ -79,7 +89,7 @@ export function AlertDetails({ alerts }: AlertDetailProps) { diff --git a/x-pack/plugins/apm/public/components/shared/AgentIcon/get_agent_icon.ts b/x-pack/plugins/apm/public/components/shared/AgentIcon/get_agent_icon.ts index f916292b7f080..d848c0d3ccd7a 100644 --- a/x-pack/plugins/apm/public/components/shared/AgentIcon/get_agent_icon.ts +++ b/x-pack/plugins/apm/public/components/shared/AgentIcon/get_agent_icon.ts @@ -14,6 +14,8 @@ import defaultIcon from '../span_icon/icons/default.svg'; import dotNetIcon from './icons/dot-net.svg'; import erlangIcon from './icons/erlang.svg'; import goIcon from './icons/go.svg'; +import iosIcon from './icons/ios.svg'; +import darkIosIcon from './icons/ios_dark.svg'; import javaIcon from './icons/java.svg'; import nodeJsIcon from './icons/nodejs.svg'; import ocamlIcon from './icons/ocaml.svg'; @@ -31,6 +33,7 @@ const agentIcons: { [key: string]: string } = { dotnet: dotNetIcon, erlang: erlangIcon, go: goIcon, + ios: iosIcon, java: javaIcon, nodejs: nodeJsIcon, ocaml: ocamlIcon, @@ -44,6 +47,7 @@ const agentIcons: { [key: string]: string } = { const darkAgentIcons: { [key: string]: string } = { ...agentIcons, + ios: darkIosIcon, php: darkPhpIcon, rum: darkRumJsIcon, rust: darkRustIcon, diff --git a/x-pack/plugins/apm/public/components/shared/AgentIcon/icons/ios.svg b/x-pack/plugins/apm/public/components/shared/AgentIcon/icons/ios.svg new file mode 100644 index 0000000000000..d5b999a7ff042 --- /dev/null +++ b/x-pack/plugins/apm/public/components/shared/AgentIcon/icons/ios.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/x-pack/plugins/apm/public/components/shared/AgentIcon/icons/ios_dark.svg b/x-pack/plugins/apm/public/components/shared/AgentIcon/icons/ios_dark.svg new file mode 100644 index 0000000000000..16ea340ba150f --- /dev/null +++ b/x-pack/plugins/apm/public/components/shared/AgentIcon/icons/ios_dark.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/x-pack/plugins/apm/public/components/shared/charts/helper/get_alert_annotations.tsx b/x-pack/plugins/apm/public/components/shared/charts/helper/get_alert_annotations.tsx index 2c086dbb17222..e906707730baa 100644 --- a/x-pack/plugins/apm/public/components/shared/charts/helper/get_alert_annotations.tsx +++ b/x-pack/plugins/apm/public/components/shared/charts/helper/get_alert_annotations.tsx @@ -9,6 +9,13 @@ import { ValuesType } from 'utility-types'; import { RectAnnotation } from '@elastic/charts'; import { EuiTheme } from 'src/plugins/kibana_react/common'; import { rgba } from 'polished'; +import { + ALERT_DURATION, + RULE_ID, + ALERT_START, + ALERT_UUID, +} from '@kbn/rule-data-utils/target/technical_field_names'; +import { parseTechnicalFields } from '../../../../../../rule_registry/common'; import { APIReturnType } from '../../../../services/rest/createCallApmApi'; type Alert = ValuesType< @@ -30,10 +37,11 @@ export function getAlertAnnotations({ theme: EuiTheme; }) { return alerts?.flatMap((alert) => { - const uuid = alert['kibana.rac.alert.uuid']!; - const start = new Date(alert['kibana.rac.alert.start']!).getTime(); - const end = start + alert['kibana.rac.alert.duration.us']! / 1000; - const color = getAlertColor({ ruleId: alert['rule.id']!, theme }); + const parsed = parseTechnicalFields(alert); + const uuid = parsed[ALERT_UUID]!; + const start = new Date(parsed[ALERT_START]!).getTime(); + const end = start + parsed[ALERT_DURATION]! / 1000; + const color = getAlertColor({ ruleId: parsed[RULE_ID]!, theme }); return [ - alert['rule.id'] === AlertType.TransactionDuration || - alert['rule.id'] === AlertType.TransactionDurationAnomaly + alert[RULE_ID]?.[0] === AlertType.TransactionDuration || + alert[RULE_ID]?.[0] === AlertType.TransactionDurationAnomaly )} /> diff --git a/x-pack/plugins/apm/public/components/shared/charts/transaction_error_rate_chart/index.tsx b/x-pack/plugins/apm/public/components/shared/charts/transaction_error_rate_chart/index.tsx index 9aefa55aaaa36..7eceaf5ca8e5d 100644 --- a/x-pack/plugins/apm/public/components/shared/charts/transaction_error_rate_chart/index.tsx +++ b/x-pack/plugins/apm/public/components/shared/charts/transaction_error_rate_chart/index.tsx @@ -9,6 +9,7 @@ import { EuiPanel, EuiTitle } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import React from 'react'; import { useParams } from 'react-router-dom'; +import { RULE_ID } from '../../../../../../rule_registry/common/technical_rule_data_field_names'; import { AlertType } from '../../../../../common/alert_types'; import { APIReturnType } from '../../../../services/rest/createCallApmApi'; import { asPercent } from '../../../../../common/utils/formatters'; @@ -152,7 +153,7 @@ export function TransactionErrorRateChart({ yDomain={{ min: 0, max: 1 }} customTheme={comparisonChartThem} alerts={alerts.filter( - (alert) => alert['rule.id'] === AlertType.TransactionErrorRate + (alert) => alert[RULE_ID]?.[0] === AlertType.TransactionErrorRate )} /> diff --git a/x-pack/plugins/apm/public/components/shared/time_comparison/get_time_range_comparison.ts b/x-pack/plugins/apm/public/components/shared/time_comparison/get_time_range_comparison.ts index 025e8c2a9935d..67c27308e6658 100644 --- a/x-pack/plugins/apm/public/components/shared/time_comparison/get_time_range_comparison.ts +++ b/x-pack/plugins/apm/public/components/shared/time_comparison/get_time_range_comparison.ts @@ -19,9 +19,9 @@ export function getComparisonChartTheme(theme: EuiTheme) { return { areaSeriesStyle: { area: { - fill: theme.eui.euiColorLightestShade, + fill: theme.eui.euiColorLightShade, visible: true, - opacity: 1, + opacity: 0.5, }, line: { stroke: theme.eui.euiColorMediumShade, diff --git a/x-pack/plugins/apm/public/context/apm_plugin/apm_plugin_context.tsx b/x-pack/plugins/apm/public/context/apm_plugin/apm_plugin_context.tsx index 175471e7ae817..ec42a11783273 100644 --- a/x-pack/plugins/apm/public/context/apm_plugin/apm_plugin_context.tsx +++ b/x-pack/plugins/apm/public/context/apm_plugin/apm_plugin_context.tsx @@ -7,8 +7,9 @@ import { AppMountParameters, CoreStart } from 'kibana/public'; import { createContext } from 'react'; +import type { ObservabilityRuleTypeRegistry } from '../../../../observability/public'; import { ConfigSchema } from '../..'; -import { ApmPluginSetupDeps, ApmRuleRegistry } from '../../plugin'; +import { ApmPluginSetupDeps } from '../../plugin'; import { MapsStartApi } from '../../../../maps/public'; export interface ApmPluginContextValue { @@ -16,7 +17,7 @@ export interface ApmPluginContextValue { config: ConfigSchema; core: CoreStart; plugins: ApmPluginSetupDeps & { maps?: MapsStartApi }; - apmRuleRegistry: ApmRuleRegistry; + observabilityRuleTypeRegistry: ObservabilityRuleTypeRegistry; } export const ApmPluginContext = createContext({} as ApmPluginContextValue); diff --git a/x-pack/plugins/apm/public/context/apm_plugin/mock_apm_plugin_context.tsx b/x-pack/plugins/apm/public/context/apm_plugin/mock_apm_plugin_context.tsx index 07da5ea7f6c1f..a16f81826636b 100644 --- a/x-pack/plugins/apm/public/context/apm_plugin/mock_apm_plugin_context.tsx +++ b/x-pack/plugins/apm/public/context/apm_plugin/mock_apm_plugin_context.tsx @@ -7,12 +7,12 @@ import React, { ReactNode } from 'react'; import { Observable, of } from 'rxjs'; +import { createObservabilityRuleTypeRegistryMock } from '../../../../observability/public'; import { ApmPluginContext, ApmPluginContextValue } from './apm_plugin_context'; import { ConfigSchema } from '../..'; import { UI_SETTINGS } from '../../../../../../src/plugins/data/common'; import { createCallApmApi } from '../../services/rest/createCallApmApi'; import { MlUrlGenerator } from '../../../../ml/public'; -import { ApmRuleRegistry } from '../../plugin'; const uiSettings: Record = { [UI_SETTINGS.TIMEPICKER_QUICK_RANGES]: [ @@ -77,11 +77,6 @@ const mockCore = { }, }; -const mockApmRuleRegistry = ({ - getTypeByRuleId: () => undefined, - registerType: () => undefined, -} as unknown) as ApmRuleRegistry; - const mockConfig: ConfigSchema = { serviceMapEnabled: true, ui: { @@ -116,7 +111,7 @@ export const mockApmPluginContextValue = { config: mockConfig, core: mockCore, plugins: mockPlugin, - apmRuleRegistry: mockApmRuleRegistry, + observabilityRuleTypeRegistry: createObservabilityRuleTypeRegistryMock(), }; export function MockApmPluginContextWrapper({ diff --git a/x-pack/plugins/apm/public/plugin.ts b/x-pack/plugins/apm/public/plugin.ts index f7bbe647d8e37..b493363d98f7a 100644 --- a/x-pack/plugins/apm/public/plugin.ts +++ b/x-pack/plugins/apm/public/plugin.ts @@ -34,19 +34,15 @@ import type { HasDataParams, ObservabilityPublicSetup, } from '../../observability/public'; -import { FormatterRuleRegistry } from '../../observability/public'; import type { TriggersAndActionsUIPublicPluginSetup, TriggersAndActionsUIPublicPluginStart, } from '../../triggers_actions_ui/public'; -import { apmRuleRegistrySettings } from '../common/rules/apm_rule_registry_settings'; -import type { APMRuleFieldMap } from '../common/rules/apm_rule_field_map'; import { registerApmAlerts } from './components/alerting/register_apm_alerts'; import { featureCatalogueEntry } from './featureCatalogueEntry'; import { toggleAppLinkInNav } from './toggleAppLinkInNav'; export type ApmPluginSetup = ReturnType; -export type ApmRuleRegistry = ApmPluginSetup['ruleRegistry']; export type ApmPluginStart = void; @@ -87,11 +83,6 @@ export class ApmPlugin implements Plugin { pluginSetupDeps.home.featureCatalogue.register(featureCatalogueEntry); } - const apmRuleRegistry = plugins.observability.ruleRegistry.create({ - ...apmRuleRegistrySettings, - fieldMap: {} as APMRuleFieldMap, - ctor: FormatterRuleRegistry, - }); const getApmDataHelper = async () => { const { fetchObservabilityOverviewPageData, @@ -127,6 +118,8 @@ export class ApmPlugin implements Plugin { return { fetchUxOverviewDate, hasRumData }; }; + const { observabilityRuleTypeRegistry } = plugins.observability; + plugins.observability.dashboard.register({ appName: 'ux', hasData: async (params?: HasDataParams) => { @@ -187,12 +180,12 @@ export class ApmPlugin implements Plugin { appMountParameters, config, pluginsStart: pluginsStart as ApmPluginStartDeps, - apmRuleRegistry, + observabilityRuleTypeRegistry, }); }, }); - registerApmAlerts(apmRuleRegistry); + registerApmAlerts(observabilityRuleTypeRegistry); core.application.register({ id: 'ux', @@ -231,14 +224,12 @@ export class ApmPlugin implements Plugin { appMountParameters, config, corePlugins: corePlugins as ApmPluginStartDeps, - apmRuleRegistry, + observabilityRuleTypeRegistry, }); }, }); - return { - ruleRegistry: apmRuleRegistry, - }; + return {}; } public start(core: CoreStart, plugins: ApmPluginStartDeps) { toggleAppLinkInNav(core, this.initializerContext.config.get()); diff --git a/x-pack/plugins/apm/server/lib/alerts/register_apm_alerts.ts b/x-pack/plugins/apm/server/lib/alerts/register_apm_alerts.ts index 9a362efa90ac0..022fad6fa7840 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_apm_alerts.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_apm_alerts.ts @@ -7,17 +7,19 @@ import { Observable } from 'rxjs'; import { Logger } from 'kibana/server'; +import { PluginSetupContract as AlertingPluginSetupContract } from '../../../../alerting/server'; +import { RuleDataClient } from '../../../../rule_registry/server'; import { registerTransactionDurationAlertType } from './register_transaction_duration_alert_type'; import { registerTransactionDurationAnomalyAlertType } from './register_transaction_duration_anomaly_alert_type'; import { registerErrorCountAlertType } from './register_error_count_alert_type'; import { APMConfig } from '../..'; import { MlPluginSetup } from '../../../../ml/server'; import { registerTransactionErrorRateAlertType } from './register_transaction_error_rate_alert_type'; -import { APMRuleRegistry } from '../../plugin'; export interface RegisterRuleDependencies { - registry: APMRuleRegistry; + ruleDataClient: RuleDataClient; ml?: MlPluginSetup; + alerting: AlertingPluginSetupContract; config$: Observable; logger: Logger; } diff --git a/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts index 15ec5d0ef0bd0..885b22ae343d8 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_error_count_alert_type.ts @@ -7,6 +7,11 @@ import { schema } from '@kbn/config-schema'; import { take } from 'rxjs/operators'; +import { + ALERT_EVALUATION_THRESHOLD, + ALERT_EVALUATION_VALUE, +} from '@kbn/rule-data-utils/target/technical_field_names'; +import { createLifecycleRuleTypeFactory } from '../../../../rule_registry/server'; import { ENVIRONMENT_NOT_DEFINED } from '../../../common/environment_filter_values'; import { asMutableArray } from '../../../common/utils/as_mutable_array'; import { AlertType, ALERT_TYPES_CONFIG } from '../../../common/alert_types'; @@ -21,7 +26,6 @@ import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; import { apmActionVariables } from './action_variables'; import { alertingEsClient } from './alerting_es_client'; import { RegisterRuleDependencies } from './register_apm_alerts'; -import { createAPMLifecycleRuleType } from './create_apm_lifecycle_rule_type'; const paramsSchema = schema.object({ windowSize: schema.number(), @@ -34,11 +38,18 @@ const paramsSchema = schema.object({ const alertTypeConfig = ALERT_TYPES_CONFIG[AlertType.ErrorCount]; export function registerErrorCountAlertType({ - registry, + alerting, + logger, + ruleDataClient, config$, }: RegisterRuleDependencies) { - registry.registerType( - createAPMLifecycleRuleType({ + const createLifecycleRuleType = createLifecycleRuleTypeFactory({ + ruleDataClient, + logger, + }); + + alerting.registerType( + createLifecycleRuleType({ id: AlertType.ErrorCount, name: alertTypeConfig.name, actionGroups: alertTypeConfig.actionGroups, @@ -146,9 +157,8 @@ export function registerErrorCountAlertType({ ? { [SERVICE_ENVIRONMENT]: environment } : {}), [PROCESSOR_EVENT]: ProcessorEvent.error, - 'kibana.observability.evaluation.value': errorCount, - 'kibana.observability.evaluation.threshold': - alertParams.threshold, + [ALERT_EVALUATION_VALUE]: errorCount, + [ALERT_EVALUATION_THRESHOLD]: alertParams.threshold, }, }) .scheduleActions(alertTypeConfig.defaultActionGroupId, { diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts index 4918a6cc892b7..f77cc3ee930b1 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_alert_type.ts @@ -8,6 +8,11 @@ import { schema } from '@kbn/config-schema'; import { take } from 'rxjs/operators'; import { QueryContainer } from '@elastic/elasticsearch/api/types'; +import { + ALERT_EVALUATION_THRESHOLD, + ALERT_EVALUATION_VALUE, +} from '@kbn/rule-data-utils/target/technical_field_names'; +import { createLifecycleRuleTypeFactory } from '../../../../rule_registry/server'; import { parseEnvironmentUrlParam } from '../../../common/environment_filter_values'; import { AlertType, ALERT_TYPES_CONFIG } from '../../../common/alert_types'; import { @@ -24,7 +29,6 @@ import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; import { apmActionVariables } from './action_variables'; import { alertingEsClient } from './alerting_es_client'; import { RegisterRuleDependencies } from './register_apm_alerts'; -import { createAPMLifecycleRuleType } from './create_apm_lifecycle_rule_type'; const paramsSchema = schema.object({ serviceName: schema.string(), @@ -43,130 +47,142 @@ const paramsSchema = schema.object({ const alertTypeConfig = ALERT_TYPES_CONFIG[AlertType.TransactionDuration]; export function registerTransactionDurationAlertType({ - registry, + alerting, + ruleDataClient, config$, + logger, }: RegisterRuleDependencies) { - registry.registerType( - createAPMLifecycleRuleType({ - id: AlertType.TransactionDuration, - name: alertTypeConfig.name, - actionGroups: alertTypeConfig.actionGroups, - defaultActionGroupId: alertTypeConfig.defaultActionGroupId, - validate: { - params: paramsSchema, - }, - actionVariables: { - context: [ - apmActionVariables.serviceName, - apmActionVariables.transactionType, - apmActionVariables.environment, - apmActionVariables.threshold, - apmActionVariables.triggerValue, - apmActionVariables.interval, - ], - }, - producer: 'apm', - minimumLicenseRequired: 'basic', - executor: async ({ services, params }) => { - const config = await config$.pipe(take(1)).toPromise(); - const alertParams = params; - const indices = await getApmIndices({ - config, - savedObjectsClient: services.savedObjectsClient, - }); - - const searchParams = { - index: indices['apm_oss.transactionIndices'], - body: { - size: 0, - query: { - bool: { - filter: [ - { - range: { - '@timestamp': { - gte: `now-${alertParams.windowSize}${alertParams.windowUnit}`, - }, + const createLifecycleRuleType = createLifecycleRuleTypeFactory({ + ruleDataClient, + logger, + }); + + const type = createLifecycleRuleType({ + id: AlertType.TransactionDuration, + name: alertTypeConfig.name, + actionGroups: alertTypeConfig.actionGroups, + defaultActionGroupId: alertTypeConfig.defaultActionGroupId, + validate: { + params: paramsSchema, + }, + actionVariables: { + context: [ + apmActionVariables.serviceName, + apmActionVariables.transactionType, + apmActionVariables.environment, + apmActionVariables.threshold, + apmActionVariables.triggerValue, + apmActionVariables.interval, + ], + }, + producer: 'apm', + minimumLicenseRequired: 'basic', + executor: async ({ services, params }) => { + const config = await config$.pipe(take(1)).toPromise(); + const alertParams = params; + const indices = await getApmIndices({ + config, + savedObjectsClient: services.savedObjectsClient, + }); + + const searchParams = { + index: indices['apm_oss.transactionIndices'], + body: { + size: 0, + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: `now-${alertParams.windowSize}${alertParams.windowUnit}`, }, }, - { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } }, - { term: { [SERVICE_NAME]: alertParams.serviceName } }, - { term: { [TRANSACTION_TYPE]: alertParams.transactionType } }, - ...environmentQuery(alertParams.environment), - ] as QueryContainer[], - }, + }, + { + term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction }, + }, + { term: { [SERVICE_NAME]: alertParams.serviceName } }, + { + term: { + [TRANSACTION_TYPE]: alertParams.transactionType, + }, + }, + ...environmentQuery(alertParams.environment), + ] as QueryContainer[], }, - aggs: { - latency: - alertParams.aggregationType === 'avg' - ? { avg: { field: TRANSACTION_DURATION } } - : { - percentiles: { - field: TRANSACTION_DURATION, - percents: [ - alertParams.aggregationType === '95th' ? 95 : 99, - ], - }, + }, + aggs: { + latency: + alertParams.aggregationType === 'avg' + ? { avg: { field: TRANSACTION_DURATION } } + : { + percentiles: { + field: TRANSACTION_DURATION, + percents: [ + alertParams.aggregationType === '95th' ? 95 : 99, + ], }, - }, + }, }, - }; - - const response = await alertingEsClient({ - scopedClusterClient: services.scopedClusterClient, - params: searchParams, - }); - - if (!response.aggregations) { - return {}; - } - - const { latency } = response.aggregations; - - const transactionDuration = - 'values' in latency - ? Object.values(latency.values)[0] - : latency?.value; - - const threshold = alertParams.threshold * 1000; - - if (transactionDuration && transactionDuration > threshold) { - const durationFormatter = getDurationFormatter(transactionDuration); - const transactionDurationFormatted = durationFormatter( - transactionDuration - ).formatted; - - const environmentParsed = parseEnvironmentUrlParam( - alertParams.environment - ); - - services - .alertWithLifecycle({ - id: `${AlertType.TransactionDuration}_${environmentParsed.text}`, - fields: { - [SERVICE_NAME]: alertParams.serviceName, - ...(environmentParsed.esFieldValue - ? { [SERVICE_ENVIRONMENT]: environmentParsed.esFieldValue } - : {}), - [TRANSACTION_TYPE]: alertParams.transactionType, - [PROCESSOR_EVENT]: ProcessorEvent.transaction, - 'kibana.observability.evaluation.value': transactionDuration, - 'kibana.observability.evaluation.threshold': - alertParams.threshold * 1000, - }, - }) - .scheduleActions(alertTypeConfig.defaultActionGroupId, { - transactionType: alertParams.transactionType, - serviceName: alertParams.serviceName, - environment: environmentParsed.text, - threshold, - triggerValue: transactionDurationFormatted, - interval: `${alertParams.windowSize}${alertParams.windowUnit}`, - }); - } + }, + }; + const response = await alertingEsClient({ + scopedClusterClient: services.scopedClusterClient, + params: searchParams, + }); + + if (!response.aggregations) { return {}; - }, - }) - ); + } + + const { latency } = response.aggregations; + + const transactionDuration = + 'values' in latency ? Object.values(latency.values)[0] : latency?.value; + + const threshold = alertParams.threshold * 1000; + + if (transactionDuration && transactionDuration > threshold) { + const durationFormatter = getDurationFormatter(transactionDuration); + const transactionDurationFormatted = durationFormatter( + transactionDuration + ).formatted; + + const environmentParsed = parseEnvironmentUrlParam( + alertParams.environment + ); + + services + .alertWithLifecycle({ + id: `${AlertType.TransactionDuration}_${environmentParsed.text}`, + fields: { + [SERVICE_NAME]: alertParams.serviceName, + ...(environmentParsed.esFieldValue + ? { + [SERVICE_ENVIRONMENT]: environmentParsed.esFieldValue, + } + : {}), + [TRANSACTION_TYPE]: alertParams.transactionType, + [PROCESSOR_EVENT]: ProcessorEvent.transaction, + [ALERT_EVALUATION_VALUE]: transactionDuration, + [ALERT_EVALUATION_THRESHOLD]: alertParams.threshold * 1000, + }, + }) + .scheduleActions(alertTypeConfig.defaultActionGroupId, { + transactionType: alertParams.transactionType, + serviceName: alertParams.serviceName, + environment: environmentParsed.text, + threshold, + triggerValue: transactionDurationFormatted, + interval: `${alertParams.windowSize}${alertParams.windowUnit}`, + }); + } + + return {}; + }, + }); + + alerting.registerType(type); } diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts index 67ff7cdb8e4e0..399fb9a216ef5 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts @@ -9,6 +9,13 @@ import { schema } from '@kbn/config-schema'; import { compact } from 'lodash'; import { ESSearchResponse } from 'typings/elasticsearch'; import { QueryContainer } from '@elastic/elasticsearch/api/types'; +import { + ALERT_EVALUATION_THRESHOLD, + ALERT_EVALUATION_VALUE, + ALERT_SEVERITY_LEVEL, + ALERT_SEVERITY_VALUE, +} from '@kbn/rule-data-utils/target/technical_field_names'; +import { createLifecycleRuleTypeFactory } from '../../../../rule_registry/server'; import { ProcessorEvent } from '../../../common/processor_event'; import { getSeverity } from '../../../common/anomaly_detection'; import { @@ -29,7 +36,6 @@ import { getMLJobs } from '../service_map/get_service_anomalies'; import { apmActionVariables } from './action_variables'; import { RegisterRuleDependencies } from './register_apm_alerts'; import { parseEnvironmentUrlParam } from '../../../common/environment_filter_values'; -import { createAPMLifecycleRuleType } from './create_apm_lifecycle_rule_type'; const paramsSchema = schema.object({ serviceName: schema.maybe(schema.string()), @@ -49,11 +55,18 @@ const alertTypeConfig = ALERT_TYPES_CONFIG[AlertType.TransactionDurationAnomaly]; export function registerTransactionDurationAnomalyAlertType({ - registry, + logger, + ruleDataClient, + alerting, ml, }: RegisterRuleDependencies) { - registry.registerType( - createAPMLifecycleRuleType({ + const createLifecycleRuleType = createLifecycleRuleTypeFactory({ + logger, + ruleDataClient, + }); + + alerting.registerType( + createLifecycleRuleType({ id: AlertType.TransactionDurationAnomaly, name: alertTypeConfig.name, actionGroups: alertTypeConfig.actionGroups, @@ -190,7 +203,7 @@ export function registerTransactionDurationAnomalyAlertType({ const job = mlJobs.find((j) => j.job_id === latest.job_id); if (!job) { - services.logger.warn( + logger.warn( `Could not find matching job for job id ${latest.job_id}` ); return undefined; @@ -231,10 +244,10 @@ export function registerTransactionDurationAnomalyAlertType({ : {}), [TRANSACTION_TYPE]: transactionType, [PROCESSOR_EVENT]: ProcessorEvent.transaction, - 'kibana.rac.alert.severity.level': severityLevel, - 'kibana.rac.alert.severity.value': score, - 'kibana.observability.evaluation.value': score, - 'kibana.observability.evaluation.threshold': threshold, + [ALERT_SEVERITY_LEVEL]: severityLevel, + [ALERT_SEVERITY_VALUE]: score, + [ALERT_EVALUATION_VALUE]: score, + [ALERT_EVALUATION_THRESHOLD]: threshold, }, }) .scheduleActions(alertTypeConfig.defaultActionGroupId, { diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts index bead17e308f06..4d6a0685fd379 100644 --- a/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts +++ b/x-pack/plugins/apm/server/lib/alerts/register_transaction_error_rate_alert_type.ts @@ -7,6 +7,11 @@ import { schema } from '@kbn/config-schema'; import { take } from 'rxjs/operators'; +import { + ALERT_EVALUATION_THRESHOLD, + ALERT_EVALUATION_VALUE, +} from '@kbn/rule-data-utils/target/technical_field_names'; +import { createLifecycleRuleTypeFactory } from '../../../../rule_registry/server'; import { AlertType, ALERT_TYPES_CONFIG } from '../../../common/alert_types'; import { EVENT_OUTCOME, @@ -22,7 +27,6 @@ import { environmentQuery } from '../../../server/utils/queries'; import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; import { apmActionVariables } from './action_variables'; import { alertingEsClient } from './alerting_es_client'; -import { createAPMLifecycleRuleType } from './create_apm_lifecycle_rule_type'; import { RegisterRuleDependencies } from './register_apm_alerts'; const paramsSchema = schema.object({ @@ -37,11 +41,18 @@ const paramsSchema = schema.object({ const alertTypeConfig = ALERT_TYPES_CONFIG[AlertType.TransactionErrorRate]; export function registerTransactionErrorRateAlertType({ - registry, + alerting, + ruleDataClient, + logger, config$, }: RegisterRuleDependencies) { - registry.registerType( - createAPMLifecycleRuleType({ + const createLifecycleRuleType = createLifecycleRuleTypeFactory({ + ruleDataClient, + logger, + }); + + alerting.registerType( + createLifecycleRuleType({ id: AlertType.TransactionErrorRate, name: alertTypeConfig.name, actionGroups: alertTypeConfig.actionGroups, @@ -183,9 +194,8 @@ export function registerTransactionErrorRateAlertType({ ...(environment ? { [SERVICE_ENVIRONMENT]: environment } : {}), [TRANSACTION_TYPE]: transactionType, [PROCESSOR_EVENT]: ProcessorEvent.transaction, - 'kibana.observability.evaluation.value': errorRate, - 'kibana.observability.evaluation.threshold': - alertParams.threshold, + [ALERT_EVALUATION_VALUE]: errorRate, + [ALERT_EVALUATION_THRESHOLD]: alertParams.threshold, }, }) .scheduleActions(alertTypeConfig.defaultActionGroupId, { diff --git a/x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts b/x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts index 37b3e282d0a59..ce1466bff01a9 100644 --- a/x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts +++ b/x-pack/plugins/apm/server/lib/alerts/test_utils/index.ts @@ -8,8 +8,9 @@ import { Logger } from 'kibana/server'; import { of } from 'rxjs'; import { elasticsearchServiceMock } from 'src/core/server/mocks'; +import type { RuleDataClient } from '../../../../../rule_registry/server'; +import { PluginSetupContract as AlertingPluginSetupContract } from '../../../../../alerting/server'; import { APMConfig } from '../../..'; -import { APMRuleRegistry } from '../../../plugin'; export const createRuleTypeMocks = () => { let alertExecutor: (...args: any[]) => Promise; @@ -27,19 +28,16 @@ export const createRuleTypeMocks = () => { error: jest.fn(), } as unknown) as Logger; - const registry = { + const alerting = { registerType: ({ executor }) => { alertExecutor = executor; }, - } as APMRuleRegistry; + } as AlertingPluginSetupContract; const scheduleActions = jest.fn(); const services = { scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), - scopedRuleRegistryClient: { - bulkIndex: jest.fn(), - }, alertInstanceFactory: jest.fn(() => ({ scheduleActions })), alertWithLifecycle: jest.fn(), logger: loggerMock, @@ -47,9 +45,21 @@ export const createRuleTypeMocks = () => { return { dependencies: { - registry, + alerting, config$: mockedConfig$, logger: loggerMock, + ruleDataClient: ({ + getReader: () => { + return { + search: jest.fn(), + }; + }, + getWriter: () => { + return { + bulk: jest.fn(), + }; + }, + } as unknown) as RuleDataClient, }, services, scheduleActions, diff --git a/x-pack/plugins/apm/server/lib/services/get_service_alerts.ts b/x-pack/plugins/apm/server/lib/services/get_service_alerts.ts index 6356731cc48d1..f58452ce4d916 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_alerts.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_alerts.ts @@ -5,33 +5,30 @@ * 2.0. */ +import { ALERT_UUID } from '@kbn/rule-data-utils/target/technical_field_names'; +import { RuleDataClient } from '../../../../rule_registry/server'; import { SERVICE_NAME, TRANSACTION_TYPE, } from '../../../common/elasticsearch_fieldnames'; -import type { PromiseReturnType } from '../../../../observability/typings/common'; -import type { APMRuleRegistry } from '../../plugin'; import { environmentQuery, rangeQuery } from '../../utils/queries'; export async function getServiceAlerts({ - apmRuleRegistryClient, + ruleDataClient, start, end, serviceName, environment, transactionType, }: { - apmRuleRegistryClient: Exclude< - PromiseReturnType, - undefined - >; + ruleDataClient: RuleDataClient; start: number; end: number; serviceName: string; environment?: string; transactionType: string; }) { - const response = await apmRuleRegistryClient.search({ + const response = await ruleDataClient.getReader().search({ body: { query: { bool: { @@ -68,13 +65,14 @@ export async function getServiceAlerts({ size: 100, fields: ['*'], collapse: { - field: 'kibana.rac.alert.uuid', + field: ALERT_UUID, }, sort: { '@timestamp': 'desc', }, }, + allow_no_indices: true, }); - return response.events; + return response.hits.hits.map((hit) => hit.fields); } diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index e12d089855834..44334889128c4 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -16,7 +16,10 @@ import { Plugin, PluginInitializerContext, } from 'src/core/server'; -import { mapValues } from 'lodash'; +import { mapValues, once } from 'lodash'; +import { TECHNICAL_COMPONENT_TEMPLATE_NAME } from '../../rule_registry/common/assets'; +import { mappingFromFieldMap } from '../../rule_registry/common/mapping_from_field_map'; +import { RuleDataClient } from '../../rule_registry/server'; import { APMConfig, APMXPackConfig } from '.'; import { mergeConfigs } from './index'; import { UI_SETTINGS } from '../../../../src/plugins/data/common'; @@ -42,10 +45,12 @@ import { } from './types'; import { registerRoutes } from './routes/register_routes'; import { getGlobalApmServerRouteRepository } from './routes/get_global_apm_server_route_repository'; -import { apmRuleRegistrySettings } from '../common/rules/apm_rule_registry_settings'; -import { apmRuleFieldMap } from '../common/rules/apm_rule_field_map'; - -export type APMRuleRegistry = ReturnType['ruleRegistry']; +import { + PROCESSOR_EVENT, + SERVICE_ENVIRONMENT, + SERVICE_NAME, + TRANSACTION_TYPE, +} from '../common/elasticsearch_fieldnames'; export class APMPlugin implements @@ -124,20 +129,81 @@ export class APMPlugin registerFeaturesUsage({ licensingPlugin: plugins.licensing }); - const apmRuleRegistry = plugins.observability.ruleRegistry.create({ - ...apmRuleRegistrySettings, - fieldMap: apmRuleFieldMap, + const getCoreStart = () => + core.getStartServices().then(([coreStart]) => coreStart); + + const ready = once(async () => { + const componentTemplateName = plugins.ruleRegistry.getFullAssetName( + 'apm-mappings' + ); + + if (!plugins.ruleRegistry.isWriteEnabled()) { + return; + } + + await plugins.ruleRegistry.createOrUpdateComponentTemplate({ + name: componentTemplateName, + body: { + template: { + settings: { + number_of_shards: 1, + }, + mappings: mappingFromFieldMap({ + [SERVICE_NAME]: { + type: 'keyword', + }, + [SERVICE_ENVIRONMENT]: { + type: 'keyword', + }, + [TRANSACTION_TYPE]: { + type: 'keyword', + }, + [PROCESSOR_EVENT]: { + type: 'keyword', + }, + }), + }, + }, + }); + + await plugins.ruleRegistry.createOrUpdateIndexTemplate({ + name: plugins.ruleRegistry.getFullAssetName('apm-index-template'), + body: { + index_patterns: [ + plugins.ruleRegistry.getFullAssetName('observability-apm*'), + ], + composed_of: [ + plugins.ruleRegistry.getFullAssetName( + TECHNICAL_COMPONENT_TEMPLATE_NAME + ), + componentTemplateName, + ], + }, + }); + }); + + ready().catch((err) => { + this.logger!.error(err); + }); + + const ruleDataClient = new RuleDataClient({ + alias: plugins.ruleRegistry.getFullAssetName('observability-apm'), + getClusterClient: async () => { + const coreStart = await getCoreStart(); + return coreStart.elasticsearch.client.asInternalUser; + }, + ready, }); registerRoutes({ core: { setup: core, - start: () => core.getStartServices().then(([coreStart]) => coreStart), + start: getCoreStart, }, logger: this.logger, config: currentConfig, repository: getGlobalApmServerRouteRepository(), - apmRuleRegistry, + ruleDataClient, plugins: mapValues(plugins, (value, key) => { return { setup: value, @@ -157,12 +223,16 @@ export class APMPlugin savedObjectsClient: await getInternalSavedObjectsClient(core), config: await mergedConfig$.pipe(take(1)).toPromise(), }); - registerApmAlerts({ - registry: apmRuleRegistry, - ml: plugins.ml, - config$: mergedConfig$, - logger: this.logger!.get('rule'), - }); + + if (plugins.alerting) { + registerApmAlerts({ + ruleDataClient, + alerting: plugins.alerting, + ml: plugins.ml, + config$: mergedConfig$, + logger: this.logger!.get('rule'), + }); + } return { config$: mergedConfig$, @@ -193,7 +263,6 @@ export class APMPlugin }, }); }, - ruleRegistry: apmRuleRegistry, }; } diff --git a/x-pack/plugins/apm/server/routes/register_routes/index.ts b/x-pack/plugins/apm/server/routes/register_routes/index.ts index f792e078c528a..c9df12fd58208 100644 --- a/x-pack/plugins/apm/server/routes/register_routes/index.ts +++ b/x-pack/plugins/apm/server/routes/register_routes/index.ts @@ -39,14 +39,14 @@ export function registerRoutes({ plugins, logger, config, - apmRuleRegistry, + ruleDataClient, }: { core: APMRouteHandlerResources['core']; plugins: APMRouteHandlerResources['plugins']; logger: APMRouteHandlerResources['logger']; repository: ServerRouteRepository; config: APMRouteHandlerResources['config']; - apmRuleRegistry: APMRouteHandlerResources['apmRuleRegistry']; + ruleDataClient: APMRouteHandlerResources['ruleDataClient']; }) { const routes = repository.getRoutes(); @@ -99,7 +99,7 @@ export function registerRoutes({ }, validatedParams ), - apmRuleRegistry, + ruleDataClient, })) as any; if (Array.isArray(data)) { diff --git a/x-pack/plugins/apm/server/routes/services.ts b/x-pack/plugins/apm/server/routes/services.ts index 54e59f2be7ae3..4384d2be78ca0 100644 --- a/x-pack/plugins/apm/server/routes/services.ts +++ b/x-pack/plugins/apm/server/routes/services.ts @@ -737,30 +737,15 @@ const serviceAlertsRoute = createApmServerRoute({ options: { tags: ['access:apm'], }, - handler: async ({ context, params, apmRuleRegistry }) => { - const alertsClient = context.alerting.getAlertsClient(); - + handler: async ({ context, params, ruleDataClient }) => { const { query: { start, end, environment, transactionType }, path: { serviceName }, } = params; - const apmRuleRegistryClient = await apmRuleRegistry.createScopedRuleRegistryClient( - { - alertsClient, - context, - } - ); - - if (!apmRuleRegistryClient) { - throw Boom.failedDependency( - 'xpack.ruleRegistry.unsafe.write.enabled is set to false' - ); - } - return { alerts: await getServiceAlerts({ - apmRuleRegistryClient, + ruleDataClient, start, end, serviceName, diff --git a/x-pack/plugins/apm/server/routes/typings.ts b/x-pack/plugins/apm/server/routes/typings.ts index 602e1f3e0edb9..13bd631085aac 100644 --- a/x-pack/plugins/apm/server/routes/typings.ts +++ b/x-pack/plugins/apm/server/routes/typings.ts @@ -12,11 +12,11 @@ import { KibanaRequest, CoreStart, } from 'src/core/server'; +import { RuleDataClient } from '../../../rule_registry/server'; import { AlertingApiRequestHandlerContext } from '../../../alerting/server'; import { LicensingApiRequestHandlerContext } from '../../../licensing/server'; import { APMConfig } from '..'; import { APMPluginDependencies } from '../types'; -import { APMRuleRegistry } from '../plugin'; export interface ApmPluginRequestHandlerContext extends RequestHandlerContext { licensing: LicensingApiRequestHandlerContext; @@ -62,5 +62,5 @@ export interface APMRouteHandlerResources { start: () => Promise[key]['start']>; }; }; - apmRuleRegistry: APMRuleRegistry; + ruleDataClient: RuleDataClient; } diff --git a/x-pack/plugins/apm/server/types.ts b/x-pack/plugins/apm/server/types.ts index dbc220f9f6b15..a5ba4f39b32b3 100644 --- a/x-pack/plugins/apm/server/types.ts +++ b/x-pack/plugins/apm/server/types.ts @@ -7,6 +7,10 @@ import { ValuesType } from 'utility-types'; import { Observable } from 'rxjs'; import { CoreSetup, CoreStart, KibanaRequest } from 'kibana/server'; +import { + RuleRegistryPluginSetupContract, + RuleRegistryPluginStartContract, +} from '../../rule_registry/server'; import { PluginSetup as DataPluginSetup, PluginStart as DataPluginStart, @@ -115,6 +119,10 @@ interface DependencyMap { setup: DataPluginSetup; start: DataPluginStart; }; + ruleRegistry: { + setup: RuleRegistryPluginSetupContract; + start: RuleRegistryPluginStartContract; + }; } const requiredDependencies = [ @@ -126,6 +134,7 @@ const requiredDependencies = [ 'embeddable', 'infra', 'observability', + 'ruleRegistry', ] as const; const optionalDependencies = [ diff --git a/x-pack/plugins/cases/common/api/helpers.ts b/x-pack/plugins/cases/common/api/helpers.ts index 43e292b91db4b..7ac686ce5c8dd 100644 --- a/x-pack/plugins/cases/common/api/helpers.ts +++ b/x-pack/plugins/cases/common/api/helpers.ts @@ -14,6 +14,7 @@ import { SUB_CASES_URL, CASE_PUSH_URL, SUB_CASE_USER_ACTIONS_URL, + CASE_ALERTS_URL, } from '../constants'; export const getCaseDetailsUrl = (id: string): string => { @@ -47,3 +48,7 @@ export const getSubCaseUserActionUrl = (caseID: string, subCaseId: string): stri export const getCasePushUrl = (caseId: string, connectorId: string): string => { return CASE_PUSH_URL.replace('{case_id}', caseId).replace('{connector_id}', connectorId); }; + +export const getCasesFromAlertsUrl = (alertId: string): string => { + return CASE_ALERTS_URL.replace('{alert_id}', alertId); +}; diff --git a/x-pack/plugins/cross_cluster_replication/public/__jest__/client_integration/follower_indices_list.test.js b/x-pack/plugins/cross_cluster_replication/public/__jest__/client_integration/follower_indices_list.test.js index c2b414ddbd845..9c881702ba3c6 100644 --- a/x-pack/plugins/cross_cluster_replication/public/__jest__/client_integration/follower_indices_list.test.js +++ b/x-pack/plugins/cross_cluster_replication/public/__jest__/client_integration/follower_indices_list.test.js @@ -12,10 +12,11 @@ at createWorker (//node_modules/brace/index.js:17992:5) */ import { stubWebWorker } from '@kbn/test/jest'; // eslint-disable-line no-unused-vars +import { act } from 'react-dom/test-utils'; import { getFollowerIndexMock } from './fixtures/follower_index'; import './mocks'; -import { setupEnvironment, pageHelpers, nextTick, delay, getRandomString } from './helpers'; +import { setupEnvironment, pageHelpers, getRandomString } from './helpers'; const { setup } = pageHelpers.followerIndexList; @@ -53,9 +54,10 @@ describe('', () => { let component; beforeEach(async () => { - ({ exists, component } = setup()); + await act(async () => { + ({ exists, component } = setup()); + }); - await nextTick(); // We need to wait next tick for the mock server response to comes in component.update(); }); @@ -91,15 +93,15 @@ describe('', () => { beforeEach(async () => { httpRequestsMockHelpers.setLoadFollowerIndicesResponse({ indices: followerIndices }); - // Mount the component - ({ component, table, actions, form } = setup()); + await act(async () => { + ({ component, table, actions, form } = setup()); + }); - await nextTick(); // Make sure that the http request is fulfilled component.update(); }); - test('pagination works', () => { - actions.clickPaginationNextButton(); + test('pagination works', async () => { + await actions.clickPaginationNextButton(); const { tableCellsValues } = table.getMetaData('followerIndexListTable'); // Pagination defaults to 20 follower indices per page. We loaded 30 follower indices, @@ -134,21 +136,16 @@ describe('', () => { httpRequestsMockHelpers.setLoadFollowerIndicesResponse({ indices: followerIndices }); // Mount the component - ({ find, exists, component, table, actions } = setup()); + await act(async () => { + ({ find, exists, component, table, actions } = setup()); + }); - await nextTick(); // Make sure that the Http request is fulfilled component.update(); // Read the index list table ({ tableCellsValues } = table.getMetaData('followerIndexListTable')); }); - afterEach(async () => { - // The updates are not all synchronouse - // We need to wait for all the updates to ran before unmounting our component - await delay(100); - }); - test('should not display the empty prompt', () => { expect(exists('emptyPrompt')).toBe(false); }); @@ -173,17 +170,17 @@ describe('', () => { }); describe('action menu', () => { - test('should be visible when a follower index is selected', () => { + test('should be visible when a follower index is selected', async () => { expect(exists('contextMenuButton')).toBe(false); - actions.selectFollowerIndexAt(0); + await actions.selectFollowerIndexAt(0); expect(exists('contextMenuButton')).toBe(true); }); test('should have a "pause", "edit" and "unfollow" action when the follower index is active', async () => { - actions.selectFollowerIndexAt(0); - actions.openContextMenu(); + await actions.selectFollowerIndexAt(0); + await actions.openContextMenu(); const contextMenu = find('contextMenu'); @@ -199,8 +196,8 @@ describe('', () => { }); test('should have a "resume", "edit" and "unfollow" action when the follower index is active', async () => { - actions.selectFollowerIndexAt(1); // Select the second follower that is "paused" - actions.openContextMenu(); + await actions.selectFollowerIndexAt(1); // Select the second follower that is "paused" + await actions.openContextMenu(); const contextMenu = find('contextMenu'); @@ -213,22 +210,22 @@ describe('', () => { ]); }); - test('should open a confirmation modal when clicking on "pause replication"', () => { + test('should open a confirmation modal when clicking on "pause replication"', async () => { expect(exists('pauseReplicationConfirmation')).toBe(false); - actions.selectFollowerIndexAt(0); - actions.openContextMenu(); - actions.clickContextMenuButtonAt(0); // first button is the "pause" action + await actions.selectFollowerIndexAt(0); + await actions.openContextMenu(); + await actions.clickContextMenuButtonAt(0); // first button is the "pause" action expect(exists('pauseReplicationConfirmation')).toBe(true); }); - test('should open a confirmation modal when clicking on "unfollow leader index"', () => { + test('should open a confirmation modal when clicking on "unfollow leader index"', async () => { expect(exists('unfollowLeaderConfirmation')).toBe(false); - actions.selectFollowerIndexAt(0); - actions.openContextMenu(); - actions.clickContextMenuButtonAt(2); // third button is the "unfollow" action + await actions.selectFollowerIndexAt(0); + await actions.openContextMenu(); + await actions.clickContextMenuButtonAt(2); // third button is the "unfollow" action expect(exists('unfollowLeaderConfirmation')).toBe(true); }); @@ -238,13 +235,13 @@ describe('', () => { test('should open a context menu when clicking on the button of each row', async () => { expect(component.find('.euiContextMenuPanel').length).toBe(0); - actions.openTableRowContextMenuAt(0); + await actions.openTableRowContextMenuAt(0); expect(component.find('.euiContextMenuPanel').length).toBe(1); }); test('should have the "pause", "edit" and "unfollow" options in the row context menu', async () => { - actions.openTableRowContextMenuAt(0); + await actions.openTableRowContextMenuAt(0); const buttonLabels = component .find('.euiContextMenuPanel') @@ -260,7 +257,7 @@ describe('', () => { test('should have the "resume", "edit" and "unfollow" options in the row context menu', async () => { // We open the context menu of the second row (index 1) as followerIndices[1].status is "paused" - actions.openTableRowContextMenuAt(1); + await actions.openTableRowContextMenuAt(1); const buttonLabels = component .find('.euiContextMenuPanel') @@ -277,8 +274,13 @@ describe('', () => { test('should open a confirmation modal when clicking on "pause replication"', async () => { expect(exists('pauseReplicationConfirmation')).toBe(false); - actions.openTableRowContextMenuAt(0); - find('pauseButton').simulate('click'); + await actions.openTableRowContextMenuAt(0); + + await act(async () => { + find('pauseButton').simulate('click'); + }); + + component.update(); expect(exists('pauseReplicationConfirmation')).toBe(true); }); @@ -286,51 +288,60 @@ describe('', () => { test('should open a confirmation modal when clicking on "resume"', async () => { expect(exists('resumeReplicationConfirmation')).toBe(false); - actions.openTableRowContextMenuAt(1); // open the second row context menu, as it is a "paused" follower index - find('resumeButton').simulate('click'); + await actions.openTableRowContextMenuAt(1); // open the second row context menu, as it is a "paused" follower index + + await act(async () => { + find('resumeButton').simulate('click'); + }); + + component.update(); expect(exists('resumeReplicationConfirmation')).toBe(true); }); - test('should open a confirmation modal when clicking on "unfollow leader index"', () => { + test('should open a confirmation modal when clicking on "unfollow leader index"', async () => { expect(exists('unfollowLeaderConfirmation')).toBe(false); - actions.openTableRowContextMenuAt(0); - find('unfollowButton').simulate('click'); + await actions.openTableRowContextMenuAt(0); + + await act(async () => { + find('unfollowButton').simulate('click'); + }); + + component.update(); expect(exists('unfollowLeaderConfirmation')).toBe(true); }); }); - // FLAKY: https://github.com/elastic/kibana/issues/75124 - describe.skip('detail panel', () => { - test('should open a detail panel when clicking on a follower index', () => { + describe('detail panel', () => { + test('should open a detail panel when clicking on a follower index', async () => { expect(exists('followerIndexDetail')).toBe(false); - actions.clickFollowerIndexAt(0); + await actions.clickFollowerIndexAt(0); expect(exists('followerIndexDetail')).toBe(true); }); - test('should set the title the index that has been selected', () => { - actions.clickFollowerIndexAt(0); // Open the detail panel + test('should set the title the index that has been selected', async () => { + await actions.clickFollowerIndexAt(0); // Open the detail panel expect(find('followerIndexDetail.title').text()).toEqual(index1.name); }); - test('should indicate the correct "status", "remote cluster" and "leader index"', () => { - actions.clickFollowerIndexAt(0); + test('should indicate the correct "status", "remote cluster" and "leader index"', async () => { + await actions.clickFollowerIndexAt(0); expect(find('followerIndexDetail.status').text()).toEqual(index1.status); expect(find('followerIndexDetail.remoteCluster').text()).toEqual(index1.remoteCluster); expect(find('followerIndexDetail.leaderIndex').text()).toEqual(index1.leaderIndex); }); - test('should have a "settings" section', () => { - actions.clickFollowerIndexAt(0); + test('should have a "settings" section', async () => { + await actions.clickFollowerIndexAt(0); expect(find('followerIndexDetail.settingsSection').find('h3').text()).toEqual('Settings'); expect(exists('followerIndexDetail.settingsValues')).toBe(true); }); - test('should set the correct follower index settings values', () => { + test('should set the correct follower index settings values', async () => { const mapSettingsToFollowerIndexProp = { maxReadReqOpCount: 'maxReadRequestOperationCount', maxOutstandingReadReq: 'maxOutstandingReadRequests', @@ -344,7 +355,7 @@ describe('', () => { readPollTimeout: 'readPollTimeout', }; - actions.clickFollowerIndexAt(0); + await actions.clickFollowerIndexAt(0); Object.entries(mapSettingsToFollowerIndexProp).forEach(([setting, prop]) => { const wrapper = find(`settingsValues.${setting}`); @@ -357,21 +368,21 @@ describe('', () => { }); }); - test('should not have settings values for a "paused" follower index', () => { - actions.clickFollowerIndexAt(1); // the second follower index is paused + test('should not have settings values for a "paused" follower index', async () => { + await actions.clickFollowerIndexAt(1); // the second follower index is paused expect(exists('followerIndexDetail.settingsValues')).toBe(false); expect(find('followerIndexDetail.settingsSection').text()).toContain( 'paused follower index does not have settings' ); }); - test('should have a section to render the follower index shards stats', () => { - actions.clickFollowerIndexAt(0); + test('should have a section to render the follower index shards stats', async () => { + await actions.clickFollowerIndexAt(0); expect(exists('followerIndexDetail.shardsStatsSection')).toBe(true); }); - test('should render a EuiCodeEditor for each shards stats', () => { - actions.clickFollowerIndexAt(0); + test('should render a EuiCodeEditor for each shards stats', async () => { + await actions.clickFollowerIndexAt(0); const codeEditors = component.find(`EuiCodeEditor`); diff --git a/x-pack/plugins/cross_cluster_replication/public/__jest__/client_integration/helpers/follower_index_list.helpers.js b/x-pack/plugins/cross_cluster_replication/public/__jest__/client_integration/helpers/follower_index_list.helpers.js index d1fd205c683c0..5e6a48413b18b 100644 --- a/x-pack/plugins/cross_cluster_replication/public/__jest__/client_integration/helpers/follower_index_list.helpers.js +++ b/x-pack/plugins/cross_cluster_replication/public/__jest__/client_integration/helpers/follower_index_list.helpers.js @@ -4,6 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import { act } from 'react-dom/test-utils'; import { registerTestBed, findTestSubject } from '@kbn/test/jest'; import { FollowerIndicesList } from '../../../app/sections/home/follower_indices_list'; @@ -37,24 +38,44 @@ export const setup = (props) => { * User Actions */ - const selectFollowerIndexAt = (index = 0) => { - const { rows } = testBed.table.getMetaData(EUI_TABLE); + const selectFollowerIndexAt = async (index = 0) => { + const { table, component } = testBed; + const { rows } = table.getMetaData(EUI_TABLE); const row = rows[index]; const checkBox = row.reactWrapper.find('input').hostNodes(); - checkBox.simulate('change', { target: { checked: true } }); + + await act(async () => { + checkBox.simulate('change', { target: { checked: true } }); + }); + + component.update(); }; - const openContextMenu = () => { - testBed.find('contextMenuButton').simulate('click'); + const openContextMenu = async () => { + const { find, component } = testBed; + + await act(async () => { + find('contextMenuButton').simulate('click'); + }); + + component.update(); }; - const clickContextMenuButtonAt = (index = 0) => { - const contextMenu = testBed.find('contextMenu'); - contextMenu.find('button').at(index).simulate('click'); + const clickContextMenuButtonAt = async (index = 0) => { + const { find, component } = testBed; + + const contextMenu = find('contextMenu'); + + await act(async () => { + contextMenu.find('button').at(index).simulate('click'); + }); + + component.update(); }; - const openTableRowContextMenuAt = (index = 0) => { - const { rows } = testBed.table.getMetaData(EUI_TABLE); + const openTableRowContextMenuAt = async (index = 0) => { + const { table, component } = testBed; + const { rows } = table.getMetaData(EUI_TABLE); const actionsColumnIndex = rows[0].columns.length - 1; // Actions are in the last column const actionsTableCell = rows[index].columns[actionsColumnIndex]; const button = actionsTableCell.reactWrapper.find('button'); @@ -63,17 +84,34 @@ export const setup = (props) => { `No button to open context menu were found on Follower index list table row ${index}` ); } - button.simulate('click'); + + await act(async () => { + button.simulate('click'); + }); + + component.update(); }; - const clickFollowerIndexAt = (index = 0) => { - const { rows } = testBed.table.getMetaData(EUI_TABLE); + const clickFollowerIndexAt = async (index = 0) => { + const { table, component } = testBed; + const { rows } = table.getMetaData(EUI_TABLE); const followerIndexLink = findTestSubject(rows[index].reactWrapper, 'followerIndexLink'); - followerIndexLink.simulate('click'); + + await act(async () => { + followerIndexLink.simulate('click'); + }); + + component.update(); }; - const clickPaginationNextButton = () => { - testBed.find('followerIndexListTable.pagination-button-next').simulate('click'); + const clickPaginationNextButton = async () => { + const { find, component } = testBed; + + await act(async () => { + find('followerIndexListTable.pagination-button-next').simulate('click'); + }); + + component.update(); }; return { diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine/engine_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine/engine_logic.test.ts index b9ec83db99f70..5435450f1bfdb 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine/engine_logic.test.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine/engine_logic.test.ts @@ -43,6 +43,7 @@ describe('EngineLogic', () => { engineName: '', isMetaEngine: false, isSampleEngine: false, + hasSchemaErrors: false, hasSchemaConflicts: false, hasUnconfirmedSchemaFields: false, engineNotFound: false, @@ -87,28 +88,6 @@ describe('EngineLogic', () => { }); }); - describe('setIndexingStatus', () => { - describe('engine', () => { - it('should set the nested obj property to the provided value', () => { - mount({ engine: mockEngineData }); - const mockReindexJob = { - percentageComplete: 50, - numDocumentsWithErrors: 2, - activeReindexJobId: '123', - }; - EngineLogic.actions.setIndexingStatus(mockReindexJob); - - expect(EngineLogic.values).toEqual({ - ...DEFAULT_VALUES, - engine: { - ...mockEngineData, - activeReindexJob: mockReindexJob, - }, - }); - }); - }); - }); - describe('setEngineNotFound', () => { describe('engineNotFound', () => { it('should be set to the provided value', () => { @@ -228,6 +207,24 @@ describe('EngineLogic', () => { }); }); + describe('hasSchemaErrors', () => { + it('should be set based on engine.activeReindexJob.numDocumentsWithErrors', () => { + const mockSchemaEngine = { + ...mockEngineData, + activeReindexJob: { + numDocumentsWithErrors: 10, + }, + }; + mount({ engine: mockSchemaEngine }); + + expect(EngineLogic.values).toEqual({ + ...DEFAULT_VALUES, + engine: mockSchemaEngine, + hasSchemaErrors: true, + }); + }); + }); + describe('hasSchemaConflicts', () => { it('should be set based on engine.schemaConflicts', () => { const mockSchemaEngine = { diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine/engine_logic.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine/engine_logic.ts index 768a9e545b878..8fbf45bc85e52 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine/engine_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine/engine_logic.ts @@ -8,7 +8,6 @@ import { kea, MakeLogicType } from 'kea'; import { HttpLogic } from '../../../shared/http'; -import { IIndexingStatus } from '../../../shared/schema/types'; import { EngineDetails, EngineTypes } from './types'; @@ -18,6 +17,7 @@ interface EngineValues { engineName: string; isMetaEngine: boolean; isSampleEngine: boolean; + hasSchemaErrors: boolean; hasSchemaConflicts: boolean; hasUnconfirmedSchemaFields: boolean; engineNotFound: boolean; @@ -26,7 +26,6 @@ interface EngineValues { interface EngineActions { setEngineData(engine: EngineDetails): { engine: EngineDetails }; setEngineName(engineName: string): { engineName: string }; - setIndexingStatus(activeReindexJob: IIndexingStatus): { activeReindexJob: IIndexingStatus }; setEngineNotFound(notFound: boolean): { notFound: boolean }; clearEngine(): void; initializeEngine(): void; @@ -37,7 +36,6 @@ export const EngineLogic = kea>({ actions: { setEngineData: (engine) => ({ engine }), setEngineName: (engineName) => ({ engineName }), - setIndexingStatus: (activeReindexJob) => ({ activeReindexJob }), setEngineNotFound: (notFound) => ({ notFound }), clearEngine: true, initializeEngine: true, @@ -55,10 +53,6 @@ export const EngineLogic = kea>({ { setEngineData: (_, { engine }) => engine, clearEngine: () => ({}), - setIndexingStatus: (state, { activeReindexJob }) => ({ - ...state, - activeReindexJob, - }), }, ], engineName: [ @@ -79,6 +73,12 @@ export const EngineLogic = kea>({ selectors: ({ selectors }) => ({ isMetaEngine: [() => [selectors.engine], (engine) => engine?.type === EngineTypes.meta], isSampleEngine: [() => [selectors.engine], (engine) => !!engine?.sample], + // Indexed engines + hasSchemaErrors: [ + () => [selectors.engine], + ({ activeReindexJob }) => activeReindexJob?.numDocumentsWithErrors > 0, + ], + // Meta engines hasSchemaConflicts: [ () => [selectors.engine], (engine) => !!(engine?.schemaConflicts && Object.keys(engine.schemaConflicts).length > 0), diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine/engine_nav.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine/engine_nav.test.tsx index 1781883aa6532..d913e3921bf1b 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine/engine_nav.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine/engine_nav.test.tsx @@ -78,6 +78,12 @@ describe('EngineNav', () => { describe('schema nav icons', () => { const myRole = { canViewEngineSchema: true }; + it('renders schema errors alert icon', () => { + setMockValues({ ...values, myRole, hasSchemaErrors: true }); + const wrapper = shallow(); + expect(wrapper.find('[data-test-subj="EngineNavSchemaErrors"]')).toHaveLength(1); + }); + it('renders unconfirmed schema fields info icon', () => { setMockValues({ ...values, myRole, hasUnconfirmedSchemaFields: true }); const wrapper = shallow(); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine/engine_nav.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine/engine_nav.tsx index 530accb501c41..4738209cee4a2 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine/engine_nav.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engine/engine_nav.tsx @@ -70,6 +70,7 @@ export const EngineNav: React.FC = () => { dataLoading, isSampleEngine, isMetaEngine, + hasSchemaErrors, hasSchemaConflicts, hasUnconfirmedSchemaFields, engine, @@ -128,9 +129,19 @@ export const EngineNav: React.FC = () => { shouldShowActiveForSubroutes data-test-subj="EngineSchemaLink" > - + {SCHEMA_TITLE} + {hasSchemaErrors && ( + + )} {hasUnconfirmedSchemaFields && ( { to={generateEnginePath(ENGINE_RELEVANCE_TUNING_PATH)} data-test-subj="EngineRelevanceTuningLink" > - + {RELEVANCE_TUNING_TITLE} {invalidBoosts && ( diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engines/components/tables/utils.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engines/components/tables/utils.ts index d9f41f8558b78..13cad349d75b0 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/engines/components/tables/utils.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/engines/components/tables/utils.ts @@ -10,7 +10,7 @@ import { EngineDetails } from '../../../engine/types'; export const getConflictingEnginesFromConflictingField = ( conflictingField: SchemaConflictFieldTypes -): string[] => Object.values(conflictingField).flat(); +) => Object.values(conflictingField).flat() as string[]; export const getConflictingEnginesFromSchemaConflicts = ( schemaConflicts: SchemaConflicts diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/components/empty_state.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/components/empty_state.test.tsx new file mode 100644 index 0000000000000..ea658c741b8a0 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/components/empty_state.test.tsx @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +import { shallow } from 'enzyme'; + +import { EuiEmptyPrompt, EuiButton } from '@elastic/eui'; + +import { EmptyState } from './'; + +describe('EmptyState', () => { + it('renders', () => { + const wrapper = shallow() + .find(EuiEmptyPrompt) + .dive(); + + expect(wrapper.find('h2').text()).toEqual('Create a schema'); + expect(wrapper.find(EuiButton).prop('href')).toEqual( + expect.stringContaining('#indexing-documents-guide-schema') + ); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/components/empty_state.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/components/empty_state.tsx new file mode 100644 index 0000000000000..6d7dd198d5eef --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/components/empty_state.tsx @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +import { EuiPanel, EuiEmptyPrompt, EuiButton } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; + +import { DOCS_PREFIX } from '../../../routes'; + +export const EmptyState: React.FC = () => { + return ( + + + {i18n.translate('xpack.enterpriseSearch.appSearch.engine.schema.empty.title', { + defaultMessage: 'Create a schema', + })} + + } + body={ +

+ {i18n.translate('xpack.enterpriseSearch.appSearch.engine.schema.empty.description', { + defaultMessage: + 'Create schema fields in advance, or index some documents and a schema will be created for you.', + })} +

+ } + actions={ + + {i18n.translate('xpack.enterpriseSearch.appSearch.engine.schema.empty.buttonLabel', { + defaultMessage: 'Read the indexing schema guide', + })} + + } + /> + + ); +}; diff --git a/x-pack/plugins/apm/server/lib/alerts/create_apm_lifecycle_rule_type.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/components/index.ts similarity index 53% rename from x-pack/plugins/apm/server/lib/alerts/create_apm_lifecycle_rule_type.ts rename to x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/components/index.ts index 8d250a5765cce..7da44849b5bc0 100644 --- a/x-pack/plugins/apm/server/lib/alerts/create_apm_lifecycle_rule_type.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/components/index.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { createLifecycleRuleTypeFactory } from '../../../../rule_registry/server'; -import { APMRuleRegistry } from '../../plugin'; - -export const createAPMLifecycleRuleType = createLifecycleRuleTypeFactory(); +export { SchemaCallouts } from './schema_callouts'; +export { SchemaTable } from './schema_table'; +export { EmptyState } from './empty_state'; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/components/schema_callouts.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/components/schema_callouts.test.tsx new file mode 100644 index 0000000000000..5bb08a6c8859a --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/components/schema_callouts.test.tsx @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { setMockValues, setMockActions } from '../../../../__mocks__'; +import '../../../__mocks__/engine_logic.mock'; + +import React from 'react'; + +import { shallow } from 'enzyme'; + +import { SchemaErrorsCallout } from '../../../../shared/schema'; + +import { + UnsearchedFieldsCallout, + UnconfirmedFieldsCallout, + ConfirmSchemaButton, +} from './schema_callouts'; + +import { SchemaCallouts } from './'; + +describe('SchemaCallouts', () => { + const values = { + hasUnconfirmedFields: false, + hasNewUnsearchedFields: false, + mostRecentIndexJob: { + hasErrors: false, + activeReindexJobId: 'some-id', + }, + }; + + beforeEach(() => { + jest.clearAllMocks(); + setMockValues(values); + }); + + it('renders nothing if there is nothing to call out', () => { + const wrapper = shallow(); + + expect(wrapper.text()).toBeFalsy(); + }); + + it('renders a schema errors callout if the most recent index job had errors', () => { + setMockValues({ + ...values, + mostRecentIndexJob: { + hasErrors: true, + activeReindexJobId: '12345', + }, + }); + const wrapper = shallow(); + + expect(wrapper.find(SchemaErrorsCallout)).toHaveLength(1); + expect(wrapper.find(SchemaErrorsCallout).prop('viewErrorsPath')).toEqual( + '/engines/some-engine/schema/reindex_job/12345' + ); + }); + + it('renders an unsearched fields callout if the schema has new unconfirmed & unsearched fields', () => { + setMockValues({ + ...values, + hasUnconfirmedFields: true, + hasNewUnsearchedFields: true, + }); + const wrapper = shallow(); + + expect(wrapper.find(UnsearchedFieldsCallout)).toHaveLength(1); + }); + + it('renders an unconfirmed fields callout if the schema has unconfirmed fields', () => { + setMockValues({ + ...values, + hasUnconfirmedFields: true, + }); + const wrapper = shallow(); + + expect(wrapper.find(UnconfirmedFieldsCallout)).toHaveLength(1); + }); + + describe('UnsearchedFieldsCallout', () => { + it('renders an info callout about unsearched fields with a link to the relevance tuning page', () => { + const wrapper = shallow(); + + expect(wrapper.prop('title')).toEqual( + 'Recently added fields are not being searched by default' + ); + expect(wrapper.find('[data-test-subj="relevanceTuningButtonLink"]').prop('to')).toEqual( + '/engines/some-engine/relevance_tuning' + ); + }); + }); + + describe('UnconfirmedFieldsCallout', () => { + it('renders an info callout about unconfirmed fields', () => { + const wrapper = shallow(); + + expect(wrapper.prop('title')).toEqual("You've recently added new schema fields"); + }); + }); + + describe('ConfirmSchemaButton', () => { + const actions = { updateSchema: jest.fn() }; + + beforeEach(() => { + setMockValues({ isUpdating: false }); + setMockActions(actions); + }); + + it('allows users to confirm schema without changes from the callouts', () => { + const wrapper = shallow(); + + wrapper.simulate('click'); + expect(actions.updateSchema).toHaveBeenCalled(); + }); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/components/schema_callouts.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/components/schema_callouts.tsx new file mode 100644 index 0000000000000..50a2ee5c83abe --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/components/schema_callouts.tsx @@ -0,0 +1,127 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +import { useValues, useActions } from 'kea'; + +import { EuiCallOut, EuiButton, EuiSpacer, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; + +import { EuiButtonTo } from '../../../../shared/react_router_helpers'; +import { SchemaErrorsCallout } from '../../../../shared/schema'; +import { ENGINE_RELEVANCE_TUNING_PATH, ENGINE_REINDEX_JOB_PATH } from '../../../routes'; +import { generateEnginePath } from '../../engine'; + +import { SchemaLogic } from '../schema_logic'; + +export const SchemaCallouts: React.FC = () => { + const { + hasUnconfirmedFields, + hasNewUnsearchedFields, + mostRecentIndexJob: { hasErrors, activeReindexJobId }, + } = useValues(SchemaLogic); + + return ( + <> + {hasErrors && ( + <> + + + + )} + {hasUnconfirmedFields && ( + <> + {hasNewUnsearchedFields ? : } + + + )} + + ); +}; + +export const UnsearchedFieldsCallout: React.FC = () => ( + +

+ {i18n.translate( + 'xpack.enterpriseSearch.appSearch.engine.schema.unsearchedFields.description', + { + defaultMessage: + 'If these new fields should be searchable, update your search settings to include them. If you want them to remain unsearchable, confirm your new field types to dismiss this alert.', + } + )} +

+ + + + {i18n.translate( + 'xpack.enterpriseSearch.appSearch.engine.schema.unsearchedFields.searchSettingsButtonLabel', + { defaultMessage: 'Update search settings' } + )} + + + + + + + +); + +export const UnconfirmedFieldsCallout: React.FC = () => ( + +

+ {i18n.translate( + 'xpack.enterpriseSearch.appSearch.engine.schema.unconfirmedFields.description', + { + defaultMessage: + 'Set your new schema field(s) to their correct or expected types, and then confirm your field types.', + } + )} +

+ + +); + +export const ConfirmSchemaButton: React.FC = () => { + const { updateSchema } = useActions(SchemaLogic); + const { isUpdating } = useValues(SchemaLogic); + + return ( + updateSchema()} + data-test-subj="confirmSchemaTypesButton" + > + {i18n.translate('xpack.enterpriseSearch.appSearch.engine.schema.confirmSchemaButtonLabel', { + defaultMessage: 'Confirm types', + })} + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/components/schema_table.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/components/schema_table.test.tsx new file mode 100644 index 0000000000000..c8b0bb7ddbac5 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/components/schema_table.test.tsx @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { setMockValues, setMockActions } from '../../../../__mocks__'; + +import React from 'react'; + +import { shallow } from 'enzyme'; + +import { EuiTable, EuiTableHeaderCell, EuiTableRow, EuiHealth } from '@elastic/eui'; + +import { SchemaFieldTypeSelect } from '../../../../shared/schema'; + +import { SchemaTable } from './'; + +describe('SchemaTable', () => { + const values = { + schema: {}, + unconfirmedFields: [], + }; + const actions = { + updateSchemaFieldType: jest.fn(), + }; + + beforeEach(() => { + jest.clearAllMocks(); + setMockValues(values); + setMockActions(actions); + }); + + it('renders', () => { + const wrapper = shallow(); + + expect(wrapper.find(EuiTable)).toHaveLength(1); + expect(wrapper.find(EuiTableHeaderCell).first().prop('children')).toEqual('Field name'); + expect(wrapper.find(EuiTableHeaderCell).last().prop('children')).toEqual('Field type'); + }); + + it('always renders an initial ID row (with no field type select)', () => { + const wrapper = shallow(); + + expect(wrapper.find(EuiTableRow)).toHaveLength(1); + expect(wrapper.find('code').text()).toEqual('id'); + expect(wrapper.find(SchemaFieldTypeSelect)).toHaveLength(0); + }); + + it('renders subsequent table rows for each schema field', () => { + setMockValues({ + ...values, + schema: { + some_text_field: 'text', + some_number_field: 'number', + some_date_field: 'date', + some_location_field: 'geolocation', + }, + }); + const wrapper = shallow(); + + expect(wrapper.find(EuiTableRow)).toHaveLength(5); + + expect(wrapper.find('code').at(1).text()).toEqual('some_text_field'); + expect(wrapper.find(SchemaFieldTypeSelect).at(0).prop('fieldType')).toEqual('text'); + + expect(wrapper.find('code').last().text()).toEqual('some_location_field'); + expect(wrapper.find(SchemaFieldTypeSelect).last().prop('fieldType')).toEqual('geolocation'); + }); + + it('renders a recently added status if a field has been recently added', () => { + setMockValues({ + ...values, + schema: { + some_new_field: 'text', + }, + unconfirmedFields: ['some_new_field'], + }); + const wrapper = shallow(); + + expect(wrapper.find(EuiHealth)).toHaveLength(1); + expect(wrapper.find(EuiHealth).childAt(0).prop('children')).toEqual('Recently added'); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/components/schema_table.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/components/schema_table.tsx new file mode 100644 index 0000000000000..d9187bb65adf0 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/components/schema_table.tsx @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +import { useValues, useActions } from 'kea'; + +import { + EuiTable, + EuiTableHeader, + EuiTableHeaderCell, + EuiTableBody, + EuiTableRow, + EuiTableRowCell, + EuiHealth, + EuiText, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; + +import { SchemaFieldTypeSelect } from '../../../../shared/schema'; +import { FIELD_NAME, FIELD_TYPE } from '../../../../shared/schema/constants'; + +import { SchemaLogic } from '../schema_logic'; + +export const SchemaTable: React.FC = () => { + const { schema, unconfirmedFields } = useValues(SchemaLogic); + const { updateSchemaFieldType } = useActions(SchemaLogic); + + return ( + + + {FIELD_NAME} + + {FIELD_TYPE} + + + + + + id + + + + + + {Object.entries(schema).map(([fieldName, fieldType]) => { + const isRecentlyAdded = unconfirmedFields.length && unconfirmedFields.includes(fieldName); + + return ( + + + {fieldName} + + {isRecentlyAdded ? ( + + + + {i18n.translate( + 'xpack.enterpriseSearch.appSearch.engine.schema.unconfirmedFieldLabel', + { defaultMessage: 'Recently added' } + )} + + + + ) : ( + + )} + + + + + ); + })} + + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/constants.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/constants.ts index b9c4e2c0454d5..0ee428e87873e 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/constants.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/constants.ts @@ -10,3 +10,18 @@ import { i18n } from '@kbn/i18n'; export const SCHEMA_TITLE = i18n.translate('xpack.enterpriseSearch.appSearch.engine.schema.title', { defaultMessage: 'Schema', }); + +export const ADD_SCHEMA_ERROR = (fieldName: string) => + i18n.translate('xpack.enterpriseSearch.appSearch.engine.schema.addSchemaErrorMessage', { + defaultMessage: 'Field name already exists: {fieldName}', + values: { fieldName }, + }); +export const ADD_SCHEMA_SUCCESS = (fieldName: string) => + i18n.translate('xpack.enterpriseSearch.appSearch.engine.schema.addSchemaSuccessMessage', { + defaultMessage: 'New field added: {fieldName}', + values: { fieldName }, + }); +export const UPDATE_SCHEMA_SUCCESS = i18n.translate( + 'xpack.enterpriseSearch.appSearch.engine.schema.updateSchemaSuccessMessage', + { defaultMessage: 'Schema updated' } +); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/reindex_job/reindex_job.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/reindex_job/reindex_job.test.tsx index 9e8386e2e8337..1755fbea5fb7b 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/reindex_job/reindex_job.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/reindex_job/reindex_job.test.tsx @@ -5,26 +5,72 @@ * 2.0. */ +import { setMockValues, setMockActions } from '../../../../__mocks__'; import '../../../../__mocks__/react_router_history.mock'; +import '../../../../__mocks__/shallow_useeffect.mock'; +import '../../../__mocks__/engine_logic.mock'; import React from 'react'; import { useParams } from 'react-router-dom'; import { shallow } from 'enzyme'; +import { Loading } from '../../../../shared/loading'; +import { SchemaErrorsAccordion } from '../../../../shared/schema'; + import { ReindexJob } from './'; describe('ReindexJob', () => { const props = { schemaBreadcrumb: ['Engines', 'some-engine', 'Schema'], }; + const values = { + dataLoading: false, + fieldCoercionErrors: {}, + engine: { + schema: { + some_field: 'text', + }, + }, + }; + const actions = { + loadReindexJob: jest.fn(), + }; beforeEach(() => { (useParams as jest.Mock).mockReturnValueOnce({ reindexJobId: 'abc1234567890' }); + setMockValues(values); + setMockActions(actions); }); it('renders', () => { + const wrapper = shallow(); + + expect(wrapper.find(SchemaErrorsAccordion)).toHaveLength(1); + expect(wrapper.find(SchemaErrorsAccordion).prop('generateViewPath')).toHaveLength(1); + }); + + it('calls loadReindexJob on page load', () => { shallow(); - // TODO: Check child components + + expect(actions.loadReindexJob).toHaveBeenCalledWith('abc1234567890'); + }); + + it('renders a loading state', () => { + setMockValues({ ...values, dataLoading: true }); + const wrapper = shallow(); + + expect(wrapper.find(Loading)).toHaveLength(1); + }); + + it('renders schema errors with links to document pages', () => { + const wrapper = shallow(); + const generateViewPath = wrapper + .find(SchemaErrorsAccordion) + .prop('generateViewPath') as Function; + + expect(generateViewPath('some-document-id')).toEqual( + '/engines/some-engine/documents/some-document-id' + ); }); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/reindex_job/reindex_job.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/reindex_job/reindex_job.tsx index 19da08d446300..576b4ae11603b 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/reindex_job/reindex_job.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/reindex_job/reindex_job.tsx @@ -5,15 +5,24 @@ * 2.0. */ -import React from 'react'; +import React, { useEffect } from 'react'; import { useParams } from 'react-router-dom'; +import { useActions, useValues } from 'kea'; + import { EuiPageHeader, EuiPageContentBody } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { FlashMessages } from '../../../../shared/flash_messages'; import { SetAppSearchChrome as SetPageChrome } from '../../../../shared/kibana_chrome'; import { BreadcrumbTrail } from '../../../../shared/kibana_chrome/generate_breadcrumbs'; +import { Loading } from '../../../../shared/loading'; +import { SchemaErrorsAccordion } from '../../../../shared/schema'; + +import { ENGINE_DOCUMENT_DETAIL_PATH } from '../../../routes'; +import { EngineLogic, generateEnginePath } from '../../engine'; + +import { ReindexJobLogic } from './reindex_job_logic'; interface Props { schemaBreadcrumb: BreadcrumbTrail; @@ -21,6 +30,17 @@ interface Props { export const ReindexJob: React.FC = ({ schemaBreadcrumb }) => { const { reindexJobId } = useParams() as { reindexJobId: string }; + const { loadReindexJob } = useActions(ReindexJobLogic); + const { dataLoading, fieldCoercionErrors } = useValues(ReindexJobLogic); + const { + engine: { schema }, + } = useValues(EngineLogic); + + useEffect(() => { + loadReindexJob(reindexJobId); + }, [reindexJobId]); + + if (dataLoading) return ; return ( <> @@ -39,7 +59,15 @@ export const ReindexJob: React.FC = ({ schemaBreadcrumb }) => { )} /> - {reindexJobId} + + + generateEnginePath(ENGINE_DOCUMENT_DETAIL_PATH, { documentId }) + } + /> + ); }; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/reindex_job/reindex_job_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/reindex_job/reindex_job_logic.test.ts new file mode 100644 index 0000000000000..b872ad3c914c6 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/reindex_job/reindex_job_logic.test.ts @@ -0,0 +1,120 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { LogicMounter, mockFlashMessageHelpers, mockHttpValues } from '../../../../__mocks__'; +import '../../../__mocks__/engine_logic.mock'; + +import { nextTick } from '@kbn/test/jest'; + +import { ReindexJobLogic } from './reindex_job_logic'; + +describe('ReindexJobLogic', () => { + const { mount } = new LogicMounter(ReindexJobLogic); + const { http } = mockHttpValues; + const { flashAPIErrors } = mockFlashMessageHelpers; + + const MOCK_RESPONSE = { + fieldCoercionErrors: { + some_erroring_field: [ + { + external_id: 'document-1', + error: "Value 'some text' cannot be parsed as a number", + }, + ], + another_erroring_field: [ + { + external_id: 'document-2', + error: "Value '123' cannot be parsed as a date", + }, + ], + }, + }; + + const DEFAULT_VALUES = { + dataLoading: true, + fieldCoercionErrors: {}, + }; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('has expected default values', () => { + mount(); + expect(ReindexJobLogic.values).toEqual(DEFAULT_VALUES); + }); + + describe('actions', () => { + describe('onLoadSuccess', () => { + it('stores fieldCoercionErrors state and sets dataLoading to false', () => { + mount({ fieldCoercionErrors: {}, dataLoading: true }); + + ReindexJobLogic.actions.onLoadSuccess(MOCK_RESPONSE); + + expect(ReindexJobLogic.values).toEqual({ + ...DEFAULT_VALUES, + dataLoading: false, + fieldCoercionErrors: MOCK_RESPONSE.fieldCoercionErrors, + }); + }); + }); + + describe('onLoadError', () => { + it('sets dataLoading to false', () => { + mount({ dataLoading: true }); + + ReindexJobLogic.actions.onLoadError(); + + expect(ReindexJobLogic.values).toEqual({ + ...DEFAULT_VALUES, + dataLoading: false, + }); + }); + }); + }); + + describe('listeners', () => { + describe('loadReindexJob', () => { + it('sets dataLoading to true', () => { + mount({ dataLoading: false }); + + ReindexJobLogic.actions.loadReindexJob('some-job-id'); + + expect(ReindexJobLogic.values).toEqual({ + ...DEFAULT_VALUES, + dataLoading: true, + }); + }); + + it('should make an API call and then set schema state', async () => { + http.get.mockReturnValueOnce(Promise.resolve(MOCK_RESPONSE)); + mount(); + jest.spyOn(ReindexJobLogic.actions, 'onLoadSuccess'); + + ReindexJobLogic.actions.loadReindexJob('some-job-id'); + await nextTick(); + + expect(http.get).toHaveBeenCalledWith( + '/api/app_search/engines/some-engine/reindex_job/some-job-id' + ); + expect(ReindexJobLogic.actions.onLoadSuccess).toHaveBeenCalledWith(MOCK_RESPONSE); + }); + + it('handles errors', async () => { + http.get.mockReturnValueOnce(Promise.reject('error')); + mount(); + jest.spyOn(ReindexJobLogic.actions, 'onLoadError'); + + ReindexJobLogic.actions.loadReindexJob('some-bad-id'); + await nextTick(); + + expect(flashAPIErrors).toHaveBeenCalledWith('error'); + expect(ReindexJobLogic.actions.onLoadError).toHaveBeenCalled(); + }); + }); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/reindex_job/reindex_job_logic.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/reindex_job/reindex_job_logic.ts new file mode 100644 index 0000000000000..8cc76f7f6e94f --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/reindex_job/reindex_job_logic.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { kea, MakeLogicType } from 'kea'; + +import { flashAPIErrors } from '../../../../shared/flash_messages'; +import { HttpLogic } from '../../../../shared/http'; +import { EngineLogic } from '../../engine'; + +import { ReindexJobApiResponse } from '../types'; + +export interface ReindexJobValues { + dataLoading: boolean; + fieldCoercionErrors: ReindexJobApiResponse['fieldCoercionErrors']; +} + +export interface ReindexJobActions { + loadReindexJob(id: string): string; + onLoadSuccess(response: ReindexJobApiResponse): ReindexJobApiResponse; + onLoadError(): void; +} + +export const ReindexJobLogic = kea>({ + path: ['enterprise_search', 'app_search', 'reindex_job_logic'], + actions: { + loadReindexJob: (id) => id, + onLoadSuccess: (response) => response, + onLoadError: true, + }, + reducers: { + dataLoading: [ + true, + { + loadReindexJob: () => true, + onLoadSuccess: () => false, + onLoadError: () => false, + }, + ], + fieldCoercionErrors: [ + {}, + { + onLoadSuccess: (_, { fieldCoercionErrors }) => fieldCoercionErrors, + }, + ], + }, + listeners: ({ actions }) => ({ + loadReindexJob: async (id) => { + const { http } = HttpLogic.values; + const { engineName } = EngineLogic.values; + + try { + const response = await http.get(`/api/app_search/engines/${engineName}/reindex_job/${id}`); + actions.onLoadSuccess(response); + } catch (e) { + flashAPIErrors(e); + actions.onLoadError(); + } + }, + }), +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/schema_base_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/schema_base_logic.test.ts new file mode 100644 index 0000000000000..2f5788278aa0b --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/schema_base_logic.test.ts @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { LogicMounter, mockFlashMessageHelpers, mockHttpValues } from '../../../__mocks__'; +import '../../__mocks__/engine_logic.mock'; + +import { nextTick } from '@kbn/test/jest'; + +import { SchemaType } from '../../../shared/schema/types'; + +import { SchemaBaseLogic } from './schema_base_logic'; + +describe('SchemaBaseLogic', () => { + const { mount } = new LogicMounter(SchemaBaseLogic); + const { http } = mockHttpValues; + const { flashAPIErrors } = mockFlashMessageHelpers; + + const MOCK_SCHEMA = { + some_text_field: SchemaType.Text, + some_number_field: SchemaType.Number, + }; + const MOCK_RESPONSE = { + schema: MOCK_SCHEMA, + } as any; + + const DEFAULT_VALUES = { + dataLoading: true, + schema: {}, + }; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('has expected default values', () => { + mount(); + expect(SchemaBaseLogic.values).toEqual(DEFAULT_VALUES); + }); + + describe('actions', () => { + describe('onSchemaLoad', () => { + it('stores schema state and sets dataLoading to false', () => { + mount({ schema: {}, dataLoading: true }); + + SchemaBaseLogic.actions.onSchemaLoad(MOCK_RESPONSE); + + expect(SchemaBaseLogic.values).toEqual({ + ...DEFAULT_VALUES, + dataLoading: false, + schema: MOCK_SCHEMA, + }); + }); + }); + + describe('setSchema', () => { + it('updates schema state', () => { + mount({ schema: {} }); + + SchemaBaseLogic.actions.setSchema(MOCK_SCHEMA); + + expect(SchemaBaseLogic.values).toEqual({ + ...DEFAULT_VALUES, + schema: MOCK_SCHEMA, + }); + }); + }); + }); + + describe('listeners', () => { + describe('loadSchema', () => { + it('sets dataLoading to true', () => { + mount({ dataLoading: false }); + + SchemaBaseLogic.actions.loadSchema(); + + expect(SchemaBaseLogic.values).toEqual({ + ...DEFAULT_VALUES, + dataLoading: true, + }); + }); + + it('should make an API call and then set schema state', async () => { + http.get.mockReturnValueOnce(Promise.resolve(MOCK_RESPONSE)); + mount(); + jest.spyOn(SchemaBaseLogic.actions, 'onSchemaLoad'); + + SchemaBaseLogic.actions.loadSchema(); + await nextTick(); + + expect(http.get).toHaveBeenCalledWith('/api/app_search/engines/some-engine/schema'); + expect(SchemaBaseLogic.actions.onSchemaLoad).toHaveBeenCalledWith(MOCK_RESPONSE); + }); + + it('handles errors', async () => { + http.get.mockReturnValueOnce(Promise.reject('error')); + mount(); + + SchemaBaseLogic.actions.loadSchema(); + await nextTick(); + + expect(flashAPIErrors).toHaveBeenCalledWith('error'); + }); + }); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/schema_base_logic.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/schema_base_logic.ts new file mode 100644 index 0000000000000..c2196c01d402b --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/schema_base_logic.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { kea, MakeLogicType } from 'kea'; + +import { flashAPIErrors } from '../../../shared/flash_messages'; +import { HttpLogic } from '../../../shared/http'; +import { Schema } from '../../../shared/schema/types'; +import { EngineLogic } from '../engine'; + +import { SchemaApiResponse, MetaEngineSchemaApiResponse } from './types'; + +export interface SchemaBaseValues { + dataLoading: boolean; + schema: Schema; +} + +export interface SchemaBaseActions { + loadSchema(): void; + onSchemaLoad( + response: SchemaApiResponse | MetaEngineSchemaApiResponse + ): SchemaApiResponse | MetaEngineSchemaApiResponse; + setSchema(schema: Schema): { schema: Schema }; +} + +export const SchemaBaseLogic = kea>({ + path: ['enterprise_search', 'app_search', 'schema_base_logic'], + actions: { + loadSchema: true, + onSchemaLoad: (response) => response, + setSchema: (schema) => ({ schema }), + }, + reducers: { + dataLoading: [ + true, + { + loadSchema: () => true, + onSchemaLoad: () => false, + }, + ], + schema: [ + {}, + { + onSchemaLoad: (_, { schema }) => schema, + setSchema: (_, { schema }) => schema, + }, + ], + }, + listeners: ({ actions }) => ({ + loadSchema: async () => { + const { http } = HttpLogic.values; + const { engineName } = EngineLogic.values; + + try { + const response = await http.get(`/api/app_search/engines/${engineName}/schema`); + actions.onSchemaLoad(response); + } catch (e) { + flashAPIErrors(e); + } + }, + }), +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/schema_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/schema_logic.test.ts new file mode 100644 index 0000000000000..123f62af4eeba --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/schema_logic.test.ts @@ -0,0 +1,291 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { LogicMounter, mockFlashMessageHelpers, mockHttpValues } from '../../../__mocks__'; +import { mockEngineActions } from '../../__mocks__/engine_logic.mock'; + +import { nextTick } from '@kbn/test/jest'; + +import { SchemaType, Schema } from '../../../shared/schema/types'; + +import { SchemaLogic } from './schema_logic'; + +describe('SchemaLogic', () => { + const { mount } = new LogicMounter(SchemaLogic); + const { http } = mockHttpValues; + const { flashAPIErrors, flashSuccessToast, setErrorMessage } = mockFlashMessageHelpers; + + const MOCK_RESPONSE = { + schema: { + some_text_field: SchemaType.Text, + some_number_field: SchemaType.Number, + }, + mostRecentIndexJob: { + percentageComplete: 100, + numDocumentsWithErrors: 10, + activeReindexJobId: 'some-id', + isActive: false, + hasErrors: true, + }, + unconfirmedFields: ['some_field'], + unsearchedUnconfirmedFields: true, + }; + + const DEFAULT_VALUES = { + dataLoading: true, + schema: {}, + isUpdating: false, + hasSchema: false, + hasSchemaChanged: false, + cachedSchema: {}, + mostRecentIndexJob: {}, + unconfirmedFields: [], + hasUnconfirmedFields: false, + hasNewUnsearchedFields: false, + isModalOpen: false, + }; + + /* + * Unfortunately, we can't mount({ schema: ... }) & have to use an action to set schema + * because of the separate connected logic file - our LogicMounter test helper sets context + * for only the currently tested file + */ + const mountAndSetSchema = ({ schema, ...values }: { schema: Schema; [key: string]: unknown }) => { + mount(values); + SchemaLogic.actions.setSchema(schema); + }; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('has expected default values', () => { + mount(); + expect(SchemaLogic.values).toEqual(DEFAULT_VALUES); + }); + + describe('actions', () => { + describe('onSchemaLoad', () => { + it('stores the API response in state and sets isUpdating & isModalOpen to false', () => { + mount({ isModalOpen: true }); + + SchemaLogic.actions.onSchemaLoad(MOCK_RESPONSE); + + expect(SchemaLogic.values).toEqual({ + ...DEFAULT_VALUES, + // SchemaBaseLogic + dataLoading: false, + schema: MOCK_RESPONSE.schema, + // SchemaLogic + isUpdating: false, + isModalOpen: false, + cachedSchema: MOCK_RESPONSE.schema, + hasSchema: true, + mostRecentIndexJob: MOCK_RESPONSE.mostRecentIndexJob, + unconfirmedFields: MOCK_RESPONSE.unconfirmedFields, + hasUnconfirmedFields: true, + hasNewUnsearchedFields: MOCK_RESPONSE.unsearchedUnconfirmedFields, + }); + }); + }); + + describe('onSchemaUpdateError', () => { + it('sets isUpdating & isModalOpen to false', () => { + mount({ isUpdating: true, isModalOpen: true }); + + SchemaLogic.actions.onSchemaUpdateError(); + + expect(SchemaLogic.values).toEqual({ + ...DEFAULT_VALUES, + isUpdating: false, + isModalOpen: false, + }); + }); + }); + + describe('openModal', () => { + it('sets isModalOpen to true', () => { + mount({ isModalOpen: false }); + + SchemaLogic.actions.openModal(); + + expect(SchemaLogic.values).toEqual({ + ...DEFAULT_VALUES, + isModalOpen: true, + }); + }); + }); + + describe('closeModal', () => { + it('sets isModalOpen to false', () => { + mount({ isModalOpen: true }); + + SchemaLogic.actions.closeModal(); + + expect(SchemaLogic.values).toEqual({ + ...DEFAULT_VALUES, + isModalOpen: false, + }); + }); + }); + }); + + describe('selectors', () => { + describe('hasSchema', () => { + it('returns true when the schema obj has items', () => { + mountAndSetSchema({ schema: { test: SchemaType.Text } }); + expect(SchemaLogic.values.hasSchema).toEqual(true); + }); + + it('returns false when the schema obj is empty', () => { + mountAndSetSchema({ schema: {} }); + expect(SchemaLogic.values.hasSchema).toEqual(false); + }); + }); + + describe('hasSchemaChanged', () => { + it('returns true when the schema state is different from the cachedSchema state', () => { + mountAndSetSchema({ + schema: { test: SchemaType.Text }, + cachedSchema: { test: SchemaType.Number }, + }); + + expect(SchemaLogic.values.hasSchemaChanged).toEqual(true); + }); + + it('returns false when the stored schema is the same as cachedSchema', () => { + mountAndSetSchema({ + schema: { test: SchemaType.Text }, + cachedSchema: { test: SchemaType.Text }, + }); + + expect(SchemaLogic.values.hasSchemaChanged).toEqual(false); + }); + }); + + describe('hasUnconfirmedFields', () => { + it('returns true when the unconfirmedFields array has items', () => { + mount({ unconfirmedFields: ['hello_world'] }); + expect(SchemaLogic.values.hasUnconfirmedFields).toEqual(true); + }); + + it('returns false when the unconfirmedFields array is empty', () => { + mount({ unconfirmedFields: [] }); + expect(SchemaLogic.values.hasUnconfirmedFields).toEqual(false); + }); + }); + }); + + describe('listeners', () => { + describe('addSchemaField', () => { + describe('if the schema field already exists', () => { + it('flashes an error and closes the modal', () => { + mountAndSetSchema({ schema: { existing_field: SchemaType.Text } }); + jest.spyOn(SchemaLogic.actions, 'closeModal'); + + SchemaLogic.actions.addSchemaField('existing_field', SchemaType.Text); + + expect(setErrorMessage).toHaveBeenCalledWith('Field name already exists: existing_field'); + expect(SchemaLogic.actions.closeModal).toHaveBeenCalled(); + }); + }); + + describe('if the schema field does not already exist', () => { + it('updates the schema state and calls updateSchema with a custom success message', () => { + mount(); + jest.spyOn(SchemaLogic.actions, 'setSchema'); + jest.spyOn(SchemaLogic.actions, 'updateSchema'); + + SchemaLogic.actions.addSchemaField('new_field', SchemaType.Date); + + expect(SchemaLogic.actions.setSchema).toHaveBeenCalledWith({ + new_field: SchemaType.Date, + }); + expect(SchemaLogic.actions.updateSchema).toHaveBeenCalledWith( + 'New field added: new_field' + ); + }); + }); + }); + + describe('updateSchemaFieldType', () => { + it("updates an existing schema key's field type value", async () => { + mountAndSetSchema({ schema: { existing_field: SchemaType.Text } }); + jest.spyOn(SchemaLogic.actions, 'setSchema'); + + SchemaLogic.actions.updateSchemaFieldType('existing_field', SchemaType.Geolocation); + + expect(SchemaLogic.actions.setSchema).toHaveBeenCalledWith({ + existing_field: SchemaType.Geolocation, + }); + }); + }); + + describe('updateSchema', () => { + it('sets isUpdating to true', () => { + mount({ isUpdating: false }); + + SchemaLogic.actions.updateSchema(); + + expect(SchemaLogic.values).toEqual({ + ...DEFAULT_VALUES, + isUpdating: true, + }); + }); + + it('should make an API call and then set schema state', async () => { + http.post.mockReturnValueOnce(Promise.resolve(MOCK_RESPONSE)); + mount(); + jest.spyOn(SchemaLogic.actions, 'onSchemaLoad'); + + SchemaLogic.actions.updateSchema(); + await nextTick(); + + expect(http.post).toHaveBeenCalledWith('/api/app_search/engines/some-engine/schema', { + body: '{}', + }); + expect(SchemaLogic.actions.onSchemaLoad).toHaveBeenCalledWith(MOCK_RESPONSE); + }); + + it('should call flashSuccessToast with a custom success message if passed', async () => { + http.post.mockReturnValueOnce(Promise.resolve(MOCK_RESPONSE)); + mount(); + + SchemaLogic.actions.updateSchema('wow it worked!!'); + await nextTick(); + + expect(flashSuccessToast).toHaveBeenCalledWith('wow it worked!!'); + }); + + it('should always call EngineLogic.actions.initializeEngine to refresh engine-wide state', async () => { + mount(); + + SchemaLogic.actions.updateSchema(); + await nextTick(); + + expect(mockEngineActions.initializeEngine).toHaveBeenCalled(); + }); + + it('handles errors and resets bad schema state back to cached/server values', async () => { + const MOCK_ERROR = 'Fields cannot contain more than 64 characters'; + const MOCK_CACHED_SCHEMA = { ok_field: SchemaType.Text }; + + http.post.mockReturnValueOnce(Promise.reject(MOCK_ERROR)); + mount({ cachedSchema: MOCK_CACHED_SCHEMA }); + jest.spyOn(SchemaLogic.actions, 'onSchemaUpdateError'); + jest.spyOn(SchemaLogic.actions, 'setSchema'); + + SchemaLogic.actions.updateSchema(); + await nextTick(); + + expect(flashAPIErrors).toHaveBeenCalledWith(MOCK_ERROR); + expect(SchemaLogic.actions.onSchemaUpdateError).toHaveBeenCalled(); + expect(SchemaLogic.actions.setSchema).toHaveBeenCalledWith(MOCK_CACHED_SCHEMA); + }); + }); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/schema_logic.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/schema_logic.ts new file mode 100644 index 0000000000000..3215a46c8e299 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/schema_logic.ts @@ -0,0 +1,164 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { kea, MakeLogicType } from 'kea'; +import { isEqual } from 'lodash'; + +import { + flashAPIErrors, + setErrorMessage, + flashSuccessToast, + clearFlashMessages, +} from '../../../shared/flash_messages'; +import { HttpLogic } from '../../../shared/http'; +import { Schema, SchemaType, IndexJob } from '../../../shared/schema/types'; +import { EngineLogic } from '../engine'; + +import { ADD_SCHEMA_ERROR, ADD_SCHEMA_SUCCESS, UPDATE_SCHEMA_SUCCESS } from './constants'; +import { SchemaBaseLogic, SchemaBaseValues, SchemaBaseActions } from './schema_base_logic'; +import { SchemaApiResponse } from './types'; + +interface SchemaValues extends SchemaBaseValues { + isUpdating: boolean; + hasSchema: boolean; + hasSchemaChanged: boolean; + cachedSchema: Schema; + mostRecentIndexJob: Partial; + unconfirmedFields: string[]; + hasUnconfirmedFields: boolean; + hasNewUnsearchedFields: boolean; + isModalOpen: boolean; +} + +interface SchemaActions extends SchemaBaseActions { + onSchemaLoad(response: SchemaApiResponse): SchemaApiResponse; + addSchemaField( + fieldName: string, + fieldType: SchemaType + ): { fieldName: string; fieldType: SchemaType }; + updateSchemaFieldType( + fieldName: string, + fieldType: SchemaType + ): { fieldName: string; fieldType: SchemaType }; + updateSchema(successMessage?: string): string | undefined; + onSchemaUpdateError(): void; + openModal(): void; + closeModal(): void; +} + +export const SchemaLogic = kea>({ + path: ['enterprise_search', 'app_search', 'schema_logic'], + connect: { + values: [SchemaBaseLogic, ['dataLoading', 'schema']], + actions: [SchemaBaseLogic, ['loadSchema', 'onSchemaLoad', 'setSchema']], + }, + actions: { + addSchemaField: (fieldName, fieldType) => ({ fieldName, fieldType }), + updateSchemaFieldType: (fieldName, fieldType) => ({ fieldName, fieldType }), + updateSchema: (successMessage) => successMessage, + onSchemaUpdateError: true, + openModal: true, + closeModal: true, + }, + reducers: { + isUpdating: [ + false, + { + updateSchema: () => true, + onSchemaLoad: () => false, + onSchemaUpdateError: () => false, + }, + ], + cachedSchema: [ + {}, + { + onSchemaLoad: (_, { schema }) => schema, + }, + ], + mostRecentIndexJob: [ + {}, + { + onSchemaLoad: (_, { mostRecentIndexJob }) => mostRecentIndexJob, + }, + ], + unconfirmedFields: [ + [], + { + onSchemaLoad: (_, { unconfirmedFields }) => unconfirmedFields, + }, + ], + hasNewUnsearchedFields: [ + false, + { + onSchemaLoad: (_, { unsearchedUnconfirmedFields }) => unsearchedUnconfirmedFields, + }, + ], + isModalOpen: [ + false, + { + openModal: () => true, + closeModal: () => false, + onSchemaLoad: () => false, + onSchemaUpdateError: () => false, + }, + ], + }, + selectors: { + hasSchema: [(selectors) => [selectors.schema], (schema) => Object.keys(schema).length > 0], + hasSchemaChanged: [ + (selectors) => [selectors.schema, selectors.cachedSchema], + (schema, cachedSchema) => !isEqual(schema, cachedSchema), + ], + hasUnconfirmedFields: [ + (selectors) => [selectors.unconfirmedFields], + (unconfirmedFields) => unconfirmedFields.length > 0, + ], + }, + listeners: ({ actions, values }) => ({ + addSchemaField: ({ fieldName, fieldType }) => { + if (values.schema.hasOwnProperty(fieldName)) { + setErrorMessage(ADD_SCHEMA_ERROR(fieldName)); + actions.closeModal(); + } else { + const updatedSchema = { ...values.schema }; + updatedSchema[fieldName] = fieldType; + actions.setSchema(updatedSchema); + actions.updateSchema(ADD_SCHEMA_SUCCESS(fieldName)); + } + }, + updateSchemaFieldType: ({ fieldName, fieldType }) => { + const updatedSchema = { ...values.schema }; + updatedSchema[fieldName] = fieldType; + actions.setSchema(updatedSchema); + }, + updateSchema: async (successMessage) => { + const { schema } = values; + const { http } = HttpLogic.values; + const { engineName } = EngineLogic.values; + + clearFlashMessages(); + + try { + const response = await http.post(`/api/app_search/engines/${engineName}/schema`, { + body: JSON.stringify(schema), + }); + actions.onSchemaLoad(response); + flashSuccessToast(successMessage || UPDATE_SCHEMA_SUCCESS); + } catch (e) { + flashAPIErrors(e); + actions.onSchemaUpdateError(); + // Restore updated schema back to server/cached schema, so we don't keep + // erroneous or bad fields in-state + actions.setSchema(values.cachedSchema); + } finally { + // Re-fetch engine data so that other views also dynamically update + // (e.g. Documents results, nav icons for invalid boosts or unconfirmed flags) + EngineLogic.actions.initializeEngine(); + } + }, + }), +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/schema_meta_engine_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/schema_meta_engine_logic.test.ts new file mode 100644 index 0000000000000..f265fb2d74113 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/schema_meta_engine_logic.test.ts @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { LogicMounter } from '../../../__mocks__'; + +import { SchemaType } from '../../../shared/schema/types'; + +import { MetaEngineSchemaLogic } from './schema_meta_engine_logic'; + +describe('MetaEngineSchemaLogic', () => { + const { mount } = new LogicMounter(MetaEngineSchemaLogic); + + const MOCK_RESPONSE = { + schema: { + some_text_field: SchemaType.Text, + some_number_field: SchemaType.Number, + }, + fields: { + some_text_field: { + text: ['source-engine-a', 'source-engine-b'], + }, + }, + conflictingFields: { + some_number_field: { + number: ['source-engine-a'], + text: ['source-engine-b'], + }, + }, + }; + + const DEFAULT_VALUES = { + dataLoading: true, + schema: {}, + fields: {}, + conflictingFields: {}, + conflictingFieldsCount: 0, + hasConflicts: false, + }; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('has expected default values', () => { + mount(); + expect(MetaEngineSchemaLogic.values).toEqual(DEFAULT_VALUES); + }); + + describe('actions', () => { + describe('onSchemaLoad', () => { + it('stores the API response in state', () => { + mount(); + + MetaEngineSchemaLogic.actions.onSchemaLoad(MOCK_RESPONSE); + + expect(MetaEngineSchemaLogic.values).toEqual({ + ...DEFAULT_VALUES, + // SchemaBaseLogic + dataLoading: false, + schema: MOCK_RESPONSE.schema, + // MetaEngineSchemaLogic + fields: MOCK_RESPONSE.fields, + conflictingFields: MOCK_RESPONSE.conflictingFields, + hasConflicts: true, + conflictingFieldsCount: 1, + }); + }); + }); + }); + + describe('selectors', () => { + describe('conflictingFieldsCount', () => { + it('returns the number of conflicting fields', () => { + mount({ conflictingFields: { field_a: {}, field_b: {} } }); + expect(MetaEngineSchemaLogic.values.conflictingFieldsCount).toEqual(2); + }); + }); + + describe('hasConflictingFields', () => { + it('returns true when the conflictingFields obj has items', () => { + mount({ conflictingFields: { field_c: {} } }); + expect(MetaEngineSchemaLogic.values.hasConflicts).toEqual(true); + }); + + it('returns false when the conflictingFields obj is empty', () => { + mount({ conflictingFields: {} }); + expect(MetaEngineSchemaLogic.values.hasConflicts).toEqual(false); + }); + }); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/schema_meta_engine_logic.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/schema_meta_engine_logic.ts new file mode 100644 index 0000000000000..5c8ab0f4662e2 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/schema_meta_engine_logic.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { kea, MakeLogicType } from 'kea'; + +import { SchemaBaseLogic, SchemaBaseValues, SchemaBaseActions } from './schema_base_logic'; +import { MetaEngineSchemaApiResponse } from './types'; + +interface MetaEngineSchemaValues extends SchemaBaseValues { + fields: MetaEngineSchemaApiResponse['fields']; + conflictingFields: MetaEngineSchemaApiResponse['conflictingFields']; + conflictingFieldsCount: number; + hasConflicts: boolean; +} + +interface MetaEngineSchemaActions extends SchemaBaseActions { + onSchemaLoad(response: MetaEngineSchemaApiResponse): MetaEngineSchemaApiResponse; +} + +export const MetaEngineSchemaLogic = kea< + MakeLogicType +>({ + path: ['enterprise_search', 'app_search', 'meta_engine_schema_logic'], + connect: { + values: [SchemaBaseLogic, ['dataLoading', 'schema']], + actions: [SchemaBaseLogic, ['loadSchema', 'onSchemaLoad']], + }, + reducers: { + fields: [ + {}, + { + onSchemaLoad: (_, { fields }) => fields, + }, + ], + conflictingFields: [ + {}, + { + onSchemaLoad: (_, { conflictingFields }) => conflictingFields, + }, + ], + }, + selectors: { + conflictingFieldsCount: [ + (selectors) => [selectors.conflictingFields], + (conflictingFields) => Object.keys(conflictingFields).length, + ], + hasConflicts: [ + (selectors) => [selectors.conflictingFieldsCount], + (conflictingFieldsCount) => conflictingFieldsCount > 0, + ], + }, +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/types.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/types.ts new file mode 100644 index 0000000000000..9d79909e5549b --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/types.ts @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + Schema, + IndexJob, + SchemaConflicts, + FieldCoercionErrors, +} from '../../../shared/schema/types'; + +export interface SchemaApiResponse { + schema: Schema; + mostRecentIndexJob: IndexJob; + unconfirmedFields: string[]; + unsearchedUnconfirmedFields: boolean; +} + +export interface MetaEngineSchemaApiResponse { + schema: Schema; + fields: SchemaConflicts; + conflictingFields: SchemaConflicts; +} + +export interface ReindexJobApiResponse { + fieldCoercionErrors: FieldCoercionErrors; +} diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/views/meta_engine_schema.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/views/meta_engine_schema.test.tsx index 8412af6455285..a6e9eef8efa70 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/views/meta_engine_schema.test.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/views/meta_engine_schema.test.tsx @@ -5,15 +5,29 @@ * 2.0. */ +import { setMockValues, setMockActions } from '../../../../__mocks__'; +import '../../../../__mocks__/shallow_useeffect.mock'; + import React from 'react'; import { shallow } from 'enzyme'; +import { Loading } from '../../../../shared/loading'; + import { MetaEngineSchema } from './'; describe('MetaEngineSchema', () => { + const values = { + dataLoading: false, + }; + const actions = { + loadSchema: jest.fn(), + }; + beforeEach(() => { jest.clearAllMocks(); + setMockValues(values); + setMockActions(actions); }); it('renders', () => { @@ -22,4 +36,17 @@ describe('MetaEngineSchema', () => { expect(wrapper.isEmptyRender()).toBe(false); // TODO: Check for schema components }); + + it('calls loadSchema on mount', () => { + shallow(); + + expect(actions.loadSchema).toHaveBeenCalled(); + }); + + it('renders a loading state', () => { + setMockValues({ ...values, dataLoading: true }); + const wrapper = shallow(); + + expect(wrapper.find(Loading)).toHaveLength(1); + }); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/views/meta_engine_schema.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/views/meta_engine_schema.tsx index d79ddae3d9b78..234fcdb5a5a50 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/views/meta_engine_schema.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/views/meta_engine_schema.tsx @@ -5,14 +5,28 @@ * 2.0. */ -import React from 'react'; +import React, { useEffect } from 'react'; + +import { useValues, useActions } from 'kea'; import { EuiPageHeader, EuiPageContentBody } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { FlashMessages } from '../../../../shared/flash_messages'; +import { Loading } from '../../../../shared/loading'; + +import { MetaEngineSchemaLogic } from '../schema_meta_engine_logic'; export const MetaEngineSchema: React.FC = () => { + const { loadSchema } = useActions(MetaEngineSchemaLogic); + const { dataLoading } = useValues(MetaEngineSchemaLogic); + + useEffect(() => { + loadSchema(); + }, []); + + if (dataLoading) return ; + return ( <> { + const values = { + dataLoading: false, + hasSchema: true, + hasSchemaChanged: false, + isUpdating: false, + isModalOpen: false, + }; + const actions = { + loadSchema: jest.fn(), + updateSchema: jest.fn(), + addSchemaField: jest.fn(), + openModal: jest.fn(), + closeModal: jest.fn(), + }; + beforeEach(() => { jest.clearAllMocks(); + setMockValues(values); + setMockActions(actions); }); it('renders', () => { const wrapper = shallow(); - expect(wrapper.isEmptyRender()).toBe(false); - // TODO: Check for schema components + expect(wrapper.find(SchemaCallouts)).toHaveLength(1); + expect(wrapper.find(SchemaTable)).toHaveLength(1); + }); + + it('calls loadSchema on mount', () => { + shallow(); + + expect(actions.loadSchema).toHaveBeenCalled(); + }); + + it('renders a loading state', () => { + setMockValues({ ...values, dataLoading: true }); + const wrapper = shallow(); + + expect(wrapper.find(Loading)).toHaveLength(1); + }); + + it('renders an empty state', () => { + setMockValues({ ...values, hasSchema: false }); + const wrapper = shallow(); + + expect(wrapper.find(EmptyState)).toHaveLength(1); }); - it('renders page action buttons', () => { - const wrapper = shallow() - .find(EuiPageHeader) - .dive() - .children() - .dive(); + describe('page action buttons', () => { + const subject = () => + shallow() + .find(EuiPageHeader) + .dive() + .children() + .dive(); + + it('renders', () => { + const wrapper = subject(); + expect(wrapper.find(EuiButton)).toHaveLength(2); + }); + + it('renders loading/disabled state when schema is updating', () => { + setMockValues({ isUpdating: true }); + const wrapper = subject(); + + expect(wrapper.find('[data-test-subj="updateSchemaButton"]').prop('isLoading')).toBe(true); + expect(wrapper.find('[data-test-subj="addSchemaFieldModalButton"]').prop('disabled')).toBe( + true + ); + }); + + describe('add button', () => { + it('opens the add schema field modal', () => { + const wrapper = subject(); + + wrapper.find('[data-test-subj="addSchemaFieldModalButton"]').simulate('click'); + expect(actions.openModal).toHaveBeenCalled(); + }); + }); + + describe('update button', () => { + describe('when nothing on the page has changed', () => { + it('is disabled', () => { + const wrapper = subject(); + + expect(wrapper.find('[data-test-subj="updateSchemaButton"]').prop('disabled')).toBe(true); + }); + }); + + describe('when schema has been changed locally', () => { + it('is enabled', () => { + setMockValues({ ...values, hasSchemaChanged: true }); + const wrapper = subject(); + + expect(wrapper.find('[data-test-subj="updateSchemaButton"]').prop('disabled')).toBe( + false + ); + }); + + it('calls updateSchema on click', () => { + setMockValues({ ...values, hasSchemaChanged: true }); + const wrapper = subject(); + + wrapper.find('[data-test-subj="updateSchemaButton"]').simulate('click'); + expect(actions.updateSchema).toHaveBeenCalled(); + }); + }); + }); + }); + + it('renders a modal that lets a user add a new schema field', () => { + setMockValues({ isModalOpen: true }); + const wrapper = shallow(); - expect(wrapper.find(EuiButton)).toHaveLength(2); - // TODO: Expect click actions + expect(wrapper.find(SchemaAddFieldModal)).toHaveLength(1); }); }); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/views/schema.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/views/schema.tsx index ad53fd2c718b2..7bc995b16468a 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/views/schema.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/schema/views/schema.tsx @@ -5,14 +5,34 @@ * 2.0. */ -import React from 'react'; +import React, { useEffect } from 'react'; + +import { useValues, useActions } from 'kea'; import { EuiPageHeader, EuiButton, EuiPageContentBody } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import { FlashMessages } from '../../../../shared/flash_messages'; +import { Loading } from '../../../../shared/loading'; +import { SchemaAddFieldModal } from '../../../../shared/schema'; + +import { SchemaCallouts, SchemaTable, EmptyState } from '../components'; +import { SchemaLogic } from '../schema_logic'; export const Schema: React.FC = () => { + const { loadSchema, updateSchema, addSchemaField, openModal, closeModal } = useActions( + SchemaLogic + ); + const { dataLoading, isUpdating, hasSchema, hasSchemaChanged, isModalOpen } = useValues( + SchemaLogic + ); + + useEffect(() => { + loadSchema(); + }, []); + + if (dataLoading) return ; + return ( <> { { defaultMessage: 'Add new fields or change the types of existing ones.' } )} rightSideItems={[ - + updateSchema()} + data-test-subj="updateSchemaButton" + > {i18n.translate( 'xpack.enterpriseSearch.appSearch.engine.schema.updateSchemaButtonLabel', { defaultMessage: 'Update types' } )} , - + {i18n.translate( 'xpack.enterpriseSearch.appSearch.engine.schema.createSchemaFieldButtonLabel', { defaultMessage: 'Create a schema field' } @@ -39,7 +70,13 @@ export const Schema: React.FC = () => { ]} /> - TODO + + + {hasSchema ? : } + {isModalOpen && ( + + )} + ); }; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/components/search_ui_form.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/components/search_ui_form.test.tsx new file mode 100644 index 0000000000000..b015c2dec6c0a --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/components/search_ui_form.test.tsx @@ -0,0 +1,202 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +jest.mock('../utils', () => ({ + generatePreviewUrl: jest.fn(), +})); + +import { setMockValues, setMockActions } from '../../../../__mocks__'; + +import React from 'react'; + +import { shallow } from 'enzyme'; + +import { ActiveField } from '../types'; +import { generatePreviewUrl } from '../utils'; + +import { SearchUIForm } from './search_ui_form'; + +describe('SearchUIForm', () => { + const values = { + validFields: ['title', 'url', 'category', 'size'], + validSortFields: ['title', 'url', 'category', 'size'], + validFacetFields: ['title', 'url', 'category', 'size'], + titleField: 'title', + urlField: 'url', + facetFields: ['category'], + sortFields: ['size'], + }; + const actions = { + onActiveFieldChange: jest.fn(), + onFacetFieldsChange: jest.fn(), + onSortFieldsChange: jest.fn(), + onTitleFieldChange: jest.fn(), + onUrlFieldChange: jest.fn(), + }; + + beforeAll(() => { + setMockValues(values); + setMockActions(actions); + }); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('renders', () => { + const wrapper = shallow(); + expect(wrapper.find('[data-test-subj="selectTitle"]').exists()).toBe(true); + expect(wrapper.find('[data-test-subj="selectFilters"]').exists()).toBe(true); + expect(wrapper.find('[data-test-subj="selectSort"]').exists()).toBe(true); + expect(wrapper.find('[data-test-subj="selectUrl"]').exists()).toBe(true); + }); + + describe('title field', () => { + const subject = () => shallow().find('[data-test-subj="selectTitle"]'); + + it('renders with its value set from state', () => { + setMockValues({ + ...values, + titleField: 'foo', + }); + + expect(subject().prop('value')).toBe('foo'); + }); + + it('updates state with new value when changed', () => { + subject().simulate('change', { target: { value: 'foo' } }); + expect(actions.onTitleFieldChange).toHaveBeenCalledWith('foo'); + }); + + it('updates active field in state on focus', () => { + subject().simulate('focus'); + expect(actions.onActiveFieldChange).toHaveBeenCalledWith(ActiveField.Title); + }); + + it('removes active field in state on blur', () => { + subject().simulate('blur'); + expect(actions.onActiveFieldChange).toHaveBeenCalledWith(ActiveField.None); + }); + }); + + describe('url field', () => { + const subject = () => shallow().find('[data-test-subj="selectUrl"]'); + + it('renders with its value set from state', () => { + setMockValues({ + ...values, + urlField: 'foo', + }); + + expect(subject().prop('value')).toBe('foo'); + }); + + it('updates state with new value when changed', () => { + subject().simulate('change', { target: { value: 'foo' } }); + expect(actions.onUrlFieldChange).toHaveBeenCalledWith('foo'); + }); + + it('updates active field in state on focus', () => { + subject().simulate('focus'); + expect(actions.onActiveFieldChange).toHaveBeenCalledWith(ActiveField.Url); + }); + + it('removes active field in state on blur', () => { + subject().simulate('blur'); + expect(actions.onActiveFieldChange).toHaveBeenCalledWith(ActiveField.None); + }); + }); + + describe('filters field', () => { + const subject = () => shallow().find('[data-test-subj="selectFilters"]'); + + it('renders with its value set from state', () => { + setMockValues({ + ...values, + facetFields: ['foo'], + }); + + expect(subject().prop('selectedOptions')).toEqual([ + { label: 'foo', text: 'foo', value: 'foo' }, + ]); + }); + + it('updates state with new value when changed', () => { + subject().simulate('change', [ + { label: 'foo', text: 'foo', value: 'foo' }, + { label: 'bar', text: 'bar', value: 'bar' }, + ]); + expect(actions.onFacetFieldsChange).toHaveBeenCalledWith(['foo', 'bar']); + }); + + it('updates active field in state on focus', () => { + subject().simulate('focus'); + expect(actions.onActiveFieldChange).toHaveBeenCalledWith(ActiveField.Filter); + }); + + it('removes active field in state on blur', () => { + subject().simulate('blur'); + expect(actions.onActiveFieldChange).toHaveBeenCalledWith(ActiveField.None); + }); + }); + + describe('sorts field', () => { + const subject = () => shallow().find('[data-test-subj="selectSort"]'); + + it('renders with its value set from state', () => { + setMockValues({ + ...values, + sortFields: ['foo'], + }); + + expect(subject().prop('selectedOptions')).toEqual([ + { label: 'foo', text: 'foo', value: 'foo' }, + ]); + }); + + it('updates state with new value when changed', () => { + subject().simulate('change', [ + { label: 'foo', text: 'foo', value: 'foo' }, + { label: 'bar', text: 'bar', value: 'bar' }, + ]); + expect(actions.onSortFieldsChange).toHaveBeenCalledWith(['foo', 'bar']); + }); + + it('updates active field in state on focus', () => { + subject().simulate('focus'); + expect(actions.onActiveFieldChange).toHaveBeenCalledWith(ActiveField.Sort); + }); + + it('removes active field in state on blur', () => { + subject().simulate('blur'); + expect(actions.onActiveFieldChange).toHaveBeenCalledWith(ActiveField.None); + }); + }); + + it('includes a link to generate the preview', () => { + (generatePreviewUrl as jest.Mock).mockReturnValue('http://www.example.com?foo=bar'); + + setMockValues({ + ...values, + urlField: 'foo', + titleField: 'bar', + facetFields: ['baz'], + sortFields: ['qux'], + }); + + const subject = () => + shallow().find('[data-test-subj="generateSearchUiPreview"]'); + + expect(subject().prop('href')).toBe('http://www.example.com?foo=bar'); + expect(generatePreviewUrl).toHaveBeenCalledWith({ + urlField: 'foo', + titleField: 'bar', + facets: ['baz'], + sortFields: ['qux'], + }); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/components/search_ui_form.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/components/search_ui_form.tsx new file mode 100644 index 0000000000000..15bd699be721c --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/components/search_ui_form.tsx @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +import { useValues, useActions } from 'kea'; + +import { EuiForm, EuiFormRow, EuiSelect, EuiComboBox, EuiButton } from '@elastic/eui'; + +import { + TITLE_FIELD_LABEL, + TITLE_FIELD_HELP_TEXT, + FILTER_FIELD_LABEL, + FILTER_FIELD_HELP_TEXT, + SORT_FIELD_LABEL, + SORT_FIELD_HELP_TEXT, + URL_FIELD_LABEL, + URL_FIELD_HELP_TEXT, + GENERATE_PREVIEW_BUTTON_LABEL, +} from '../i18n'; +import { SearchUILogic } from '../search_ui_logic'; +import { ActiveField } from '../types'; +import { generatePreviewUrl } from '../utils'; + +export const SearchUIForm: React.FC = () => { + const { + validFields, + validSortFields, + validFacetFields, + titleField, + urlField, + facetFields, + sortFields, + } = useValues(SearchUILogic); + const { + onActiveFieldChange, + onFacetFieldsChange, + onSortFieldsChange, + onTitleFieldChange, + onUrlFieldChange, + } = useActions(SearchUILogic); + + const previewHref = generatePreviewUrl({ + titleField, + urlField, + facets: facetFields, + sortFields, + }); + + const formatSelectOption = (fieldName: string) => { + return { text: fieldName, value: fieldName }; + }; + const formatMultiOptions = (fieldNames: string[]) => + fieldNames.map((fieldName) => ({ label: fieldName, text: fieldName, value: fieldName })); + const formatMultiOptionsWithEmptyOption = (fieldNames: string[]) => [ + { label: '', text: '', value: '' }, + ...formatMultiOptions(fieldNames), + ]; + + const optionFields = formatMultiOptionsWithEmptyOption(validFields); + const sortOptionFields = formatMultiOptions(validSortFields); + const facetOptionFields = formatMultiOptions(validFacetFields); + const selectedTitleOption = formatSelectOption(titleField); + const selectedURLOption = formatSelectOption(urlField); + const selectedSortOptions = formatMultiOptions(sortFields); + const selectedFacetOptions = formatMultiOptions(facetFields); + + return ( + + + onTitleFieldChange(e.target.value)} + fullWidth + onFocus={() => onActiveFieldChange(ActiveField.Title)} + onBlur={() => onActiveFieldChange(ActiveField.None)} + hasNoInitialSelection + data-test-subj="selectTitle" + /> + + + onFacetFieldsChange(newValues.map((field) => field.value!))} + onFocus={() => onActiveFieldChange(ActiveField.Filter)} + onBlur={() => onActiveFieldChange(ActiveField.None)} + fullWidth + data-test-subj="selectFilters" + /> + + + onSortFieldsChange(newValues.map((field) => field.value!))} + onFocus={() => onActiveFieldChange(ActiveField.Sort)} + onBlur={() => onActiveFieldChange(ActiveField.None)} + fullWidth + data-test-subj="selectSort" + /> + + + + onUrlFieldChange(e.target.value)} + fullWidth + onFocus={() => onActiveFieldChange(ActiveField.Url)} + onBlur={() => onActiveFieldChange(ActiveField.None)} + hasNoInitialSelection + data-test-subj="selectUrl" + /> + + + {GENERATE_PREVIEW_BUTTON_LABEL} + + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/components/search_ui_graphic.scss b/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/components/search_ui_graphic.scss new file mode 100644 index 0000000000000..cc49789358a28 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/components/search_ui_graphic.scss @@ -0,0 +1,188 @@ +.searchUIGraphic { + transform-style: preserve-3d; + transform: rotate3d(0, 1, 0, -8deg); + + #search-area { + .outerBox { + fill: $euiColorLightShade; + } + .field { + fill: $euiColorEmptyShade; + } + .searchIcon { + fill: $euiColorDarkShade; + } + .type { + fill: $euiColorDarkShade; + } + } + #background { + fill: $euiColorLightestShade; + } + #results { + .outerBox { + fill: $euiColorEmptyShade; + stroke: $euiColorLightShade; + stroke-width: $euiBorderWidthThin; + } + .shoe { + fill: $euiColorMediumShade; + } + .url { + fill: $euiColorDarkShade; + transform: translateY(5px); + } + .titleCopy { + fill: $euiColorDarkShade; + } + .titleBox { + fill: $euiColorEmptyShade; + } + .blockIn { + fill: $euiColorLightShade; + } + } + #filter { + .outerBox { + fill: $euiColorEmptyShade; + stroke: $euiColorLightShade; + stroke-width: 1px; + } + .header { + fill: $euiColorDarkShade; + } + .checkbox { + fill: $euiColorDarkShade; + } + .check { + fill: $euiColorEmptyShade; + } + .filterCopy { + fill: $euiColorDarkShade; + } + } + #sort { + .outerBox { + fill: $euiColorEmptyShade; + stroke: $euiColorLightShade; + stroke-width: 1px; + } + .header { + fill: $euiColorDarkShade; + } + .selectCopy { + fill: $euiColorDarkShade; + } + .selectBox { + fill: $euiColorEmptyShade; + stroke: $euiColorLightShade; + stroke-width: 1px; + } + .selectControl { + fill: $euiColorDarkShade; + } + } + #pagination { + .outerBox { + fill: $euiColorLightShade; + } + .arrow { + fill: $euiColorEmptyShade; + } + } + + &.activeTitle { + #results { + .titleBox { + fill: $euiColorEmptyShade; + stroke: $euiColorPrimary; + stroke-width: 1px; + } + .titleCopy { + fill: $euiColorPrimary; + } + .outerBox { + fill: $euiColorEmptyShade; + stroke: $euiColorPrimary; + } + .url { + fill: $euiColorPrimary; + opacity: .1; + } + .shoe { + fill: $euiColorPrimary; + opacity: .1; + } + } + } + + &.activeUrl { + #results { + .outerBox { + fill: $euiColorEmptyShade; + stroke: $euiColorPrimary; + stroke-width: 1px; + } + .url { + fill: $euiColorPrimary; + } + .titleBox { + fill: $euiColorEmptyShade; + } + .titleCopy { + fill: $euiColorPrimary; + opacity: .1; + } + .shoe { + fill: $euiColorPrimary; + opacity: .1; + } + } + } + + &.activeFilter { + #filter { + .outerBox { + fill: $euiColorEmptyShade; + stroke: $euiColorPrimary; + stroke-width: $euiBorderWidthThick; + } + .header { + fill: $euiColorPrimary; + } + .checkbox { + fill: $euiColorPrimary; + } + .check { + fill: $euiColorEmptyShade; + } + .filterCopy { + fill: $euiColorPrimary; + } + } + } + + &.activeSort { + #sort { + .outerBox { + fill: $euiColorEmptyShade; + stroke: $euiColorPrimary; + stroke-width: 2px; + } + .header { + fill: $euiColorPrimary; + } + .selectCopy { + fill: $euiColorPrimary; + } + .selectBox { + fill: $euiColorEmptyShade; + stroke: $euiColorPrimary; + stroke-width: 1px; + } + .selectControl { + fill: $euiColorPrimary; + } + } + } +} diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/components/search_ui_graphic.test.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/components/search_ui_graphic.test.tsx new file mode 100644 index 0000000000000..0326cef9a2455 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/components/search_ui_graphic.test.tsx @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { setMockValues } from '../../../../__mocks__'; + +import React from 'react'; + +import { shallow } from 'enzyme'; + +import { ActiveField } from '../types'; + +import { SearchUIGraphic } from './search_ui_graphic'; + +describe('SearchUIGraphic', () => { + const values = { + activeField: ActiveField.Sort, + }; + + beforeEach(() => { + jest.clearAllMocks(); + setMockValues(values); + }); + + it('renders an svg with a className determined by the currently active field', () => { + const wrapper = shallow(); + expect(wrapper.hasClass('activeSort')).toBe(true); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/components/search_ui_graphic.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/components/search_ui_graphic.tsx new file mode 100644 index 0000000000000..02b7ac3423227 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/components/search_ui_graphic.tsx @@ -0,0 +1,232 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +import { useValues } from 'kea'; + +import { SearchUILogic } from '../search_ui_logic'; + +import './search_ui_graphic.scss'; + +export const SearchUIGraphic: React.FC = () => { + const { activeField } = useValues(SearchUILogic); + const svgClass = 'searchUIGraphic'; + + return ( + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/i18n.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/i18n.ts new file mode 100644 index 0000000000000..dfdc1d1db4c39 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/i18n.ts @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; + +export const SEARCH_UI_TITLE = i18n.translate( + 'xpack.enterpriseSearch.appSearch.engine.searchUI.title', + { defaultMessage: 'Search UI' } +); + +export const TITLE_FIELD_LABEL = i18n.translate( + 'xpack.enterpriseSearch.appSearch.engine.searchUI.titleFieldLabel', + { defaultMessage: 'Title field (Optional)' } +); +export const TITLE_FIELD_HELP_TEXT = i18n.translate( + 'xpack.enterpriseSearch.appSearch.engine.searchUI.titleFieldHelpText', + { defaultMessage: 'Used as the top-level visual identifier for every rendered result' } +); +export const FILTER_FIELD_LABEL = i18n.translate( + 'xpack.enterpriseSearch.appSearch.engine.searchUI.filterFieldLabel', + { defaultMessage: 'Filter fields (Optional)' } +); +export const FILTER_FIELD_HELP_TEXT = i18n.translate( + 'xpack.enterpriseSearch.appSearch.engine.searchUI.filterFieldHelpText', + { defaultMessage: 'Faceted values rendered as filters and available as query refinement' } +); +export const SORT_FIELD_LABEL = i18n.translate( + 'xpack.enterpriseSearch.appSearch.engine.searchUI.sortFieldLabel', + { defaultMessage: 'Sort fields (Optional)' } +); +export const SORT_FIELD_HELP_TEXT = i18n.translate( + 'xpack.enterpriseSearch.appSearch.engine.searchUI.sortHelpText', + { defaultMessage: 'Used to display result sorting options, ascending and descending' } +); +export const URL_FIELD_LABEL = i18n.translate( + 'xpack.enterpriseSearch.appSearch.engine.searchUI.urlFieldLabel', + { defaultMessage: 'URL field (Optional)' } +); +export const URL_FIELD_HELP_TEXT = i18n.translate( + 'xpack.enterpriseSearch.appSearch.engine.searchUI.urlFieldHelpText', + { defaultMessage: "Used as a result's link target, if applicable" } +); +export const GENERATE_PREVIEW_BUTTON_LABEL = i18n.translate( + 'xpack.enterpriseSearch.appSearch.engine.searchUI.generatePreviewButtonLabel', + { defaultMessage: 'Generate search experience' } +); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/index.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/index.ts index 5de1224a9f28a..e87606fa04c65 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/index.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/index.ts @@ -5,6 +5,6 @@ * 2.0. */ -export { SEARCH_UI_TITLE } from './constants'; +export { SEARCH_UI_TITLE } from './i18n'; export { SearchUI } from './search_ui'; export { SearchUILogic } from './search_ui_logic'; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/search_ui.tsx b/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/search_ui.tsx index d4e4d72e4740a..e75bc36177151 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/search_ui.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/search_ui.tsx @@ -9,14 +9,26 @@ import React, { useEffect } from 'react'; import { useActions } from 'kea'; -import { EuiPageHeader, EuiPageContentBody } from '@elastic/eui'; +import { + EuiPageHeader, + EuiPageContentBody, + EuiText, + EuiFlexItem, + EuiFlexGroup, + EuiSpacer, + EuiLink, +} from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; import { FlashMessages } from '../../../shared/flash_messages'; import { SetAppSearchChrome as SetPageChrome } from '../../../shared/kibana_chrome'; +import { DOCS_PREFIX } from '../../routes'; import { getEngineBreadcrumbs } from '../engine'; -import { SEARCH_UI_TITLE } from './constants'; +import { SearchUIForm } from './components/search_ui_form'; +import { SearchUIGraphic } from './components/search_ui_graphic'; +import { SEARCH_UI_TITLE } from './i18n'; import { SearchUILogic } from './search_ui_logic'; export const SearchUI: React.FC = () => { @@ -31,7 +43,51 @@ export const SearchUI: React.FC = () => { - TODO + + + + +

+ + + + ), + }} + /> +

+

+ + + + ), + }} + /> +

+ + + + + + + + + ); }; diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/search_ui_logic.test.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/search_ui_logic.test.ts index 29261f3a4031f..2191c633131bb 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/search_ui_logic.test.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/search_ui_logic.test.ts @@ -78,10 +78,10 @@ describe('SearchUILogic', () => { }); }); - describe('onURLFieldChange', () => { + describe('onUrlFieldChange', () => { it('sets the urlField value', () => { mount({ urlField: '' }); - SearchUILogic.actions.onURLFieldChange('foo'); + SearchUILogic.actions.onUrlFieldChange('foo'); expect(SearchUILogic.values).toEqual({ ...DEFAULT_VALUES, urlField: 'foo', @@ -128,6 +128,10 @@ describe('SearchUILogic', () => { validFields: ['test'], validSortFields: ['test'], validFacetFields: ['test'], + defaultValues: { + urlField: 'url', + titleField: 'title', + }, }; describe('loadFieldData', () => { @@ -142,7 +146,13 @@ describe('SearchUILogic', () => { expect(http.get).toHaveBeenCalledWith( '/api/app_search/engines/engine1/search_ui/field_config' ); - expect(SearchUILogic.actions.onFieldDataLoaded).toHaveBeenCalledWith(MOCK_RESPONSE); + expect(SearchUILogic.actions.onFieldDataLoaded).toHaveBeenCalledWith({ + validFields: ['test'], + validSortFields: ['test'], + validFacetFields: ['test'], + urlField: 'url', + titleField: 'title', + }); }); it('handles errors', async () => { diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/search_ui_logic.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/search_ui_logic.ts index 7b3454c9e8413..c9e2e5623d9fd 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/search_ui_logic.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/search_ui_logic.ts @@ -17,6 +17,8 @@ interface InitialFieldValues { validFields: string[]; validSortFields: string[]; validFacetFields: string[]; + urlField?: string; + titleField?: string; } interface SearchUIActions { loadFieldData(): void; @@ -25,7 +27,7 @@ interface SearchUIActions { onFacetFieldsChange(facetFields: string[]): { facetFields: string[] }; onSortFieldsChange(sortFields: string[]): { sortFields: string[] }; onTitleFieldChange(titleField: string): { titleField: string }; - onURLFieldChange(urlField: string): { urlField: string }; + onUrlFieldChange(urlField: string): { urlField: string }; } interface SearchUIValues { @@ -49,7 +51,7 @@ export const SearchUILogic = kea> onFacetFieldsChange: (facetFields) => ({ facetFields }), onSortFieldsChange: (sortFields) => ({ sortFields }), onTitleFieldChange: (titleField) => ({ titleField }), - onURLFieldChange: (urlField) => ({ urlField }), + onUrlFieldChange: (urlField) => ({ urlField }), }), reducers: () => ({ dataLoading: [ @@ -61,8 +63,20 @@ export const SearchUILogic = kea> validFields: [[], { onFieldDataLoaded: (_, { validFields }) => validFields }], validSortFields: [[], { onFieldDataLoaded: (_, { validSortFields }) => validSortFields }], validFacetFields: [[], { onFieldDataLoaded: (_, { validFacetFields }) => validFacetFields }], - titleField: ['', { onTitleFieldChange: (_, { titleField }) => titleField }], - urlField: ['', { onURLFieldChange: (_, { urlField }) => urlField }], + titleField: [ + '', + { + onTitleFieldChange: (_, { titleField }) => titleField, + onFieldDataLoaded: (_, { titleField }) => titleField || '', + }, + ], + urlField: [ + '', + { + onUrlFieldChange: (_, { urlField }) => urlField, + onFieldDataLoaded: (_, { urlField }) => urlField || '', + }, + ], facetFields: [[], { onFacetFieldsChange: (_, { facetFields }) => facetFields }], sortFields: [[], { onSortFieldsChange: (_, { sortFields }) => sortFields }], activeField: [ActiveField.None, { onActiveFieldChange: (_, { activeField }) => activeField }], @@ -76,8 +90,20 @@ export const SearchUILogic = kea> try { const initialFieldValues = await http.get(url); + const { + defaultValues: { urlField, titleField }, + validFields, + validSortFields, + validFacetFields, + } = initialFieldValues; - actions.onFieldDataLoaded(initialFieldValues); + actions.onFieldDataLoaded({ + validFields, + validSortFields, + validFacetFields, + urlField, + titleField, + }); } catch (e) { flashAPIErrors(e); } diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/types.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/types.ts index 132ce46bc13fb..224aeab05af35 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/types.ts +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/types.ts @@ -6,9 +6,9 @@ */ export enum ActiveField { - Title, - Filter, - Sort, - Url, - None, + Title = 'Title', + Filter = 'Filter', + Sort = 'Sort', + Url = 'Url', + None = '', } diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/utils.test.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/utils.test.ts new file mode 100644 index 0000000000000..8d99ffc514b3f --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/utils.test.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import '../../../__mocks__/enterprise_search_url.mock'; + +import { generatePreviewUrl } from './utils'; + +jest.mock('../engine', () => ({ + EngineLogic: { + values: { + engineName: 'national-parks-demo', + }, + }, +})); + +describe('generatePreviewUrl', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('generates a url to the preview application from state', () => { + expect( + generatePreviewUrl({ + titleField: 'foo', + urlField: 'bar', + facets: ['baz', 'qux'], + sortFields: ['quux', 'quuz'], + empty: '', // Empty fields should be stripped + empty2: [''], // Empty fields should be stripped + }) + ).toEqual( + 'http://localhost:3002/as/engines/national-parks-demo/reference_application/preview?facets[]=baz&facets[]=qux&sortFields[]=quux&sortFields[]=quuz&titleField=foo&urlField=bar' + ); + }); +}); diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/utils.ts b/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/utils.ts new file mode 100644 index 0000000000000..72d90514ea0a0 --- /dev/null +++ b/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/utils.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import queryString, { ParsedQuery } from 'query-string'; + +import { getAppSearchUrl } from '../../../shared/enterprise_search_url'; +import { EngineLogic } from '../engine'; + +export const generatePreviewUrl = (query: ParsedQuery) => { + const { engineName } = EngineLogic.values; + return queryString.stringifyUrl( + { + query, + url: getAppSearchUrl(`/engines/${engineName}/reference_application/preview`), + }, + { arrayFormat: 'bracket', skipEmptyString: true } + ); +}; diff --git a/x-pack/plugins/enterprise_search/public/applications/shared/flash_messages/flash_messages.tsx b/x-pack/plugins/enterprise_search/public/applications/shared/flash_messages/flash_messages.tsx index 10f80d9a6345a..cf4e1a2d6cabc 100644 --- a/x-pack/plugins/enterprise_search/public/applications/shared/flash_messages/flash_messages.tsx +++ b/x-pack/plugins/enterprise_search/public/applications/shared/flash_messages/flash_messages.tsx @@ -10,6 +10,7 @@ import React, { Fragment } from 'react'; import { useValues, useActions } from 'kea'; import { EuiCallOut, EuiSpacer, EuiGlobalToastList } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; import { FLASH_MESSAGE_TYPES, DEFAULT_TOAST_TIMEOUT } from './constants'; import { FlashMessagesLogic } from './flash_messages_logic'; @@ -25,7 +26,14 @@ export const Callouts: React.FC = ({ children }) => { const { messages } = useValues(FlashMessagesLogic); return ( -
+
{messages.map(({ type, message, description }, index) => ( ; // This is a mapping of schema field types ("text", "number", "geolocation", "date") // to the names of source engines which utilize that type -export type SchemaConflictFieldTypes = Record; +export type SchemaConflictFieldTypes = Partial>; export interface SchemaConflict { fieldTypes: SchemaConflictFieldTypes; diff --git a/x-pack/plugins/enterprise_search/server/routes/app_search/index.ts b/x-pack/plugins/enterprise_search/server/routes/app_search/index.ts index 6ecdb8d8857c6..99aaaeeec38b3 100644 --- a/x-pack/plugins/enterprise_search/server/routes/app_search/index.ts +++ b/x-pack/plugins/enterprise_search/server/routes/app_search/index.ts @@ -16,6 +16,7 @@ import { registerEnginesRoutes } from './engines'; import { registerOnboardingRoutes } from './onboarding'; import { registerResultSettingsRoutes } from './result_settings'; import { registerRoleMappingsRoutes } from './role_mappings'; +import { registerSchemaRoutes } from './schema'; import { registerSearchSettingsRoutes } from './search_settings'; import { registerSearchUIRoutes } from './search_ui'; import { registerSettingsRoutes } from './settings'; @@ -28,6 +29,7 @@ export const registerAppSearchRoutes = (dependencies: RouteDependencies) => { registerAnalyticsRoutes(dependencies); registerDocumentsRoutes(dependencies); registerDocumentRoutes(dependencies); + registerSchemaRoutes(dependencies); registerCurationsRoutes(dependencies); registerSynonymsRoutes(dependencies); registerSearchSettingsRoutes(dependencies); diff --git a/x-pack/plugins/enterprise_search/server/routes/app_search/schema.test.ts b/x-pack/plugins/enterprise_search/server/routes/app_search/schema.test.ts new file mode 100644 index 0000000000000..408838a4de31b --- /dev/null +++ b/x-pack/plugins/enterprise_search/server/routes/app_search/schema.test.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { MockRouter, mockRequestHandler, mockDependencies } from '../../__mocks__'; + +import { registerSchemaRoutes } from './schema'; + +describe('schema routes', () => { + describe('GET /api/app_search/engines/{engineName}/schema', () => { + let mockRouter: MockRouter; + + beforeEach(() => { + jest.clearAllMocks(); + mockRouter = new MockRouter({ + method: 'get', + path: '/api/app_search/engines/{engineName}/schema', + }); + + registerSchemaRoutes({ + ...mockDependencies, + router: mockRouter.router, + }); + }); + + it('creates a request handler', () => { + expect(mockRequestHandler.createRequest).toHaveBeenCalledWith({ + path: '/as/engines/:engineName/schema', + }); + }); + }); + + describe('POST /api/app_search/engines/{engineName}/schema', () => { + let mockRouter: MockRouter; + + beforeEach(() => { + jest.clearAllMocks(); + mockRouter = new MockRouter({ + method: 'post', + path: '/api/app_search/engines/{engineName}/schema', + }); + + registerSchemaRoutes({ + ...mockDependencies, + router: mockRouter.router, + }); + }); + + it('creates a request handler', () => { + expect(mockRequestHandler.createRequest).toHaveBeenCalledWith({ + path: '/as/engines/:engineName/schema', + }); + }); + }); + + describe('GET /api/app_search/engines/{engineName}/reindex_job/{reindexJobId}', () => { + let mockRouter: MockRouter; + + beforeEach(() => { + jest.clearAllMocks(); + mockRouter = new MockRouter({ + method: 'get', + path: '/api/app_search/engines/{engineName}/reindex_job/{reindexJobId}', + }); + + registerSchemaRoutes({ + ...mockDependencies, + router: mockRouter.router, + }); + }); + + it('creates a request handler', () => { + expect(mockRequestHandler.createRequest).toHaveBeenCalledWith({ + path: '/as/engines/:engineName/reindex_job/:reindexJobId', + }); + }); + }); +}); diff --git a/x-pack/plugins/enterprise_search/server/routes/app_search/schema.ts b/x-pack/plugins/enterprise_search/server/routes/app_search/schema.ts new file mode 100644 index 0000000000000..74d07bd2bf75d --- /dev/null +++ b/x-pack/plugins/enterprise_search/server/routes/app_search/schema.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema } from '@kbn/config-schema'; + +import { skipBodyValidation } from '../../lib/route_config_helpers'; +import { RouteDependencies } from '../../plugin'; + +export function registerSchemaRoutes({ + router, + enterpriseSearchRequestHandler, +}: RouteDependencies) { + router.get( + { + path: '/api/app_search/engines/{engineName}/schema', + validate: { + params: schema.object({ + engineName: schema.string(), + }), + }, + }, + enterpriseSearchRequestHandler.createRequest({ + path: '/as/engines/:engineName/schema', + }) + ); + + router.post( + skipBodyValidation({ + path: '/api/app_search/engines/{engineName}/schema', + validate: { + params: schema.object({ + engineName: schema.string(), + }), + }, + }), + enterpriseSearchRequestHandler.createRequest({ + path: '/as/engines/:engineName/schema', + }) + ); + + router.get( + { + path: '/api/app_search/engines/{engineName}/reindex_job/{reindexJobId}', + validate: { + params: schema.object({ + engineName: schema.string(), + reindexJobId: schema.string(), + }), + }, + }, + enterpriseSearchRequestHandler.createRequest({ + path: '/as/engines/:engineName/reindex_job/:reindexJobId', + }) + ); +} diff --git a/x-pack/plugins/file_data_visualizer/public/application/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx b/x-pack/plugins/file_data_visualizer/public/application/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx new file mode 100644 index 0000000000000..acbb77a7e0cac --- /dev/null +++ b/x-pack/plugins/file_data_visualizer/public/application/components/stats_table/components/field_data_expanded_row/choropleth_map.tsx @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC, useMemo } from 'react'; +import { EuiFlexItem, EuiSpacer, EuiText, htmlIdGenerator } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { + FIELD_ORIGIN, + SOURCE_TYPES, + STYLE_TYPE, + COLOR_MAP_TYPE, +} from '../../../../../../../maps/common/constants'; +import { EMSTermJoinConfig } from '../../../../../../../maps/public'; +import { FieldVisStats } from '../../types'; +import { VectorLayerDescriptor } from '../../../../../../../maps/common/descriptor_types'; +import { EmbeddedMapComponent } from '../../../embedded_map'; + +export const getChoroplethTopValuesLayer = ( + fieldName: string, + topValues: Array<{ key: any; doc_count: number }>, + { layerId, field }: EMSTermJoinConfig +): VectorLayerDescriptor => { + return { + id: htmlIdGenerator()(), + label: i18n.translate('xpack.fileDataVisualizer.choroplethMap.topValuesCount', { + defaultMessage: 'Top values count for {fieldName}', + values: { fieldName }, + }), + joins: [ + { + // Left join is the id from the type of field (e.g. world_countries) + leftField: field, + right: { + id: 'anomaly_count', + type: SOURCE_TYPES.TABLE_SOURCE, + __rows: topValues, + __columns: [ + { + name: 'key', + type: 'string', + }, + { + name: 'doc_count', + type: 'number', + }, + ], + // Right join/term is the field in the doc you’re trying to join it to (foreign key - e.g. US) + term: 'key', + }, + }, + ], + sourceDescriptor: { + type: 'EMS_FILE', + id: layerId, + }, + style: { + type: 'VECTOR', + // @ts-ignore missing style properties. Remove once 'VectorLayerDescriptor' type is updated + properties: { + icon: { type: STYLE_TYPE.STATIC, options: { value: 'marker' } }, + fillColor: { + type: STYLE_TYPE.DYNAMIC, + options: { + color: 'Blue to Red', + colorCategory: 'palette_0', + fieldMetaOptions: { isEnabled: true, sigma: 3 }, + type: COLOR_MAP_TYPE.ORDINAL, + field: { + name: 'doc_count', + origin: FIELD_ORIGIN.JOIN, + }, + useCustomColorRamp: false, + }, + }, + lineColor: { + type: STYLE_TYPE.DYNAMIC, + options: { fieldMetaOptions: { isEnabled: true } }, + }, + lineWidth: { type: STYLE_TYPE.STATIC, options: { size: 1 } }, + }, + isTimeAware: true, + }, + type: 'VECTOR', + }; +}; + +interface Props { + stats: FieldVisStats | undefined; + suggestion: EMSTermJoinConfig; +} + +export const ChoroplethMap: FC = ({ stats, suggestion }) => { + const { fieldName, isTopValuesSampled, topValues, topValuesSamplerShardSize } = stats!; + + const layerList: VectorLayerDescriptor[] = useMemo( + () => [getChoroplethTopValuesLayer(fieldName || '', topValues || [], suggestion)], + [suggestion, fieldName, topValues] + ); + + return ( + +
+ +
+ {isTopValuesSampled === true && ( + <> + + + + + + )} + + ); +}; diff --git a/x-pack/plugins/file_data_visualizer/public/application/components/stats_table/components/field_data_expanded_row/keyword_content.tsx b/x-pack/plugins/file_data_visualizer/public/application/components/stats_table/components/field_data_expanded_row/keyword_content.tsx index 3f1a7aad5463f..6448883bfce73 100644 --- a/x-pack/plugins/file_data_visualizer/public/application/components/stats_table/components/field_data_expanded_row/keyword_content.tsx +++ b/x-pack/plugins/file_data_visualizer/public/application/components/stats_table/components/field_data_expanded_row/keyword_content.tsx @@ -5,21 +5,55 @@ * 2.0. */ -import React, { FC } from 'react'; +import React, { FC, useCallback, useEffect, useState } from 'react'; import type { FieldDataRowProps } from '../../types/field_data_row'; import { TopValues } from '../../../top_values'; +import { EMSTermJoinConfig } from '../../../../../../../maps/public'; +import { useFileDataVisualizerKibana } from '../../../../kibana_context'; import { DocumentStatsTable } from './document_stats'; import { ExpandedRowContent } from './expanded_row_content'; +import { ChoroplethMap } from './choropleth_map'; + +const COMMON_EMS_LAYER_IDS = [ + 'world_countries', + 'administrative_regions_lvl2', + 'usa_zip_codes', + 'usa_states', +]; export const KeywordContent: FC = ({ config }) => { - const { stats } = config; + const [EMSSuggestion, setEMSSuggestion] = useState(); + const { stats, fieldName } = config; const fieldFormat = 'fieldFormat' in config ? config.fieldFormat : undefined; + const { + services: { maps: mapsPlugin }, + } = useFileDataVisualizerKibana(); + + const loadEMSTermSuggestions = useCallback(async () => { + if (!mapsPlugin) return; + const suggestion: EMSTermJoinConfig | null = await mapsPlugin.suggestEMSTermJoinConfig({ + emsLayerIds: COMMON_EMS_LAYER_IDS, + sampleValues: Array.isArray(stats?.topValues) + ? stats?.topValues.map((value) => value.key) + : [], + sampleValuesColumnName: fieldName || '', + }); + setEMSSuggestion(suggestion); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [fieldName]); + + useEffect( + function getInitialEMSTermSuggestion() { + loadEMSTermSuggestions(); + }, + [loadEMSTermSuggestions] + ); return ( - + {EMSSuggestion && stats && } ); }; diff --git a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/agent_logs.tsx b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/agent_logs.tsx index 6bc1db2fc4c7d..fa3b6b56a18b9 100644 --- a/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/agent_logs.tsx +++ b/x-pack/plugins/fleet/public/applications/fleet/sections/agents/agent_details_page/components/agent_logs/agent_logs.tsx @@ -6,11 +6,11 @@ */ import url from 'url'; +import { stringify } from 'querystring'; import React, { memo, useMemo, useState, useCallback, useEffect } from 'react'; import styled from 'styled-components'; import { encode } from 'rison-node'; -import { stringify } from 'query-string'; import { EuiFlexGroup, EuiFlexItem, @@ -170,20 +170,17 @@ export const AgentLogsUI: React.FunctionComponent = memo(({ agen http.basePath.prepend( url.format({ pathname: '/app/logs/stream', - search: stringify( - { - logPosition: encode({ - start: state.start, - end: state.end, - streamLive: false, - }), - logFilter: encode({ - expression: logStreamQuery, - kind: 'kuery', - }), - }, - { sort: false, encode: false } - ), + search: stringify({ + logPosition: encode({ + start: state.start, + end: state.end, + streamLive: false, + }), + logFilter: encode({ + expression: logStreamQuery, + kind: 'kuery', + }), + }), }) ), [http.basePath, state.start, state.end, logStreamQuery] diff --git a/x-pack/plugins/fleet/server/services/epm/registry/index.ts b/x-pack/plugins/fleet/server/services/epm/registry/index.ts index 0a755228e0b98..9e04d75f36a16 100644 --- a/x-pack/plugins/fleet/server/services/epm/registry/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/registry/index.ts @@ -77,8 +77,6 @@ export const pkgToPkgKey = ({ name, version }: { name: string; version: string } export async function fetchList(params?: SearchParams): Promise { const registryUrl = getRegistryUrl(); const url = new URL(`${registryUrl}/search`); - const kibanaVersion = appContextService.getKibanaVersion().split('-')[0]; // may be x.y.z-SNAPSHOT - const kibanaBranch = appContextService.getKibanaBranch(); if (params) { if (params.category) { url.searchParams.set('category', params.category); @@ -88,10 +86,7 @@ export async function fetchList(params?: SearchParams): Promise { return getResponse(`${registryUrl}${filePath}`); } +function setKibanaVersion(url: URL) { + const kibanaVersion = appContextService.getKibanaVersion().split('-')[0]; // may be x.y.z-SNAPSHOT + const kibanaBranch = appContextService.getKibanaBranch(); + + // on master, request all packages regardless of version + if (kibanaVersion && kibanaBranch !== 'master') { + url.searchParams.set('kibana.version', kibanaVersion); + } +} + export async function fetchCategories(params?: CategoriesParams): Promise { const registryUrl = getRegistryUrl(); const url = new URL(`${registryUrl}/categories`); @@ -154,6 +159,8 @@ export async function fetchCategories(params?: CategoriesParams): Promise { type: 'logs', dataset: 'package.dataset1', streams: [{ input: 'log', template_path: 'some_template_path.yml' }], + path: 'dataset1', }, ], policy_templates: [ @@ -151,6 +163,57 @@ describe('Package policy service', () => { ]); }); + it('should work with a two level dataset name', async () => { + const inputs = await packagePolicyService.compilePackagePolicyInputs( + ({ + data_streams: [ + { + type: 'logs', + dataset: 'package.dataset1.level1', + streams: [{ input: 'log', template_path: 'some_template_path.yml' }], + path: 'dataset1_level1', + }, + ], + policy_templates: [ + { + inputs: [{ type: 'log' }], + }, + ], + } as unknown) as PackageInfo, + [ + { + type: 'log', + enabled: true, + streams: [ + { + id: 'datastream01', + data_stream: { dataset: 'package.dataset1.level1', type: 'logs' }, + enabled: true, + }, + ], + }, + ] + ); + + expect(inputs).toEqual([ + { + type: 'log', + enabled: true, + streams: [ + { + id: 'datastream01', + data_stream: { dataset: 'package.dataset1.level1', type: 'logs' }, + enabled: true, + compiled_stream: { + metricset: ['dataset1.level1'], + type: 'log', + }, + }, + ], + }, + ]); + }); + it('should work with config variables at the input level', async () => { const inputs = await packagePolicyService.compilePackagePolicyInputs( ({ @@ -159,6 +222,7 @@ describe('Package policy service', () => { dataset: 'package.dataset1', type: 'logs', streams: [{ input: 'log', template_path: 'some_template_path.yml' }], + path: 'dataset1', }, ], policy_templates: [ @@ -261,6 +325,7 @@ describe('Package policy service', () => { dataset: 'package.dataset1', type: 'logs', streams: [{ input: 'log', template_path: 'some_template_path.yml' }], + path: 'dataset1', }, ], policy_templates: [ diff --git a/x-pack/plugins/fleet/server/services/package_policy.ts b/x-pack/plugins/fleet/server/services/package_policy.ts index e4d34e551c95b..7c870415bc521 100644 --- a/x-pack/plugins/fleet/server/services/package_policy.ts +++ b/x-pack/plugins/fleet/server/services/package_policy.ts @@ -56,10 +56,6 @@ import { appContextService } from '.'; const SAVED_OBJECT_TYPE = PACKAGE_POLICY_SAVED_OBJECT_TYPE; -function getDataset(st: string) { - return st.split('.')[1]; -} - class PackagePolicyService { public async create( soClient: SavedObjectsClientContract, @@ -562,7 +558,7 @@ async function _compilePackageStream( if (!stream.enabled) { return { ...stream, compiled_stream: undefined }; } - const datasetPath = getDataset(stream.data_stream.dataset); + const packageDataStreams = pkgInfo.data_streams; if (!packageDataStreams) { throw new Error('Stream template not found, no data streams'); @@ -571,8 +567,11 @@ async function _compilePackageStream( const packageDataStream = packageDataStreams.find( (pkgDataStream) => pkgDataStream.dataset === stream.data_stream.dataset ); + if (!packageDataStream) { - throw new Error(`Stream template not found, unable to find dataset ${datasetPath}`); + throw new Error( + `Stream template not found, unable to find dataset ${stream.data_stream.dataset}` + ); } const streamFromPkg = (packageDataStream.streams || []).find( @@ -583,9 +582,11 @@ async function _compilePackageStream( } if (!streamFromPkg.template_path) { - throw new Error(`Stream template path not found for dataset ${datasetPath}`); + throw new Error(`Stream template path not found for dataset ${stream.data_stream.dataset}`); } + const datasetPath = packageDataStream.path; + const [pkgStreamTemplate] = await getAssetsData( registryPkgInfo, (path: string) => path.endsWith(streamFromPkg.template_path), @@ -594,7 +595,7 @@ async function _compilePackageStream( if (!pkgStreamTemplate || !pkgStreamTemplate.buffer) { throw new Error( - `Unable to load stream template ${streamFromPkg.template_path} for dataset ${datasetPath}` + `Unable to load stream template ${streamFromPkg.template_path} for dataset ${stream.data_stream.dataset}` ); } diff --git a/x-pack/plugins/global_search_bar/public/components/search_bar.test.tsx b/x-pack/plugins/global_search_bar/public/components/search_bar.test.tsx index 94855ab0aa4ab..c8bd54540e6a6 100644 --- a/x-pack/plugins/global_search_bar/public/components/search_bar.test.tsx +++ b/x-pack/plugins/global_search_bar/public/components/search_bar.test.tsx @@ -66,9 +66,7 @@ describe('SearchBar', () => { }; const simulateTypeChar = async (text: string) => { - await waitFor(() => - getSearchProps(component).onKeyUpCapture({ currentTarget: { value: text } }) - ); + await waitFor(() => getSearchProps(component).onInput({ currentTarget: { value: text } })); }; const getDisplayedOptionsTitle = () => { diff --git a/x-pack/plugins/global_search_bar/public/components/search_bar.tsx b/x-pack/plugins/global_search_bar/public/components/search_bar.tsx index dfe73512279ed..5234b4e7b0ad5 100644 --- a/x-pack/plugins/global_search_bar/public/components/search_bar.tsx +++ b/x-pack/plugins/global_search_bar/public/components/search_bar.tsx @@ -398,8 +398,7 @@ export function SearchBar({ } searchProps={{ - onKeyUpCapture: (e: React.KeyboardEvent) => - setSearchValue(e.currentTarget.value), + onInput: (e: React.UIEvent) => setSearchValue(e.currentTarget.value), 'data-test-subj': 'nav-search-input', inputRef: setSearchRef, compressed: true, diff --git a/x-pack/plugins/index_management/__jest__/components/index_table.test.js b/x-pack/plugins/index_management/__jest__/components/index_table.test.js index 6b8aa5b445af2..4ac94319d4711 100644 --- a/x-pack/plugins/index_management/__jest__/components/index_table.test.js +++ b/x-pack/plugins/index_management/__jest__/components/index_table.test.js @@ -196,6 +196,20 @@ describe('index table', () => { button = findTestSubject(rendered, 'indexActionsContextMenuButton'); expect(button.length).toEqual(1); }); + test('should update the Actions menu button text when more than one row is selected', () => { + const rendered = mountWithIntl(component); + let button = findTestSubject(rendered, 'indexTableContextMenuButton'); + expect(button.length).toEqual(0); + const checkboxes = findTestSubject(rendered, 'indexTableRowCheckbox'); + checkboxes.at(0).simulate('change', { target: { checked: true } }); + rendered.update(); + button = findTestSubject(rendered, 'indexActionsContextMenuButton'); + expect(button.text()).toEqual('Manage index'); + checkboxes.at(1).simulate('change', { target: { checked: true } }); + rendered.update(); + button = findTestSubject(rendered, 'indexActionsContextMenuButton'); + expect(button.text()).toEqual('Manage 2 indices'); + }); test('should show system indices only when the switch is turned on', () => { const rendered = mountWithIntl(component); snapshot(rendered.find('.euiPagination li').map((item) => item.text())); diff --git a/x-pack/plugins/index_management/public/application/sections/home/index_list/index_actions_context_menu/index_actions_context_menu.js b/x-pack/plugins/index_management/public/application/sections/home/index_list/index_actions_context_menu/index_actions_context_menu.js index 20a4af59bab11..944992fbc9146 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/index_list/index_actions_context_menu/index_actions_context_menu.js +++ b/x-pack/plugins/index_management/public/application/sections/home/index_list/index_actions_context_menu/index_actions_context_menu.js @@ -696,7 +696,8 @@ export class IndexActionsContextMenu extends Component { iconSide = 'right', anchorPosition = 'rightUp', label = i18n.translate('xpack.idxMgmt.indexActionsMenu.manageButtonLabel', { - defaultMessage: 'Manage {selectedIndexCount, plural, one {index} other {indices}}', + defaultMessage: + 'Manage {selectedIndexCount, plural, one {index} other {{selectedIndexCount} indices}}', values: { selectedIndexCount }, }), iconType = 'arrowDown', diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/layout.tsx b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/layout.tsx index 534132eb75fa1..2e79e7e4ab8c4 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/layout.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/layout.tsx @@ -9,6 +9,7 @@ import React, { useCallback, useEffect, useState } from 'react'; import useInterval from 'react-use/lib/useInterval'; import { EuiFlexGroup, EuiFlexItem, EuiSpacer } from '@elastic/eui'; +import { SavedView } from '../../../../containers/saved_view/saved_view'; import { AutoSizer } from '../../../../components/auto_sizer'; import { convertIntervalToString } from '../../../../utils/convert_interval_to_string'; import { NodesOverview } from './nodes_overview'; @@ -26,182 +27,191 @@ import { ViewSwitcher } from './waffle/view_switcher'; import { IntervalLabel } from './waffle/interval_label'; import { createInventoryMetricFormatter } from '../lib/create_inventory_metric_formatter'; import { createLegend } from '../lib/create_legend'; -import { useSavedViewContext } from '../../../../containers/saved_view/saved_view'; import { useWaffleViewState } from '../hooks/use_waffle_view_state'; import { SavedViewsToolbarControls } from '../../../../components/saved_views/toolbar_control'; import { BottomDrawer } from './bottom_drawer'; import { Legend } from './waffle/legend'; -export const Layout = () => { - const [showLoading, setShowLoading] = useState(true); - const { sourceId, source } = useSourceContext(); - const { currentView, shouldLoadDefault } = useSavedViewContext(); - const { - metric, - groupBy, - sort, - nodeType, - accountId, - region, - changeView, - view, - autoBounds, - boundsOverride, - legend, - } = useWaffleOptionsContext(); - const { currentTime, jumpToTime, isAutoReloading } = useWaffleTimeContext(); - const { filterQueryAsJson, applyFilterQuery } = useWaffleFiltersContext(); - const { loading, nodes, reload, interval } = useSnapshot( - filterQueryAsJson, - [metric], - groupBy, - nodeType, - sourceId, - currentTime, - accountId, - region, - false - ); - - const legendPalette = legend?.palette ?? DEFAULT_LEGEND.palette; - const legendSteps = legend?.steps ?? DEFAULT_LEGEND.steps; - const legendReverseColors = legend?.reverseColors ?? DEFAULT_LEGEND.reverseColors; - - const options = { - formatter: InfraFormatterType.percent, - formatTemplate: '{{value}}', - legend: createLegend(legendPalette, legendSteps, legendReverseColors), - metric, - sort, - fields: source?.configuration?.fields, - groupBy, - }; - - useInterval( - () => { - if (!loading) { - jumpToTime(Date.now()); - } - }, - isAutoReloading ? 5000 : null - ); - - const intervalAsString = convertIntervalToString(interval); - const dataBounds = calculateBoundsFromNodes(nodes); - const bounds = autoBounds ? dataBounds : boundsOverride; - /* eslint-disable-next-line react-hooks/exhaustive-deps */ - const formatter = useCallback(createInventoryMetricFormatter(options.metric), [options.metric]); - const { viewState, onViewChange } = useWaffleViewState(); - - useEffect(() => { - if (currentView) { - onViewChange(currentView); - } - }, [currentView, onViewChange]); - - useEffect(() => { - // load snapshot data after default view loaded, unless we're not loading a view - if (currentView != null || !shouldLoadDefault) { - reload(); - } - - /** - * INFO: why disable exhaustive-deps - * We need to wait on the currentView not to be null because it is loaded async and could change the view state. - * We don't actually need to watch the value of currentView though, since the view state will be synched up by the - * changing params in the reload method so we should only "watch" the reload method. - * - * TODO: Should refactor this in the future to make it more clear where all the view state is coming - * from and it's precedence [query params, localStorage, defaultView, out of the box view] - */ +export const Layout = React.memo( + ({ + shouldLoadDefault, + currentView, + }: { + shouldLoadDefault: boolean; + currentView: SavedView | null; + }) => { + const [showLoading, setShowLoading] = useState(true); + const { sourceId, source } = useSourceContext(); + const { + metric, + groupBy, + sort, + nodeType, + accountId, + region, + changeView, + view, + autoBounds, + boundsOverride, + legend, + } = useWaffleOptionsContext(); + const { currentTime, jumpToTime, isAutoReloading } = useWaffleTimeContext(); + const { filterQueryAsJson, applyFilterQuery } = useWaffleFiltersContext(); + const { loading, nodes, reload, interval } = useSnapshot( + filterQueryAsJson, + [metric], + groupBy, + nodeType, + sourceId, + currentTime, + accountId, + region, + false + ); + + const legendPalette = legend?.palette ?? DEFAULT_LEGEND.palette; + const legendSteps = legend?.steps ?? DEFAULT_LEGEND.steps; + const legendReverseColors = legend?.reverseColors ?? DEFAULT_LEGEND.reverseColors; + + const options = { + formatter: InfraFormatterType.percent, + formatTemplate: '{{value}}', + legend: createLegend(legendPalette, legendSteps, legendReverseColors), + metric, + sort, + fields: source?.configuration?.fields, + groupBy, + }; + + useInterval( + () => { + if (!loading) { + jumpToTime(Date.now()); + } + }, + isAutoReloading ? 5000 : null + ); + + const intervalAsString = convertIntervalToString(interval); + const dataBounds = calculateBoundsFromNodes(nodes); + const bounds = autoBounds ? dataBounds : boundsOverride; /* eslint-disable-next-line react-hooks/exhaustive-deps */ - }, [reload, shouldLoadDefault]); - - useEffect(() => { - setShowLoading(true); - }, [options.metric, nodeType]); - - useEffect(() => { - const hasNodes = nodes && nodes.length; - // Don't show loading screen when we're auto-reloading - setShowLoading(!hasNodes); - }, [nodes]); - - return ( - <> - - - {({ measureRef: pageMeasureRef, bounds: { width = 0 } }) => ( - - - {({ measureRef: topActionMeasureRef, bounds: { height: topActionHeight = 0 } }) => ( - <> - - - - - - - - - - - - - - - - - {({ measureRef, bounds: { height = 0 } }) => ( - <> - - {view === 'map' && ( - { + if (currentView) { + onViewChange(currentView); + } + }, [currentView, onViewChange]); + + useEffect(() => { + // load snapshot data after default view loaded, unless we're not loading a view + if (currentView != null || !shouldLoadDefault) { + reload(); + } + + /** + * INFO: why disable exhaustive-deps + * We need to wait on the currentView not to be null because it is loaded async and could change the view state. + * We don't actually need to watch the value of currentView though, since the view state will be synched up by the + * changing params in the reload method so we should only "watch" the reload method. + * + * TODO: Should refactor this in the future to make it more clear where all the view state is coming + * from and it's precedence [query params, localStorage, defaultView, out of the box view] + */ + /* eslint-disable-next-line react-hooks/exhaustive-deps */ + }, [reload, shouldLoadDefault]); + + useEffect(() => { + setShowLoading(true); + }, [options.metric, nodeType]); + + useEffect(() => { + const hasNodes = nodes && nodes.length; + // Don't show loading screen when we're auto-reloading + setShowLoading(!hasNodes); + }, [nodes]); + + return ( + <> + + + {({ measureRef: pageMeasureRef, bounds: { width = 0 } }) => ( + + + {({ + measureRef: topActionMeasureRef, + bounds: { height: topActionHeight = 0 }, + }) => ( + <> + + + + + + + + + + + + + + + + + {({ measureRef, bounds: { height = 0 } }) => ( + <> + - + {view === 'map' && ( + - - )} - - )} - - - )} - - - )} - - - - ); -}; + width={width} + > + + + )} + + )} + + + )} + + + )} + + + + ); + } +); const MainContainer = euiStyled.div` position: relative; diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/layout_view.tsx b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/layout_view.tsx new file mode 100644 index 0000000000000..1e66fe22ac45e --- /dev/null +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/layout_view.tsx @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { useSavedViewContext } from '../../../../containers/saved_view/saved_view'; +import { Layout } from './layout'; + +export const LayoutView = () => { + const { shouldLoadDefault, currentView } = useSavedViewContext(); + return ; +}; diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/index.tsx b/x-pack/plugins/infra/public/pages/metrics/inventory_view/index.tsx index 57073fee13c18..81a8d1ae8d294 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/index.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/index.tsx @@ -21,7 +21,7 @@ import { ViewSourceConfigurationButton } from '../../../components/source_config import { Source } from '../../../containers/metrics_source'; import { useTrackPageview } from '../../../../../observability/public'; import { useKibana } from '../../../../../../../src/plugins/kibana_react/public'; -import { Layout } from './components/layout'; +import { LayoutView } from './components/layout_view'; import { useLinkProps } from '../../../hooks/use_link_props'; import { SavedViewProvider } from '../../../containers/saved_view/saved_view'; import { DEFAULT_WAFFLE_VIEW_STATE } from './hooks/use_waffle_view_state'; @@ -69,7 +69,7 @@ export const SnapshotPage = () => { viewType={'inventory-view'} defaultViewState={DEFAULT_WAFFLE_VIEW_STATE} > - + ) : hasFailedLoadingSource ? ( diff --git a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.ts b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.ts index fdf96a583479a..42ba918694482 100644 --- a/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.ts +++ b/x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.ts @@ -13,6 +13,7 @@ import { createPercentileAggregation } from './create_percentile_aggregation'; import { calculateDateHistogramOffset } from '../../../metrics/lib/calculate_date_histogram_offset'; const MINIMUM_BUCKETS = 5; +const COMPOSITE_RESULTS_PER_PAGE = 100; const getParsedFilterQuery: (filterQuery: string | undefined) => Record | null = ( filterQuery @@ -83,7 +84,7 @@ export const getElasticsearchMetricQuery = ( ? { groupings: { composite: { - size: 10, + size: COMPOSITE_RESULTS_PER_PAGE, sources: Array.isArray(groupBy) ? groupBy.map((field, index) => ({ [`groupBy${index}`]: { diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/append.test.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/append.test.tsx new file mode 100644 index 0000000000000..af3651525d6a3 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/append.test.tsx @@ -0,0 +1,118 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { act } from 'react-dom/test-utils'; +import { setup, SetupResult, getProcessorValue } from './processor.helpers'; + +const APPEND_TYPE = 'append'; + +describe('Processor: Append', () => { + let onUpdate: jest.Mock; + let testBed: SetupResult; + + beforeAll(() => { + jest.useFakeTimers(); + }); + + afterAll(() => { + jest.useRealTimers(); + }); + + beforeEach(async () => { + onUpdate = jest.fn(); + + await act(async () => { + testBed = await setup({ + value: { + processors: [], + }, + onFlyoutOpen: jest.fn(), + onUpdate, + }); + }); + testBed.component.update(); + const { + actions: { addProcessor, addProcessorType }, + } = testBed; + // Open the processor flyout + addProcessor(); + + // Add type (the other fields are not visible until a type is selected) + await addProcessorType(APPEND_TYPE); + }); + + test('prevents form submission if required fields are not provided', async () => { + const { + actions: { saveNewProcessor }, + form, + } = testBed; + + // Click submit button with only the type defined + await saveNewProcessor(); + + // Expect form error as "field" and "value" are required parameters + expect(form.getErrorsMessages()).toEqual([ + 'A field value is required.', + 'A value is required.', + ]); + }); + + test('saves with required parameter values', async () => { + const { + actions: { saveNewProcessor }, + form, + find, + component, + } = testBed; + + // Add "field" value (required) + form.setInputValue('fieldNameField.input', 'field_1'); + + await act(async () => { + find('appendValueField.input').simulate('change', [{ label: 'Some_Value' }]); + }); + component.update(); + + // Save the field + await saveNewProcessor(); + + const processors = getProcessorValue(onUpdate, APPEND_TYPE); + expect(processors[0].append).toEqual({ + field: 'field_1', + value: ['Some_Value'], + }); + }); + + test('allows optional parameters to be set', async () => { + const { + actions: { saveNewProcessor }, + form, + find, + component, + } = testBed; + + // Add "field" value (required) + form.setInputValue('fieldNameField.input', 'field_1'); + + // Set optional parameteres + await act(async () => { + find('appendValueField.input').simulate('change', [{ label: 'Some_Value' }]); + component.update(); + }); + + form.toggleEuiSwitch('ignoreFailureSwitch.input'); + // Save the field with new changes + await saveNewProcessor(); + + const processors = getProcessorValue(onUpdate, APPEND_TYPE); + expect(processors[0].append).toEqual({ + field: 'field_1', + ignore_failure: true, + value: ['Some_Value'], + }); + }); +}); diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/bytes.test.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/bytes.test.tsx index af4f6e468ca36..51117c3b517f9 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/bytes.test.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/bytes.test.tsx @@ -35,24 +35,21 @@ describe('Processor: Bytes', () => { }); }); testBed.component.update(); - }); - - test('prevents form submission if required fields are not provided', async () => { const { - actions: { addProcessor, saveNewProcessor, addProcessorType }, - form, + actions: { addProcessor, addProcessorType }, } = testBed; - - // Open flyout to add new processor + // Open the processor flyout addProcessor(); - // Click submit button without entering any fields - await saveNewProcessor(); - - // Expect form error as a processor type is required - expect(form.getErrorsMessages()).toEqual(['A type is required.']); // Add type (the other fields are not visible until a type is selected) await addProcessorType(BYTES_TYPE); + }); + + test('prevents form submission if required fields are not provided', async () => { + const { + actions: { saveNewProcessor }, + form, + } = testBed; // Click submit button with only the type defined await saveNewProcessor(); @@ -61,16 +58,12 @@ describe('Processor: Bytes', () => { expect(form.getErrorsMessages()).toEqual(['A field value is required.']); }); - test('saves with default parameter values', async () => { + test('saves with required parameter values', async () => { const { - actions: { addProcessor, saveNewProcessor, addProcessorType }, + actions: { saveNewProcessor }, form, } = testBed; - // Open flyout to add new processor - addProcessor(); - // Add type (the other fields are not visible until a type is selected) - await addProcessorType(BYTES_TYPE); // Add "field" value (required) form.setInputValue('fieldNameField.input', 'field_1'); // Save the field @@ -84,14 +77,10 @@ describe('Processor: Bytes', () => { test('allows optional parameters to be set', async () => { const { - actions: { addProcessor, addProcessorType, saveNewProcessor }, + actions: { saveNewProcessor }, form, } = testBed; - // Open flyout to add new processor - addProcessor(); - // Add type (the other fields are not visible until a type is selected) - await addProcessorType(BYTES_TYPE); // Add "field" value (required) form.setInputValue('fieldNameField.input', 'field_1'); diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/common_processor_fields.test.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/common_processor_fields.test.tsx index af4c187df75e6..af5c8c50abf7b 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/common_processor_fields.test.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/common_processor_fields.test.tsx @@ -37,6 +37,21 @@ describe('Processor: Common Fields For All Processors', () => { testBed.component.update(); }); + test('prevents form submission if required type field is not provided', async () => { + const { + actions: { addProcessor, saveNewProcessor }, + form, + } = testBed; + + // Open flyout to add new processor + addProcessor(); + // Click submit button without entering any fields + await saveNewProcessor(); + + // Expect form error as a processor type is required + expect(form.getErrorsMessages()).toEqual(['A type is required.']); + }); + test('saves with common fields set', async () => { const { actions: { addProcessor, saveNewProcessor, addProcessorType }, diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/fail.test.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/fail.test.tsx new file mode 100644 index 0000000000000..db5840379536a --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/fail.test.tsx @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { act } from 'react-dom/test-utils'; +import { setup, SetupResult, getProcessorValue } from './processor.helpers'; + +const FAIL_TYPE = 'fail'; +describe('Processor: Fail', () => { + let onUpdate: jest.Mock; + let testBed: SetupResult; + + beforeAll(() => { + jest.useFakeTimers(); + }); + + afterAll(() => { + jest.useRealTimers(); + }); + + beforeEach(async () => { + onUpdate = jest.fn(); + + await act(async () => { + testBed = await setup({ + value: { + processors: [], + }, + onFlyoutOpen: jest.fn(), + onUpdate, + }); + }); + testBed.component.update(); + + const { + actions: { addProcessor, addProcessorType }, + } = testBed; + // Open the processor flyout + addProcessor(); + + // Add type (the other fields are not visible until a type is selected) + await addProcessorType(FAIL_TYPE); + }); + + test('prevents form submission if required message field is not provided', async () => { + const { + actions: { saveNewProcessor }, + form, + } = testBed; + + // Click submit button with only the type defined + await saveNewProcessor(); + + // Expect form error as "field" is required parameter + expect(form.getErrorsMessages()).toEqual(['A message is required.']); + }); + + test('saves with required parameter value', async () => { + const { + actions: { saveNewProcessor }, + form, + } = testBed; + + // Add "message" value (required) + form.setInputValue('messageField.input', 'Test Error Message'); + // Save the field + await saveNewProcessor(); + + const processors = getProcessorValue(onUpdate, FAIL_TYPE); + expect(processors[0].fail).toEqual({ + message: 'Test Error Message', + }); + }); +}); diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/processor.helpers.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/processor.helpers.tsx index 8616241a43d8d..797d26f2404ab 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/processor.helpers.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/processor.helpers.tsx @@ -137,8 +137,10 @@ type TestSubject = | 'addProcessorForm.submitButton' | 'addProcessorButton' | 'addProcessorForm.submitButton' + | 'appendValueField.input' | 'processorTypeSelector.input' | 'fieldNameField.input' + | 'messageField.input' | 'mockCodeEditor' | 'tagField.input' | 'ignoreMissingSwitch.input' diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/uri_parts.test.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/uri_parts.test.tsx index 573adad3247f5..442788a7f75aa 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/uri_parts.test.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/uri_parts.test.tsx @@ -12,8 +12,6 @@ import { setup, SetupResult, getProcessorValue } from './processor.helpers'; const defaultUriPartsParameters = { keep_original: undefined, remove_if_successful: undefined, - ignore_failure: undefined, - description: undefined, }; const URI_PARTS_TYPE = 'uri_parts'; @@ -43,19 +41,22 @@ describe('Processor: URI parts', () => { }); }); testBed.component.update(); - }); - test('prevents form submission if required fields are not provided', async () => { const { - actions: { addProcessor, saveNewProcessor, addProcessorType }, - form, + actions: { addProcessor, addProcessorType }, } = testBed; - - // Open flyout to add new processor + // Open the processor flyout addProcessor(); // Add type (the other fields are not visible until a type is selected) await addProcessorType(URI_PARTS_TYPE); + }); + + test('prevents form submission if required fields are not provided', async () => { + const { + actions: { saveNewProcessor }, + form, + } = testBed; // Click submit button with only the type defined await saveNewProcessor(); @@ -64,16 +65,12 @@ describe('Processor: URI parts', () => { expect(form.getErrorsMessages()).toEqual(['A field value is required.']); }); - test('saves with default parameter values', async () => { + test('saves with required parameter values', async () => { const { - actions: { addProcessor, saveNewProcessor, addProcessorType }, + actions: { saveNewProcessor }, form, } = testBed; - // Open flyout to add new processor - addProcessor(); - // Add type (the other fields are not visible until a type is selected) - await addProcessorType(URI_PARTS_TYPE); // Add "field" value (required) form.setInputValue('fieldNameField.input', 'field_1'); // Save the field @@ -88,14 +85,10 @@ describe('Processor: URI parts', () => { test('allows optional parameters to be set', async () => { const { - actions: { addProcessor, addProcessorType, saveNewProcessor }, + actions: { saveNewProcessor }, form, } = testBed; - // Open flyout to add new processor - addProcessor(); - // Add type (the other fields are not visible until a type is selected) - await addProcessorType(URI_PARTS_TYPE); // Add "field" value (required) form.setInputValue('fieldNameField.input', 'field_1'); @@ -109,9 +102,7 @@ describe('Processor: URI parts', () => { const processors = getProcessorValue(onUpdate, URI_PARTS_TYPE); expect(processors[0].uri_parts).toEqual({ - description: undefined, field: 'field_1', - ignore_failure: undefined, keep_original: false, remove_if_successful: true, target_field: 'target_field', diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/append.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/append.tsx index d10347b4d9655..fde39e7462009 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/append.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/append.tsx @@ -52,7 +52,12 @@ export const Append: FunctionComponent = () => { })} /> - + ); }; diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/fail.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/fail.tsx index f2fe6d6ef95f6..156aa9ae1545a 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/fail.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/fail.tsx @@ -37,5 +37,12 @@ const fieldsConfig: FieldsConfig = { }; export const Fail: FunctionComponent = () => { - return ; + return ( + + ); }; diff --git a/x-pack/plugins/lens/public/assets/globe_illustration.tsx b/x-pack/plugins/lens/public/assets/globe_illustration.tsx new file mode 100644 index 0000000000000..af2f2c7a48e46 --- /dev/null +++ b/x-pack/plugins/lens/public/assets/globe_illustration.tsx @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as React from 'react'; +import { EuiIconProps } from '@elastic/eui'; + +export const GlobeIllustration = ({ title, titleId, ...props }: Omit) => ( + + + + + +); diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/data_panel_wrapper.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/data_panel_wrapper.tsx index 3296ddac7bc29..9bf03025e400f 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/data_panel_wrapper.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/data_panel_wrapper.tsx @@ -15,6 +15,7 @@ import { Action } from './state_management'; import { DragContext, DragDropIdentifier } from '../../drag_drop'; import { StateSetter, FramePublicAPI, DatasourceDataPanelProps, Datasource } from '../../types'; import { Query, Filter } from '../../../../../../src/plugins/data/public'; +import { UiActionsStart } from '../../../../../../src/plugins/ui_actions/public'; interface DataPanelWrapperProps { datasourceState: unknown; @@ -29,6 +30,7 @@ interface DataPanelWrapperProps { filters: Filter[]; dropOntoWorkspace: (field: DragDropIdentifier) => void; hasSuggestionForField: (field: DragDropIdentifier) => boolean; + plugins: { uiActions: UiActionsStart }; } export const DataPanelWrapper = memo((props: DataPanelWrapperProps) => { @@ -56,6 +58,7 @@ export const DataPanelWrapper = memo((props: DataPanelWrapperProps) => { showNoDataPopover: props.showNoDataPopover, dropOntoWorkspace: props.dropOntoWorkspace, hasSuggestionForField: props.hasSuggestionForField, + uiActions: props.plugins.uiActions, }; const [showDatasourceSwitcher, setDatasourceSwitcher] = useState(false); diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx index 362787ea91c4f..91b59664ada83 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/editor_frame.tsx @@ -332,6 +332,7 @@ export function EditorFrame(props: EditorFrameProps) { showNoDataPopover={props.showNoDataPopover} dropOntoWorkspace={dropOntoWorkspace} hasSuggestionForField={hasSuggestionForField} + plugins={props.plugins} /> } configPanel={ diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/geo_field_workspace_panel.scss b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/geo_field_workspace_panel.scss new file mode 100644 index 0000000000000..ca1a62cae64d6 --- /dev/null +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/geo_field_workspace_panel.scss @@ -0,0 +1,18 @@ +@import '../../../mixins'; + +.lnsVisualizeGeoFieldWorkspacePanel__dragDrop { + padding: $euiSizeXXL ($euiSizeXL * 2); + border: $euiBorderThin; + border-radius: $euiBorderRadius; + + &.lnsDragDrop-isDropTarget { + @include lnsDroppable; + @include lnsDroppableActive; + + } + + &.lnsDragDrop-isActiveDropTarget { + @include lnsDroppableActiveHover; + + } +} diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/geo_field_workspace_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/geo_field_workspace_panel.tsx new file mode 100644 index 0000000000000..b50b6463c5a25 --- /dev/null +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/geo_field_workspace_panel.tsx @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { EuiPageContentBody, EuiText } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { i18n } from '@kbn/i18n'; +import { + UiActionsStart, + VISUALIZE_GEO_FIELD_TRIGGER, +} from '../../../../../../../src/plugins/ui_actions/public'; +import { getVisualizeGeoFieldMessage } from '../../../utils'; +import { DragDrop } from '../../../drag_drop'; +import { GlobeIllustration } from '../../../assets/globe_illustration'; +import './geo_field_workspace_panel.scss'; + +interface Props { + fieldType: string; + fieldName: string; + indexPatternId: string; + uiActions: UiActionsStart; +} + +const dragDropIdentifier = { + id: 'lnsGeoFieldWorkspace', + humanData: { + label: i18n.translate('xpack.lens.geoFieldWorkspace.dropZoneLabel', { + defaultMessage: 'drop zone to open in maps', + }), + }, +}; + +const dragDropOrder = [1, 0, 0, 0]; + +export function GeoFieldWorkspacePanel(props: Props) { + function onDrop() { + props.uiActions.getTrigger(VISUALIZE_GEO_FIELD_TRIGGER).exec({ + indexPatternId: props.indexPatternId, + fieldName: props.fieldName, + }); + } + + return ( + + +

+ {getVisualizeGeoFieldMessage(props.fieldType)} +

+ + +

+ + + +

+ + + + ); +} diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx index a31146e500434..3d5d9a6d84d81 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx @@ -338,17 +338,22 @@ export const InnerWorkspacePanel = React.memo(function InnerWorkspacePanel({ ); }; - return ( - + const dragDropContext = useContext(DragContext); + + const renderDragDrop = () => { + const customWorkspaceRenderer = + activeDatasourceId && + datasourceMap[activeDatasourceId]?.getCustomWorkspaceRenderer && + dragDropContext.dragging + ? datasourceMap[activeDatasourceId].getCustomWorkspaceRenderer!( + datasourceStates[activeDatasourceId].state, + dragDropContext.dragging + ) + : undefined; + + return customWorkspaceRenderer ? ( + customWorkspaceRenderer() + ) : ( + ); + }; + + return ( + + {renderDragDrop()} ); }); diff --git a/x-pack/plugins/lens/public/editor_frame_service/service.tsx b/x-pack/plugins/lens/public/editor_frame_service/service.tsx index 849baa93652cc..46dc326a015a8 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/service.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/service.tsx @@ -44,7 +44,7 @@ export interface EditorFrameStartPlugins { embeddable?: EmbeddableStart; dashboard?: DashboardStart; expressions: ExpressionsStart; - uiActions?: UiActionsStart; + uiActions: UiActionsStart; charts: ChartsPluginSetup; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx index 6c5116436dddb..eeec7871a262c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.test.tsx @@ -22,6 +22,7 @@ import { documentField } from './document_field'; import { chartPluginMock } from '../../../../../src/plugins/charts/public/mocks'; import { indexPatternFieldEditorPluginMock } from '../../../../../src/plugins/index_pattern_field_editor/public/mocks'; import { getFieldByNameFactory } from './pure_helpers'; +import { uiActionsPluginMock } from '../../../../../src/plugins/ui_actions/public/mocks'; const fieldsOne = [ { @@ -267,6 +268,7 @@ describe('IndexPattern Data Panel', () => { showNoDataPopover: jest.fn(), dropOntoWorkspace: jest.fn(), hasSuggestionForField: jest.fn(() => false), + uiActions: uiActionsPluginMock.createStartContract(), }; }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx index 4839d9388253b..a0a6b30e541a7 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/datapanel.tsx @@ -41,6 +41,7 @@ import { fieldExists } from './pure_helpers'; import { Loader } from '../loader'; import { esQuery, IIndexPattern } from '../../../../../src/plugins/data/public'; import { IndexPatternFieldEditorStart } from '../../../../../src/plugins/index_pattern_field_editor/public'; +import { VISUALIZE_GEO_FIELD_TRIGGER } from '../../../../../src/plugins/ui_actions/public'; export type Props = Omit, 'core'> & { data: DataPublicPluginStart; @@ -73,6 +74,8 @@ const supportedFieldTypes = new Set([ 'ip_range', 'histogram', 'document', + 'geo_point', + 'geo_shape', ]); const fieldTypeNames: Record = { @@ -83,6 +86,8 @@ const fieldTypeNames: Record = { date: i18n.translate('xpack.lens.datatypes.date', { defaultMessage: 'date' }), ip: i18n.translate('xpack.lens.datatypes.ipAddress', { defaultMessage: 'IP' }), histogram: i18n.translate('xpack.lens.datatypes.histogram', { defaultMessage: 'histogram' }), + geo_point: i18n.translate('xpack.lens.datatypes.geoPoint', { defaultMessage: 'geo_point' }), + geo_shape: i18n.translate('xpack.lens.datatypes.geoShape', { defaultMessage: 'geo_shape' }), }; // Wrapper around esQuery.buildEsQuery, handling errors (e.g. because a query can't be parsed) by @@ -121,6 +126,7 @@ export function IndexPatternDataPanel({ showNoDataPopover, dropOntoWorkspace, hasSuggestionForField, + uiActions, }: Props) { const { indexPatternRefs, indexPatterns, currentIndexPatternId } = state; const onChangeIndexPattern = useCallback( @@ -233,6 +239,7 @@ export function IndexPatternDataPanel({ existenceFetchTimeout={state.existenceFetchTimeout} dropOntoWorkspace={dropOntoWorkspace} hasSuggestionForField={hasSuggestionForField} + uiActions={uiActions} /> )} @@ -286,6 +293,7 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ charts, dropOntoWorkspace, hasSuggestionForField, + uiActions, }: Omit & { data: DataPublicPluginStart; core: CoreStart; @@ -310,7 +318,10 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ isMetaAccordionOpen: false, }); const currentIndexPattern = indexPatterns[currentIndexPatternId]; - const allFields = currentIndexPattern.fields; + const visualizeGeoFieldTrigger = uiActions.getTrigger(VISUALIZE_GEO_FIELD_TRIGGER); + const allFields = visualizeGeoFieldTrigger + ? currentIndexPattern.fields + : currentIndexPattern.fields.filter(({ type }) => type !== 'geo_point' && type !== 'geo_shape'); const clearLocalState = () => setLocalState((s) => ({ ...s, nameFilter: '', typeFilter: [] })); const hasSyncedExistingFields = existingFields[currentIndexPattern.title]; const availableFieldTypes = uniq(allFields.map(({ type }) => type)).filter( @@ -807,6 +818,7 @@ export const InnerIndexPatternDataPanel = function InnerIndexPatternDataPanel({ hasSuggestionForField={hasSuggestionForField} editField={editField} removeField={removeField} + uiActions={uiActions} /> diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.test.tsx index dcc11ea426117..cf9f7c0c559e4 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.test.tsx @@ -17,6 +17,7 @@ import { dataPluginMock } from '../../../../../src/plugins/data/public/mocks'; import { IndexPattern } from './types'; import { chartPluginMock } from '../../../../../src/plugins/charts/public/mocks'; import { documentField } from './document_field'; +import { uiActionsPluginMock } from '../../../../../src/plugins/ui_actions/public/mocks'; const chartsThemeService = chartPluginMock.createSetupContract().theme; @@ -109,6 +110,7 @@ describe('IndexPattern Field Item', () => { itemIndex: 0, dropOntoWorkspace: () => {}, hasSuggestionForField: () => false, + uiActions: uiActionsPluginMock.createStartContract(), }; data.fieldFormats = ({ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx index fce4fcda14cfc..013bb46500d0d 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/field_item.tsx @@ -54,6 +54,9 @@ import { BucketedAggregation, FieldStatsResponse } from '../../common'; import { IndexPattern, IndexPatternField, DraggedField } from './types'; import { LensFieldIcon } from './lens_field_icon'; import { trackUiEvent } from '../lens_ui_telemetry'; +import { UiActionsStart } from '../../../../../src/plugins/ui_actions/public'; +import { VisualizeGeoFieldButton } from './visualize_geo_field_button'; +import { getVisualizeGeoFieldMessage } from '../utils'; import { debouncedComponent } from '../debounced_component'; @@ -75,6 +78,7 @@ export interface FieldItemProps { editField?: (name: string) => void; removeField?: (name: string) => void; hasSuggestionForField: DatasourceDataPanelProps['hasSuggestionForField']; + uiActions: UiActionsStart; } interface State { @@ -149,7 +153,13 @@ export const InnerFieldItem = function InnerFieldItem(props: FieldItemProps) { function fetchData() { // Range types don't have any useful stats we can show - if (state.isLoading || field.type === 'document' || field.type.includes('range')) { + if ( + state.isLoading || + field.type === 'document' || + field.type.includes('range') || + field.type === 'geo_point' || + field.type === 'geo_shape' + ) { return; } @@ -392,6 +402,7 @@ function FieldItemPopoverContents(props: State & FieldItemProps) { removeField, hasSuggestionForField, hideDetails, + uiActions, } = props; const chartTheme = chartsThemeService.useChartsTheme(); @@ -467,6 +478,21 @@ function FieldItemPopoverContents(props: State & FieldItemProps) { ); + } else if (field.type === 'geo_point' || field.type === 'geo_shape') { + return ( + <> + {panelHeader} + + {getVisualizeGeoFieldMessage(field.type)} + + + + + ); } else if ( (!props.histogram || props.histogram.buckets.length === 0) && (!props.topValues || props.topValues.buckets.length === 0) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/field_list.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/field_list.tsx index ee0011ad0390c..13d5d25bc2ea2 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/field_list.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/field_list.tsx @@ -14,6 +14,7 @@ import { NoFieldsCallout } from './no_fields_callout'; import { IndexPatternField } from './types'; import { FieldItemSharedProps, FieldsAccordion } from './fields_accordion'; import { DatasourceDataPanelProps } from '../types'; +import { UiActionsStart } from '../../../../../src/plugins/ui_actions/public'; const PAGINATION_SIZE = 50; export type FieldGroups = Record< @@ -55,6 +56,7 @@ export const FieldList = React.memo(function FieldList({ hasSuggestionForField, editField, removeField, + uiActions, }: { exists: (field: IndexPatternField) => boolean; fieldGroups: FieldGroups; @@ -72,6 +74,7 @@ export const FieldList = React.memo(function FieldList({ hasSuggestionForField: DatasourceDataPanelProps['hasSuggestionForField']; editField?: (name: string) => void; removeField?: (name: string) => void; + uiActions: UiActionsStart; }) { const [pageSize, setPageSize] = useState(PAGINATION_SIZE); const [scrollContainer, setScrollContainer] = useState(undefined); @@ -155,6 +158,7 @@ export const FieldList = React.memo(function FieldList({ groupIndex={0} dropOntoWorkspace={dropOntoWorkspace} hasSuggestionForField={hasSuggestionForField} + uiActions={uiActions} /> )) )} @@ -206,6 +210,7 @@ export const FieldList = React.memo(function FieldList({ defaultNoFieldsMessage={fieldGroup.defaultNoFieldsMessage} /> } + uiActions={uiActions} /> diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/fields_accordion.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/fields_accordion.test.tsx index c8c48e2accf9b..6270b94abf565 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/fields_accordion.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/fields_accordion.test.tsx @@ -15,6 +15,7 @@ import { IndexPattern } from './types'; import { FieldItem } from './field_item'; import { FieldsAccordion, FieldsAccordionProps, FieldItemSharedProps } from './fields_accordion'; import { chartPluginMock } from '../../../../../src/plugins/charts/public/mocks'; +import { uiActionsPluginMock } from '../../../../../src/plugins/ui_actions/public/mocks'; describe('Fields Accordion', () => { let defaultProps: FieldsAccordionProps; @@ -76,6 +77,7 @@ describe('Fields Accordion', () => { groupIndex: 0, dropOntoWorkspace: () => {}, hasSuggestionForField: () => false, + uiActions: uiActionsPluginMock.createStartContract(), }; }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/fields_accordion.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/fields_accordion.tsx index 42de7cb328b13..9f5409f9837f4 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/fields_accordion.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/fields_accordion.tsx @@ -24,6 +24,7 @@ import { Query, Filter } from '../../../../../src/plugins/data/public'; import { DatasourceDataPanelProps } from '../types'; import { IndexPattern } from './types'; import { ChartsPluginSetup } from '../../../../../src/plugins/charts/public'; +import { UiActionsStart } from '../../../../../src/plugins/ui_actions/public'; export interface FieldItemSharedProps { core: DatasourceDataPanelProps['core']; @@ -57,6 +58,7 @@ export interface FieldsAccordionProps { hasSuggestionForField: DatasourceDataPanelProps['hasSuggestionForField']; editField?: (name: string) => void; removeField?: (name: string) => void; + uiActions: UiActionsStart; } export const FieldsAccordion = memo(function InnerFieldsAccordion({ @@ -80,6 +82,7 @@ export const FieldsAccordion = memo(function InnerFieldsAccordion({ hasSuggestionForField, editField, removeField, + uiActions, }: FieldsAccordionProps) { const renderField = useCallback( (field: IndexPatternField, index) => ( @@ -95,6 +98,7 @@ export const FieldsAccordion = memo(function InnerFieldsAccordion({ hasSuggestionForField={hasSuggestionForField} editField={editField} removeField={removeField} + uiActions={uiActions} /> ), [ @@ -106,6 +110,7 @@ export const FieldsAccordion = memo(function InnerFieldsAccordion({ groupIndex, editField, removeField, + uiActions, ] ); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/index.ts index a556c6ce0c095..f8bc84643bcab 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/index.ts @@ -15,6 +15,7 @@ import { DataPublicPluginStart, } from '../../../../../src/plugins/data/public'; import { Datasource, EditorFrameSetup } from '../types'; +import { UiActionsStart } from '../../../../../src/plugins/ui_actions/public'; export interface IndexPatternDatasourceSetupPlugins { expressions: ExpressionsSetup; @@ -26,6 +27,7 @@ export interface IndexPatternDatasourceSetupPlugins { export interface IndexPatternDatasourceStartPlugins { data: DataPublicPluginStart; indexPatternFieldEditor: IndexPatternFieldEditorStart; + uiActions: UiActionsStart; } export class IndexPatternDatasource { @@ -44,20 +46,23 @@ export class IndexPatternDatasource { getTimeScaleFunction, getSuffixFormatter, } = await import('../async_services'); - return core.getStartServices().then(([coreStart, { data, indexPatternFieldEditor }]) => { - data.fieldFormats.register([getSuffixFormatter(data.fieldFormats.deserialize)]); - expressions.registerFunction(getTimeScaleFunction(data)); - expressions.registerFunction(counterRate); - expressions.registerFunction(renameColumns); - expressions.registerFunction(formatColumn); - return getIndexPatternDatasource({ - core: coreStart, - storage: new Storage(localStorage), - data, - charts, - indexPatternFieldEditor, - }); - }) as Promise; + return core + .getStartServices() + .then(([coreStart, { data, indexPatternFieldEditor, uiActions }]) => { + data.fieldFormats.register([getSuffixFormatter(data.fieldFormats.deserialize)]); + expressions.registerFunction(getTimeScaleFunction(data)); + expressions.registerFunction(counterRate); + expressions.registerFunction(renameColumns); + expressions.registerFunction(formatColumn); + return getIndexPatternDatasource({ + core: coreStart, + storage: new Storage(localStorage), + data, + charts, + indexPatternFieldEditor, + uiActions, + }); + }) as Promise; }); } } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index c291c7ab3eac0..b1ff7b36b47a3 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -17,6 +17,7 @@ import { getFieldByNameFactory } from './pure_helpers'; import { operationDefinitionMap, getErrorMessages } from './operations'; import { createMockedReferenceOperation } from './operations/mocks'; import { indexPatternFieldEditorPluginMock } from 'src/plugins/index_pattern_field_editor/public/mocks'; +import { uiActionsPluginMock } from '../../../../../src/plugins/ui_actions/public/mocks'; jest.mock('./loader'); jest.mock('../id_generator'); @@ -172,6 +173,7 @@ describe('IndexPattern Data Source', () => { data: dataPluginMock.createStartContract(), charts: chartPluginMock.createSetupContract(), indexPatternFieldEditor: indexPatternFieldEditorPluginMock.createStartContract(), + uiActions: uiActionsPluginMock.createStartContract(), }); baseState = { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index 81eb46e816715..8fb0994c42fb9 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -45,7 +45,7 @@ import { import { isDraggedField, normalizeOperationDataType } from './utils'; import { LayerPanel } from './layerpanel'; import { IndexPatternColumn, getErrorMessages, IncompleteColumn } from './operations'; -import { IndexPatternPrivateState, IndexPatternPersistedState } from './types'; +import { IndexPatternField, IndexPatternPrivateState, IndexPatternPersistedState } from './types'; import { KibanaContextProvider } from '../../../../../src/plugins/kibana_react/public'; import { DataPublicPluginStart } from '../../../../../src/plugins/data/public'; import { VisualizeFieldContext } from '../../../../../src/plugins/ui_actions/public'; @@ -53,6 +53,9 @@ import { mergeLayer } from './state_helpers'; import { Datasource, StateSetter } from '../types'; import { ChartsPluginSetup } from '../../../../../src/plugins/charts/public'; import { deleteColumn, isReferenced } from './operations'; +import { UiActionsStart } from '../../../../../src/plugins/ui_actions/public'; +import { GeoFieldWorkspacePanel } from '../editor_frame_service/editor_frame/workspace_panel/geo_field_workspace_panel'; +import { DraggingIdentifier } from '../drag_drop'; export { OperationType, IndexPatternColumn, deleteColumn } from './operations'; @@ -78,12 +81,14 @@ export function getIndexPatternDatasource({ data, charts, indexPatternFieldEditor, + uiActions, }: { core: CoreStart; storage: IStorageWrapper; data: DataPublicPluginStart; charts: ChartsPluginSetup; indexPatternFieldEditor: IndexPatternFieldEditorStart; + uiActions: UiActionsStart; }) { const uiSettings = core.uiSettings; const onIndexPatternLoadError = (err: Error) => @@ -197,6 +202,7 @@ export function getIndexPatternDatasource({ indexPatternFieldEditor={indexPatternFieldEditor} {...props} core={core} + uiActions={uiActions} /> , domElement @@ -320,6 +326,34 @@ export function getIndexPatternDatasource({ getDropProps, onDrop, + getCustomWorkspaceRenderer: (state: IndexPatternPrivateState, dragging: DraggingIdentifier) => { + if (dragging.field === undefined || dragging.indexPatternId === undefined) { + return undefined; + } + + const draggedField = dragging as DraggingIdentifier & { + field: IndexPatternField; + indexPatternId: string; + }; + const geoFieldType = + draggedField.field.esTypes && + draggedField.field.esTypes.find((esType) => { + return ['geo_point', 'geo_shape'].includes(esType); + }); + return geoFieldType + ? () => { + return ( + + ); + } + : undefined; + }, + // Reset the temporary invalid state when closing the editor, but don't // update the state if it's not needed updateStateOnCloseDimension: ({ state, layerId, columnId }) => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/visualize_geo_field_button.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/visualize_geo_field_button.tsx new file mode 100644 index 0000000000000..3101fb12b933a --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/visualize_geo_field_button.tsx @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { MouseEvent, useEffect, useState } from 'react'; +import { EuiButton } from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { + visualizeGeoFieldTrigger, + VISUALIZE_GEO_FIELD_TRIGGER, + UiActionsStart, +} from '../../../../../src/plugins/ui_actions/public'; + +interface Props { + indexPatternId: string; + fieldName: string; + uiActions: UiActionsStart; +} + +export function VisualizeGeoFieldButton(props: Props) { + const [href, setHref] = useState(undefined); + + async function loadHref() { + const actions = await props.uiActions.getTriggerCompatibleActions(VISUALIZE_GEO_FIELD_TRIGGER, { + indexPatternId: props.indexPatternId, + fieldName: props.fieldName, + }); + const triggerOptions = { + indexPatternId: props.indexPatternId, + fieldName: props.fieldName, + trigger: visualizeGeoFieldTrigger, + }; + const loadedHref = actions.length ? await actions[0].getHref?.(triggerOptions) : undefined; + setHref(loadedHref); + } + + useEffect( + () => { + loadHref(); + }, + /* eslint-disable-next-line react-hooks/exhaustive-deps */ + [] + ); + + function onClick(event: MouseEvent) { + event.preventDefault(); + props.uiActions.getTrigger(VISUALIZE_GEO_FIELD_TRIGGER).exec({ + indexPatternId: props.indexPatternId, + fieldName: props.fieldName, + }); + } + + return ( + <> + {/* eslint-disable-next-line @elastic/eui/href-or-on-click */} + + + + + ); +} diff --git a/x-pack/plugins/lens/public/pie_visualization/render_function.test.tsx b/x-pack/plugins/lens/public/pie_visualization/render_function.test.tsx index 4dcd9772b61b4..7191da0af6bfe 100644 --- a/x-pack/plugins/lens/public/pie_visualization/render_function.test.tsx +++ b/x-pack/plugins/lens/public/pie_visualization/render_function.test.tsx @@ -13,6 +13,7 @@ import { NodeColorAccessor, ShapeTreeNode, HierarchyOfArrays, + Chart, } from '@elastic/charts'; import { shallow } from 'enzyme'; import { LensMultiTable } from '../types'; @@ -42,12 +43,12 @@ describe('PieVisualization component', () => { type: 'datatable', columns: [ { id: 'a', name: 'a', meta: { type: 'number' } }, - { id: 'b', name: 'b', meta: { type: 'number' } }, - { id: 'c', name: 'c', meta: { type: 'string' } }, + { id: 'b', name: 'b', meta: { type: 'string' } }, + { id: 'c', name: 'c', meta: { type: 'number' } }, ], rows: [ - { a: 6, b: 2, c: 'I', d: 'Row 1' }, - { a: 1, b: 5, c: 'J', d: 'Row 2' }, + { a: 6, b: 'I', c: 2, d: 'Row 1' }, + { a: 1, b: 'J', c: 5, d: 'Row 2' }, ], }, }, @@ -249,14 +250,14 @@ describe('PieVisualization component', () => { Object { "id": "b", "meta": Object { - "type": "number", + "type": "string", }, "name": "b", }, Object { "id": "c", "meta": Object { - "type": "string", + "type": "number", }, "name": "c", }, @@ -264,14 +265,14 @@ describe('PieVisualization component', () => { "rows": Array [ Object { "a": 6, - "b": 2, - "c": "I", + "b": "I", + "c": 2, "d": "Row 1", }, Object { "a": 1, - "b": 5, - "c": "J", + "b": "J", + "c": 5, "d": "Row 2", }, ], @@ -292,6 +293,48 @@ describe('PieVisualization component', () => { expect(component.find(Settings).first().prop('onElementClick')).toBeUndefined(); }); + test('it renders the empty placeholder when metric contains only falsy data', () => { + const defaultData = getDefaultArgs().data; + const emptyData: LensMultiTable = { + ...defaultData, + tables: { + first: { + ...defaultData.tables.first, + rows: [ + { a: 0, b: 'I', c: 0, d: 'Row 1' }, + { a: 0, b: 'J', c: null, d: 'Row 2' }, + ], + }, + }, + }; + + const component = shallow( + + ); + expect(component.find(EmptyPlaceholder)).toHaveLength(1); + }); + + test('it renders the chart when metric contains truthy data and buckets contain only falsy data', () => { + const defaultData = getDefaultArgs().data; + const emptyData: LensMultiTable = { + ...defaultData, + tables: { + first: { + ...defaultData.tables.first, + // a and b are buckets, c is a metric + rows: [{ a: 0, b: undefined, c: 12 }], + }, + }, + }; + + const component = shallow( + + ); + + expect(component.find(EmptyPlaceholder)).toHaveLength(0); + expect(component.find(Chart)).toHaveLength(1); + }); + test('it shows emptyPlaceholder for undefined grouped data', () => { const defaultData = getDefaultArgs().data; const emptyData: LensMultiTable = { @@ -300,8 +343,8 @@ describe('PieVisualization component', () => { first: { ...defaultData.tables.first, rows: [ - { a: undefined, b: undefined, c: 'I', d: 'Row 1' }, - { a: undefined, b: undefined, c: 'J', d: 'Row 2' }, + { a: undefined, b: 'I', c: undefined, d: 'Row 1' }, + { a: undefined, b: 'J', c: undefined, d: 'Row 2' }, ], }, }, diff --git a/x-pack/plugins/lens/public/pie_visualization/render_function.tsx b/x-pack/plugins/lens/public/pie_visualization/render_function.tsx index b6bb2908b9ed2..cc31222f6b9ab 100644 --- a/x-pack/plugins/lens/public/pie_visualization/render_function.tsx +++ b/x-pack/plugins/lens/public/pie_visualization/render_function.tsx @@ -222,11 +222,15 @@ export function PieComponent( const value = row[metricColumn.id]; return typeof value === 'number' && value < 0; }); + + const isMetricEmpty = firstTable.rows.every((row) => { + return !row[metricColumn.id]; + }); + const isEmpty = firstTable.rows.length === 0 || - firstTable.rows.every((row) => - groups.every((colId) => !row[colId] || typeof row[colId] === 'undefined') - ); + firstTable.rows.every((row) => groups.every((colId) => typeof row[colId] === 'undefined')) || + isMetricEmpty; if (isEmpty) { return ; diff --git a/x-pack/plugins/lens/public/types.ts b/x-pack/plugins/lens/public/types.ts index 94b4433a82551..51d679e7c40e5 100644 --- a/x-pack/plugins/lens/public/types.ts +++ b/x-pack/plugins/lens/public/types.ts @@ -33,6 +33,7 @@ import type { LensResizeActionData, LensToggleActionData, } from './datatable_visualization/components/types'; +import { UiActionsStart } from '../../../../src/plugins/ui_actions/public'; export type ErrorCallback = (e: { message: string }) => void; @@ -213,6 +214,10 @@ export interface Datasource { } ) => { dropTypes: DropType[]; nextLabel?: string } | undefined; onDrop: (props: DatasourceDimensionDropHandlerProps) => false | true | { deleted: string }; + getCustomWorkspaceRenderer?: ( + state: T, + dragging: DraggingIdentifier + ) => undefined | (() => JSX.Element); updateStateOnCloseDimension?: (props: { layerId: string; columnId: string; @@ -267,6 +272,7 @@ export interface DatasourceDataPanelProps { filters: Filter[]; dropOntoWorkspace: (field: DragDropIdentifier) => void; hasSuggestionForField: (field: DragDropIdentifier) => boolean; + uiActions: UiActionsStart; } interface SharedDimensionProps { @@ -343,7 +349,7 @@ export type DatasourceDimensionDropHandlerProps = DatasourceDimensionDropProp dropType: DropType; }; -export type FieldOnlyDataType = 'document' | 'ip' | 'histogram'; +export type FieldOnlyDataType = 'document' | 'ip' | 'histogram' | 'geo_point' | 'geo_shape'; export type DataType = 'string' | 'number' | 'date' | 'boolean' | FieldOnlyDataType; // An operation represents a column in a table, not any information diff --git a/x-pack/plugins/lens/public/utils.ts b/x-pack/plugins/lens/public/utils.ts index ab95141431541..2d8cfee2185fa 100644 --- a/x-pack/plugins/lens/public/utils.ts +++ b/x-pack/plugins/lens/public/utils.ts @@ -42,3 +42,10 @@ export const desanitizeFilterContext = ( } return result; }; + +export function getVisualizeGeoFieldMessage(fieldType: string) { + return i18n.translate('xpack.lens.visualizeGeoFieldMessage', { + defaultMessage: `Lens cannot visualize {fieldType} fields`, + values: { fieldType }, + }); +} diff --git a/x-pack/plugins/lens/public/xy_visualization/xy_suggestions.ts b/x-pack/plugins/lens/public/xy_visualization/xy_suggestions.ts index 1616a4c141476..d5120cfae973c 100644 --- a/x-pack/plugins/lens/public/xy_visualization/xy_suggestions.ts +++ b/x-pack/plugins/lens/public/xy_visualization/xy_suggestions.ts @@ -27,6 +27,8 @@ const columnSortOrder = { boolean: 4, number: 5, histogram: 6, + geo_point: 7, + geo_shape: 8, }; /** diff --git a/x-pack/plugins/lists/server/routes/create_endpoint_list_item_route.ts b/x-pack/plugins/lists/server/routes/create_endpoint_list_item_route.ts index 76ff8b1728922..9c42df6abd7bd 100644 --- a/x-pack/plugins/lists/server/routes/create_endpoint_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/create_endpoint_list_item_route.ts @@ -6,17 +6,17 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { ENDPOINT_LIST_ID, ENDPOINT_LIST_ITEM_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { CreateEndpointListItemSchemaDecoded, createEndpointListItemSchema, exceptionListItemSchema, } from '../../common/schemas'; -import { getExceptionListClient } from './utils/get_exception_list_client'; +import { buildRouteValidation, buildSiemResponse, getExceptionListClient } from './utils'; import { validateExceptionListSize } from './validate'; export const createEndpointListItemRoute = (router: ListsPluginRouter): void => { diff --git a/x-pack/plugins/lists/server/routes/create_endpoint_list_route.ts b/x-pack/plugins/lists/server/routes/create_endpoint_list_route.ts index 23f098f7e9457..599870c226564 100644 --- a/x-pack/plugins/lists/server/routes/create_endpoint_list_route.ts +++ b/x-pack/plugins/lists/server/routes/create_endpoint_list_route.ts @@ -6,12 +6,13 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { ENDPOINT_LIST_URL } from '../../common/constants'; -import { buildSiemResponse, transformError } from '../siem_server_deps'; import { createEndpointListSchema } from '../../common/schemas'; +import { buildSiemResponse } from './utils'; import { getExceptionListClient } from './utils/get_exception_list_client'; /** diff --git a/x-pack/plugins/lists/server/routes/create_exception_list_item_route.ts b/x-pack/plugins/lists/server/routes/create_exception_list_item_route.ts index 4bcb41c666f56..81260584e8a50 100644 --- a/x-pack/plugins/lists/server/routes/create_exception_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/create_exception_list_item_route.ts @@ -6,16 +6,17 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { EXCEPTION_LIST_ITEM_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { CreateExceptionListItemSchemaDecoded, createExceptionListItemSchema, exceptionListItemSchema, } from '../../common/schemas'; +import { buildRouteValidation, buildSiemResponse } from './utils'; import { getExceptionListClient } from './utils/get_exception_list_client'; import { endpointDisallowedFields } from './endpoint_disallowed_fields'; import { validateEndpointExceptionItemEntries, validateExceptionListSize } from './validate'; diff --git a/x-pack/plugins/lists/server/routes/create_exception_list_route.ts b/x-pack/plugins/lists/server/routes/create_exception_list_route.ts index 12c887a16c318..1a35bdb008662 100644 --- a/x-pack/plugins/lists/server/routes/create_exception_list_route.ts +++ b/x-pack/plugins/lists/server/routes/create_exception_list_route.ts @@ -6,16 +6,17 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { EXCEPTION_LIST_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { CreateExceptionListSchemaDecoded, createExceptionListSchema, exceptionListSchema, } from '../../common/schemas'; +import { buildRouteValidation, buildSiemResponse } from './utils'; import { getExceptionListClient } from './utils/get_exception_list_client'; export const createExceptionListRoute = (router: ListsPluginRouter): void => { diff --git a/x-pack/plugins/lists/server/routes/create_list_index_route.ts b/x-pack/plugins/lists/server/routes/create_list_index_route.ts index 12fe586a07cc0..3b0d34b8952a1 100644 --- a/x-pack/plugins/lists/server/routes/create_list_index_route.ts +++ b/x-pack/plugins/lists/server/routes/create_list_index_route.ts @@ -6,12 +6,14 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; -import { buildSiemResponse, transformError } from '../siem_server_deps'; import { LIST_INDEX } from '../../common/constants'; import { acknowledgeSchema } from '../../common/schemas'; +import { buildSiemResponse } from './utils'; + import { getListClient } from '.'; export const createListIndexRoute = (router: ListsPluginRouter): void => { diff --git a/x-pack/plugins/lists/server/routes/create_list_item_route.ts b/x-pack/plugins/lists/server/routes/create_list_item_route.ts index 2e3c944af0df8..4df121af4c1ba 100644 --- a/x-pack/plugins/lists/server/routes/create_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/create_list_item_route.ts @@ -6,12 +6,14 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { LIST_ITEM_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { createListItemSchema, listItemSchema } from '../../common/schemas'; +import { buildRouteValidation, buildSiemResponse } from './utils'; + import { getListClient } from '.'; export const createListItemRoute = (router: ListsPluginRouter): void => { diff --git a/x-pack/plugins/lists/server/routes/create_list_route.ts b/x-pack/plugins/lists/server/routes/create_list_route.ts index 4346d519c9003..dabbd690bba21 100644 --- a/x-pack/plugins/lists/server/routes/create_list_route.ts +++ b/x-pack/plugins/lists/server/routes/create_list_route.ts @@ -6,12 +6,14 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { LIST_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { CreateListSchemaDecoded, createListSchema, listSchema } from '../../common/schemas'; +import { buildRouteValidation, buildSiemResponse } from './utils'; + import { getListClient } from '.'; export const createListRoute = (router: ListsPluginRouter): void => { diff --git a/x-pack/plugins/lists/server/routes/delete_endpoint_list_item_route.ts b/x-pack/plugins/lists/server/routes/delete_endpoint_list_item_route.ts index 195384356f40b..59d91f6234176 100644 --- a/x-pack/plugins/lists/server/routes/delete_endpoint_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/delete_endpoint_list_item_route.ts @@ -6,17 +6,22 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { ENDPOINT_LIST_ITEM_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { DeleteEndpointListItemSchemaDecoded, deleteEndpointListItemSchema, exceptionListItemSchema, } from '../../common/schemas'; -import { getErrorMessageExceptionListItem, getExceptionListClient } from './utils'; +import { + buildRouteValidation, + buildSiemResponse, + getErrorMessageExceptionListItem, + getExceptionListClient, +} from './utils'; export const deleteEndpointListItemRoute = (router: ListsPluginRouter): void => { router.delete( diff --git a/x-pack/plugins/lists/server/routes/delete_exception_list_item_route.ts b/x-pack/plugins/lists/server/routes/delete_exception_list_item_route.ts index ddcd1cf9b7180..ce4f91ffc671a 100644 --- a/x-pack/plugins/lists/server/routes/delete_exception_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/delete_exception_list_item_route.ts @@ -6,17 +6,22 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { EXCEPTION_LIST_ITEM_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { DeleteExceptionListItemSchemaDecoded, deleteExceptionListItemSchema, exceptionListItemSchema, } from '../../common/schemas'; -import { getErrorMessageExceptionListItem, getExceptionListClient } from './utils'; +import { + buildRouteValidation, + buildSiemResponse, + getErrorMessageExceptionListItem, + getExceptionListClient, +} from './utils'; export const deleteExceptionListItemRoute = (router: ListsPluginRouter): void => { router.delete( diff --git a/x-pack/plugins/lists/server/routes/delete_exception_list_route.ts b/x-pack/plugins/lists/server/routes/delete_exception_list_route.ts index f11deef5cb0c8..eeeb5fb44c16a 100644 --- a/x-pack/plugins/lists/server/routes/delete_exception_list_route.ts +++ b/x-pack/plugins/lists/server/routes/delete_exception_list_route.ts @@ -6,17 +6,22 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { EXCEPTION_LIST_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { DeleteExceptionListSchemaDecoded, deleteExceptionListSchema, exceptionListSchema, } from '../../common/schemas'; -import { getErrorMessageExceptionList, getExceptionListClient } from './utils'; +import { + buildRouteValidation, + buildSiemResponse, + getErrorMessageExceptionList, + getExceptionListClient, +} from './utils'; export const deleteExceptionListRoute = (router: ListsPluginRouter): void => { router.delete( diff --git a/x-pack/plugins/lists/server/routes/delete_list_index_route.ts b/x-pack/plugins/lists/server/routes/delete_list_index_route.ts index efad16c37a2dc..22c56a21df419 100644 --- a/x-pack/plugins/lists/server/routes/delete_list_index_route.ts +++ b/x-pack/plugins/lists/server/routes/delete_list_index_route.ts @@ -6,12 +6,14 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { LIST_INDEX } from '../../common/constants'; -import { buildSiemResponse, transformError } from '../siem_server_deps'; import { acknowledgeSchema } from '../../common/schemas'; +import { buildSiemResponse } from './utils'; + import { getListClient } from '.'; /** diff --git a/x-pack/plugins/lists/server/routes/delete_list_item_route.ts b/x-pack/plugins/lists/server/routes/delete_list_item_route.ts index a07035fc50d9c..197590ecb142c 100644 --- a/x-pack/plugins/lists/server/routes/delete_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/delete_list_item_route.ts @@ -6,12 +6,14 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { LIST_ITEM_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { deleteListItemSchema, listItemArraySchema, listItemSchema } from '../../common/schemas'; +import { buildRouteValidation, buildSiemResponse } from './utils'; + import { getListClient } from '.'; export const deleteListItemRoute = (router: ListsPluginRouter): void => { diff --git a/x-pack/plugins/lists/server/routes/delete_list_route.ts b/x-pack/plugins/lists/server/routes/delete_list_route.ts index 65faa54b20cc7..033c49aa7b235 100644 --- a/x-pack/plugins/lists/server/routes/delete_list_route.ts +++ b/x-pack/plugins/lists/server/routes/delete_list_route.ts @@ -6,10 +6,10 @@ */ import { EntriesArray, validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { LIST_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { ExceptionListItemSchema, FoundExceptionListSchema, @@ -21,6 +21,8 @@ import { getSavedObjectType } from '../services/exception_lists/utils'; import { ExceptionListClient } from '../services/exception_lists/exception_list_client'; import { escapeQuotes } from '../services/utils/escape_query'; +import { buildRouteValidation, buildSiemResponse } from './utils'; + import { getExceptionListClient, getListClient } from '.'; export const deleteListRoute = (router: ListsPluginRouter): void => { diff --git a/x-pack/plugins/lists/server/routes/export_exception_list_route.ts b/x-pack/plugins/lists/server/routes/export_exception_list_route.ts index 30f7a16e6100d..3d82cbac47a88 100644 --- a/x-pack/plugins/lists/server/routes/export_exception_list_route.ts +++ b/x-pack/plugins/lists/server/routes/export_exception_list_route.ts @@ -5,12 +5,13 @@ * 2.0. */ +import { transformError } from '@kbn/securitysolution-es-utils'; + import type { ListsPluginRouter } from '../types'; import { EXCEPTION_LIST_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { exportExceptionListQuerySchema } from '../../common/schemas'; -import { getExceptionListClient } from './utils'; +import { buildRouteValidation, buildSiemResponse, getExceptionListClient } from './utils'; export const exportExceptionListRoute = (router: ListsPluginRouter): void => { router.get( diff --git a/x-pack/plugins/lists/server/routes/export_list_item_route.ts b/x-pack/plugins/lists/server/routes/export_list_item_route.ts index e07b78a23c7e0..13a2aa9beea05 100644 --- a/x-pack/plugins/lists/server/routes/export_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/export_list_item_route.ts @@ -7,11 +7,14 @@ import { Stream } from 'stream'; +import { transformError } from '@kbn/securitysolution-es-utils'; + import type { ListsPluginRouter } from '../types'; import { LIST_ITEM_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { exportListItemQuerySchema } from '../../common/schemas'; +import { buildRouteValidation, buildSiemResponse } from './utils'; + import { getListClient } from '.'; export const exportListItemRoute = (router: ListsPluginRouter): void => { diff --git a/x-pack/plugins/lists/server/routes/find_endpoint_list_item_route.ts b/x-pack/plugins/lists/server/routes/find_endpoint_list_item_route.ts index ee5245982dc0b..cbf3c320c407a 100644 --- a/x-pack/plugins/lists/server/routes/find_endpoint_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/find_endpoint_list_item_route.ts @@ -6,17 +6,17 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { ENDPOINT_LIST_ID, ENDPOINT_LIST_ITEM_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { FindEndpointListItemSchemaDecoded, findEndpointListItemSchema, foundExceptionListItemSchema, } from '../../common/schemas'; -import { getExceptionListClient } from './utils'; +import { buildRouteValidation, buildSiemResponse, getExceptionListClient } from './utils'; export const findEndpointListItemRoute = (router: ListsPluginRouter): void => { router.get( diff --git a/x-pack/plugins/lists/server/routes/find_exception_list_item_route.ts b/x-pack/plugins/lists/server/routes/find_exception_list_item_route.ts index 82988a7cbeb76..45ce1dbb87fba 100644 --- a/x-pack/plugins/lists/server/routes/find_exception_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/find_exception_list_item_route.ts @@ -6,17 +6,17 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { EXCEPTION_LIST_ITEM_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { FindExceptionListItemSchemaDecoded, findExceptionListItemSchema, foundExceptionListItemSchema, } from '../../common/schemas'; -import { getExceptionListClient } from './utils'; +import { buildRouteValidation, buildSiemResponse, getExceptionListClient } from './utils'; export const findExceptionListItemRoute = (router: ListsPluginRouter): void => { router.get( diff --git a/x-pack/plugins/lists/server/routes/find_exception_list_route.ts b/x-pack/plugins/lists/server/routes/find_exception_list_route.ts index 4b188b4dca4e2..0181bfed5b857 100644 --- a/x-pack/plugins/lists/server/routes/find_exception_list_route.ts +++ b/x-pack/plugins/lists/server/routes/find_exception_list_route.ts @@ -6,17 +6,17 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { EXCEPTION_LIST_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { FindExceptionListSchemaDecoded, findExceptionListSchema, foundExceptionListSchema, } from '../../common/schemas'; -import { getExceptionListClient } from './utils'; +import { buildRouteValidation, buildSiemResponse, getExceptionListClient } from './utils'; export const findExceptionListRoute = (router: ListsPluginRouter): void => { router.get( diff --git a/x-pack/plugins/lists/server/routes/find_list_item_route.ts b/x-pack/plugins/lists/server/routes/find_list_item_route.ts index a904d7f84733d..c64dfd561e0e3 100644 --- a/x-pack/plugins/lists/server/routes/find_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/find_list_item_route.ts @@ -6,10 +6,10 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { LIST_ITEM_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { FindListItemSchemaDecoded, findListItemSchema, @@ -17,7 +17,7 @@ import { } from '../../common/schemas'; import { decodeCursor } from '../services/utils'; -import { getListClient } from './utils'; +import { buildRouteValidation, buildSiemResponse, getListClient } from './utils'; export const findListItemRoute = (router: ListsPluginRouter): void => { router.get( diff --git a/x-pack/plugins/lists/server/routes/find_list_route.ts b/x-pack/plugins/lists/server/routes/find_list_route.ts index c5f1b58c1e957..19c20515ef5f2 100644 --- a/x-pack/plugins/lists/server/routes/find_list_route.ts +++ b/x-pack/plugins/lists/server/routes/find_list_route.ts @@ -6,14 +6,14 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { LIST_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { findListSchema, foundListSchema } from '../../common/schemas'; import { decodeCursor } from '../services/utils'; -import { getListClient } from './utils'; +import { buildRouteValidation, buildSiemResponse, getListClient } from './utils'; export const findListRoute = (router: ListsPluginRouter): void => { router.get( diff --git a/x-pack/plugins/lists/server/routes/import_list_item_route.ts b/x-pack/plugins/lists/server/routes/import_list_item_route.ts index 070764b0e1e77..77d9623f40a23 100644 --- a/x-pack/plugins/lists/server/routes/import_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/import_list_item_route.ts @@ -7,13 +7,14 @@ import { schema } from '@kbn/config-schema'; import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { LIST_ITEM_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { importListItemQuerySchema, listSchema } from '../../common/schemas'; import { ConfigType } from '../config'; +import { buildRouteValidation, buildSiemResponse } from './utils'; import { createStreamFromBuffer } from './utils/create_stream_from_buffer'; import { getListClient } from '.'; diff --git a/x-pack/plugins/lists/server/routes/patch_list_item_route.ts b/x-pack/plugins/lists/server/routes/patch_list_item_route.ts index 53fd7c65c8ab8..ce4ff71a1d886 100644 --- a/x-pack/plugins/lists/server/routes/patch_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/patch_list_item_route.ts @@ -6,12 +6,14 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { LIST_ITEM_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { listItemSchema, patchListItemSchema } from '../../common/schemas'; +import { buildRouteValidation, buildSiemResponse } from './utils'; + import { getListClient } from '.'; export const patchListItemRoute = (router: ListsPluginRouter): void => { diff --git a/x-pack/plugins/lists/server/routes/patch_list_route.ts b/x-pack/plugins/lists/server/routes/patch_list_route.ts index f139fb72c3066..3f2427b30f2be 100644 --- a/x-pack/plugins/lists/server/routes/patch_list_route.ts +++ b/x-pack/plugins/lists/server/routes/patch_list_route.ts @@ -6,12 +6,14 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { LIST_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { listSchema, patchListSchema } from '../../common/schemas'; +import { buildRouteValidation, buildSiemResponse } from './utils'; + import { getListClient } from '.'; export const patchListRoute = (router: ListsPluginRouter): void => { diff --git a/x-pack/plugins/lists/server/routes/read_endpoint_list_item_route.ts b/x-pack/plugins/lists/server/routes/read_endpoint_list_item_route.ts index c78a4a435e5b4..72cfe38090cd8 100644 --- a/x-pack/plugins/lists/server/routes/read_endpoint_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/read_endpoint_list_item_route.ts @@ -6,17 +6,22 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { ENDPOINT_LIST_ITEM_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { ReadEndpointListItemSchemaDecoded, exceptionListItemSchema, readEndpointListItemSchema, } from '../../common/schemas'; -import { getErrorMessageExceptionListItem, getExceptionListClient } from './utils'; +import { + buildRouteValidation, + buildSiemResponse, + getErrorMessageExceptionListItem, + getExceptionListClient, +} from './utils'; export const readEndpointListItemRoute = (router: ListsPluginRouter): void => { router.get( diff --git a/x-pack/plugins/lists/server/routes/read_exception_list_item_route.ts b/x-pack/plugins/lists/server/routes/read_exception_list_item_route.ts index fd92543fa85a7..3563645f554bb 100644 --- a/x-pack/plugins/lists/server/routes/read_exception_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/read_exception_list_item_route.ts @@ -6,17 +6,22 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { EXCEPTION_LIST_ITEM_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { ReadExceptionListItemSchemaDecoded, exceptionListItemSchema, readExceptionListItemSchema, } from '../../common/schemas'; -import { getErrorMessageExceptionListItem, getExceptionListClient } from './utils'; +import { + buildRouteValidation, + buildSiemResponse, + getErrorMessageExceptionListItem, + getExceptionListClient, +} from './utils'; export const readExceptionListItemRoute = (router: ListsPluginRouter): void => { router.get( diff --git a/x-pack/plugins/lists/server/routes/read_exception_list_route.ts b/x-pack/plugins/lists/server/routes/read_exception_list_route.ts index 3d4e831f4a2da..f82c397e67d2b 100644 --- a/x-pack/plugins/lists/server/routes/read_exception_list_route.ts +++ b/x-pack/plugins/lists/server/routes/read_exception_list_route.ts @@ -6,17 +6,22 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { EXCEPTION_LIST_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { ReadExceptionListSchemaDecoded, exceptionListSchema, readExceptionListSchema, } from '../../common/schemas'; -import { getErrorMessageExceptionList, getExceptionListClient } from './utils'; +import { + buildRouteValidation, + buildSiemResponse, + getErrorMessageExceptionList, + getExceptionListClient, +} from './utils'; export const readExceptionListRoute = (router: ListsPluginRouter): void => { router.get( diff --git a/x-pack/plugins/lists/server/routes/read_list_index_route.ts b/x-pack/plugins/lists/server/routes/read_list_index_route.ts index 467348669bc0b..619600f3a7ee1 100644 --- a/x-pack/plugins/lists/server/routes/read_list_index_route.ts +++ b/x-pack/plugins/lists/server/routes/read_list_index_route.ts @@ -6,12 +6,14 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { LIST_INDEX } from '../../common/constants'; -import { buildSiemResponse, transformError } from '../siem_server_deps'; import { listItemIndexExistSchema } from '../../common/schemas'; +import { buildSiemResponse } from './utils'; + import { getListClient } from '.'; export const readListIndexRoute = (router: ListsPluginRouter): void => { diff --git a/x-pack/plugins/lists/server/routes/read_list_item_route.ts b/x-pack/plugins/lists/server/routes/read_list_item_route.ts index fd216197f91b5..2355a393d4a77 100644 --- a/x-pack/plugins/lists/server/routes/read_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/read_list_item_route.ts @@ -6,12 +6,14 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { LIST_ITEM_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { listItemArraySchema, listItemSchema, readListItemSchema } from '../../common/schemas'; +import { buildRouteValidation, buildSiemResponse } from './utils'; + import { getListClient } from '.'; export const readListItemRoute = (router: ListsPluginRouter): void => { diff --git a/x-pack/plugins/lists/server/routes/read_list_route.ts b/x-pack/plugins/lists/server/routes/read_list_route.ts index 56acb1e043bd5..e66774998d554 100644 --- a/x-pack/plugins/lists/server/routes/read_list_route.ts +++ b/x-pack/plugins/lists/server/routes/read_list_route.ts @@ -6,12 +6,14 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { LIST_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { listSchema, readListSchema } from '../../common/schemas'; +import { buildRouteValidation, buildSiemResponse } from './utils'; + import { getListClient } from '.'; export const readListRoute = (router: ListsPluginRouter): void => { diff --git a/x-pack/plugins/lists/server/routes/read_privileges_route.ts b/x-pack/plugins/lists/server/routes/read_privileges_route.ts index 798f8631668a9..8c7faa7f7eb9d 100644 --- a/x-pack/plugins/lists/server/routes/read_privileges_route.ts +++ b/x-pack/plugins/lists/server/routes/read_privileges_route.ts @@ -5,13 +5,13 @@ * 2.0. */ +import { readPrivileges, transformError } from '@kbn/securitysolution-es-utils'; import { merge } from 'lodash/fp'; import type { ListsPluginRouter } from '../types'; import { LIST_PRIVILEGES_URL } from '../../common/constants'; -import { buildSiemResponse, readPrivileges, transformError } from '../siem_server_deps'; -import { getListClient } from './utils'; +import { buildSiemResponse, getListClient } from './utils'; export const readPrivilegesRoute = (router: ListsPluginRouter): void => { router.get( diff --git a/x-pack/plugins/lists/server/routes/update_endpoint_list_item_route.ts b/x-pack/plugins/lists/server/routes/update_endpoint_list_item_route.ts index 9f445f4e3c114..9468fd2e8c226 100644 --- a/x-pack/plugins/lists/server/routes/update_endpoint_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/update_endpoint_list_item_route.ts @@ -6,16 +6,18 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { ENDPOINT_LIST_ITEM_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { UpdateEndpointListItemSchemaDecoded, exceptionListItemSchema, updateEndpointListItemSchema, } from '../../common/schemas'; +import { buildRouteValidation, buildSiemResponse } from './utils'; + import { getExceptionListClient } from '.'; export const updateEndpointListItemRoute = (router: ListsPluginRouter): void => { diff --git a/x-pack/plugins/lists/server/routes/update_exception_list_item_route.ts b/x-pack/plugins/lists/server/routes/update_exception_list_item_route.ts index 6a87af6c666bb..6fbb1b7de80af 100644 --- a/x-pack/plugins/lists/server/routes/update_exception_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/update_exception_list_item_route.ts @@ -6,10 +6,10 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { EXCEPTION_LIST_ITEM_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { UpdateExceptionListItemSchemaDecoded, exceptionListItemSchema, @@ -17,6 +17,8 @@ import { } from '../../common/schemas'; import { updateExceptionListItemValidate } from '../../common/schemas/request/update_exception_list_item_validation'; +import { buildRouteValidation, buildSiemResponse } from './utils'; + import { getExceptionListClient } from '.'; export const updateExceptionListItemRoute = (router: ListsPluginRouter): void => { diff --git a/x-pack/plugins/lists/server/routes/update_exception_list_route.ts b/x-pack/plugins/lists/server/routes/update_exception_list_route.ts index a6b99579d87ad..cf670b28cee56 100644 --- a/x-pack/plugins/lists/server/routes/update_exception_list_route.ts +++ b/x-pack/plugins/lists/server/routes/update_exception_list_route.ts @@ -6,17 +6,22 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { EXCEPTION_LIST_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { UpdateExceptionListSchemaDecoded, exceptionListSchema, updateExceptionListSchema, } from '../../common/schemas'; -import { getErrorMessageExceptionList, getExceptionListClient } from './utils'; +import { + buildRouteValidation, + buildSiemResponse, + getErrorMessageExceptionList, + getExceptionListClient, +} from './utils'; export const updateExceptionListRoute = (router: ListsPluginRouter): void => { router.put( diff --git a/x-pack/plugins/lists/server/routes/update_list_item_route.ts b/x-pack/plugins/lists/server/routes/update_list_item_route.ts index e2905c1a00a11..f806b3f5d09d7 100644 --- a/x-pack/plugins/lists/server/routes/update_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/update_list_item_route.ts @@ -6,12 +6,14 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { LIST_ITEM_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { listItemSchema, updateListItemSchema } from '../../common/schemas'; +import { buildRouteValidation, buildSiemResponse } from './utils'; + import { getListClient } from '.'; export const updateListItemRoute = (router: ListsPluginRouter): void => { diff --git a/x-pack/plugins/lists/server/routes/update_list_route.ts b/x-pack/plugins/lists/server/routes/update_list_route.ts index d69c110aa129b..25457d7cdb333 100644 --- a/x-pack/plugins/lists/server/routes/update_list_route.ts +++ b/x-pack/plugins/lists/server/routes/update_list_route.ts @@ -6,12 +6,14 @@ */ import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; import type { ListsPluginRouter } from '../types'; import { LIST_URL } from '../../common/constants'; -import { buildRouteValidation, buildSiemResponse, transformError } from '../siem_server_deps'; import { listSchema, updateListSchema } from '../../common/schemas'; +import { buildRouteValidation, buildSiemResponse } from './utils'; + import { getListClient } from '.'; export const updateListRoute = (router: ListsPluginRouter): void => { diff --git a/x-pack/plugins/lists/server/routes/utils/build_siem_response.ts b/x-pack/plugins/lists/server/routes/utils/build_siem_response.ts new file mode 100644 index 0000000000000..fe76de24aa7b9 --- /dev/null +++ b/x-pack/plugins/lists/server/routes/utils/build_siem_response.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CustomHttpResponseOptions, KibanaResponseFactory } from 'src/core/server'; + +/** + * Copied from x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.ts + * We cannot put this in kbn package just yet as the types from 'src/core/server' aren't a kbn package yet and this would pull in a lot of copied things. + * TODO: Once more types are moved into kbn package we can move this into a kbn package. + */ +const statusToErrorMessage = ( + statusCode: number +): + | 'Bad Request' + | 'Unauthorized' + | 'Forbidden' + | 'Not Found' + | 'Conflict' + | 'Internal Error' + | '(unknown error)' => { + switch (statusCode) { + case 400: + return 'Bad Request'; + case 401: + return 'Unauthorized'; + case 403: + return 'Forbidden'; + case 404: + return 'Not Found'; + case 409: + return 'Conflict'; + case 500: + return 'Internal Error'; + default: + return '(unknown error)'; + } +}; + +/** + * Copied from x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.ts + * We cannot put this in kbn package just yet as the types from 'src/core/server' aren't a kbn package yet and this would pull in a lot of copied things. + * TODO: Once more types are moved into kbn package we can move this into a kbn package. + */ +export class SiemResponseFactory { + constructor(private response: KibanaResponseFactory) {} + + // eslint-disable-next-line @typescript-eslint/explicit-function-return-type + error({ statusCode, body, headers }: CustomHttpResponseOptions) { + // KibanaResponse is not exported so we cannot use a return type here and that is why the linter is turned off above + const contentType: CustomHttpResponseOptions['headers'] = { + 'content-type': 'application/json', + }; + const defaultedHeaders: CustomHttpResponseOptions['headers'] = { + ...contentType, + ...(headers ?? {}), + }; + + return this.response.custom({ + body: Buffer.from( + JSON.stringify({ + message: body ?? statusToErrorMessage(statusCode), + status_code: statusCode, + }) + ), + headers: defaultedHeaders, + statusCode, + }); + } +} + +/** + * Copied from x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.ts + * We cannot put this in kbn package just yet as the types from 'src/core/server' aren't a kbn package yet and this would pull in a lot of copied things. + * TODO: Once more types are moved into kbn package we can move this into a kbn package. + */ +export const buildSiemResponse = (response: KibanaResponseFactory): SiemResponseFactory => + new SiemResponseFactory(response); diff --git a/x-pack/plugins/lists/server/routes/utils/index.ts b/x-pack/plugins/lists/server/routes/utils/index.ts index 0149770e695b9..f035ae5dbfe9b 100644 --- a/x-pack/plugins/lists/server/routes/utils/index.ts +++ b/x-pack/plugins/lists/server/routes/utils/index.ts @@ -9,3 +9,5 @@ export * from './get_error_message_exception_list_item'; export * from './get_error_message_exception_list'; export * from './get_list_client'; export * from './get_exception_list_client'; +export * from './route_validation'; +export * from './build_siem_response'; diff --git a/x-pack/plugins/lists/server/routes/utils/route_validation.test.ts b/x-pack/plugins/lists/server/routes/utils/route_validation.test.ts new file mode 100644 index 0000000000000..9e6064d192d40 --- /dev/null +++ b/x-pack/plugins/lists/server/routes/utils/route_validation.test.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as rt from 'io-ts'; +import { RouteValidationResultFactory } from 'src/core/server'; + +import { buildRouteValidation } from './route_validation'; + +/** + * Copied from x-pack/plugins/security_solution/server/utils/build_validation/route_validation.test.ts + * TODO: Once we can move this into a kbn package because the types such as RouteValidationResultFactory are in packages, then please do. + */ +describe('Route Validation with ', () => { + describe('buildRouteValidation', () => { + const schema = rt.exact( + rt.type({ + ids: rt.array(rt.string), + }) + ); + type Schema = rt.TypeOf; + + /** + * If your schema is using exact all the way down then the validation will + * catch any additional keys that should not be present within the validation + * when the route_validation uses the exact check. + */ + const deepSchema = rt.exact( + rt.type({ + topLevel: rt.exact( + rt.type({ + secondLevel: rt.exact( + rt.type({ + thirdLevel: rt.string, + }) + ), + }) + ), + }) + ); + type DeepSchema = rt.TypeOf; + + const validationResult: RouteValidationResultFactory = { + badRequest: jest.fn().mockImplementation((e) => e), + ok: jest.fn().mockImplementation((validatedInput) => validatedInput), + }; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + test('return validation error', () => { + const input: Omit & { id: string } = { id: 'someId' }; + const result = buildRouteValidation(schema)(input, validationResult); + + expect(result).toEqual('Invalid value "undefined" supplied to "ids"'); + }); + + test('return validated input', () => { + const input: Schema = { ids: ['someId'] }; + const result = buildRouteValidation(schema)(input, validationResult); + + expect(result).toEqual(input); + }); + + test('returns validation error if given extra keys on input for an array', () => { + const input: Schema & { somethingExtra: string } = { + ids: ['someId'], + somethingExtra: 'hello', + }; + const result = buildRouteValidation(schema)(input, validationResult); + expect(result).toEqual('invalid keys "somethingExtra"'); + }); + + test('return validation input for a deep 3rd level object', () => { + const input: DeepSchema = { topLevel: { secondLevel: { thirdLevel: 'hello' } } }; + const result = buildRouteValidation(deepSchema)(input, validationResult); + expect(result).toEqual(input); + }); + + test('return validation error for a deep 3rd level object that has an extra key value of "somethingElse"', () => { + const input: DeepSchema & { + topLevel: { secondLevel: { thirdLevel: string; somethingElse: string } }; + } = { + topLevel: { secondLevel: { somethingElse: 'extraKey', thirdLevel: 'hello' } }, + }; + const result = buildRouteValidation(deepSchema)(input, validationResult); + expect(result).toEqual('invalid keys "somethingElse"'); + }); + }); +}); diff --git a/x-pack/plugins/lists/server/routes/utils/route_validation.ts b/x-pack/plugins/lists/server/routes/utils/route_validation.ts new file mode 100644 index 0000000000000..8e74760d6d15f --- /dev/null +++ b/x-pack/plugins/lists/server/routes/utils/route_validation.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { fold } from 'fp-ts/lib/Either'; +import { pipe } from 'fp-ts/lib/pipeable'; +import * as rt from 'io-ts'; +import { exactCheck, formatErrors } from '@kbn/securitysolution-io-ts-utils'; + +import { + RouteValidationError, + RouteValidationFunction, + RouteValidationResultFactory, +} from '../../../../../../src/core/server'; + +type RequestValidationResult = + | { + value: T; + error?: undefined; + } + | { + value?: undefined; + error: RouteValidationError; + }; + +/** + * Copied from x-pack/plugins/security_solution/server/utils/build_validation/route_validation.ts + * This really should be in @kbn/securitysolution-io-ts-utils rather than copied yet again, however, this has types + * from a lot of places such as RouteValidationResultFactory from core/server which in turn can pull in @kbn/schema + * which cannot work on the front end and @kbn/securitysolution-io-ts-utils works on both front and backend. + * + * TODO: Figure out a way to move this function into a package rather than copying it/forking it within plugins + */ +export const buildRouteValidation = >( + schema: T +): RouteValidationFunction => ( + inputValue: unknown, + validationResult: RouteValidationResultFactory +): RequestValidationResult => + pipe( + schema.decode(inputValue), + (decoded) => exactCheck(inputValue, decoded), + fold>( + (errors: rt.Errors) => validationResult.badRequest(formatErrors(errors).join()), + (validatedInput: A) => validationResult.ok(validatedInput) + ) + ); diff --git a/x-pack/plugins/lists/server/services/lists/list_client.ts b/x-pack/plugins/lists/server/services/lists/list_client.ts index 0b9bfbed28d83..a602bcf943808 100644 --- a/x-pack/plugins/lists/server/services/lists/list_client.ts +++ b/x-pack/plugins/lists/server/services/lists/list_client.ts @@ -6,6 +6,17 @@ */ import { ElasticsearchClient } from 'kibana/server'; +import { + createBootstrapIndex, + deleteAllIndex, + deletePolicy, + deleteTemplate, + getIndexExists, + getPolicyExists, + getTemplateExists, + setPolicy, + setTemplate, +} from '@kbn/securitysolution-es-utils'; import { FoundListItemSchema, @@ -40,17 +51,6 @@ import { searchListItemByValues, updateListItem, } from '../../services/items'; -import { - createBootstrapIndex, - deleteAllIndex, - deletePolicy, - deleteTemplate, - getIndexExists, - getPolicyExists, - getTemplateExists, - setPolicy, - setTemplate, -} from '../../siem_server_deps'; import listsItemsPolicy from '../items/list_item_policy.json'; import listPolicy from './list_policy.json'; diff --git a/x-pack/plugins/lists/server/siem_server_deps.ts b/x-pack/plugins/lists/server/siem_server_deps.ts deleted file mode 100644 index a4263d089e84c..0000000000000 --- a/x-pack/plugins/lists/server/siem_server_deps.ts +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export { - transformError, - deleteTemplate, - deletePolicy, - deleteAllIndex, - setPolicy, - setTemplate, - buildSiemResponse, - getTemplateExists, - getPolicyExists, - createBootstrapIndex, - getIndexExists, - buildRouteValidation, - readPrivileges, -} from '../../security_solution/server'; - -export { formatErrors } from '../../security_solution/common'; diff --git a/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.tsx b/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.tsx index db781344eb6f8..a1474237f6c9c 100644 --- a/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.tsx +++ b/x-pack/plugins/maps/public/classes/layers/vector_layer/vector_layer.tsx @@ -782,7 +782,7 @@ export class VectorLayer extends AbstractLayer implements IVectorLayer { } const filterExpr = getPointFilterExpression(this.hasJoins()); - if (filterExpr !== mbMap.getFilter(pointLayerId)) { + if (!_.isEqual(filterExpr, mbMap.getFilter(pointLayerId))) { mbMap.setFilter(pointLayerId, filterExpr); mbMap.setFilter(textLayerId, filterExpr); } @@ -818,7 +818,7 @@ export class VectorLayer extends AbstractLayer implements IVectorLayer { } const filterExpr = getPointFilterExpression(this.hasJoins()); - if (filterExpr !== mbMap.getFilter(symbolLayerId)) { + if (!_.isEqual(filterExpr, mbMap.getFilter(symbolLayerId))) { mbMap.setFilter(symbolLayerId, filterExpr); } @@ -900,14 +900,14 @@ export class VectorLayer extends AbstractLayer implements IVectorLayer { this.syncVisibilityWithMb(mbMap, fillLayerId); mbMap.setLayerZoomRange(fillLayerId, this.getMinZoom(), this.getMaxZoom()); const fillFilterExpr = getFillFilterExpression(hasJoins); - if (fillFilterExpr !== mbMap.getFilter(fillLayerId)) { + if (!_.isEqual(fillFilterExpr, mbMap.getFilter(fillLayerId))) { mbMap.setFilter(fillLayerId, fillFilterExpr); } this.syncVisibilityWithMb(mbMap, lineLayerId); mbMap.setLayerZoomRange(lineLayerId, this.getMinZoom(), this.getMaxZoom()); const lineFilterExpr = getLineFilterExpression(hasJoins); - if (lineFilterExpr !== mbMap.getFilter(lineLayerId)) { + if (!_.isEqual(lineFilterExpr, mbMap.getFilter(lineLayerId))) { mbMap.setFilter(lineLayerId, lineFilterExpr); } @@ -931,7 +931,7 @@ export class VectorLayer extends AbstractLayer implements IVectorLayer { } const filterExpr = getCentroidFilterExpression(this.hasJoins()); - if (filterExpr !== mbMap.getFilter(centroidLayerId)) { + if (!_.isEqual(filterExpr, mbMap.getFilter(centroidLayerId))) { mbMap.setFilter(centroidLayerId, filterExpr); } diff --git a/x-pack/plugins/maps/public/classes/util/mb_filter_expressions.ts b/x-pack/plugins/maps/public/classes/util/mb_filter_expressions.ts index 4526340d3d865..c798f05df9813 100644 --- a/x-pack/plugins/maps/public/classes/util/mb_filter_expressions.ts +++ b/x-pack/plugins/maps/public/classes/util/mb_filter_expressions.ts @@ -15,61 +15,55 @@ import { export const EXCLUDE_TOO_MANY_FEATURES_BOX = ['!=', ['get', KBN_TOO_MANY_FEATURES_PROPERTY], true]; const EXCLUDE_CENTROID_FEATURES = ['!=', ['get', KBN_IS_CENTROID_FEATURE], true]; -const VISIBILITY_FILTER_CLAUSE = ['all', ['==', ['get', FEATURE_VISIBLE_PROPERTY_NAME], true]]; -// Kibana features are features added by kibana that do not exist in real data -const EXCLUDE_KBN_FEATURES = ['all', EXCLUDE_TOO_MANY_FEATURES_BOX, EXCLUDE_CENTROID_FEATURES]; +function getFilterExpression(geometryFilter: unknown[], hasJoins: boolean) { + const filters: unknown[] = [ + EXCLUDE_TOO_MANY_FEATURES_BOX, + EXCLUDE_CENTROID_FEATURES, + geometryFilter, + ]; -const CLOSED_SHAPE_MB_FILTER = [ - ...EXCLUDE_KBN_FEATURES, - [ - 'any', - ['==', ['geometry-type'], GEO_JSON_TYPE.POLYGON], - ['==', ['geometry-type'], GEO_JSON_TYPE.MULTI_POLYGON], - ], -]; + if (hasJoins) { + filters.push(['==', ['get', FEATURE_VISIBLE_PROPERTY_NAME], true]); + } -const VISIBLE_CLOSED_SHAPE_MB_FILTER = [...VISIBILITY_FILTER_CLAUSE, CLOSED_SHAPE_MB_FILTER]; - -const ALL_SHAPE_MB_FILTER = [ - ...EXCLUDE_KBN_FEATURES, - [ - 'any', - ['==', ['geometry-type'], GEO_JSON_TYPE.POLYGON], - ['==', ['geometry-type'], GEO_JSON_TYPE.MULTI_POLYGON], - ['==', ['geometry-type'], GEO_JSON_TYPE.LINE_STRING], - ['==', ['geometry-type'], GEO_JSON_TYPE.MULTI_LINE_STRING], - ], -]; - -const VISIBLE_ALL_SHAPE_MB_FILTER = [...VISIBILITY_FILTER_CLAUSE, ALL_SHAPE_MB_FILTER]; - -const POINT_MB_FILTER = [ - ...EXCLUDE_KBN_FEATURES, - [ - 'any', - ['==', ['geometry-type'], GEO_JSON_TYPE.POINT], - ['==', ['geometry-type'], GEO_JSON_TYPE.MULTI_POINT], - ], -]; - -const VISIBLE_POINT_MB_FILTER = [...VISIBILITY_FILTER_CLAUSE, POINT_MB_FILTER]; - -const CENTROID_MB_FILTER = ['all', ['==', ['get', KBN_IS_CENTROID_FEATURE], true]]; - -const VISIBLE_CENTROID_MB_FILTER = [...VISIBILITY_FILTER_CLAUSE, CENTROID_MB_FILTER]; + return ['all', ...filters]; +} export function getFillFilterExpression(hasJoins: boolean): unknown[] { - return hasJoins ? VISIBLE_CLOSED_SHAPE_MB_FILTER : CLOSED_SHAPE_MB_FILTER; + return getFilterExpression( + [ + 'any', + ['==', ['geometry-type'], GEO_JSON_TYPE.POLYGON], + ['==', ['geometry-type'], GEO_JSON_TYPE.MULTI_POLYGON], + ], + hasJoins + ); } export function getLineFilterExpression(hasJoins: boolean): unknown[] { - return hasJoins ? VISIBLE_ALL_SHAPE_MB_FILTER : ALL_SHAPE_MB_FILTER; + return getFilterExpression( + [ + 'any', + ['==', ['geometry-type'], GEO_JSON_TYPE.POLYGON], + ['==', ['geometry-type'], GEO_JSON_TYPE.MULTI_POLYGON], + ['==', ['geometry-type'], GEO_JSON_TYPE.LINE_STRING], + ['==', ['geometry-type'], GEO_JSON_TYPE.MULTI_LINE_STRING], + ], + hasJoins + ); } export function getPointFilterExpression(hasJoins: boolean): unknown[] { - return hasJoins ? VISIBLE_POINT_MB_FILTER : POINT_MB_FILTER; + return getFilterExpression( + [ + 'any', + ['==', ['geometry-type'], GEO_JSON_TYPE.POINT], + ['==', ['geometry-type'], GEO_JSON_TYPE.MULTI_POINT], + ], + hasJoins + ); } export function getCentroidFilterExpression(hasJoins: boolean): unknown[] { - return hasJoins ? VISIBLE_CENTROID_MB_FILTER : CENTROID_MB_FILTER; + return getFilterExpression(['==', ['get', KBN_IS_CENTROID_FEATURE], true], hasJoins); } diff --git a/x-pack/plugins/maps/public/routes/map_page/map_app/map_app.tsx b/x-pack/plugins/maps/public/routes/map_page/map_app/map_app.tsx index aa1527d48654e..6a31e25197210 100644 --- a/x-pack/plugins/maps/public/routes/map_page/map_app/map_app.tsx +++ b/x-pack/plugins/maps/public/routes/map_page/map_app/map_app.tsx @@ -6,7 +6,6 @@ */ import React from 'react'; -import 'mapbox-gl/dist/mapbox-gl.css'; import _ from 'lodash'; import { i18n } from '@kbn/i18n'; import { AppLeaveAction, AppMountParameters } from 'kibana/public'; diff --git a/x-pack/plugins/ml/common/constants/embeddable_map.ts b/x-pack/plugins/ml/common/constants/embeddable_map.ts new file mode 100644 index 0000000000000..6cb345bae630e --- /dev/null +++ b/x-pack/plugins/ml/common/constants/embeddable_map.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const COMMON_EMS_LAYER_IDS = [ + 'world_countries', + 'administrative_regions_lvl2', + 'usa_zip_codes', + 'usa_states', +]; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/index_based/components/field_data_row/choropleth_map/choropleth_map.tsx b/x-pack/plugins/ml/public/application/datavisualizer/index_based/components/field_data_row/choropleth_map/choropleth_map.tsx new file mode 100644 index 0000000000000..8b7cbf83f7996 --- /dev/null +++ b/x-pack/plugins/ml/public/application/datavisualizer/index_based/components/field_data_row/choropleth_map/choropleth_map.tsx @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC, useMemo } from 'react'; +import { EuiFlexItem, EuiSpacer, EuiText, htmlIdGenerator } from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { + FIELD_ORIGIN, + SOURCE_TYPES, + STYLE_TYPE, + COLOR_MAP_TYPE, +} from '../../../../../../../../maps/common/constants'; +import { EMSTermJoinConfig } from '../../../../../../../../maps/public'; +import { FieldVisStats } from '../../../../stats_table/types'; +import { VectorLayerDescriptor } from '../../../../../../../../maps/common/descriptor_types'; +import { MlEmbeddedMapComponent } from '../../../../../components/ml_embedded_map'; + +export const getChoroplethTopValuesLayer = ( + fieldName: string, + topValues: Array<{ key: any; doc_count: number }>, + { layerId, field }: EMSTermJoinConfig +): VectorLayerDescriptor => { + return { + id: htmlIdGenerator()(), + label: i18n.translate('xpack.ml.dataviz.choroplethMap.topValuesCount', { + defaultMessage: 'Top values count for {fieldName}', + values: { fieldName }, + }), + joins: [ + { + // Left join is the id from the type of field (e.g. world_countries) + leftField: field, + right: { + id: 'anomaly_count', + type: SOURCE_TYPES.TABLE_SOURCE, + __rows: topValues, + __columns: [ + { + name: 'key', + type: 'string', + }, + { + name: 'doc_count', + type: 'number', + }, + ], + // Right join/term is the field in the doc you’re trying to join it to (foreign key - e.g. US) + term: 'key', + }, + }, + ], + sourceDescriptor: { + type: 'EMS_FILE', + id: layerId, + }, + style: { + type: 'VECTOR', + // @ts-ignore missing style properties. Remove once 'VectorLayerDescriptor' type is updated + properties: { + icon: { type: STYLE_TYPE.STATIC, options: { value: 'marker' } }, + fillColor: { + type: STYLE_TYPE.DYNAMIC, + options: { + color: 'Blue to Red', + colorCategory: 'palette_0', + fieldMetaOptions: { isEnabled: true, sigma: 3 }, + type: COLOR_MAP_TYPE.ORDINAL, + field: { + name: 'doc_count', + origin: FIELD_ORIGIN.JOIN, + }, + useCustomColorRamp: false, + }, + }, + lineColor: { + type: STYLE_TYPE.DYNAMIC, + options: { fieldMetaOptions: { isEnabled: true } }, + }, + lineWidth: { type: STYLE_TYPE.STATIC, options: { size: 1 } }, + }, + isTimeAware: true, + }, + type: 'VECTOR', + }; +}; + +interface Props { + stats: FieldVisStats | undefined; + suggestion: EMSTermJoinConfig; +} + +export const ChoroplethMap: FC = ({ stats, suggestion }) => { + const { fieldName, isTopValuesSampled, topValues, topValuesSamplerShardSize } = stats!; + + const layerList: VectorLayerDescriptor[] = useMemo( + () => [getChoroplethTopValuesLayer(fieldName || '', topValues || [], suggestion)], + [suggestion, stats] + ); + + return ( + +
+ +
+ {isTopValuesSampled === true && ( + <> + + + + + + )} + + ); +}; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/index_based/components/field_data_row/choropleth_map/index.ts b/x-pack/plugins/ml/public/application/datavisualizer/index_based/components/field_data_row/choropleth_map/index.ts new file mode 100644 index 0000000000000..6159b5e2ad9bb --- /dev/null +++ b/x-pack/plugins/ml/public/application/datavisualizer/index_based/components/field_data_row/choropleth_map/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { ChoroplethMap } from './choropleth_map'; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/stats_table/components/field_data_expanded_row/keyword_content.tsx b/x-pack/plugins/ml/public/application/datavisualizer/stats_table/components/field_data_expanded_row/keyword_content.tsx index 9d4e13c291656..9239632a3f909 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/stats_table/components/field_data_expanded_row/keyword_content.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/stats_table/components/field_data_expanded_row/keyword_content.tsx @@ -5,21 +5,50 @@ * 2.0. */ -import React, { FC } from 'react'; +import React, { FC, useEffect, useState } from 'react'; import type { FieldDataRowProps } from '../../types/field_data_row'; import { TopValues } from '../../../index_based/components/field_data_row/top_values'; +import { ChoroplethMap } from '../../../index_based/components/field_data_row/choropleth_map'; +import { useMlKibana } from '../../../../../application/contexts/kibana'; +import { EMSTermJoinConfig } from '../../../../../../../maps/public'; +import { COMMON_EMS_LAYER_IDS } from '../../../../../../common/constants/embeddable_map'; import { DocumentStatsTable } from './document_stats'; import { ExpandedRowContent } from './expanded_row_content'; export const KeywordContent: FC = ({ config }) => { - const { stats } = config; + const [EMSSuggestion, setEMSSuggestion] = useState(); + const { stats, fieldName } = config; const fieldFormat = 'fieldFormat' in config ? config.fieldFormat : undefined; + const { + services: { maps: mapsPlugin }, + } = useMlKibana(); + + const loadEMSTermSuggestions = async () => { + if (!mapsPlugin) return; + const suggestion: EMSTermJoinConfig | null = await mapsPlugin.suggestEMSTermJoinConfig({ + emsLayerIds: COMMON_EMS_LAYER_IDS, + sampleValues: Array.isArray(stats?.topValues) + ? stats?.topValues.map((value) => value.key) + : [], + sampleValuesColumnName: fieldName || '', + }); + setEMSSuggestion(suggestion); + }; + + useEffect( + function getInitialEMSTermSuggestion() { + loadEMSTermSuggestions(); + }, + [config?.fieldName] + ); return ( - - + {EMSSuggestion && stats && } + {EMSSuggestion === null && ( + + )} ); }; diff --git a/x-pack/plugins/ml/public/application/explorer/anomalies_map.tsx b/x-pack/plugins/ml/public/application/explorer/anomalies_map.tsx index 7914061dc81c7..73a6a9d64b60e 100644 --- a/x-pack/plugins/ml/public/application/explorer/anomalies_map.tsx +++ b/x-pack/plugins/ml/public/application/explorer/anomalies_map.tsx @@ -28,14 +28,9 @@ import { isDefined } from '../../../common/types/guards'; import { MlEmbeddedMapComponent } from '../components/ml_embedded_map'; import { EMSTermJoinConfig } from '../../../../maps/public'; import { AnomaliesTableRecord } from '../../../common/types/anomalies'; +import { COMMON_EMS_LAYER_IDS } from '../../../common/constants/embeddable_map'; const MAX_ENTITY_VALUES = 3; -const COMMON_EMS_LAYER_IDS = [ - 'world_countries', - 'administrative_regions_lvl2', - 'usa_zip_codes', - 'usa_states', -]; function getAnomalyRows(anomalies: AnomaliesTableRecord[], jobId: string) { const anomalyRows: Record< diff --git a/x-pack/plugins/ml/server/routes/saved_objects.ts b/x-pack/plugins/ml/server/routes/saved_objects.ts index b1e1aaf30b12b..c93730517cc11 100644 --- a/x-pack/plugins/ml/server/routes/saved_objects.ts +++ b/x-pack/plugins/ml/server/routes/saved_objects.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { schema } from '@kbn/config-schema'; import { wrapError } from '../client/error_wrapper'; import { RouteInitialization, SavedObjectsRouteDeps } from '../types'; import { checksFactory, syncSavedObjectsFactory } from '../saved_objects'; @@ -13,8 +12,8 @@ import { jobsAndSpaces, jobsAndCurrentSpace, syncJobObjects, - jobTypeSchema, canDeleteJobSchema, + jobTypeSchema, } from './schemas/saved_objects'; import { spacesUtilsProvider } from '../lib/spaces_utils'; @@ -308,7 +307,7 @@ export function savedObjectsRoutes( { path: '/api/ml/saved_objects/can_delete_job/{jobType}', validate: { - params: schema.object({ jobType: jobTypeSchema }), + params: jobTypeSchema, body: canDeleteJobSchema, }, options: { diff --git a/x-pack/plugins/ml/server/routes/schemas/saved_objects.ts b/x-pack/plugins/ml/server/routes/schemas/saved_objects.ts index 94ee71a1bae19..85f56c1ffb412 100644 --- a/x-pack/plugins/ml/server/routes/schemas/saved_objects.ts +++ b/x-pack/plugins/ml/server/routes/schemas/saved_objects.ts @@ -7,19 +7,21 @@ import { schema } from '@kbn/config-schema'; -export const jobTypeSchema = schema.oneOf([ +export const jobTypeLiterals = schema.oneOf([ schema.literal('anomaly-detector'), schema.literal('data-frame-analytics'), ]); +export const jobTypeSchema = schema.object({ jobType: jobTypeLiterals }); + export const jobsAndSpaces = schema.object({ - jobType: jobTypeSchema, + jobType: jobTypeLiterals, jobIds: schema.arrayOf(schema.string()), spaces: schema.arrayOf(schema.string()), }); export const jobsAndCurrentSpace = schema.object({ - jobType: jobTypeSchema, + jobType: jobTypeLiterals, jobIds: schema.arrayOf(schema.string()), }); diff --git a/x-pack/plugins/monitoring/common/types/es.ts b/x-pack/plugins/monitoring/common/types/es.ts index 81a45695002e2..f324164b09302 100644 --- a/x-pack/plugins/monitoring/common/types/es.ts +++ b/x-pack/plugins/monitoring/common/types/es.ts @@ -266,6 +266,19 @@ export interface ElasticsearchLegacySource { }; metrics?: { beat?: { + cgroup?: { + memory: { + id: string; + mem: { + limit: { + bytes: number; + }; + usage: { + bytes: number; + }; + }; + }; + }; memstats?: { memory_alloc?: number; }; diff --git a/x-pack/plugins/monitoring/public/components/apm/instances/instances.js b/x-pack/plugins/monitoring/public/components/apm/instances/instances.js index 37731f4994d2b..f58047ad6e253 100644 --- a/x-pack/plugins/monitoring/public/components/apm/instances/instances.js +++ b/x-pack/plugins/monitoring/public/components/apm/instances/instances.js @@ -30,7 +30,22 @@ import { FormattedMessage } from '@kbn/i18n/react'; import { isSetupModeFeatureEnabled } from '../../../lib/setup_mode'; import { SetupModeFeature } from '../../../../common/enums'; -function getColumns(alerts, setupMode) { +function getColumns(alerts, setupMode, cgroup) { + const memoryField = cgroup + ? { + name: i18n.translate('xpack.monitoring.apm.instances.cgroupMemoryUsageTitle', { + defaultMessage: 'Memory Usage (cgroup)', + }), + field: 'cgroup_memory', + render: (value) => formatMetric(value, 'byte'), + } + : { + name: i18n.translate('xpack.monitoring.apm.instances.allocatedMemoryTitle', { + defaultMessage: 'Allocated Memory', + }), + field: 'memory', + render: (value) => formatMetric(value, 'byte'), + }; return [ { name: i18n.translate('xpack.monitoring.apm.instances.nameTitle', { @@ -112,13 +127,7 @@ function getColumns(alerts, setupMode) { }, }), }, - { - name: i18n.translate('xpack.monitoring.apm.instances.allocatedMemoryTitle', { - defaultMessage: 'Allocated Memory', - }), - field: 'memory', - render: (value) => formatMetric(value, 'byte'), - }, + memoryField, { name: i18n.translate('xpack.monitoring.apm.instances.versionTitle', { defaultMessage: 'Version', @@ -166,7 +175,7 @@ export function ApmServerInstances({ apms, alerts, setupMode }) { diff --git a/x-pack/plugins/monitoring/server/lib/apm/_apm_stats.js b/x-pack/plugins/monitoring/server/lib/apm/_apm_stats.js index 0dfcbfff834d8..64450405b3268 100644 --- a/x-pack/plugins/monitoring/server/lib/apm/_apm_stats.js +++ b/x-pack/plugins/monitoring/server/lib/apm/_apm_stats.js @@ -7,6 +7,11 @@ import { get } from 'lodash'; +const getMemPath = (cgroup) => + cgroup + ? 'beats_stats.metrics.beat.cgroup.memory.mem.usage.bytes' + : 'beats_stats.metrics.beat.memstats.rss'; + export const getDiffCalculation = (max, min) => { // no need to test max >= 0, but min <= 0 which is normal for a derivative after restart // because we are aggregating/collapsing on ephemeral_ids @@ -21,13 +26,11 @@ export const apmAggFilterPath = [ 'aggregations.total', 'aggregations.min_events_total.value', 'aggregations.max_events_total.value', - 'aggregations.min_mem_rss_total.value', - 'aggregations.max_mem_rss_total.value', - 'aggregations.max_mem_total_total.value', + 'aggregations.min_mem_total.value', + 'aggregations.max_mem_total.value', 'aggregations.versions.buckets', ]; - -export const apmUuidsAgg = (maxBucketSize) => ({ +export const apmUuidsAgg = (maxBucketSize, cgroup) => ({ total: { cardinality: { field: 'beats_stats.beat.uuid', @@ -55,19 +58,14 @@ export const apmUuidsAgg = (maxBucketSize) => ({ field: 'beats_stats.metrics.libbeat.pipeline.events.total', }, }, - min_mem_rss: { + min_mem: { min: { - field: 'beats_stats.metrics.beat.memstats.rss', - }, - }, - max_mem_rss: { - max: { - field: 'beats_stats.metrics.beat.memstats.rss', + field: getMemPath(cgroup), }, }, - max_mem_total: { + max_mem: { max: { - field: 'beats_stats.metrics.beat.memstats.memory_total', + field: getMemPath(cgroup), }, }, }, @@ -82,19 +80,14 @@ export const apmUuidsAgg = (maxBucketSize) => ({ buckets_path: 'ephemeral_ids>max_events', }, }, - min_mem_rss_total: { - sum_bucket: { - buckets_path: 'ephemeral_ids>min_mem_rss', - }, - }, - max_mem_rss_total: { + min_mem_total: { sum_bucket: { - buckets_path: 'ephemeral_ids>max_mem_rss', + buckets_path: 'ephemeral_ids>min_mem', }, }, - max_mem_total_total: { + max_mem_total: { sum_bucket: { - buckets_path: 'ephemeral_ids>max_mem_total', + buckets_path: 'ephemeral_ids>max_mem', }, }, }); @@ -104,16 +97,14 @@ export const apmAggResponseHandler = (response) => { const eventsTotalMax = get(response, 'aggregations.max_events_total.value', 0); const eventsTotalMin = get(response, 'aggregations.min_events_total.value', 0); - const memRssMax = get(response, 'aggregations.max_mem_rss_total.value', 0); - const memRssMin = get(response, 'aggregations.min_mem_rss_total.value', 0); - const memTotal = get(response, 'aggregations.max_mem_total_total.value', 0); + const memMax = get(response, 'aggregations.max_mem_total.value', 0); + const memMin = get(response, 'aggregations.min_mem_total.value', 0); const versions = get(response, 'aggregations.versions.buckets', []).map(({ key }) => key); return { apmTotal, totalEvents: getDiffCalculation(eventsTotalMax, eventsTotalMin), - memRss: getDiffCalculation(memRssMax, memRssMin), - memTotal, + memRss: getDiffCalculation(memMax, memMin), versions, }; }; diff --git a/x-pack/plugins/monitoring/server/lib/apm/get_apms.ts b/x-pack/plugins/monitoring/server/lib/apm/get_apms.ts index 05c52a56da930..b606eaf5fe793 100644 --- a/x-pack/plugins/monitoring/server/lib/apm/get_apms.ts +++ b/x-pack/plugins/monitoring/server/lib/apm/get_apms.ts @@ -88,6 +88,7 @@ export function handleResponse(response: ElasticsearchResponse, start: number, e memory: hit._source.beats_stats?.metrics?.beat?.memstats?.memory_alloc ?? hit._source.beat?.stats?.memstats?.memory?.alloc, + cgroup_memory: hit._source.beats_stats?.metrics?.beat?.cgroup?.memory.mem.usage.bytes, version: stats?.beat?.version, time_of_last_event: hit._source.beats_stats?.timestamp ?? hit._source['@timestamp'], }); @@ -122,6 +123,7 @@ export async function getApms(req: LegacyRequest, apmIndexPattern: string, clust 'hits.hits._source.beats_stats.metrics.libbeat.output.read.errors', 'hits.hits._source.beats_stats.metrics.libbeat.output.write.errors', 'hits.hits._source.beats_stats.metrics.beat.memstats.memory_alloc', + 'hits.hits._source.beats_stats.metrics.beat.cgroup.memory.mem.usage.bytes', 'hits.hits._source.beat.stats.beat.uuid', 'hits.hits._source.beat.stats.beat.name', 'hits.hits._source.beat.stats.beat.host', diff --git a/x-pack/plugins/monitoring/server/lib/apm/get_apms_for_clusters.js b/x-pack/plugins/monitoring/server/lib/apm/get_apms_for_clusters.js index 3ece0af0369fd..45bbe35ebd59d 100644 --- a/x-pack/plugins/monitoring/server/lib/apm/get_apms_for_clusters.js +++ b/x-pack/plugins/monitoring/server/lib/apm/get_apms_for_clusters.js @@ -13,13 +13,12 @@ import { apmAggResponseHandler, apmUuidsAgg, apmAggFilterPath } from './_apm_sta import { getTimeOfLastEvent } from './_get_time_of_last_event'; export function handleResponse(clusterUuid, response) { - const { apmTotal, totalEvents, memRss, memTotal, versions } = apmAggResponseHandler(response); + const { apmTotal, totalEvents, memRss, versions } = apmAggResponseHandler(response); // combine stats const stats = { totalEvents, memRss, - memTotal, apms: { total: apmTotal, }, @@ -39,6 +38,7 @@ export function getApmsForClusters(req, apmIndexPattern, clusters) { const end = req.payload.timeRange.max; const config = req.server.config(); const maxBucketSize = config.get('monitoring.ui.max_bucket_size'); + const cgroup = config.get('monitoring.ui.container.apm.enabled'); return Promise.all( clusters.map(async (cluster) => { @@ -55,7 +55,7 @@ export function getApmsForClusters(req, apmIndexPattern, clusters) { clusterUuid, metric: ApmMetric.getMetricFields(), // override default of BeatMetric.getMetricFields }), - aggs: apmUuidsAgg(maxBucketSize), + aggs: apmUuidsAgg(maxBucketSize, cgroup), }, }; diff --git a/x-pack/plugins/monitoring/server/lib/apm/get_stats.js b/x-pack/plugins/monitoring/server/lib/apm/get_stats.js index df199fe3c3685..7dd1b652254cb 100644 --- a/x-pack/plugins/monitoring/server/lib/apm/get_stats.js +++ b/x-pack/plugins/monitoring/server/lib/apm/get_stats.js @@ -30,6 +30,7 @@ export async function getStats(req, apmIndexPattern, clusterUuid) { const start = moment.utc(req.payload.timeRange.min).valueOf(); const end = moment.utc(req.payload.timeRange.max).valueOf(); const maxBucketSize = config.get('monitoring.ui.max_bucket_size'); + const cgroup = config.get('monitoring.ui.container.apm.enabled'); const params = { index: apmIndexPattern, @@ -42,7 +43,7 @@ export async function getStats(req, apmIndexPattern, clusterUuid) { end, clusterUuid, }), - aggs: apmUuidsAgg(maxBucketSize), + aggs: apmUuidsAgg(maxBucketSize, cgroup), }, }; diff --git a/x-pack/plugins/monitoring/server/routes/api/v1/apm/instances.js b/x-pack/plugins/monitoring/server/routes/api/v1/apm/instances.js index bae4edb201428..53afa4c3f01b4 100644 --- a/x-pack/plugins/monitoring/server/routes/api/v1/apm/instances.js +++ b/x-pack/plugins/monitoring/server/routes/api/v1/apm/instances.js @@ -44,6 +44,7 @@ export function apmInstancesRoute(server) { return { stats, apms, + cgroup: req.server.config().get('monitoring.ui.container.apm.enabled'), }; } catch (err) { return handleError(err, req); diff --git a/x-pack/plugins/observability/common/rules/observability_rule_field_map.ts b/x-pack/plugins/observability/common/rules/observability_rule_field_map.ts deleted file mode 100644 index 370f5d4ef79f2..0000000000000 --- a/x-pack/plugins/observability/common/rules/observability_rule_field_map.ts +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ecsFieldMap, pickWithPatterns } from '../../../rule_registry/common'; - -export const observabilityRuleFieldMap = { - ...pickWithPatterns(ecsFieldMap, 'host.name', 'service.name'), - 'kibana.observability.evaluation.value': { - type: 'scaled_float' as const, - scaling_factor: 1000, - }, - 'kibana.observability.evaluation.threshold': { - type: 'scaled_float' as const, - scaling_factor: 1000, - }, -}; - -export type ObservabilityRuleFieldMap = typeof observabilityRuleFieldMap; diff --git a/x-pack/plugins/observability/common/utils/formatters/duration.ts b/x-pack/plugins/observability/common/utils/formatters/duration.ts index 6bbeb44ef06af..481005332cc30 100644 --- a/x-pack/plugins/observability/common/utils/formatters/duration.ts +++ b/x-pack/plugins/observability/common/utils/formatters/duration.ts @@ -201,6 +201,9 @@ export function asDuration( const formatter = getDurationFormatter(value); return formatter(value, { defaultValue, extended }).formatted; } + +export type AsDuration = typeof asDuration; + /** * Convert a microsecond value to decimal milliseconds. Normally we use * `asDuration`, but this is used in places like tables where we always want diff --git a/x-pack/plugins/observability/common/utils/formatters/formatters.ts b/x-pack/plugins/observability/common/utils/formatters/formatters.ts index 3c307f64fa0a9..9bdccc7e9edfe 100644 --- a/x-pack/plugins/observability/common/utils/formatters/formatters.ts +++ b/x-pack/plugins/observability/common/utils/formatters/formatters.ts @@ -47,6 +47,8 @@ export function asPercent( return numeral(decimal).format('0.0%'); } +export type AsPercent = typeof asPercent; + export function asDecimalOrInteger(value: number) { // exact 0 or above 10 should not have decimal if (value === 0 || value >= 10) { diff --git a/x-pack/plugins/observability/kibana.json b/x-pack/plugins/observability/kibana.json index 0ee978c75d6c0..52d5493ae69a4 100644 --- a/x-pack/plugins/observability/kibana.json +++ b/x-pack/plugins/observability/kibana.json @@ -15,7 +15,8 @@ "requiredPlugins": [ "data", "alerting", - "ruleRegistry" + "ruleRegistry", + "triggersActionsUi" ], "ui": true, "server": true, diff --git a/x-pack/plugins/observability/public/application/application.test.tsx b/x-pack/plugins/observability/public/application/application.test.tsx index c0b51652a7d0e..9182a0e8196c8 100644 --- a/x-pack/plugins/observability/public/application/application.test.tsx +++ b/x-pack/plugins/observability/public/application/application.test.tsx @@ -10,7 +10,7 @@ import React from 'react'; import { Observable } from 'rxjs'; import { AppMountParameters, CoreStart } from 'src/core/public'; import { ObservabilityPublicPluginsStart } from '../plugin'; -import { createObservabilityRuleRegistryMock } from '../rules/observability_rule_registry_mock'; +import { createObservabilityRuleTypeRegistryMock } from '../rules/observability_rule_type_registry_mock'; import { renderApp } from './'; describe('renderApp', () => { @@ -58,7 +58,7 @@ describe('renderApp', () => { core, plugins, appMountParameters: params, - observabilityRuleRegistry: createObservabilityRuleRegistryMock(), + observabilityRuleTypeRegistry: createObservabilityRuleTypeRegistryMock(), }); unmount(); }).not.toThrowError(); diff --git a/x-pack/plugins/observability/public/application/index.tsx b/x-pack/plugins/observability/public/application/index.tsx index 8607b57b42666..460aa6c35bdb8 100644 --- a/x-pack/plugins/observability/public/application/index.tsx +++ b/x-pack/plugins/observability/public/application/index.tsx @@ -18,11 +18,12 @@ import { import { PluginContext } from '../context/plugin_context'; import { usePluginContext } from '../hooks/use_plugin_context'; import { useRouteParams } from '../hooks/use_route_params'; -import { ObservabilityPublicPluginsStart, ObservabilityRuleRegistry } from '../plugin'; +import { ObservabilityPublicPluginsStart } from '../plugin'; import { HasDataContextProvider } from '../context/has_data_context'; import { Breadcrumbs, routes } from '../routes'; import { Storage } from '../../../../../src/plugins/kibana_utils/public'; import { ConfigSchema } from '..'; +import { ObservabilityRuleTypeRegistry } from '../rules/create_observability_rule_type_registry'; function getTitleFromBreadCrumbs(breadcrumbs: Breadcrumbs) { return breadcrumbs.map(({ text }) => text).reverse(); @@ -72,12 +73,12 @@ export const renderApp = ({ core, plugins, appMountParameters, - observabilityRuleRegistry, + observabilityRuleTypeRegistry, }: { config: ConfigSchema; core: CoreStart; plugins: ObservabilityPublicPluginsStart; - observabilityRuleRegistry: ObservabilityRuleRegistry; + observabilityRuleTypeRegistry: ObservabilityRuleTypeRegistry; appMountParameters: AppMountParameters; }) => { const { element, history } = appMountParameters; @@ -94,7 +95,7 @@ export const renderApp = ({ ReactDOM.render( diff --git a/x-pack/plugins/observability/public/components/app/section/apm/index.test.tsx b/x-pack/plugins/observability/public/components/app/section/apm/index.test.tsx index d41f131ef521b..e2669d87d6776 100644 --- a/x-pack/plugins/observability/public/components/app/section/apm/index.test.tsx +++ b/x-pack/plugins/observability/public/components/app/section/apm/index.test.tsx @@ -14,7 +14,8 @@ import * as hasDataHook from '../../../../hooks/use_has_data'; import * as pluginContext from '../../../../hooks/use_plugin_context'; import { HasDataContextValue } from '../../../../context/has_data_context'; import { AppMountParameters, CoreStart } from 'kibana/public'; -import { ObservabilityPublicPluginsStart, ObservabilityRuleRegistry } from '../../../../plugin'; +import { ObservabilityPublicPluginsStart } from '../../../../plugin'; +import { createObservabilityRuleTypeRegistryMock } from '../../../../rules/observability_rule_type_registry_mock'; jest.mock('react-router-dom', () => ({ useLocation: () => ({ @@ -41,10 +42,7 @@ describe('APMSection', () => { } as unknown) as CoreStart, appMountParameters: {} as AppMountParameters, config: { unsafe: { alertingExperience: { enabled: true } } }, - observabilityRuleRegistry: ({ - registerType: jest.fn(), - getTypeByRuleId: jest.fn(), - } as unknown) as ObservabilityRuleRegistry, + observabilityRuleTypeRegistry: createObservabilityRuleTypeRegistryMock(), plugins: ({ data: { query: { diff --git a/x-pack/plugins/observability/public/components/app/section/ux/index.test.tsx b/x-pack/plugins/observability/public/components/app/section/ux/index.test.tsx index fa4d1a744e3ea..b4227cc122dde 100644 --- a/x-pack/plugins/observability/public/components/app/section/ux/index.test.tsx +++ b/x-pack/plugins/observability/public/components/app/section/ux/index.test.tsx @@ -7,7 +7,6 @@ import { AppMountParameters, CoreStart } from 'kibana/public'; import React from 'react'; -import { createObservabilityRuleRegistryMock } from '../../../../rules/observability_rule_registry_mock'; import { HasDataContextValue } from '../../../../context/has_data_context'; import * as fetcherHook from '../../../../hooks/use_fetcher'; import * as hasDataHook from '../../../../hooks/use_has_data'; @@ -16,6 +15,7 @@ import { ObservabilityPublicPluginsStart } from '../../../../plugin'; import { render } from '../../../../utils/test_helper'; import { UXSection } from './'; import { response } from './mock_data/ux.mock'; +import { createObservabilityRuleTypeRegistryMock } from '../../../../rules/observability_rule_type_registry_mock'; jest.mock('react-router-dom', () => ({ useLocation: () => ({ @@ -55,7 +55,7 @@ describe('UXSection', () => { }, }, } as unknown) as ObservabilityPublicPluginsStart, - observabilityRuleRegistry: createObservabilityRuleRegistryMock(), + observabilityRuleTypeRegistry: createObservabilityRuleTypeRegistryMock(), })); }); it('renders with core web vitals', () => { diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/exploratory_view.test.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/exploratory_view.test.tsx index d95fad758565b..cf51c4614e543 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/exploratory_view.test.tsx +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/exploratory_view.test.tsx @@ -43,7 +43,7 @@ describe('ExploratoryView', () => { await waitFor(() => { screen.getByText(/open in lens/i); - screen.getByRole('heading', { name: /exploratory view/i }); + screen.getByRole('heading', { name: /analyze data/i }); }); }); diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/header/header.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/header/header.tsx index 9d051e89e1a38..8f2f30185d37f 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/header/header.tsx +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/header/header.tsx @@ -33,7 +33,7 @@ export function ExploratoryViewHeader({ seriesId, lensAttributes }: Props) {

{DataViewLabels[series.reportType] ?? i18n.translate('xpack.observability.expView.heading.label', { - defaultMessage: 'Exploratory view', + defaultMessage: 'Analyze data', })}{' '} ({ useLocation: () => ({ @@ -39,7 +39,7 @@ describe('useTimeRange', () => { }, }, } as unknown) as ObservabilityPublicPluginsStart, - observabilityRuleRegistry: createObservabilityRuleRegistryMock(), + observabilityRuleTypeRegistry: createObservabilityRuleTypeRegistryMock(), })); jest.spyOn(kibanaUISettings, 'useKibanaUISettings').mockImplementation(() => ({ from: '2020-10-08T05:00:00.000Z', @@ -81,7 +81,7 @@ describe('useTimeRange', () => { }, }, } as unknown) as ObservabilityPublicPluginsStart, - observabilityRuleRegistry: createObservabilityRuleRegistryMock(), + observabilityRuleTypeRegistry: createObservabilityRuleTypeRegistryMock(), })); }); it('returns ranges and absolute times from kibana default settings', () => { diff --git a/x-pack/plugins/observability/public/index.ts b/x-pack/plugins/observability/public/index.ts index a011d1fc2c414..8dd2f6a57eefe 100644 --- a/x-pack/plugins/observability/public/index.ts +++ b/x-pack/plugins/observability/public/index.ts @@ -62,4 +62,5 @@ export { getApmTraceUrl } from './utils/get_apm_trace_url'; export { createExploratoryViewUrl } from './components/shared/exploratory_view/configurations/utils'; export type { SeriesUrl } from './components/shared/exploratory_view/types'; -export { FormatterRuleRegistry } from './rules/formatter_rule_registry'; +export type { ObservabilityRuleTypeRegistry } from './rules/create_observability_rule_type_registry'; +export { createObservabilityRuleTypeRegistryMock } from './rules/observability_rule_type_registry_mock'; diff --git a/x-pack/plugins/observability/public/pages/alerts/alerts.stories.tsx b/x-pack/plugins/observability/public/pages/alerts/alerts.stories.tsx index 6940f6aaad692..0d47f3da89d36 100644 --- a/x-pack/plugins/observability/public/pages/alerts/alerts.stories.tsx +++ b/x-pack/plugins/observability/public/pages/alerts/alerts.stories.tsx @@ -13,7 +13,7 @@ import { AlertsPage } from '.'; import { HttpSetup } from '../../../../../../src/core/public'; import { KibanaContextProvider } from '../../../../../../src/plugins/kibana_react/public'; import { PluginContext, PluginContextValue } from '../../context/plugin_context'; -import { createObservabilityRuleRegistryMock } from '../../rules/observability_rule_registry_mock'; +import { createObservabilityRuleTypeRegistryMock } from '../../rules/observability_rule_type_registry_mock'; import { createCallObservabilityApi } from '../../services/call_observability_api'; import type { ObservabilityAPIReturnType } from '../../services/call_observability_api/types'; import { apmAlertResponseExample, dynamicIndexPattern } from './example_data'; @@ -62,7 +62,7 @@ export default { core: { http: { basePath: { prepend: (_: string) => '' } }, }, - observabilityRuleRegistry: createObservabilityRuleRegistryMock(), + observabilityRuleTypeRegistry: createObservabilityRuleTypeRegistryMock(), } as unknown) as PluginContextValue } > diff --git a/x-pack/plugins/observability/public/pages/alerts/alerts_flyout/alerts_flyout.stories.tsx b/x-pack/plugins/observability/public/pages/alerts/alerts_flyout/alerts_flyout.stories.tsx index 96d3c1fc9c390..90c75a70c0813 100644 --- a/x-pack/plugins/observability/public/pages/alerts/alerts_flyout/alerts_flyout.stories.tsx +++ b/x-pack/plugins/observability/public/pages/alerts/alerts_flyout/alerts_flyout.stories.tsx @@ -62,25 +62,26 @@ Example.args = { reason: 'Error count for opbeans-java was above the threshold', active: true, start: 1618235449493, - - 'rule.id': 'apm.error_rate', - 'service.environment': 'production', - 'service.name': 'opbeans-java', - 'rule.name': 'Error count threshold | opbeans-java (smith test)', - 'kibana.rac.alert.duration.us': 61787000, - 'kibana.observability.evaluation.threshold': 0, - 'kibana.rac.alert.status': 'open', - tags: ['apm', 'service.name:opbeans-java'], - 'kibana.rac.alert.uuid': 'c50fbc70-0d77-462d-ac0a-f2bd0b8512e4', - 'rule.uuid': '474920d0-93e9-11eb-ac86-0b455460de81', - 'event.action': 'active', - '@timestamp': '2021-04-14T21:43:42.966Z', - 'kibana.rac.alert.id': 'apm.error_rate_opbeans-java_production', - 'processor.event': 'error', - 'kibana.rac.alert.start': '2021-04-14T21:42:41.179Z', - 'kibana.rac.producer': 'apm', - 'event.kind': 'state', - 'rule.category': 'Error count threshold', - 'kibana.observability.evaluation.value': 1, + fields: { + 'rule.id': 'apm.error_rate', + 'service.environment': ['production'], + 'service.name': ['opbeans-java'], + 'rule.name': 'Error count threshold | opbeans-java (smith test)', + 'kibana.rac.alert.duration.us': 61787000, + 'kibana.rac.alert.evaluation.threshold': 0, + 'kibana.rac.alert.status': 'open', + tags: ['apm', 'service.name:opbeans-java'], + 'kibana.rac.alert.uuid': 'c50fbc70-0d77-462d-ac0a-f2bd0b8512e4', + 'rule.uuid': '474920d0-93e9-11eb-ac86-0b455460de81', + 'event.action': 'active', + '@timestamp': '2021-04-14T21:43:42.966Z', + 'kibana.rac.alert.id': 'apm.error_rate_opbeans-java_production', + 'processor.event': ['error'], + 'kibana.rac.alert.start': '2021-04-14T21:42:41.179Z', + 'kibana.rac.producer': 'apm', + 'event.kind': 'state', + 'rule.category': 'Error count threshold', + 'kibana.rac.alert.evaluation.value': 1, + }, }, } as Args; diff --git a/x-pack/plugins/observability/public/pages/alerts/alerts_flyout/index.tsx b/x-pack/plugins/observability/public/pages/alerts/alerts_flyout/index.tsx index 09fe464aa8cb5..b4bf96bcc6905 100644 --- a/x-pack/plugins/observability/public/pages/alerts/alerts_flyout/index.tsx +++ b/x-pack/plugins/observability/public/pages/alerts/alerts_flyout/index.tsx @@ -22,6 +22,14 @@ import { import { i18n } from '@kbn/i18n'; import moment from 'moment-timezone'; import React from 'react'; +import { + ALERT_DURATION, + ALERT_EVALUATION_THRESHOLD, + ALERT_EVALUATION_VALUE, + ALERT_SEVERITY_LEVEL, + RULE_CATEGORY, + RULE_NAME, +} from '@kbn/rule-data-utils/target/technical_field_names'; import { TopAlert } from '../'; import { useUiSetting } from '../../../../../../../src/plugins/kibana_react/public'; import { asDuration } from '../../../../common/utils/formatters'; @@ -46,7 +54,7 @@ export function AlertsFlyout({ onClose, alert }: AlertsFlyoutProps) { title: i18n.translate('xpack.observability.alertsFlyout.severityLabel', { defaultMessage: 'Severity', }), - description: , + description: , }, { title: i18n.translate('xpack.observability.alertsFlyout.triggeredLabel', { @@ -60,25 +68,25 @@ export function AlertsFlyout({ onClose, alert }: AlertsFlyoutProps) { title: i18n.translate('xpack.observability.alertsFlyout.durationLabel', { defaultMessage: 'Duration', }), - description: asDuration(alert['kibana.rac.alert.duration.us'], { extended: true }), + description: asDuration(alert.fields[ALERT_DURATION], { extended: true }), }, { title: i18n.translate('xpack.observability.alertsFlyout.expectedValueLabel', { defaultMessage: 'Expected value', }), - description: alert['kibana.observability.evaluation.threshold'] ?? '-', + description: alert.fields[ALERT_EVALUATION_THRESHOLD] ?? '-', }, { title: i18n.translate('xpack.observability.alertsFlyout.actualValueLabel', { defaultMessage: 'Actual value', }), - description: alert['kibana.observability.evaluation.value'] ?? '-', + description: alert.fields[ALERT_EVALUATION_VALUE] ?? '-', }, { title: i18n.translate('xpack.observability.alertsFlyout.ruleTypeLabel', { defaultMessage: 'Rule type', }), - description: alert['rule.category'] ?? '-', + description: alert.fields[RULE_CATEGORY] ?? '-', }, ]; @@ -86,7 +94,7 @@ export function AlertsFlyout({ onClose, alert }: AlertsFlyoutProps) { -

{alert['rule.name']}

+

{alert.fields[RULE_NAME]}

 {alert.reason} diff --git a/x-pack/plugins/observability/public/pages/alerts/alerts_table.tsx b/x-pack/plugins/observability/public/pages/alerts/alerts_table.tsx index b0ff156fde377..f377186623a03 100644 --- a/x-pack/plugins/observability/public/pages/alerts/alerts_table.tsx +++ b/x-pack/plugins/observability/public/pages/alerts/alerts_table.tsx @@ -16,6 +16,10 @@ import { } from '@elastic/eui'; import { i18n } from '@kbn/i18n'; import React, { useState } from 'react'; +import { + ALERT_DURATION, + ALERT_SEVERITY_LEVEL, +} from '@kbn/rule-data-utils/target/technical_field_names'; import { asDuration } from '../../../common/utils/formatters'; import { TimestampTooltip } from '../../components/shared/timestamp_tooltip'; import { usePluginContext } from '../../hooks/use_plugin_context'; @@ -94,9 +98,7 @@ export function AlertsTable(props: AlertsTableProps) { }), render: (_, alert) => { const { active } = alert; - return active - ? null - : asDuration(alert['kibana.rac.alert.duration.us'], { extended: true }); + return active ? null : asDuration(alert.fields[ALERT_DURATION], { extended: true }); }, }, { @@ -105,7 +107,7 @@ export function AlertsTable(props: AlertsTableProps) { defaultMessage: 'Severity', }), render: (_, alert) => { - return ; + return ; }, }, { diff --git a/x-pack/plugins/observability/public/pages/alerts/example_data.ts b/x-pack/plugins/observability/public/pages/alerts/example_data.ts index dba6f1e9aaa2f..5318fce82c1d3 100644 --- a/x-pack/plugins/observability/public/pages/alerts/example_data.ts +++ b/x-pack/plugins/observability/public/pages/alerts/example_data.ts @@ -7,42 +7,42 @@ export const apmAlertResponseExample = [ { - 'rule.id': 'apm.error_rate', - 'service.name': 'opbeans-java', - 'rule.name': 'Error count threshold | opbeans-java (smith test)', - 'kibana.rac.alert.duration.us': 180057000, - 'kibana.rac.alert.status': 'open', - 'kibana.rac.alert.severity.level': 'warning', + 'rule.id': ['apm.error_rate'], + 'service.name': ['opbeans-java'], + 'rule.name': ['Error count threshold | opbeans-java (smith test)'], + 'kibana.rac.alert.duration.us': [180057000], + 'kibana.rac.alert.status': ['open'], + 'kibana.rac.alert.severity.level': ['warning'], tags: ['apm', 'service.name:opbeans-java'], - 'kibana.rac.alert.uuid': '0175ec0a-a3b1-4d41-b557-e21c2d024352', - 'rule.uuid': '474920d0-93e9-11eb-ac86-0b455460de81', - 'event.action': 'active', - '@timestamp': '2021-04-12T13:53:49.550Z', - 'kibana.rac.alert.id': 'apm.error_rate_opbeans-java_production', - 'kibana.rac.alert.start': '2021-04-12T13:50:49.493Z', - 'kibana.rac.producer': 'apm', - 'event.kind': 'state', - 'rule.category': 'Error count threshold', + 'kibana.rac.alert.uuid': ['0175ec0a-a3b1-4d41-b557-e21c2d024352'], + 'rule.uuid': ['474920d0-93e9-11eb-ac86-0b455460de81'], + 'event.action': ['active'], + '@timestamp': ['2021-04-12T13:53:49.550Z'], + 'kibana.rac.alert.id': ['apm.error_rate_opbeans-java_production'], + 'kibana.rac.alert.start': ['2021-04-12T13:50:49.493Z'], + 'kibana.rac.producer': ['apm'], + 'event.kind': ['state'], + 'rule.category': ['Error count threshold'], 'service.environment': ['production'], 'processor.event': ['error'], }, { - 'rule.id': 'apm.error_rate', - 'service.name': 'opbeans-java', - 'rule.name': 'Error count threshold | opbeans-java (smith test)', - 'kibana.rac.alert.duration.us': 2419005000, - 'kibana.rac.alert.end': '2021-04-12T13:49:49.446Z', - 'kibana.rac.alert.status': 'closed', + 'rule.id': ['apm.error_rate'], + 'service.name': ['opbeans-java'], + 'rule.name': ['Error count threshold | opbeans-java (smith test)'], + 'kibana.rac.alert.duration.us': [2419005000], + 'kibana.rac.alert.end': ['2021-04-12T13:49:49.446Z'], + 'kibana.rac.alert.status': ['closed'], tags: ['apm', 'service.name:opbeans-java'], - 'kibana.rac.alert.uuid': '32b940e1-3809-4c12-8eee-f027cbb385e2', - 'rule.uuid': '474920d0-93e9-11eb-ac86-0b455460de81', - 'event.action': 'close', - '@timestamp': '2021-04-12T13:49:49.446Z', - 'kibana.rac.alert.id': 'apm.error_rate_opbeans-java_production', - 'kibana.rac.alert.start': '2021-04-12T13:09:30.441Z', - 'kibana.rac.producer': 'apm', - 'event.kind': 'state', - 'rule.category': 'Error count threshold', + 'kibana.rac.alert.uuid': ['32b940e1-3809-4c12-8eee-f027cbb385e2'], + 'rule.uuid': ['474920d0-93e9-11eb-ac86-0b455460de81'], + 'event.action': ['close'], + '@timestamp': ['2021-04-12T13:49:49.446Z'], + 'kibana.rac.alert.id': ['apm.error_rate_opbeans-java_production'], + 'kibana.rac.alert.start': ['2021-04-12T13:09:30.441Z'], + 'kibana.rac.producer': ['apm'], + 'event.kind': ['state'], + 'rule.category': ['Error count threshold'], 'service.environment': ['production'], 'processor.event': ['error'], }, diff --git a/x-pack/plugins/observability/public/pages/alerts/index.tsx b/x-pack/plugins/observability/public/pages/alerts/index.tsx index a6d5c6926973e..1f468a70d0976 100644 --- a/x-pack/plugins/observability/public/pages/alerts/index.tsx +++ b/x-pack/plugins/observability/public/pages/alerts/index.tsx @@ -17,6 +17,16 @@ import { i18n } from '@kbn/i18n'; import React from 'react'; import { useHistory } from 'react-router-dom'; import { format, parse } from 'url'; +import { + ALERT_START, + EVENT_ACTION, + RULE_ID, + RULE_NAME, +} from '@kbn/rule-data-utils/target/technical_field_names'; +import { + ParsedTechnicalFields, + parseTechnicalFields, +} from '../../../../rule_registry/common/parse_technical_fields'; import { asDuration, asPercent } from '../../../common/utils/formatters'; import { ExperimentalBadge } from '../../components/shared/experimental_badge'; import { useFetcher } from '../../hooks/use_fetcher'; @@ -30,7 +40,8 @@ import { AlertsTable } from './alerts_table'; export type TopAlertResponse = ObservabilityAPIReturnType<'GET /api/observability/rules/alerts/top'>[number]; -export interface TopAlert extends TopAlertResponse { +export interface TopAlert { + fields: ParsedTechnicalFields; start: number; reason: string; link?: string; @@ -42,7 +53,7 @@ interface AlertsPageProps { } export function AlertsPage({ routeParams }: AlertsPageProps) { - const { core, observabilityRuleRegistry } = usePluginContext(); + const { core, observabilityRuleTypeRegistry } = usePluginContext(); const { prepend } = core.http.basePath; const history = useHistory(); const { @@ -74,18 +85,19 @@ export function AlertsPage({ routeParams }: AlertsPageProps) { }, }).then((alerts) => { return alerts.map((alert) => { - const ruleType = observabilityRuleRegistry.getTypeByRuleId(alert['rule.id']); + const parsedFields = parseTechnicalFields(alert); + const formatter = observabilityRuleTypeRegistry.getFormatter(parsedFields[RULE_ID]!); const formatted = { link: undefined, - reason: alert['rule.name'], - ...(ruleType?.format?.({ alert, formatters: { asDuration, asPercent } }) ?? {}), + reason: parsedFields[RULE_NAME]!, + ...(formatter?.({ fields: parsedFields, formatters: { asDuration, asPercent } }) ?? {}), }; const parsedLink = formatted.link ? parse(formatted.link, true) : undefined; return { - ...alert, ...formatted, + fields: parsedFields, link: parsedLink ? format({ ...parsedLink, @@ -96,13 +108,13 @@ export function AlertsPage({ routeParams }: AlertsPageProps) { }, }) : undefined, - active: alert['event.action'] !== 'close', - start: new Date(alert['kibana.rac.alert.start']).getTime(), + active: parsedFields[EVENT_ACTION] !== 'close', + start: new Date(parsedFields[ALERT_START]!).getTime(), }; }); }); }, - [kuery, observabilityRuleRegistry, rangeFrom, rangeTo] + [kuery, observabilityRuleTypeRegistry, rangeFrom, rangeTo] ); return ( diff --git a/x-pack/plugins/observability/public/pages/overview/overview.stories.tsx b/x-pack/plugins/observability/public/pages/overview/overview.stories.tsx index 559aa8d5884a9..ebd1c73859169 100644 --- a/x-pack/plugins/observability/public/pages/overview/overview.stories.tsx +++ b/x-pack/plugins/observability/public/pages/overview/overview.stories.tsx @@ -23,7 +23,7 @@ import { emptyResponse as emptyLogsResponse, fetchLogsData } from './mock/logs.m import { emptyResponse as emptyMetricsResponse, fetchMetricsData } from './mock/metrics.mock'; import { newsFeedFetchData } from './mock/news_feed.mock'; import { emptyResponse as emptyUptimeResponse, fetchUptimeData } from './mock/uptime.mock'; -import { createObservabilityRuleRegistryMock } from '../../rules/observability_rule_registry_mock'; +import { createObservabilityRuleTypeRegistryMock } from '../../rules/observability_rule_type_registry_mock'; function unregisterAll() { unregisterDataHandler({ appName: 'apm' }); @@ -54,7 +54,7 @@ const withCore = makeDecorator({ }, }, } as unknown) as ObservabilityPublicPluginsStart, - observabilityRuleRegistry: createObservabilityRuleRegistryMock(), + observabilityRuleTypeRegistry: createObservabilityRuleTypeRegistryMock(), }} > diff --git a/x-pack/plugins/observability/public/plugin.ts b/x-pack/plugins/observability/public/plugin.ts index 517675fe1d525..6856bc97b4a36 100644 --- a/x-pack/plugins/observability/public/plugin.ts +++ b/x-pack/plugins/observability/public/plugin.ts @@ -7,6 +7,10 @@ import { i18n } from '@kbn/i18n'; import { BehaviorSubject } from 'rxjs'; +import { + TriggersAndActionsUIPublicPluginSetup, + TriggersAndActionsUIPublicPluginStart, +} from '../../triggers_actions_ui/public'; import { AppMountParameters, AppUpdater, @@ -25,26 +29,23 @@ import type { HomePublicPluginStart, } from '../../../../src/plugins/home/public'; import type { LensPublicStart } from '../../lens/public'; -import type { RuleRegistryPublicPluginSetupContract } from '../../rule_registry/public'; -import type { ObservabilityRuleFieldMap } from '../common/rules/observability_rule_field_map'; -import { observabilityRuleRegistrySettings } from '../common/rules/observability_rule_registry_settings'; import { registerDataHandler } from './data_handler'; -import { FormatterRuleRegistry } from './rules/formatter_rule_registry'; import { createCallObservabilityApi } from './services/call_observability_api'; import { toggleOverviewLinkInNav } from './toggle_overview_link_in_nav'; import { ConfigSchema } from '.'; +import { createObservabilityRuleTypeRegistry } from './rules/create_observability_rule_type_registry'; export type ObservabilityPublicSetup = ReturnType; -export type ObservabilityRuleRegistry = ObservabilityPublicSetup['ruleRegistry']; export interface ObservabilityPublicPluginsSetup { data: DataPublicPluginSetup; - ruleRegistry: RuleRegistryPublicPluginSetupContract; + triggersActionsUi: TriggersAndActionsUIPublicPluginSetup; home?: HomePublicPluginSetup; } export interface ObservabilityPublicPluginsStart { home?: HomePublicPluginStart; + triggersActionsUi: TriggersAndActionsUIPublicPluginStart; data: DataPublicPluginStart; lens: LensPublicStart; } @@ -75,11 +76,9 @@ export class Plugin createCallObservabilityApi(coreSetup.http); - const observabilityRuleRegistry = pluginsSetup.ruleRegistry.registry.create({ - ...observabilityRuleRegistrySettings, - fieldMap: {} as ObservabilityRuleFieldMap, - ctor: FormatterRuleRegistry, - }); + const observabilityRuleTypeRegistry = createObservabilityRuleTypeRegistry( + pluginsSetup.triggersActionsUi.alertTypeRegistry + ); const mount = async (params: AppMountParameters) => { // Load application bundle @@ -92,7 +91,7 @@ export class Plugin core: coreStart, plugins: pluginsStart, appMountParameters: params, - observabilityRuleRegistry, + observabilityRuleTypeRegistry, }); }; @@ -165,7 +164,7 @@ export class Plugin return { dashboard: { register: registerDataHandler }, - ruleRegistry: observabilityRuleRegistry, + observabilityRuleTypeRegistry, isAlertingExperienceEnabled: () => config.unsafe.alertingExperience.enabled, }; } diff --git a/x-pack/plugins/observability/public/routes/index.tsx b/x-pack/plugins/observability/public/routes/index.tsx index 3e5c3ddc553ef..0bdb03995ad46 100644 --- a/x-pack/plugins/observability/public/routes/index.tsx +++ b/x-pack/plugins/observability/public/routes/index.tsx @@ -132,7 +132,7 @@ export const routes = { breadcrumb: [ { text: i18n.translate('xpack.observability.overview.exploratoryView', { - defaultMessage: 'Exploratory view', + defaultMessage: 'Analyze data', }), }, ], diff --git a/x-pack/plugins/observability/public/rules/create_observability_rule_type_registry.ts b/x-pack/plugins/observability/public/rules/create_observability_rule_type_registry.ts new file mode 100644 index 0000000000000..cba9df83c6fe3 --- /dev/null +++ b/x-pack/plugins/observability/public/rules/create_observability_rule_type_registry.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { AlertTypeModel, AlertTypeRegistryContract } from '../../../triggers_actions_ui/public'; +import { ParsedTechnicalFields } from '../../../rule_registry/common/parse_technical_fields'; +import { AsDuration, AsPercent } from '../../common/utils/formatters'; + +type Formatter = (options: { + fields: ParsedTechnicalFields & Record; + formatters: { asDuration: AsDuration; asPercent: AsPercent }; +}) => { reason: string; link: string }; + +export function createObservabilityRuleTypeRegistry(alertTypeRegistry: AlertTypeRegistryContract) { + const formatters: Array<{ typeId: string; fn: Formatter }> = []; + return { + register: (type: AlertTypeModel & { format: Formatter }) => { + const { format, ...rest } = type; + formatters.push({ typeId: type.id, fn: format }); + alertTypeRegistry.register(rest); + }, + getFormatter: (typeId: string) => { + return formatters.find((formatter) => formatter.typeId === typeId)?.fn; + }, + }; +} + +export type ObservabilityRuleTypeRegistry = ReturnType; diff --git a/x-pack/plugins/observability/public/rules/formatter_rule_registry.ts b/x-pack/plugins/observability/public/rules/formatter_rule_registry.ts deleted file mode 100644 index 0d0d22cf750fb..0000000000000 --- a/x-pack/plugins/observability/public/rules/formatter_rule_registry.ts +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import type { RuleType } from '../../../rule_registry/public'; -import type { BaseRuleFieldMap, OutputOfFieldMap } from '../../../rule_registry/common'; -import { RuleRegistry } from '../../../rule_registry/public'; -import type { asDuration, asPercent } from '../../common/utils/formatters'; - -type AlertTypeOf = OutputOfFieldMap; - -type FormattableRuleType = RuleType & { - format?: (options: { - alert: AlertTypeOf; - formatters: { - asDuration: typeof asDuration; - asPercent: typeof asPercent; - }; - }) => { - reason?: string; - link?: string; - }; -}; - -export class FormatterRuleRegistry extends RuleRegistry< - TFieldMap, - FormattableRuleType -> {} diff --git a/x-pack/plugins/observability/public/rules/observability_rule_registry_mock.ts b/x-pack/plugins/observability/public/rules/observability_rule_registry_mock.ts deleted file mode 100644 index 389b581b5fb60..0000000000000 --- a/x-pack/plugins/observability/public/rules/observability_rule_registry_mock.ts +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ObservabilityRuleRegistry } from '../plugin'; - -const createRuleRegistryMock = () => ({ - registerType: () => {}, - getTypeByRuleId: () => ({ format: () => ({ link: '/test/example' }) }), - create: () => createRuleRegistryMock(), -}); - -export const createObservabilityRuleRegistryMock = () => - createRuleRegistryMock() as ObservabilityRuleRegistry & ReturnType; diff --git a/x-pack/plugins/observability/public/rules/observability_rule_type_registry_mock.ts b/x-pack/plugins/observability/public/rules/observability_rule_type_registry_mock.ts new file mode 100644 index 0000000000000..b2cf48f8e1c32 --- /dev/null +++ b/x-pack/plugins/observability/public/rules/observability_rule_type_registry_mock.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ObservabilityRuleTypeRegistry } from './create_observability_rule_type_registry'; + +const createRuleTypeRegistryMock = () => ({ + registerFormatter: () => {}, +}); + +export const createObservabilityRuleTypeRegistryMock = () => + createRuleTypeRegistryMock() as ObservabilityRuleTypeRegistry & + ReturnType; diff --git a/x-pack/plugins/observability/public/utils/test_helper.tsx b/x-pack/plugins/observability/public/utils/test_helper.tsx index 63e34b018aed0..ef7c62a143f25 100644 --- a/x-pack/plugins/observability/public/utils/test_helper.tsx +++ b/x-pack/plugins/observability/public/utils/test_helper.tsx @@ -15,7 +15,7 @@ import translations from '../../../translations/translations/ja-JP.json'; import { PluginContext } from '../context/plugin_context'; import { ObservabilityPublicPluginsStart } from '../plugin'; import { EuiThemeProvider } from '../../../../../src/plugins/kibana_react/common'; -import { createObservabilityRuleRegistryMock } from '../rules/observability_rule_registry_mock'; +import { createObservabilityRuleTypeRegistryMock } from '../rules/observability_rule_type_registry_mock'; const appMountParameters = ({ setHeaderActionMenu: () => {} } as unknown) as AppMountParameters; @@ -37,14 +37,14 @@ const plugins = ({ data: { query: { timefilter: { timefilter: { setTime: jest.fn() } } } }, } as unknown) as ObservabilityPublicPluginsStart; -const observabilityRuleRegistry = createObservabilityRuleRegistryMock(); +const observabilityRuleTypeRegistry = createObservabilityRuleTypeRegistryMock(); export const render = (component: React.ReactNode) => { return testLibRender( {component} diff --git a/x-pack/plugins/observability/server/lib/rules/get_top_alerts.ts b/x-pack/plugins/observability/server/lib/rules/get_top_alerts.ts index 0045c0f0c6757..ddfc112ab1452 100644 --- a/x-pack/plugins/observability/server/lib/rules/get_top_alerts.ts +++ b/x-pack/plugins/observability/server/lib/rules/get_top_alerts.ts @@ -4,24 +4,24 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { Required } from 'utility-types'; -import { ObservabilityRuleRegistryClient } from '../../types'; +import { ALERT_UUID, TIMESTAMP } from '@kbn/rule-data-utils/target/technical_field_names'; +import { RuleDataClient } from '../../../../rule_registry/server'; import { kqlQuery, rangeQuery } from '../../utils/queries'; export async function getTopAlerts({ - ruleRegistryClient, + ruleDataClient, start, end, kuery, size, }: { - ruleRegistryClient: ObservabilityRuleRegistryClient; + ruleDataClient: RuleDataClient; start: number; end: number; kuery?: string; size: number; }) { - const response = await ruleRegistryClient.search({ + const response = await ruleDataClient.getReader().search({ body: { query: { bool: { @@ -30,26 +30,18 @@ export async function getTopAlerts({ }, fields: ['*'], collapse: { - field: 'kibana.rac.alert.uuid', + field: ALERT_UUID, }, size, sort: { - '@timestamp': 'desc', + [TIMESTAMP]: 'desc', }, _source: false, }, + allow_no_indices: true, }); - return response.events.map((event) => { - return event as Required< - typeof event, - | 'rule.id' - | 'rule.name' - | 'kibana.rac.alert.start' - | 'event.action' - | 'rule.category' - | 'rule.name' - | 'kibana.rac.alert.duration.us' - >; + return response.hits.hits.map((hit) => { + return hit.fields; }); } diff --git a/x-pack/plugins/observability/server/plugin.ts b/x-pack/plugins/observability/server/plugin.ts index b5208260297d0..046a9a62d5fa7 100644 --- a/x-pack/plugins/observability/server/plugin.ts +++ b/x-pack/plugins/observability/server/plugin.ts @@ -6,6 +6,7 @@ */ import { PluginInitializerContext, Plugin, CoreSetup } from 'src/core/server'; +import { RuleDataClient } from '../../rule_registry/server'; import { ObservabilityConfig } from '.'; import { bootstrapAnnotations, @@ -16,11 +17,8 @@ import type { RuleRegistryPluginSetupContract } from '../../rule_registry/server import { uiSettings } from './ui_settings'; import { registerRoutes } from './routes/register_routes'; import { getGlobalObservabilityServerRouteRepository } from './routes/get_global_observability_server_route_repository'; -import { observabilityRuleRegistrySettings } from '../common/rules/observability_rule_registry_settings'; -import { observabilityRuleFieldMap } from '../common/rules/observability_rule_field_map'; export type ObservabilityPluginSetup = ReturnType; -export type ObservabilityRuleRegistry = ObservabilityPluginSetup['ruleRegistry']; export class ObservabilityPlugin implements Plugin { constructor(private readonly initContext: PluginInitializerContext) { @@ -51,19 +49,25 @@ export class ObservabilityPlugin implements Plugin { }); } - const observabilityRuleRegistry = plugins.ruleRegistry.create({ - ...observabilityRuleRegistrySettings, - fieldMap: observabilityRuleFieldMap, + const start = () => core.getStartServices().then(([coreStart]) => coreStart); + + const ruleDataClient = new RuleDataClient({ + getClusterClient: async () => { + const coreStart = await start(); + return coreStart.elasticsearch.client.asInternalUser; + }, + ready: () => Promise.resolve(), + alias: plugins.ruleRegistry.getFullAssetName(), }); registerRoutes({ core: { setup: core, - start: () => core.getStartServices().then(([coreStart]) => coreStart), + start, }, - ruleRegistry: observabilityRuleRegistry, logger: this.initContext.logger.get(), repository: getGlobalObservabilityServerRouteRepository(), + ruleDataClient, }); return { @@ -71,7 +75,6 @@ export class ObservabilityPlugin implements Plugin { const api = await annotationsApiPromise; return api?.getScopedAnnotationsClient(...args); }, - ruleRegistry: observabilityRuleRegistry, }; } diff --git a/x-pack/plugins/observability/server/routes/register_routes.ts b/x-pack/plugins/observability/server/routes/register_routes.ts index 85ee456b812b8..75b6703cc64de 100644 --- a/x-pack/plugins/observability/server/routes/register_routes.ts +++ b/x-pack/plugins/observability/server/routes/register_routes.ts @@ -13,23 +13,23 @@ import { import { CoreSetup, CoreStart, Logger, RouteRegistrar } from 'kibana/server'; import Boom from '@hapi/boom'; import { RequestAbortedError } from '@elastic/elasticsearch/lib/errors'; -import { ObservabilityRuleRegistry } from '../plugin'; +import { RuleDataClient } from '../../../rule_registry/server'; import { ObservabilityRequestHandlerContext } from '../types'; import { AbstractObservabilityServerRouteRepository } from './types'; export function registerRoutes({ - ruleRegistry, repository, core, logger, + ruleDataClient, }: { core: { setup: CoreSetup; start: () => Promise; }; - ruleRegistry: ObservabilityRuleRegistry; repository: AbstractObservabilityServerRouteRepository; logger: Logger; + ruleDataClient: RuleDataClient; }) { const routes = repository.getRoutes(); @@ -59,10 +59,10 @@ export function registerRoutes({ const data = (await handler({ context, request, - ruleRegistry, core, logger, params: decodedParams, + ruleDataClient, })) as any; return response.ok({ body: data }); diff --git a/x-pack/plugins/observability/server/routes/rules.ts b/x-pack/plugins/observability/server/routes/rules.ts index cd3f4976e0af3..1f500adff5dcf 100644 --- a/x-pack/plugins/observability/server/routes/rules.ts +++ b/x-pack/plugins/observability/server/routes/rules.ts @@ -6,7 +6,6 @@ */ import * as t from 'io-ts'; import { isoToEpochRt, toNumberRt } from '@kbn/io-ts-utils'; -import Boom from '@hapi/boom'; import { createObservabilityServerRoute } from './create_observability_server_route'; import { createObservabilityServerRouteRepository } from './create_observability_server_route_repository'; import { getTopAlerts } from '../lib/rules/get_top_alerts'; @@ -28,22 +27,13 @@ const alertsListRoute = createObservabilityServerRoute({ }), ]), }), - handler: async ({ ruleRegistry, context, params }) => { - const ruleRegistryClient = await ruleRegistry.createScopedRuleRegistryClient({ - context, - alertsClient: context.alerting.getAlertsClient(), - }); - - if (!ruleRegistryClient) { - throw Boom.failedDependency('xpack.ruleRegistry.unsafe.write.enabled is set to false'); - } - + handler: async ({ ruleDataClient, context, params }) => { const { query: { start, end, kuery, size = 100 }, } = params; return getTopAlerts({ - ruleRegistryClient, + ruleDataClient, start, end, kuery, @@ -57,17 +47,10 @@ const alertsDynamicIndexPatternRoute = createObservabilityServerRoute({ options: { tags: [], }, - handler: async ({ ruleRegistry, context }) => { - const ruleRegistryClient = await ruleRegistry.createScopedRuleRegistryClient({ - context, - alertsClient: context.alerting.getAlertsClient(), - }); - - if (!ruleRegistryClient) { - throw Boom.failedDependency(); - } + handler: async ({ ruleDataClient }) => { + const reader = ruleDataClient.getReader({ namespace: 'observability' }); - return ruleRegistryClient.getDynamicIndexPattern(); + return reader.getDynamicIndexPattern(); }, }); diff --git a/x-pack/plugins/observability/server/routes/types.ts b/x-pack/plugins/observability/server/routes/types.ts index 0588bf8df2292..1fa7229c6cf62 100644 --- a/x-pack/plugins/observability/server/routes/types.ts +++ b/x-pack/plugins/observability/server/routes/types.ts @@ -12,7 +12,7 @@ import type { ServerRouteRepository, } from '@kbn/server-route-repository'; import { CoreSetup, CoreStart, KibanaRequest, Logger } from 'kibana/server'; -import { ObservabilityRuleRegistry } from '../plugin'; +import { RuleDataClient } from '../../../rule_registry/server'; import { ObservabilityServerRouteRepository } from './get_global_observability_server_route_repository'; import { ObservabilityRequestHandlerContext } from '../types'; @@ -24,7 +24,7 @@ export interface ObservabilityRouteHandlerResources { start: () => Promise; setup: CoreSetup; }; - ruleRegistry: ObservabilityRuleRegistry; + ruleDataClient: RuleDataClient; request: KibanaRequest; context: ObservabilityRequestHandlerContext; logger: Logger; diff --git a/x-pack/plugins/observability/server/types.ts b/x-pack/plugins/observability/server/types.ts index 81b32b3f8db7b..da13e60804a60 100644 --- a/x-pack/plugins/observability/server/types.ts +++ b/x-pack/plugins/observability/server/types.ts @@ -7,9 +7,7 @@ import type { IRouter, RequestHandlerContext } from 'src/core/server'; import type { AlertingApiRequestHandlerContext } from '../../alerting/server'; -import type { ScopedRuleRegistryClient, FieldMapOf } from '../../rule_registry/server'; import type { LicensingApiRequestHandlerContext } from '../../licensing/server'; -import type { ObservabilityRuleRegistry } from './plugin'; export type { ObservabilityRouteCreateOptions, @@ -31,7 +29,3 @@ export interface ObservabilityRequestHandlerContext extends RequestHandlerContex * @internal */ export type ObservabilityPluginRouter = IRouter; - -export type ObservabilityRuleRegistryClient = ScopedRuleRegistryClient< - FieldMapOf ->; diff --git a/x-pack/plugins/reporting/public/components/reporting_panel_content.test.tsx b/x-pack/plugins/reporting/public/components/reporting_panel_content.test.tsx new file mode 100644 index 0000000000000..6c5b8df104ecd --- /dev/null +++ b/x-pack/plugins/reporting/public/components/reporting_panel_content.test.tsx @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { mountWithIntl } from '@kbn/test/jest'; +import { notificationServiceMock } from 'src/core/public/mocks'; + +import { ReportingPanelContent, Props } from './reporting_panel_content'; + +describe('ReportingPanelContent', () => { + const mountComponent = (props: Partial) => + mountWithIntl( + 'test' } as any} + toasts={notificationServiceMock.createSetupContract().toasts} + {...props} + /> + ); + describe('saved state', () => { + it('prevents generating reports when saving is required and we have unsaved changes', () => { + const wrapper = mountComponent({ + requiresSavedState: true, + isDirty: true, + objectId: undefined, + }); + wrapper.update(); + expect(wrapper.find('[data-test-subj="generateReportButton"]').last().props().disabled).toBe( + true + ); + }); + + it('allows generating reports when saving is not required', () => { + const wrapper = mountComponent({ + requiresSavedState: false, + isDirty: true, + objectId: undefined, + }); + wrapper.update(); + expect(wrapper.find('[data-test-subj="generateReportButton"]').last().props().disabled).toBe( + false + ); + }); + }); +}); diff --git a/x-pack/plugins/reporting/public/components/reporting_panel_content.tsx b/x-pack/plugins/reporting/public/components/reporting_panel_content.tsx index 399b503fe48d3..a38c37c808689 100644 --- a/x-pack/plugins/reporting/public/components/reporting_panel_content.tsx +++ b/x-pack/plugins/reporting/public/components/reporting_panel_content.tsx @@ -19,6 +19,11 @@ export interface Props { apiClient: ReportingAPIClient; toasts: ToastsSetup; reportType: string; + + /** + * Whether the report to be generated requires saved state that is not captured in the URL submitted to the report generator. + */ + requiresSavedState: boolean; layoutId: string | undefined; objectId?: string; getJobParams: () => BaseParams; @@ -85,7 +90,10 @@ class ReportingPanelContentUi extends Component { } public render() { - if (this.isNotSaved() || this.props.isDirty || this.state.isStale) { + if ( + this.props.requiresSavedState && + (this.isNotSaved() || this.props.isDirty || this.state.isStale) + ) { return ( { public render() { return ( { + const coreStart = await getCoreStart(); + return coreStart.elasticsearch.client.asInternalUser; + }, + ready, +}); -The rule registry client can either be injected in the executor, or created in the scope of a request. It exposes a `search` method and a `bulkIndex` method. When `search` is called, it first gets all the rules the current user has access to, and adds these ids to the search request that it executes. This means that the user can only see data from rules they have access to. +// to start writing data, call `getWriter().bulk()`. It supports a `namespace` +// property as well, that for instance can be used to write data to a space-specific +// index. +await ruleDataClient.getWriter().bulk({ + body: eventsToIndex.flatMap((event) => [{ index: {} }, event]), +}); -Both `search` and `bulkIndex` are fully typed, in the sense that they reflect the mappings defined for the registry. +// to read data, simply call ruleDataClient.getReader().search: +const response = await ruleDataClient.getReader().search({ + body: { + query: { + }, + size: 100, + fields: ['*'], + collapse: { + field: ALERT_UUID, + }, + sort: { + '@timestamp': 'desc', + }, + }, + allow_no_indices: true, +}); +``` ## Schema -The following fields are available in the root rule registry: +The following fields are defined in the technical field component template and should always be used: - `@timestamp`: the ISO timestamp of the alert event. For the lifecycle rule type helper, it is always the value of `startedAt` that is injected by the Kibana alerting framework. - `event.kind`: signal (for the changeable alert document), state (for the state changes of the alert, e.g. when it opens, recovers, or changes in severity), or metric (individual evaluations that might be related to an alert). @@ -67,7 +134,7 @@ The following fields are available in the root rule registry: - `rule.uuid`: the saved objects id of the rule. - `rule.name`: the name of the rule (as specified by the user). - `rule.category`: the name of the rule type (as defined by the rule type producer) -- `kibana.rac.producer`: the producer of the rule type. Usually a Kibana plugin. e.g., `APM`. +- `kibana.rac.alert.producer`: the producer of the rule type. Usually a Kibana plugin. e.g., `APM`. - `kibana.rac.alert.id`: the id of the alert, that is unique within the context of the rule execution it was created in. E.g., for a rule that monitors latency for all services in all environments, this might be `opbeans-java:production`. - `kibana.rac.alert.uuid`: the unique identifier for the alert during its lifespan. If an alert recovers (or closes), this identifier is re-generated when it is opened again. - `kibana.rac.alert.status`: the status of the alert. Can be `open` or `closed`. @@ -76,5 +143,5 @@ The following fields are available in the root rule registry: - `kibana.rac.alert.duration.us`: the duration of the alert, in microseconds. This is always the difference between either the current time, or the time when the alert recovered. - `kibana.rac.alert.severity.level`: the severity of the alert, as a keyword (e.g. critical). - `kibana.rac.alert.severity.value`: the severity of the alert, as a numerical value, which allows sorting. - -This list is not final - just a start. Field names might change or moved to a scoped registry. If we implement log and sequence based rule types the list of fields will grow. If a rule type needs additional fields, the recommendation would be to have the field in its own registry first (or in its producer’s registry), and if usage is more broadly adopted, it can be moved to the root registry. +- `kibana.rac.alert.evaluation.value`: The measured (numerical value). +- `kibana.rac.alert.threshold.value`: The threshold that was defined (or, in case of multiple thresholds, the one that was exceeded). diff --git a/x-pack/plugins/rule_registry/common/assets.ts b/x-pack/plugins/rule_registry/common/assets.ts new file mode 100644 index 0000000000000..1a5b14c605ea5 --- /dev/null +++ b/x-pack/plugins/rule_registry/common/assets.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const TECHNICAL_COMPONENT_TEMPLATE_NAME = `technical-mappings`; +export const ECS_COMPONENT_TEMPLATE_NAME = `ecs-mappings`; +export const DEFAULT_ILM_POLICY_ID = 'ilm-policy'; diff --git a/x-pack/plugins/rule_registry/common/assets/component_templates/ecs_component_template.ts b/x-pack/plugins/rule_registry/common/assets/component_templates/ecs_component_template.ts new file mode 100644 index 0000000000000..7acbe0bc1227b --- /dev/null +++ b/x-pack/plugins/rule_registry/common/assets/component_templates/ecs_component_template.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { merge } from 'lodash'; +import { mappingFromFieldMap } from '../../mapping_from_field_map'; +import { ClusterPutComponentTemplateBody } from '../../types'; +import { ecsFieldMap } from '../field_maps/ecs_field_map'; +import { technicalRuleFieldMap } from '../field_maps/technical_rule_field_map'; + +export const ecsComponentTemplate: ClusterPutComponentTemplateBody = { + template: { + settings: { + number_of_shards: 1, + }, + mappings: merge( + {}, + mappingFromFieldMap(ecsFieldMap), + mappingFromFieldMap(technicalRuleFieldMap) + ), + }, +}; diff --git a/x-pack/plugins/rule_registry/common/assets/component_templates/technical_component_template.ts b/x-pack/plugins/rule_registry/common/assets/component_templates/technical_component_template.ts new file mode 100644 index 0000000000000..cc096faba387e --- /dev/null +++ b/x-pack/plugins/rule_registry/common/assets/component_templates/technical_component_template.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { mappingFromFieldMap } from '../../mapping_from_field_map'; +import { ClusterPutComponentTemplateBody } from '../../types'; +import { technicalRuleFieldMap } from '../field_maps/technical_rule_field_map'; + +export const technicalComponentTemplate: ClusterPutComponentTemplateBody = { + template: { + settings: { + number_of_shards: 1, + }, + mappings: mappingFromFieldMap(technicalRuleFieldMap), + }, +}; diff --git a/x-pack/plugins/rule_registry/common/field_map/ecs_field_map.ts b/x-pack/plugins/rule_registry/common/assets/field_maps/ecs_field_map.ts similarity index 100% rename from x-pack/plugins/rule_registry/common/field_map/ecs_field_map.ts rename to x-pack/plugins/rule_registry/common/assets/field_maps/ecs_field_map.ts diff --git a/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.ts b/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.ts new file mode 100644 index 0000000000000..a946e9523548c --- /dev/null +++ b/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { pickWithPatterns } from '../../../common/pick_with_patterns'; +import { + ALERT_DURATION, + ALERT_END, + ALERT_EVALUATION_THRESHOLD, + ALERT_EVALUATION_VALUE, + ALERT_ID, + ALERT_SEVERITY_LEVEL, + ALERT_SEVERITY_VALUE, + ALERT_START, + ALERT_STATUS, + ALERT_UUID, + EVENT_ACTION, + EVENT_KIND, + PRODUCER, + RULE_CATEGORY, + RULE_ID, + RULE_NAME, + RULE_UUID, + TAGS, + TIMESTAMP, +} from '../../../common/technical_rule_data_field_names'; +import { ecsFieldMap } from './ecs_field_map'; + +export const technicalRuleFieldMap = { + ...pickWithPatterns( + ecsFieldMap, + TIMESTAMP, + EVENT_KIND, + EVENT_ACTION, + RULE_UUID, + RULE_ID, + RULE_NAME, + RULE_CATEGORY, + TAGS + ), + [PRODUCER]: { type: 'keyword' }, + [ALERT_UUID]: { type: 'keyword' }, + [ALERT_ID]: { type: 'keyword' }, + [ALERT_START]: { type: 'date' }, + [ALERT_END]: { type: 'date' }, + [ALERT_DURATION]: { type: 'long' }, + [ALERT_SEVERITY_LEVEL]: { type: 'keyword' }, + [ALERT_SEVERITY_VALUE]: { type: 'long' }, + [ALERT_STATUS]: { type: 'keyword' }, + [ALERT_EVALUATION_THRESHOLD]: { type: 'scaled_float', scaling_factor: 100 }, + [ALERT_EVALUATION_VALUE]: { type: 'scaled_float', scaling_factor: 100 }, +} as const; + +export type TechnicalRuleFieldMaps = typeof technicalRuleFieldMap; diff --git a/x-pack/plugins/rule_registry/common/assets/index_templates/base_index_template.ts b/x-pack/plugins/rule_registry/common/assets/index_templates/base_index_template.ts new file mode 100644 index 0000000000000..ee2e45640c149 --- /dev/null +++ b/x-pack/plugins/rule_registry/common/assets/index_templates/base_index_template.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const baseIndexTemplate = { + template: { + settings: { + number_of_shards: 1, + number_of_replicas: 0, + }, + }, +}; diff --git a/x-pack/plugins/rule_registry/server/rule_registry/defaults/ilm_policy.ts b/x-pack/plugins/rule_registry/common/assets/lifecycle_policies/default_lifecycle_policy.ts similarity index 86% rename from x-pack/plugins/rule_registry/server/rule_registry/defaults/ilm_policy.ts rename to x-pack/plugins/rule_registry/common/assets/lifecycle_policies/default_lifecycle_policy.ts index c80f7e772f308..f207087f7aa19 100644 --- a/x-pack/plugins/rule_registry/server/rule_registry/defaults/ilm_policy.ts +++ b/x-pack/plugins/rule_registry/common/assets/lifecycle_policies/default_lifecycle_policy.ts @@ -5,9 +5,7 @@ * 2.0. */ -import { ILMPolicy } from '../types'; - -export const defaultIlmPolicy: ILMPolicy = { +export const defaultLifecyclePolicy = { policy: { phases: { hot: { diff --git a/x-pack/plugins/rule_registry/common/field_map/base_rule_field_map.ts b/x-pack/plugins/rule_registry/common/field_map/base_rule_field_map.ts deleted file mode 100644 index 22a74212d2ce0..0000000000000 --- a/x-pack/plugins/rule_registry/common/field_map/base_rule_field_map.ts +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { ecsFieldMap } from './ecs_field_map'; -import { pickWithPatterns } from '../pick_with_patterns'; - -export const baseRuleFieldMap = { - ...pickWithPatterns( - ecsFieldMap, - '@timestamp', - 'event.kind', - 'event.action', - 'rule.uuid', - 'rule.id', - 'rule.name', - 'rule.category', - 'tags' - ), - 'kibana.rac.producer': { type: 'keyword' }, - 'kibana.rac.alert.uuid': { type: 'keyword' }, - 'kibana.rac.alert.id': { type: 'keyword' }, - 'kibana.rac.alert.start': { type: 'date' }, - 'kibana.rac.alert.end': { type: 'date' }, - 'kibana.rac.alert.duration.us': { type: 'long' }, - 'kibana.rac.alert.severity.level': { type: 'keyword' }, - 'kibana.rac.alert.severity.value': { type: 'long' }, - 'kibana.rac.alert.status': { type: 'keyword' }, -} as const; - -export type BaseRuleFieldMap = typeof baseRuleFieldMap; diff --git a/x-pack/plugins/rule_registry/common/field_map/index.ts b/x-pack/plugins/rule_registry/common/field_map/index.ts index 8db5c2738439b..fac8575b8af48 100644 --- a/x-pack/plugins/rule_registry/common/field_map/index.ts +++ b/x-pack/plugins/rule_registry/common/field_map/index.ts @@ -5,8 +5,6 @@ * 2.0. */ -export * from './base_rule_field_map'; -export * from './ecs_field_map'; export * from './merge_field_maps'; export * from './runtime_type_from_fieldmap'; export * from './types'; diff --git a/x-pack/plugins/rule_registry/common/index.ts b/x-pack/plugins/rule_registry/common/index.ts index b614feebc974a..5d36cd8cad7be 100644 --- a/x-pack/plugins/rule_registry/common/index.ts +++ b/x-pack/plugins/rule_registry/common/index.ts @@ -4,5 +4,4 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -export * from './field_map'; -export * from './pick_with_patterns'; +export { parseTechnicalFields } from './parse_technical_fields'; diff --git a/x-pack/plugins/rule_registry/server/rule_registry/field_map/mapping_from_field_map.ts b/x-pack/plugins/rule_registry/common/mapping_from_field_map.ts similarity index 79% rename from x-pack/plugins/rule_registry/server/rule_registry/field_map/mapping_from_field_map.ts rename to x-pack/plugins/rule_registry/common/mapping_from_field_map.ts index f1d7126906431..17eb5ae8967af 100644 --- a/x-pack/plugins/rule_registry/server/rule_registry/field_map/mapping_from_field_map.ts +++ b/x-pack/plugins/rule_registry/common/mapping_from_field_map.ts @@ -5,11 +5,11 @@ * 2.0. */ +import { TypeMapping } from '@elastic/elasticsearch/api/types'; import { set } from '@elastic/safer-lodash-set'; -import { FieldMap } from '../../../common'; -import { Mappings } from '../types'; +import { FieldMap } from './field_map/types'; -export function mappingFromFieldMap(fieldMap: FieldMap): Mappings { +export function mappingFromFieldMap(fieldMap: FieldMap): TypeMapping { const mappings = { dynamic: 'strict' as const, properties: {}, diff --git a/x-pack/plugins/rule_registry/common/parse_technical_fields.ts b/x-pack/plugins/rule_registry/common/parse_technical_fields.ts new file mode 100644 index 0000000000000..9d92c657468a3 --- /dev/null +++ b/x-pack/plugins/rule_registry/common/parse_technical_fields.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isLeft } from 'fp-ts/lib/Either'; +import { PathReporter } from 'io-ts/lib/PathReporter'; +import { technicalRuleFieldMap } from './assets/field_maps/technical_rule_field_map'; +import { runtimeTypeFromFieldMap } from './field_map'; + +const technicalFieldRuntimeType = runtimeTypeFromFieldMap(technicalRuleFieldMap); + +export const parseTechnicalFields = (input: unknown) => { + const validate = technicalFieldRuntimeType.decode(input); + + if (isLeft(validate)) { + throw new Error(PathReporter.report(validate).join('\n')); + } + + return technicalFieldRuntimeType.encode(validate.right); +}; + +export type ParsedTechnicalFields = ReturnType; diff --git a/x-pack/plugins/rule_registry/common/technical_rule_data_field_names.ts b/x-pack/plugins/rule_registry/common/technical_rule_data_field_names.ts new file mode 100644 index 0000000000000..5c954a31e79ac --- /dev/null +++ b/x-pack/plugins/rule_registry/common/technical_rule_data_field_names.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from '@kbn/rule-data-utils/target/technical_field_names'; diff --git a/x-pack/plugins/rule_registry/common/types.ts b/x-pack/plugins/rule_registry/common/types.ts new file mode 100644 index 0000000000000..299d2c300ab49 --- /dev/null +++ b/x-pack/plugins/rule_registry/common/types.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { estypes } from '@elastic/elasticsearch'; + +export type PutIndexTemplateRequest = estypes.PutIndexTemplateRequest & { + body?: { composed_of?: string[] }; +}; + +export interface ClusterPutComponentTemplateBody { + template: { + settings: { + number_of_shards: number; + }; + mappings: estypes.TypeMapping; + }; +} diff --git a/x-pack/plugins/rule_registry/kibana.json b/x-pack/plugins/rule_registry/kibana.json index ec2b366f739e6..7e3f8bf6afb72 100644 --- a/x-pack/plugins/rule_registry/kibana.json +++ b/x-pack/plugins/rule_registry/kibana.json @@ -10,6 +10,5 @@ "alerting", "triggersActionsUi" ], - "server": true, - "ui": true + "server": true } diff --git a/x-pack/plugins/rule_registry/public/index.ts b/x-pack/plugins/rule_registry/public/index.ts deleted file mode 100644 index 59697261ff20b..0000000000000 --- a/x-pack/plugins/rule_registry/public/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { PluginInitializerContext } from 'kibana/public'; -import { Plugin } from './plugin'; - -export type { RuleRegistryPublicPluginSetupContract } from './plugin'; -export { RuleRegistry } from './rule_registry'; -export type { IRuleRegistry, RuleType } from './rule_registry/types'; - -export const plugin = (context: PluginInitializerContext) => { - return new Plugin(context); -}; diff --git a/x-pack/plugins/rule_registry/public/plugin.ts b/x-pack/plugins/rule_registry/public/plugin.ts deleted file mode 100644 index 7f0bceefb6797..0000000000000 --- a/x-pack/plugins/rule_registry/public/plugin.ts +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { - CoreSetup, - CoreStart, - Plugin as PluginClass, - PluginInitializerContext, -} from '../../../../src/core/public'; -import type { - PluginSetupContract as AlertingPluginPublicSetupContract, - PluginStartContract as AlertingPluginPublicStartContract, -} from '../../alerting/public'; -import type { - TriggersAndActionsUIPublicPluginSetup, - TriggersAndActionsUIPublicPluginStart, -} from '../../triggers_actions_ui/public'; -import type { BaseRuleFieldMap } from '../common'; -import { RuleRegistry } from './rule_registry'; - -interface RuleRegistrySetupPlugins { - alerting: AlertingPluginPublicSetupContract; - triggersActionsUi: TriggersAndActionsUIPublicPluginSetup; -} - -interface RuleRegistryStartPlugins { - alerting: AlertingPluginPublicStartContract; - triggersActionsUi: TriggersAndActionsUIPublicPluginStart; -} - -export type RuleRegistryPublicPluginSetupContract = ReturnType; - -export class Plugin - implements PluginClass { - constructor(context: PluginInitializerContext) {} - - public setup(core: CoreSetup, plugins: RuleRegistrySetupPlugins) { - const rootRegistry = new RuleRegistry({ - fieldMap: {} as BaseRuleFieldMap, - alertTypeRegistry: plugins.triggersActionsUi.alertTypeRegistry, - }); - return { - registry: rootRegistry, - }; - } - - start(core: CoreStart, plugins: RuleRegistryStartPlugins) { - return { - registerType: plugins.triggersActionsUi.alertTypeRegistry, - }; - } -} diff --git a/x-pack/plugins/rule_registry/public/rule_registry/index.ts b/x-pack/plugins/rule_registry/public/rule_registry/index.ts deleted file mode 100644 index ea47fe2e26aad..0000000000000 --- a/x-pack/plugins/rule_registry/public/rule_registry/index.ts +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { BaseRuleFieldMap } from '../../common'; -import type { RuleType, CreateRuleRegistry, RuleRegistryConstructorOptions } from './types'; - -export class RuleRegistry { - protected types: TRuleType[] = []; - - constructor(private readonly options: RuleRegistryConstructorOptions) {} - - getTypes(): TRuleType[] { - return this.types; - } - - getTypeByRuleId(id: string): TRuleType | undefined { - return this.types.find((type) => type.id === id); - } - - registerType(type: TRuleType) { - this.types.push(type); - if (this.options.parent) { - this.options.parent.registerType(type); - } else { - this.options.alertTypeRegistry.register(type); - } - } - - create: CreateRuleRegistry = ({ fieldMap, ctor }) => { - const createOptions = { - fieldMap: { - ...this.options.fieldMap, - ...fieldMap, - }, - alertTypeRegistry: this.options.alertTypeRegistry, - parent: this, - }; - - const registry = ctor ? new ctor(createOptions) : new RuleRegistry(createOptions); - - return registry as any; - }; -} diff --git a/x-pack/plugins/rule_registry/public/rule_registry/types.ts b/x-pack/plugins/rule_registry/public/rule_registry/types.ts deleted file mode 100644 index 7c186385ebd35..0000000000000 --- a/x-pack/plugins/rule_registry/public/rule_registry/types.ts +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import type { AlertTypeRegistryContract } from '../../../triggers_actions_ui/public'; -import type { BaseRuleFieldMap, FieldMap } from '../../common'; - -export interface RuleRegistryConstructorOptions { - fieldMap: TFieldMap; - alertTypeRegistry: AlertTypeRegistryContract; - parent?: IRuleRegistry; -} - -export type RuleType = Parameters[0]; - -export type RegisterRuleType< - TFieldMap extends BaseRuleFieldMap, - TAdditionalRegisterOptions = {} -> = (type: RuleType & TAdditionalRegisterOptions) => void; - -export type RuleRegistryExtensions = Record< - T, - (...args: any[]) => any ->; - -export type CreateRuleRegistry< - TFieldMap extends BaseRuleFieldMap, - TRuleType extends RuleType, - TInstanceType = undefined -> = < - TNextFieldMap extends FieldMap, - TRuleRegistryInstance extends IRuleRegistry< - TFieldMap & TNextFieldMap, - any - > = TInstanceType extends IRuleRegistry - ? TInstanceType - : IRuleRegistry ->(options: { - fieldMap: TNextFieldMap; - ctor?: new ( - options: RuleRegistryConstructorOptions - ) => TRuleRegistryInstance; -}) => TRuleRegistryInstance; - -export interface IRuleRegistry< - TFieldMap extends BaseRuleFieldMap, - TRuleType extends RuleType, - TInstanceType = undefined -> { - create: CreateRuleRegistry; - registerType(type: TRuleType): void; - getTypeByRuleId(ruleId: string): TRuleType; - getTypes(): TRuleType[]; -} - -export type FieldMapOfRuleRegistry = TRuleRegistry extends IRuleRegistry< - infer TFieldMap, - any -> - ? TFieldMap - : never; diff --git a/x-pack/plugins/rule_registry/server/index.ts b/x-pack/plugins/rule_registry/server/index.ts index 9fd1408fcdb21..b51ba3e10f91a 100644 --- a/x-pack/plugins/rule_registry/server/index.ts +++ b/x-pack/plugins/rule_registry/server/index.ts @@ -9,21 +9,23 @@ import { schema, TypeOf } from '@kbn/config-schema'; import { PluginInitializerContext } from 'src/core/server'; import { RuleRegistryPlugin } from './plugin'; -export { RuleRegistryPluginSetupContract } from './plugin'; -export { createLifecycleRuleTypeFactory } from './rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory'; -export { FieldMapOf } from './types'; -export { ScopedRuleRegistryClient } from './rule_registry/create_scoped_rule_registry_client/types'; +export type { RuleRegistryPluginSetupContract, RuleRegistryPluginStartContract } from './plugin'; +export { RuleDataClient } from './rule_data_client'; +export { IRuleDataClient } from './rule_data_client/types'; +export { getRuleExecutorData, RuleExecutorData } from './utils/get_rule_executor_data'; +export { createLifecycleRuleTypeFactory } from './utils/create_lifecycle_rule_type_factory'; export const config = { schema: schema.object({ enabled: schema.boolean({ defaultValue: true }), - unsafe: schema.object({ - write: schema.object({ enabled: schema.boolean({ defaultValue: false }) }), + write: schema.object({ + enabled: schema.boolean({ defaultValue: true }), }), + index: schema.string({ defaultValue: '.alerts' }), }), }; -export type RuleRegistryConfig = TypeOf; +export type RuleRegistryPluginConfig = TypeOf; export const plugin = (initContext: PluginInitializerContext) => new RuleRegistryPlugin(initContext); diff --git a/x-pack/plugins/rule_registry/server/plugin.ts b/x-pack/plugins/rule_registry/server/plugin.ts index 09df47c40a394..3c645f98f5c71 100644 --- a/x-pack/plugins/rule_registry/server/plugin.ts +++ b/x-pack/plugins/rule_registry/server/plugin.ts @@ -6,44 +6,44 @@ */ import { PluginInitializerContext, Plugin, CoreSetup } from 'src/core/server'; -import { PluginSetupContract as AlertingPluginSetupContract } from '../../alerting/server'; -import { RuleRegistry } from './rule_registry'; -import { defaultIlmPolicy } from './rule_registry/defaults/ilm_policy'; -import { BaseRuleFieldMap, baseRuleFieldMap } from '../common'; -import { RuleRegistryConfig } from '.'; +import { RuleDataPluginService } from './rule_data_plugin_service'; +import { RuleRegistryPluginConfig } from '.'; -export type RuleRegistryPluginSetupContract = RuleRegistry; +export type RuleRegistryPluginSetupContract = RuleDataPluginService; +export type RuleRegistryPluginStartContract = void; export class RuleRegistryPlugin implements Plugin { constructor(private readonly initContext: PluginInitializerContext) { this.initContext = initContext; } - public setup( - core: CoreSetup, - plugins: { alerting: AlertingPluginSetupContract } - ): RuleRegistryPluginSetupContract { - const globalConfig = this.initContext.config.legacy.get(); - const config = this.initContext.config.get(); + public setup(core: CoreSetup): RuleRegistryPluginSetupContract { + const config = this.initContext.config.get(); const logger = this.initContext.logger.get(); - const rootRegistry = new RuleRegistry({ - coreSetup: core, - ilmPolicy: defaultIlmPolicy, - fieldMap: baseRuleFieldMap, - kibanaIndex: globalConfig.kibana.index, - name: 'alerts', - kibanaVersion: this.initContext.env.packageInfo.version, - logger: logger.get('root'), - alertingPluginSetupContract: plugins.alerting, - writeEnabled: config.unsafe.write.enabled, + const service = new RuleDataPluginService({ + logger, + isWriteEnabled: config.write.enabled, + index: config.index, + getClusterClient: async () => { + const [coreStart] = await core.getStartServices(); + + return coreStart.elasticsearch.client.asInternalUser; + }, + }); + + service.init().catch((originalError) => { + const error = new Error('Failed installing assets'); + // @ts-ignore + error.stack = originalError.stack; + logger.error(error); }); - return rootRegistry; + return service; } - public start() {} + public start(): RuleRegistryPluginStartContract {} public stop() {} } diff --git a/x-pack/plugins/rule_registry/server/rule_data_client/index.ts b/x-pack/plugins/rule_registry/server/rule_data_client/index.ts new file mode 100644 index 0000000000000..135c870f20727 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_data_client/index.ts @@ -0,0 +1,132 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { TypeMapping } from '@elastic/elasticsearch/api/types'; +import { ResponseError } from '@elastic/elasticsearch/lib/errors'; +import { IndexPatternsFetcher } from '../../../../../src/plugins/data/server'; +import { + IRuleDataClient, + RuleDataClientConstructorOptions, + RuleDataReader, + RuleDataWriter, +} from './types'; + +function getNamespacedAlias(options: { alias: string; namespace?: string }) { + return [options.alias, options.namespace].filter(Boolean).join('-'); +} + +export class RuleDataClient implements IRuleDataClient { + constructor(private readonly options: RuleDataClientConstructorOptions) {} + + private async getClusterClient() { + await this.options.ready(); + return await this.options.getClusterClient(); + } + + getReader(options: { namespace?: string } = {}): RuleDataReader { + const index = `${[this.options.alias, options.namespace].filter(Boolean).join('-')}*`; + + return { + search: async (request) => { + const clusterClient = await this.getClusterClient(); + + const { body } = (await clusterClient.search({ + ...request, + index, + })) as { body: any }; + + return body; + }, + getDynamicIndexPattern: async () => { + const clusterClient = await this.getClusterClient(); + const indexPatternsFetcher = new IndexPatternsFetcher(clusterClient); + + const fields = await indexPatternsFetcher.getFieldsForWildcard({ + pattern: index, + }); + + return { + fields, + timeFieldName: '@timestamp', + title: index, + }; + }, + }; + } + + getWriter(options: { namespace?: string } = {}): RuleDataWriter { + const { namespace } = options; + const alias = getNamespacedAlias({ alias: this.options.alias, namespace }); + return { + bulk: async (request) => { + const clusterClient = await this.getClusterClient(); + + const requestWithDefaultParameters = { + ...request, + require_alias: true, + index: alias, + }; + + return clusterClient.bulk(requestWithDefaultParameters).then((response) => { + if (response.body.errors) { + if ( + response.body.items.length === 1 && + response.body.items[0]?.index?.error?.type === 'index_not_found_exception' + ) { + return this.createOrUpdateWriteTarget({ namespace }).then(() => { + return clusterClient.bulk(requestWithDefaultParameters); + }); + } + const error = new ResponseError(response); + throw error; + } + return response; + }); + }, + }; + } + + async createOrUpdateWriteTarget({ namespace }: { namespace?: string }) { + const alias = getNamespacedAlias({ alias: this.options.alias, namespace }); + + const clusterClient = await this.getClusterClient(); + + const { body: aliasExists } = await clusterClient.indices.existsAlias({ + name: alias, + }); + + const concreteIndexName = `${alias}-000001`; + + if (!aliasExists) { + try { + await clusterClient.indices.create({ + index: concreteIndexName, + body: { + aliases: { + [alias]: { + is_write_index: true, + }, + }, + }, + }); + } catch (err) { + // something might have created the index already, that sounds OK + if (err?.meta?.body?.type !== 'resource_already_exists_exception') { + throw err; + } + } + } + + const { body: simulateResponse } = await clusterClient.transport.request({ + method: 'POST', + path: `/_index_template/_simulate_index/${concreteIndexName}`, + }); + + const mappings: TypeMapping = simulateResponse.template.mappings; + + await clusterClient.indices.putMapping({ index: `${alias}*`, body: mappings }); + } +} diff --git a/x-pack/plugins/rule_registry/server/rule_data_client/types.ts b/x-pack/plugins/rule_registry/server/rule_data_client/types.ts new file mode 100644 index 0000000000000..348fca6a58188 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_data_client/types.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ApiResponse } from '@elastic/elasticsearch'; +import { BulkRequest, BulkResponse } from '@elastic/elasticsearch/api/types'; +import { ElasticsearchClient } from 'kibana/server'; +import { FieldDescriptor } from 'src/plugins/data/server'; +import { ESSearchRequest, ESSearchResponse } from 'typings/elasticsearch'; +import { TechnicalRuleDataFieldName } from '../../common/technical_rule_data_field_names'; + +export interface RuleDataReader { + search( + request: TSearchRequest + ): Promise< + ESSearchResponse>, TSearchRequest> + >; + getDynamicIndexPattern( + target?: string + ): Promise<{ + title: string; + timeFieldName: string; + fields: FieldDescriptor[]; + }>; +} + +export interface RuleDataWriter { + bulk(request: BulkRequest): Promise>; +} + +export interface IRuleDataClient { + getReader(options?: { namespace?: string }): RuleDataReader; + getWriter(options?: { namespace?: string }): RuleDataWriter; + createOrUpdateWriteTarget(options: { namespace?: string }): Promise; +} + +export interface RuleDataClientConstructorOptions { + getClusterClient: () => Promise; + ready: () => Promise; + alias: string; +} diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index.ts new file mode 100644 index 0000000000000..159e9b8152597 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/index.ts @@ -0,0 +1,158 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { ClusterPutComponentTemplate } from '@elastic/elasticsearch/api/requestParams'; +import { estypes } from '@elastic/elasticsearch'; +import { ElasticsearchClient, Logger } from 'kibana/server'; +import { technicalComponentTemplate } from '../../common/assets/component_templates/technical_component_template'; +import { + DEFAULT_ILM_POLICY_ID, + ECS_COMPONENT_TEMPLATE_NAME, + TECHNICAL_COMPONENT_TEMPLATE_NAME, +} from '../../common/assets'; +import { ecsComponentTemplate } from '../../common/assets/component_templates/ecs_component_template'; +import { defaultLifecyclePolicy } from '../../common/assets/lifecycle_policies/default_lifecycle_policy'; +import { ClusterPutComponentTemplateBody, PutIndexTemplateRequest } from '../../common/types'; + +const BOOTSTRAP_TIMEOUT = 60000; + +interface RuleDataPluginServiceConstructorOptions { + getClusterClient: () => Promise; + logger: Logger; + isWriteEnabled: boolean; + index: string; +} + +function createSignal() { + let resolver: () => void; + + let ready: boolean = false; + + const promise = new Promise((resolve) => { + resolver = resolve; + }); + + function wait(): Promise { + return promise.then(() => { + ready = true; + }); + } + + function complete() { + resolver(); + } + + return { wait, complete, isReady: () => ready }; +} + +export class RuleDataPluginService { + signal = createSignal(); + + constructor(private readonly options: RuleDataPluginServiceConstructorOptions) {} + + private assertWriteEnabled() { + if (!this.isWriteEnabled) { + throw new Error('Write operations are disabled'); + } + } + + private async getClusterClient() { + return await this.options.getClusterClient(); + } + + async init() { + if (!this.isWriteEnabled) { + this.options.logger.info('Write is disabled, not installing assets'); + this.signal.complete(); + return; + } + + this.options.logger.info(`Installing assets in namespace ${this.getFullAssetName()}`); + + await this._createOrUpdateLifecyclePolicy({ + policy: this.getFullAssetName(DEFAULT_ILM_POLICY_ID), + body: defaultLifecyclePolicy, + }); + + await this._createOrUpdateComponentTemplate({ + name: this.getFullAssetName(TECHNICAL_COMPONENT_TEMPLATE_NAME), + body: technicalComponentTemplate, + }); + + await this._createOrUpdateComponentTemplate({ + name: this.getFullAssetName(ECS_COMPONENT_TEMPLATE_NAME), + body: ecsComponentTemplate, + }); + + this.options.logger.info(`Installed all assets`); + + this.signal.complete(); + } + + private async _createOrUpdateComponentTemplate( + template: ClusterPutComponentTemplate + ) { + this.assertWriteEnabled(); + + const clusterClient = await this.getClusterClient(); + this.options.logger.debug(`Installing component template ${template.name}`); + return clusterClient.cluster.putComponentTemplate(template); + } + + private async _createOrUpdateIndexTemplate(template: PutIndexTemplateRequest) { + this.assertWriteEnabled(); + + const clusterClient = await this.getClusterClient(); + this.options.logger.debug(`Installing index template ${template.name}`); + return clusterClient.indices.putIndexTemplate(template); + } + + private async _createOrUpdateLifecyclePolicy(policy: estypes.PutLifecycleRequest) { + this.assertWriteEnabled(); + const clusterClient = await this.getClusterClient(); + + this.options.logger.debug(`Installing lifecycle policy ${policy.policy}`); + return clusterClient.ilm.putLifecycle(policy); + } + + async createOrUpdateComponentTemplate( + template: ClusterPutComponentTemplate + ) { + await this.wait(); + return this._createOrUpdateComponentTemplate(template); + } + + async createOrUpdateIndexTemplate(template: PutIndexTemplateRequest) { + await this.wait(); + return this._createOrUpdateIndexTemplate(template); + } + + async createOrUpdateLifecyclePolicy(policy: estypes.PutLifecycleRequest) { + await this.wait(); + return this._createOrUpdateLifecyclePolicy(policy); + } + + isReady() { + return this.signal.isReady(); + } + + wait() { + return Promise.race([ + this.signal.wait(), + new Promise((resolve, reject) => { + setTimeout(reject, BOOTSTRAP_TIMEOUT); + }), + ]); + } + + isWriteEnabled(): boolean { + return this.options.isWriteEnabled; + } + + getFullAssetName(assetName?: string) { + return [this.options.index, assetName].filter(Boolean).join('-'); + } +} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts b/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts deleted file mode 100644 index 0d7735380b640..0000000000000 --- a/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/index.ts +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { Either, isLeft, isRight } from 'fp-ts/lib/Either'; -import { Errors } from 'io-ts'; -import { PathReporter } from 'io-ts/lib/PathReporter'; -import { Logger } from 'kibana/server'; -import { IScopedClusterClient as ScopedClusterClient } from 'src/core/server'; -import { castArray, compact } from 'lodash'; -import { ESSearchRequest } from 'typings/elasticsearch'; -import { IndexPatternsFetcher } from '../../../../../../src/plugins/data/server'; -import { ClusterClientAdapter } from '../../../../event_log/server'; -import { TypeOfFieldMap } from '../../../common'; -import { ScopedRuleRegistryClient, EventsOf } from './types'; -import { BaseRuleFieldMap } from '../../../common'; -import { RuleRegistry } from '..'; - -const createPathReporterError = (either: Either) => { - const error = new Error(`Failed to validate alert event`); - error.stack += '\n' + PathReporter.report(either).join('\n'); - return error; -}; - -export function createScopedRuleRegistryClient({ - ruleUuids, - scopedClusterClient, - clusterClientAdapter, - indexAliasName, - indexTarget, - logger, - registry, - ruleData, -}: { - ruleUuids: string[]; - scopedClusterClient: ScopedClusterClient; - clusterClientAdapter: ClusterClientAdapter<{ - body: TypeOfFieldMap; - index: string; - }>; - indexAliasName: string; - indexTarget: string; - logger: Logger; - registry: RuleRegistry; - ruleData?: { - rule: { - id: string; - uuid: string; - category: string; - name: string; - }; - producer: string; - tags: string[]; - }; -}): ScopedRuleRegistryClient { - const fieldmapType = registry.getFieldMapType(); - - const defaults = ruleData - ? { - 'rule.uuid': ruleData.rule.uuid, - 'rule.id': ruleData.rule.id, - 'rule.name': ruleData.rule.name, - 'rule.category': ruleData.rule.category, - 'kibana.rac.producer': ruleData.producer, - tags: ruleData.tags, - } - : {}; - - const client: ScopedRuleRegistryClient = { - search: async (searchRequest) => { - const fields = [ - 'rule.id', - ...(searchRequest.body?.fields ? castArray(searchRequest.body.fields) : []), - ]; - - const response = await scopedClusterClient.asInternalUser.search({ - ...searchRequest, - index: indexTarget, - body: { - ...searchRequest.body, - query: { - bool: { - filter: [ - { terms: { 'rule.uuid': ruleUuids } }, - ...compact([searchRequest.body?.query]), - ], - }, - }, - fields, - }, - }); - - return { - body: response.body as any, - events: compact( - response.body.hits.hits.map((hit) => { - const ruleTypeId: string = hit.fields!['rule.id'][0]; - - const registryOfType = registry.getRegistryByRuleTypeId(ruleTypeId); - - if (ruleTypeId && !registryOfType) { - logger.warn( - `Could not find type ${ruleTypeId} in registry, decoding with default type` - ); - } - - const type = registryOfType?.getFieldMapType() ?? fieldmapType; - - const validation = type.decode(hit.fields); - if (isLeft(validation)) { - const error = createPathReporterError(validation); - logger.error(error); - return undefined; - } - return type.encode(validation.right); - }) - ) as EventsOf, - }; - }, - getDynamicIndexPattern: async () => { - const indexPatternsFetcher = new IndexPatternsFetcher(scopedClusterClient.asInternalUser); - - const fields = await indexPatternsFetcher.getFieldsForWildcard({ - pattern: indexTarget, - }); - - return { - fields, - timeFieldName: '@timestamp', - title: indexTarget, - }; - }, - index: (doc) => { - const validation = fieldmapType.decode({ - ...doc, - ...defaults, - }); - - if (isLeft(validation)) { - throw createPathReporterError(validation); - } - - clusterClientAdapter.indexDocument({ - body: validation.right, - index: indexAliasName, - }); - }, - bulkIndex: (docs) => { - const validations = docs.map((doc) => { - return fieldmapType.decode({ - ...doc, - ...defaults, - }); - }); - - const errors = compact( - validations.map((validation) => - isLeft(validation) ? createPathReporterError(validation) : null - ) - ); - - errors.forEach((error) => { - logger.error(error); - }); - - const operations = compact( - validations.map((validation) => (isRight(validation) ? validation.right : null)) - ).map((doc) => ({ body: doc, index: indexAliasName })); - - return clusterClientAdapter.indexDocuments(operations); - }, - }; - - // @ts-expect-error: We can't use ScopedRuleRegistryClient - // when creating the client, due to #41693 which will be fixed in 4.2 - return client; -} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/types.ts b/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/types.ts deleted file mode 100644 index f7b2394fe3a32..0000000000000 --- a/x-pack/plugins/rule_registry/server/rule_registry/create_scoped_rule_registry_client/types.ts +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { FieldDescriptor } from 'src/plugins/data/server'; -import { ESSearchRequest, ESSearchResponse } from 'typings/elasticsearch'; -import { - PatternsUnionOf, - PickWithPatterns, - OutputOfFieldMap, - BaseRuleFieldMap, -} from '../../../common'; - -export type PrepopulatedRuleEventFields = keyof Pick< - BaseRuleFieldMap, - 'rule.uuid' | 'rule.id' | 'rule.name' | 'rule.category' | 'kibana.rac.producer' ->; - -type FieldsOf = - | Array<{ field: PatternsUnionOf } | PatternsUnionOf> - | PatternsUnionOf; - -type Fields = Array<{ field: TPattern } | TPattern> | TPattern; - -type FieldsESSearchRequest = ESSearchRequest & { - body?: { fields: FieldsOf }; -}; - -export type EventsOf< - TFieldsESSearchRequest extends ESSearchRequest, - TFieldMap extends BaseRuleFieldMap -> = TFieldsESSearchRequest extends { body: { fields: infer TFields } } - ? TFields extends Fields - ? Array>> - : never - : never; - -export interface ScopedRuleRegistryClient { - search>( - request: TSearchRequest - ): Promise<{ - body: ESSearchResponse; - events: EventsOf; - }>; - getDynamicIndexPattern(): Promise<{ - title: string; - timeFieldName: string; - fields: FieldDescriptor[]; - }>; - index(doc: Omit, PrepopulatedRuleEventFields>): void; - bulkIndex( - doc: Array, PrepopulatedRuleEventFields>> - ): Promise; -} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/index.ts b/x-pack/plugins/rule_registry/server/rule_registry/index.ts deleted file mode 100644 index bbc381f60a809..0000000000000 --- a/x-pack/plugins/rule_registry/server/rule_registry/index.ts +++ /dev/null @@ -1,328 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { CoreSetup, Logger, RequestHandlerContext } from 'kibana/server'; -import { inspect } from 'util'; -import { AlertsClient } from '../../../alerting/server'; -import { SpacesServiceStart } from '../../../spaces/server'; -import { - ActionVariable, - AlertInstanceState, - AlertTypeParams, - AlertTypeState, -} from '../../../alerting/common'; -import { createReadySignal, ClusterClientAdapter } from '../../../event_log/server'; -import { ILMPolicy } from './types'; -import { RuleParams, RuleType } from '../types'; -import { - mergeFieldMaps, - TypeOfFieldMap, - FieldMap, - FieldMapType, - BaseRuleFieldMap, - runtimeTypeFromFieldMap, -} from '../../common'; -import { mappingFromFieldMap } from './field_map/mapping_from_field_map'; -import { PluginSetupContract as AlertingPluginSetupContract } from '../../../alerting/server'; -import { createScopedRuleRegistryClient } from './create_scoped_rule_registry_client'; -import { ScopedRuleRegistryClient } from './create_scoped_rule_registry_client/types'; - -interface RuleRegistryOptions { - kibanaIndex: string; - kibanaVersion: string; - name: string; - logger: Logger; - coreSetup: CoreSetup; - spacesStart?: SpacesServiceStart; - fieldMap: TFieldMap; - ilmPolicy: ILMPolicy; - alertingPluginSetupContract: AlertingPluginSetupContract; - writeEnabled: boolean; -} - -export class RuleRegistry { - private readonly esAdapter: ClusterClientAdapter<{ - body: TypeOfFieldMap; - index: string; - }>; - private readonly children: Array> = []; - private readonly types: Array> = []; - - private readonly fieldmapType: FieldMapType; - - constructor(private readonly options: RuleRegistryOptions) { - const { logger, coreSetup } = options; - - this.fieldmapType = runtimeTypeFromFieldMap(options.fieldMap); - - const { wait, signal } = createReadySignal(); - - this.esAdapter = new ClusterClientAdapter<{ - body: TypeOfFieldMap; - index: string; - }>({ - wait, - elasticsearchClientPromise: coreSetup - .getStartServices() - .then(([{ elasticsearch }]) => elasticsearch.client.asInternalUser), - logger: logger.get('esAdapter'), - }); - - if (this.options.writeEnabled) { - this.initialize() - .then(() => { - this.options.logger.debug('Bootstrapped alerts index'); - signal(true); - }) - .catch((err) => { - logger.error(inspect(err, { depth: null })); - signal(false); - }); - } else { - logger.debug('Write disabled, indices are not being bootstrapped'); - } - } - - private getEsNames() { - const base = [this.options.kibanaIndex, this.options.name]; - const indexTarget = `${base.join('-')}*`; - const indexAliasName = [...base, this.options.kibanaVersion.toLowerCase()].join('-'); - const policyName = [...base, 'policy'].join('-'); - - return { - indexAliasName, - indexTarget, - policyName, - }; - } - - private async initialize() { - const { indexAliasName, policyName } = this.getEsNames(); - - const ilmPolicyExists = await this.esAdapter.doesIlmPolicyExist(policyName); - - if (!ilmPolicyExists) { - await this.esAdapter.createIlmPolicy( - policyName, - (this.options.ilmPolicy as unknown) as Record - ); - } - - const templateExists = await this.esAdapter.doesIndexTemplateExist(indexAliasName); - - const mappings = mappingFromFieldMap(this.options.fieldMap); - - const esClient = (await this.options.coreSetup.getStartServices())[0].elasticsearch.client - .asInternalUser; - - if (!templateExists) { - await this.esAdapter.createIndexTemplate(indexAliasName, { - index_patterns: [`${indexAliasName}-*`], - settings: { - number_of_shards: 1, - auto_expand_replicas: '0-1', - 'index.lifecycle.name': policyName, - 'index.lifecycle.rollover_alias': indexAliasName, - 'sort.field': '@timestamp', - 'sort.order': 'desc', - }, - mappings, - }); - } else { - await esClient.indices.putTemplate({ - name: indexAliasName, - body: { - index_patterns: [`${indexAliasName}-*`], - mappings, - }, - create: false, - }); - } - - const aliasExists = await this.esAdapter.doesAliasExist(indexAliasName); - - if (!aliasExists) { - await this.esAdapter.createIndex(`${indexAliasName}-000001`, { - aliases: { - [indexAliasName]: { - is_write_index: true, - }, - }, - }); - } else { - const { body: aliases } = (await esClient.indices.getAlias({ - index: indexAliasName, - })) as { body: Record }> }; - - const writeIndex = Object.entries(aliases).find( - ([indexName, alias]) => alias.aliases[indexAliasName]?.is_write_index === true - )![0]; - - const { body: fieldsInWriteIndex } = await esClient.fieldCaps({ - index: writeIndex, - fields: '*', - }); - - const fieldsNotOrDifferentInIndex = Object.entries(this.options.fieldMap).filter( - ([fieldName, descriptor]) => { - return ( - !fieldsInWriteIndex.fields[fieldName] || - !fieldsInWriteIndex.fields[fieldName][descriptor.type] - ); - } - ); - - if (fieldsNotOrDifferentInIndex.length > 0) { - this.options.logger.debug( - `Some fields were not found in write index mapping: ${Object.keys( - Object.fromEntries(fieldsNotOrDifferentInIndex) - ).join(',')}` - ); - this.options.logger.info(`Updating index mapping due to new fields`); - - await esClient.indices.putMapping({ - index: indexAliasName, - body: mappings, - }); - } - } - } - - getFieldMapType() { - return this.fieldmapType; - } - - getRuleTypeById(ruleTypeId: string) { - return this.types.find((type) => type.id === ruleTypeId); - } - - getRegistryByRuleTypeId(ruleTypeId: string): RuleRegistry | undefined { - if (this.getRuleTypeById(ruleTypeId)) { - return this; - } - - return this.children.find((child) => child.getRegistryByRuleTypeId(ruleTypeId)); - } - - async createScopedRuleRegistryClient({ - context, - alertsClient, - }: { - context: RequestHandlerContext; - alertsClient: AlertsClient; - }): Promise | undefined> { - if (!this.options.writeEnabled) { - return undefined; - } - const { indexAliasName, indexTarget } = this.getEsNames(); - - const frameworkAlerts = ( - await alertsClient.find({ - options: { - perPage: 1000, - }, - }) - ).data; - - return createScopedRuleRegistryClient({ - ruleUuids: frameworkAlerts.map((frameworkAlert) => frameworkAlert.id), - scopedClusterClient: context.core.elasticsearch.client, - clusterClientAdapter: this.esAdapter, - registry: this, - indexAliasName, - indexTarget, - logger: this.options.logger, - }); - } - - registerType( - type: RuleType - ) { - const logger = this.options.logger.get(type.id); - - const { indexAliasName, indexTarget } = this.getEsNames(); - - this.types.push(type); - - this.options.alertingPluginSetupContract.registerType< - AlertTypeParams, - AlertTypeState, - AlertInstanceState, - { [key in TActionVariable['name']]: any }, - string - >({ - ...type, - executor: async (executorOptions) => { - const { services, alertId, name, tags } = executorOptions; - - const rule = { - id: type.id, - uuid: alertId, - category: type.name, - name, - }; - - const producer = type.producer; - - return type.executor({ - ...executorOptions, - rule, - producer, - services: { - ...services, - logger, - ...(this.options.writeEnabled - ? { - scopedRuleRegistryClient: createScopedRuleRegistryClient({ - scopedClusterClient: services.scopedClusterClient, - ruleUuids: [rule.uuid], - clusterClientAdapter: this.esAdapter, - registry: this, - indexAliasName, - indexTarget, - ruleData: { - producer, - rule, - tags, - }, - logger: this.options.logger, - }), - } - : {}), - }, - }); - }, - }); - } - - create({ - name, - fieldMap, - ilmPolicy, - }: { - name: string; - fieldMap: TNextFieldMap; - ilmPolicy?: ILMPolicy; - }): RuleRegistry { - const mergedFieldMap = fieldMap - ? mergeFieldMaps(this.options.fieldMap, fieldMap) - : this.options.fieldMap; - - const child = new RuleRegistry({ - ...this.options, - logger: this.options.logger.get(name), - name: [this.options.name, name].filter(Boolean).join('-'), - fieldMap: mergedFieldMap, - ...(ilmPolicy ? { ilmPolicy } : {}), - }); - - this.children.push(child); - - // @ts-expect-error could be instantiated with a different subtype of constraint - return child; - } -} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory.ts b/x-pack/plugins/rule_registry/server/rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory.ts deleted file mode 100644 index 65eaf0964cfca..0000000000000 --- a/x-pack/plugins/rule_registry/server/rule_registry/rule_type_helpers/create_lifecycle_rule_type_factory.ts +++ /dev/null @@ -1,235 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import * as t from 'io-ts'; -import { isLeft } from 'fp-ts/lib/Either'; -import v4 from 'uuid/v4'; -import { Mutable } from 'utility-types'; -import { AlertInstance } from '../../../../alerting/server'; -import { ActionVariable, AlertInstanceState } from '../../../../alerting/common'; -import { RuleParams, RuleType } from '../../types'; -import { BaseRuleFieldMap, OutputOfFieldMap } from '../../../common'; -import { PrepopulatedRuleEventFields } from '../create_scoped_rule_registry_client/types'; -import { RuleRegistry } from '..'; - -type UserDefinedAlertFields = Omit< - OutputOfFieldMap, - PrepopulatedRuleEventFields | 'kibana.rac.alert.id' | 'kibana.rac.alert.uuid' | '@timestamp' ->; - -type LifecycleAlertService< - TFieldMap extends BaseRuleFieldMap, - TActionVariable extends ActionVariable -> = (alert: { - id: string; - fields: UserDefinedAlertFields; -}) => AlertInstance; - -type CreateLifecycleRuleType = < - TRuleParams extends RuleParams, - TActionVariable extends ActionVariable ->( - type: RuleType< - TFieldMap, - TRuleParams, - TActionVariable, - { alertWithLifecycle: LifecycleAlertService } - > -) => RuleType; - -const trackedAlertStateRt = t.type({ - alertId: t.string, - alertUuid: t.string, - started: t.string, -}); - -const wrappedStateRt = t.type({ - wrapped: t.record(t.string, t.unknown), - trackedAlerts: t.record(t.string, trackedAlertStateRt), -}); - -export function createLifecycleRuleTypeFactory< - TRuleRegistry extends RuleRegistry ->(): TRuleRegistry extends RuleRegistry - ? CreateLifecycleRuleType - : never; - -export function createLifecycleRuleTypeFactory(): CreateLifecycleRuleType { - return (type) => { - return { - ...type, - executor: async (options) => { - const { - services: { scopedRuleRegistryClient, alertInstanceFactory, logger }, - state: previousState, - rule, - } = options; - - const decodedState = wrappedStateRt.decode(previousState); - - const state = isLeft(decodedState) - ? { - wrapped: previousState, - trackedAlerts: {}, - } - : decodedState.right; - - const currentAlerts: Record< - string, - UserDefinedAlertFields & { 'kibana.rac.alert.id': string } - > = {}; - - const timestamp = options.startedAt.toISOString(); - - const nextWrappedState = await type.executor({ - ...options, - state: state.wrapped, - services: { - ...options.services, - alertWithLifecycle: ({ id, fields }) => { - currentAlerts[id] = { - ...fields, - 'kibana.rac.alert.id': id, - }; - return alertInstanceFactory(id); - }, - }, - }); - - const currentAlertIds = Object.keys(currentAlerts); - const trackedAlertIds = Object.keys(state.trackedAlerts); - const newAlertIds = currentAlertIds.filter((alertId) => !trackedAlertIds.includes(alertId)); - - const allAlertIds = [...new Set(currentAlertIds.concat(trackedAlertIds))]; - - const trackedAlertStatesOfRecovered = Object.values(state.trackedAlerts).filter( - (trackedAlertState) => !currentAlerts[trackedAlertState.alertId] - ); - - logger.debug( - `Tracking ${allAlertIds.length} alerts (${newAlertIds.length} new, ${trackedAlertStatesOfRecovered.length} recovered)` - ); - - const alertsDataMap: Record> = { - ...currentAlerts, - }; - - if (scopedRuleRegistryClient && trackedAlertStatesOfRecovered.length) { - const { events } = await scopedRuleRegistryClient.search({ - body: { - query: { - bool: { - filter: [ - { - term: { - 'rule.uuid': rule.uuid, - }, - }, - { - terms: { - 'kibana.rac.alert.uuid': trackedAlertStatesOfRecovered.map( - (trackedAlertState) => trackedAlertState.alertUuid - ), - }, - }, - ], - }, - }, - size: trackedAlertStatesOfRecovered.length, - collapse: { - field: 'kibana.rac.alert.uuid', - }, - _source: false, - fields: ['*'], - sort: { - '@timestamp': 'desc' as const, - }, - }, - }); - - events.forEach((event) => { - const alertId = event['kibana.rac.alert.id']!; - alertsDataMap[alertId] = event; - }); - } - - const eventsToIndex: Array> = allAlertIds.map( - (alertId) => { - const alertData = alertsDataMap[alertId]; - - if (!alertData) { - logger.warn(`Could not find alert data for ${alertId}`); - } - - const event: Mutable> = { - ...alertData, - '@timestamp': timestamp, - 'event.kind': 'state', - 'kibana.rac.alert.id': alertId, - }; - - const isNew = !state.trackedAlerts[alertId]; - const isRecovered = !currentAlerts[alertId]; - const isActiveButNotNew = !isNew && !isRecovered; - const isActive = !isRecovered; - - const { alertUuid, started } = state.trackedAlerts[alertId] ?? { - alertUuid: v4(), - started: timestamp, - }; - - event['kibana.rac.alert.start'] = started; - event['kibana.rac.alert.uuid'] = alertUuid; - - if (isNew) { - event['event.action'] = 'open'; - } - - if (isRecovered) { - event['kibana.rac.alert.end'] = timestamp; - event['event.action'] = 'close'; - event['kibana.rac.alert.status'] = 'closed'; - } - - if (isActiveButNotNew) { - event['event.action'] = 'active'; - } - - if (isActive) { - event['kibana.rac.alert.status'] = 'open'; - } - - event['kibana.rac.alert.duration.us'] = - (options.startedAt.getTime() - new Date(event['kibana.rac.alert.start']!).getTime()) * - 1000; - - return event; - } - ); - - if (eventsToIndex.length && scopedRuleRegistryClient) { - await scopedRuleRegistryClient.bulkIndex(eventsToIndex); - } - - const nextTrackedAlerts = Object.fromEntries( - eventsToIndex - .filter((event) => event['kibana.rac.alert.status'] !== 'closed') - .map((event) => { - const alertId = event['kibana.rac.alert.id']!; - const alertUuid = event['kibana.rac.alert.uuid']!; - const started = new Date(event['kibana.rac.alert.start']!).toISOString(); - return [alertId, { alertId, alertUuid, started }]; - }) - ); - - return { - wrapped: nextWrappedState, - trackedAlerts: nextTrackedAlerts, - }; - }, - }; - }; -} diff --git a/x-pack/plugins/rule_registry/server/rule_registry/types.ts b/x-pack/plugins/rule_registry/server/rule_registry/types.ts deleted file mode 100644 index ec7293d1c1d4c..0000000000000 --- a/x-pack/plugins/rule_registry/server/rule_registry/types.ts +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export interface Mappings { - dynamic: 'strict' | boolean; - properties: Record; -} - -enum ILMPolicyPhase { - hot = 'hot', - delete = 'delete', -} - -enum ILMPolicyAction { - rollover = 'rollover', - delete = 'delete', -} - -interface ILMActionOptions { - [ILMPolicyAction.rollover]: { - max_size: string; - max_age: string; - }; - [ILMPolicyAction.delete]: {}; -} - -export interface ILMPolicy { - policy: { - phases: Record< - ILMPolicyPhase, - { - actions: { - [key in keyof ILMActionOptions]?: ILMActionOptions[key]; - }; - } - >; - }; -} diff --git a/x-pack/plugins/rule_registry/server/types.ts b/x-pack/plugins/rule_registry/server/types.ts index dd54046365d98..959c05fd1334e 100644 --- a/x-pack/plugins/rule_registry/server/types.ts +++ b/x-pack/plugins/rule_registry/server/types.ts @@ -4,97 +4,37 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { Type, TypeOf } from '@kbn/config-schema'; -import { Logger } from 'kibana/server'; + import { - ActionVariable, AlertInstanceContext, AlertInstanceState, AlertTypeParams, AlertTypeState, } from '../../alerting/common'; -import { ActionGroup, AlertExecutorOptions } from '../../alerting/server'; -import { RuleRegistry } from './rule_registry'; -import { ScopedRuleRegistryClient } from './rule_registry/create_scoped_rule_registry_client/types'; -import { BaseRuleFieldMap } from '../common'; - -export type RuleParams = Type; +import { AlertType } from '../../alerting/server'; -type TypeOfRuleParams = TypeOf; +type SimpleAlertType< + TParams extends AlertTypeParams = {}, + TAlertInstanceContext extends AlertInstanceContext = {} +> = AlertType; -type RuleExecutorServices< - TFieldMap extends BaseRuleFieldMap, - TActionVariable extends ActionVariable -> = AlertExecutorOptions< - AlertTypeParams, - AlertTypeState, - AlertInstanceState, - { [key in TActionVariable['name']]: any }, - string ->['services'] & { - logger: Logger; - scopedRuleRegistryClient?: ScopedRuleRegistryClient; -}; - -type PassthroughAlertExecutorOptions = Pick< - AlertExecutorOptions< - AlertTypeParams, - AlertTypeState, - AlertInstanceState, - AlertInstanceContext, - string - >, - 'previousStartedAt' | 'startedAt' | 'state' ->; - -type RuleExecutorFunction< - TFieldMap extends BaseRuleFieldMap, - TRuleParams extends RuleParams, - TActionVariable extends ActionVariable, - TAdditionalRuleExecutorServices extends Record +export type AlertTypeExecutor< + TParams extends AlertTypeParams = {}, + TAlertInstanceContext extends AlertInstanceContext = {}, + TServices extends Record = {} > = ( - options: PassthroughAlertExecutorOptions & { - services: RuleExecutorServices & TAdditionalRuleExecutorServices; - params: TypeOfRuleParams; - rule: { - id: string; - uuid: string; - name: string; - category: string; - }; - producer: string; + options: Parameters['executor']>[0] & { + services: TServices; } -) => Promise>; - -interface RuleTypeBase { - id: string; - name: string; - actionGroups: Array>; - defaultActionGroupId: string; - producer: string; - minimumLicenseRequired: 'basic' | 'gold' | 'trial'; -} - -export type RuleType< - TFieldMap extends BaseRuleFieldMap, - TRuleParams extends RuleParams, - TActionVariable extends ActionVariable, - TAdditionalRuleExecutorServices extends Record = {} -> = RuleTypeBase & { - validate: { - params: TRuleParams; - }; - actionVariables: { - context: TActionVariable[]; - }; - executor: RuleExecutorFunction< - TFieldMap, - TRuleParams, - TActionVariable, - TAdditionalRuleExecutorServices - >; +) => Promise; + +export type AlertTypeWithExecutor< + TParams extends AlertTypeParams = {}, + TAlertInstanceContext extends AlertInstanceContext = {}, + TServices extends Record = {} +> = Omit< + AlertType, + 'executor' +> & { + executor: AlertTypeExecutor; }; - -export type FieldMapOf< - TRuleRegistry extends RuleRegistry -> = TRuleRegistry extends RuleRegistry ? TFieldMap : never; diff --git a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type_factory.ts b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type_factory.ts new file mode 100644 index 0000000000000..b523dd6770b9f --- /dev/null +++ b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type_factory.ts @@ -0,0 +1,246 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { Logger } from '@kbn/logging'; +import { isLeft } from 'fp-ts/lib/Either'; +import * as t from 'io-ts'; +import { Mutable } from 'utility-types'; +import v4 from 'uuid/v4'; +import { AlertInstance } from '../../../alerting/server'; +import { RuleDataClient } from '..'; +import { + AlertInstanceContext, + AlertInstanceState, + AlertTypeParams, +} from '../../../alerting/common'; +import { + ALERT_DURATION, + ALERT_END, + ALERT_ID, + ALERT_START, + ALERT_STATUS, + ALERT_UUID, + EVENT_ACTION, + EVENT_KIND, + RULE_UUID, + TIMESTAMP, +} from '../../common/technical_rule_data_field_names'; +import { AlertTypeWithExecutor } from '../types'; +import { ParsedTechnicalFields, parseTechnicalFields } from '../../common/parse_technical_fields'; +import { getRuleExecutorData } from './get_rule_executor_data'; + +type LifecycleAlertService> = (alert: { + id: string; + fields: Record; +}) => AlertInstance; + +const trackedAlertStateRt = t.type({ + alertId: t.string, + alertUuid: t.string, + started: t.string, +}); + +const wrappedStateRt = t.type({ + wrapped: t.record(t.string, t.unknown), + trackedAlerts: t.record(t.string, trackedAlertStateRt), +}); + +type CreateLifecycleRuleTypeFactory = (options: { + ruleDataClient: RuleDataClient; + logger: Logger; +}) => < + TParams extends AlertTypeParams, + TAlertInstanceContext extends AlertInstanceContext, + TServices extends { alertWithLifecycle: LifecycleAlertService } +>( + type: AlertTypeWithExecutor +) => AlertTypeWithExecutor; + +export const createLifecycleRuleTypeFactory: CreateLifecycleRuleTypeFactory = ({ + logger, + ruleDataClient, +}) => (type) => { + return { + ...type, + executor: async (options) => { + const { + services: { alertInstanceFactory }, + state: previousState, + } = options; + + const ruleExecutorData = getRuleExecutorData(type, options); + + const decodedState = wrappedStateRt.decode(previousState); + + const state = isLeft(decodedState) + ? { + wrapped: previousState, + trackedAlerts: {}, + } + : decodedState.right; + + const currentAlerts: Record = {}; + + const timestamp = options.startedAt.toISOString(); + + const nextWrappedState = await type.executor({ + ...options, + state: state.wrapped, + services: { + ...options.services, + alertWithLifecycle: ({ id, fields }) => { + currentAlerts[id] = { + ...fields, + [ALERT_ID]: id, + }; + return alertInstanceFactory(id); + }, + }, + }); + + const currentAlertIds = Object.keys(currentAlerts); + const trackedAlertIds = Object.keys(state.trackedAlerts); + const newAlertIds = currentAlertIds.filter((alertId) => !trackedAlertIds.includes(alertId)); + + const allAlertIds = [...new Set(currentAlertIds.concat(trackedAlertIds))]; + + const trackedAlertStatesOfRecovered = Object.values(state.trackedAlerts).filter( + (trackedAlertState) => !currentAlerts[trackedAlertState.alertId] + ); + + logger.debug( + `Tracking ${allAlertIds.length} alerts (${newAlertIds.length} new, ${trackedAlertStatesOfRecovered.length} recovered)` + ); + + const alertsDataMap: Record< + string, + { + [ALERT_ID]: string; + } + > = { + ...currentAlerts, + }; + + if (trackedAlertStatesOfRecovered.length) { + const { hits } = await ruleDataClient.getReader().search({ + body: { + query: { + bool: { + filter: [ + { + term: { + [RULE_UUID]: ruleExecutorData[RULE_UUID], + }, + }, + { + terms: { + [ALERT_UUID]: trackedAlertStatesOfRecovered.map( + (trackedAlertState) => trackedAlertState.alertUuid + ), + }, + }, + ], + }, + }, + size: trackedAlertStatesOfRecovered.length, + collapse: { + field: ALERT_UUID, + }, + _source: false, + fields: [{ field: '*', include_unmapped: true }], + sort: { + [TIMESTAMP]: 'desc' as const, + }, + }, + allow_no_indices: true, + }); + + hits.hits.forEach((hit) => { + const fields = parseTechnicalFields(hit.fields); + const alertId = fields[ALERT_ID]!; + alertsDataMap[alertId] = { + ...fields, + [ALERT_ID]: alertId, + }; + }); + } + + const eventsToIndex = allAlertIds.map((alertId) => { + const alertData = alertsDataMap[alertId]; + + if (!alertData) { + logger.warn(`Could not find alert data for ${alertId}`); + } + + const event: Mutable = { + ...alertData, + ...ruleExecutorData, + [TIMESTAMP]: timestamp, + [EVENT_KIND]: 'state', + [ALERT_ID]: alertId, + }; + + const isNew = !state.trackedAlerts[alertId]; + const isRecovered = !currentAlerts[alertId]; + const isActiveButNotNew = !isNew && !isRecovered; + const isActive = !isRecovered; + + const { alertUuid, started } = state.trackedAlerts[alertId] ?? { + alertUuid: v4(), + started: timestamp, + }; + + event[ALERT_START] = started; + event[ALERT_UUID] = alertUuid; + + if (isNew) { + event[EVENT_ACTION] = 'open'; + } + + if (isRecovered) { + event[ALERT_END] = timestamp; + event[EVENT_ACTION] = 'close'; + event[ALERT_STATUS] = 'closed'; + } + + if (isActiveButNotNew) { + event[EVENT_ACTION] = 'active'; + } + + if (isActive) { + event[ALERT_STATUS] = 'open'; + } + + event[ALERT_DURATION] = + (options.startedAt.getTime() - new Date(event[ALERT_START]!).getTime()) * 1000; + + return event; + }); + + if (eventsToIndex.length) { + await ruleDataClient.getWriter().bulk({ + body: eventsToIndex.flatMap((event) => [{ index: {} }, event]), + }); + } + + const nextTrackedAlerts = Object.fromEntries( + eventsToIndex + .filter((event) => event[ALERT_STATUS] !== 'closed') + .map((event) => { + const alertId = event[ALERT_ID]!; + const alertUuid = event[ALERT_UUID]!; + const started = new Date(event[ALERT_START]!).toISOString(); + return [alertId, { alertId, alertUuid, started }]; + }) + ); + + return { + wrapped: nextWrappedState, + trackedAlerts: nextTrackedAlerts, + }; + }, + }; +}; diff --git a/x-pack/plugins/rule_registry/server/utils/get_rule_executor_data.ts b/x-pack/plugins/rule_registry/server/utils/get_rule_executor_data.ts new file mode 100644 index 0000000000000..1ea640add7b48 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/utils/get_rule_executor_data.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + PRODUCER, + RULE_CATEGORY, + RULE_ID, + RULE_NAME, + RULE_UUID, + TAGS, +} from '../../common/technical_rule_data_field_names'; +import { AlertTypeExecutor, AlertTypeWithExecutor } from '../types'; + +export interface RuleExecutorData { + [RULE_CATEGORY]: string; + [RULE_ID]: string; + [RULE_UUID]: string; + [RULE_NAME]: string; + [PRODUCER]: string; + [TAGS]: string[]; +} + +export function getRuleExecutorData( + type: AlertTypeWithExecutor, + options: Parameters[0] +) { + return { + [RULE_ID]: type.id, + [RULE_UUID]: options.alertId, + [RULE_CATEGORY]: type.name, + [RULE_NAME]: options.name, + [TAGS]: options.tags, + [PRODUCER]: type.producer, + }; +} diff --git a/x-pack/plugins/rule_registry/server/utils/with_rule_data_client_factory.ts b/x-pack/plugins/rule_registry/server/utils/with_rule_data_client_factory.ts new file mode 100644 index 0000000000000..02ff6b10f74cf --- /dev/null +++ b/x-pack/plugins/rule_registry/server/utils/with_rule_data_client_factory.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { AlertInstanceContext, AlertTypeParams } from '../../../alerting/common'; +import { RuleDataClient } from '../rule_data_client'; +import { AlertTypeWithExecutor } from '../types'; + +export const withRuleDataClientFactory = (ruleDataClient: RuleDataClient) => < + TParams extends AlertTypeParams, + TAlertInstanceContext extends AlertInstanceContext, + TServices extends Record = {} +>( + type: AlertTypeWithExecutor< + TParams, + TAlertInstanceContext, + TServices & { ruleDataClient: RuleDataClient } + > +): AlertTypeWithExecutor< + TParams, + TAlertInstanceContext, + TServices & { ruleDataClient: RuleDataClient } +> => { + return { + ...type, + executor: (options) => { + return type.executor({ + ...options, + services: { + ...options.services, + ruleDataClient, + }, + }); + }, + }; +}; diff --git a/x-pack/plugins/security/common/constants.ts b/x-pack/plugins/security/common/constants.ts index 0ff04e4f731d0..6d808f2305c91 100644 --- a/x-pack/plugins/security/common/constants.ts +++ b/x-pack/plugins/security/common/constants.ts @@ -57,3 +57,31 @@ export const NAME_REGEX = /^(?! )[a-zA-Z0-9 !"#$%&'()*+,\-./\\:;<=>?@\[\]^_`{|}~ * Maximum length of usernames and role names. */ export const MAX_NAME_LENGTH = 1024; + +/** + * Client session timeout is decreased by this number so that Kibana server can still access session + * content during logout request to properly clean user session up (invalidate access tokens, + * redirect to logout portal etc.). + */ +export const SESSION_GRACE_PERIOD_MS = 5 * 1000; + +/** + * Duration we'll normally display the warning toast + */ +export const SESSION_EXPIRATION_WARNING_MS = 5 * 60 * 1000; + +/** + * Current session info is checked this number of milliseconds before the warning toast shows. This + * will prevent the toast from being shown if the session has already been extended. + */ +export const SESSION_CHECK_MS = 1000; + +/** + * Session will be extended at most once this number of milliseconds while user activity is detected. + */ +export const SESSION_EXTENSION_THROTTLE_MS = 60 * 1000; + +/** + * Route to get session info and extend session expiration + */ +export const SESSION_ROUTE = '/internal/security/session'; diff --git a/x-pack/plugins/security/common/types.ts b/x-pack/plugins/security/common/types.ts index 94e4fef6517a6..690aced63b187 100644 --- a/x-pack/plugins/security/common/types.ts +++ b/x-pack/plugins/security/common/types.ts @@ -8,8 +8,7 @@ import type { AuthenticationProvider } from './model'; export interface SessionInfo { - now: number; - idleTimeoutExpiration: number | null; - lifespanExpiration: number | null; + expiresInMs: number | null; + canBeExtended: boolean; provider: AuthenticationProvider; } diff --git a/x-pack/plugins/security/public/plugin.tsx b/x-pack/plugins/security/public/plugin.tsx index c805d9f1caf79..7d7ab36f1dadb 100644 --- a/x-pack/plugins/security/public/plugin.tsx +++ b/x-pack/plugins/security/public/plugin.tsx @@ -27,13 +27,7 @@ import type { ConfigType } from './config'; import { ManagementService } from './management'; import { SecurityNavControlService } from './nav_control'; import { SecurityCheckupService } from './security_checkup'; -import type { ISessionTimeout } from './session'; -import { - SessionExpired, - SessionTimeout, - SessionTimeoutHttpInterceptor, - UnauthorizedResponseHttpInterceptor, -} from './session'; +import { SessionExpired, SessionTimeout, UnauthorizedResponseHttpInterceptor } from './session'; export interface PluginSetupDependencies { licensing: LicensingPluginSetup; @@ -58,7 +52,7 @@ export class SecurityPlugin PluginSetupDependencies, PluginStartDependencies > { - private sessionTimeout!: ISessionTimeout; + private sessionTimeout!: SessionTimeout; private readonly authenticationService = new AuthenticationService(); private readonly navControlService = new SecurityNavControlService(); private readonly securityLicenseService = new SecurityLicenseService(); @@ -84,7 +78,6 @@ export class SecurityPlugin const sessionExpired = new SessionExpired(logoutUrl, tenant); http.intercept(new UnauthorizedResponseHttpInterceptor(sessionExpired, anonymousPaths)); this.sessionTimeout = new SessionTimeout(notifications, sessionExpired, http, tenant); - http.intercept(new SessionTimeoutHttpInterceptor(this.sessionTimeout, anonymousPaths)); const { license } = this.securityLicenseService.setup({ license$: licensing.license$ }); diff --git a/x-pack/plugins/security/public/session/index.ts b/x-pack/plugins/security/public/session/index.ts index 30e73eacd9264..9679e05fd1bd9 100644 --- a/x-pack/plugins/security/public/session/index.ts +++ b/x-pack/plugins/security/public/session/index.ts @@ -6,6 +6,5 @@ */ export { SessionExpired } from './session_expired'; -export { SessionTimeout, ISessionTimeout } from './session_timeout'; -export { SessionTimeoutHttpInterceptor } from './session_timeout_http_interceptor'; +export { SessionTimeout } from './session_timeout'; export { UnauthorizedResponseHttpInterceptor } from './unauthorized_response_http_interceptor'; diff --git a/x-pack/plugins/security/public/session/session_expiration_toast.test.tsx b/x-pack/plugins/security/public/session/session_expiration_toast.test.tsx new file mode 100644 index 0000000000000..a78bed34b2088 --- /dev/null +++ b/x-pack/plugins/security/public/session/session_expiration_toast.test.tsx @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { fireEvent, render } from '@testing-library/react'; +import React from 'react'; +import { of } from 'rxjs'; + +import { I18nProvider } from '@kbn/i18n/react'; + +import { createSessionExpirationToast, SessionExpirationToast } from './session_expiration_toast'; +import type { SessionState } from './session_timeout'; + +describe('createSessionExpirationToast', () => { + it('creates a toast', () => { + const sessionState$ = of({ + lastExtensionTime: Date.now(), + expiresInMs: 60 * 1000, + canBeExtended: true, + }); + const onExtend = jest.fn(); + const onClose = jest.fn(); + const toast = createSessionExpirationToast(sessionState$, onExtend, onClose); + + expect(toast).toEqual( + expect.objectContaining({ + color: 'warning', + iconType: 'clock', + onClose: expect.any(Function), + text: expect.any(Function), + title: expect.any(String), + toastLifeTimeMs: 2147483647, + }) + ); + }); +}); + +describe('SessionExpirationToast', () => { + it('renders session expiration time', () => { + const sessionState$ = of({ + lastExtensionTime: Date.now(), + expiresInMs: 60 * 1000, + canBeExtended: true, + }); + + const { getByText } = render( + + + + ); + getByText(/You will be logged out in [0-9]+ seconds/); + }); + + it('renders extend button if session can be extended', () => { + const sessionState$ = of({ + lastExtensionTime: Date.now(), + expiresInMs: 60 * 1000, + canBeExtended: true, + }); + const onExtend = jest.fn().mockReturnValue(new Promise(() => {})); + + const { getByRole } = render( + + + + ); + fireEvent.click(getByRole('button', { name: 'Stay logged in' })); + + expect(onExtend).toHaveBeenCalled(); + }); + + it('does not render extend button if session cannot be extended', () => { + const sessionState$ = of({ + lastExtensionTime: Date.now(), + expiresInMs: 60 * 1000, + canBeExtended: false, + }); + const onExtend = jest.fn(); + + const { queryByRole } = render( + + + + ); + expect(queryByRole('button', { name: 'Stay logged in' })).toBeNull(); + }); +}); diff --git a/x-pack/plugins/security/public/session/session_expiration_toast.tsx b/x-pack/plugins/security/public/session/session_expiration_toast.tsx new file mode 100644 index 0000000000000..ad7aa67b64b3f --- /dev/null +++ b/x-pack/plugins/security/public/session/session_expiration_toast.tsx @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { EuiButton, EuiFlexGroup, EuiFlexItem, EuiSpacer } from '@elastic/eui'; +import type { FunctionComponent } from 'react'; +import React from 'react'; +import useAsyncFn from 'react-use/lib/useAsyncFn'; +import useObservable from 'react-use/lib/useObservable'; +import type { Observable } from 'rxjs'; + +import { i18n } from '@kbn/i18n'; +import { FormattedMessage, FormattedRelative } from '@kbn/i18n/react'; +import type { ToastInput } from 'src/core/public'; + +import { toMountPoint } from '../../../../../src/plugins/kibana_react/public'; +import { SESSION_GRACE_PERIOD_MS } from '../../common/constants'; +import type { SessionState } from './session_timeout'; + +export interface SessionExpirationToastProps { + sessionState$: Observable; + onExtend: () => Promise; +} + +export const SessionExpirationToast: FunctionComponent = ({ + sessionState$, + onExtend, +}) => { + const state = useObservable(sessionState$); + const [{ loading }, extend] = useAsyncFn(onExtend); + + if (!state || !state.expiresInMs) { + return null; + } + + const expirationWarning = ( + + ), + }} + /> + ); + + if (state.canBeExtended) { + return ( + <> + {expirationWarning} + + + + + + + + + + ); + } + + return expirationWarning; +}; + +export const createSessionExpirationToast = ( + sessionState$: Observable, + onExtend: () => Promise, + onClose: () => void +): ToastInput => { + return { + color: 'warning', + iconType: 'clock', + title: i18n.translate('xpack.security.sessionExpirationToast.title', { + defaultMessage: 'Session timeout', + }), + text: toMountPoint( + + ), + onClose, + toastLifeTimeMs: 0x7fffffff, // Toast is hidden based on observable so using maximum possible timeout + }; +}; diff --git a/x-pack/plugins/security/public/session/session_idle_timeout_warning.test.tsx b/x-pack/plugins/security/public/session/session_idle_timeout_warning.test.tsx deleted file mode 100644 index 48a3bb482948e..0000000000000 --- a/x-pack/plugins/security/public/session/session_idle_timeout_warning.test.tsx +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import React from 'react'; - -import { mountWithIntl } from '@kbn/test/jest'; - -import { SessionIdleTimeoutWarning } from './session_idle_timeout_warning'; - -describe('SessionIdleTimeoutWarning', () => { - it('fires its callback when the OK button is clicked', () => { - const handler = jest.fn(); - const wrapper = mountWithIntl( - - ); - - expect(handler).toBeCalledTimes(0); - wrapper.find('EuiButton[data-test-subj="refreshSessionButton"]').simulate('click'); - expect(handler).toBeCalledTimes(1); - }); -}); diff --git a/x-pack/plugins/security/public/session/session_idle_timeout_warning.tsx b/x-pack/plugins/security/public/session/session_idle_timeout_warning.tsx deleted file mode 100644 index 47dce0aa9993e..0000000000000 --- a/x-pack/plugins/security/public/session/session_idle_timeout_warning.tsx +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EuiButton, EuiProgress } from '@elastic/eui'; -import React from 'react'; - -import { i18n } from '@kbn/i18n'; -import { FormattedMessage, FormattedRelative } from '@kbn/i18n/react'; -import type { ToastInput } from 'src/core/public'; - -import { toMountPoint } from '../../../../../src/plugins/kibana_react/public'; - -interface Props { - onRefreshSession: () => void; - timeout: number; -} - -export const SessionIdleTimeoutWarning = (props: Props) => { - return ( - <> - -

- - ), - }} - /> -

-
- - - -
- - ); -}; - -export const createToast = (toastLifeTimeMs: number, onRefreshSession: () => void): ToastInput => { - const timeout = toastLifeTimeMs + Date.now(); - return { - color: 'warning', - text: toMountPoint( - - ), - title: i18n.translate('xpack.security.components.sessionIdleTimeoutWarning.title', { - defaultMessage: 'Warning', - }), - iconType: 'clock', - toastLifeTimeMs, - }; -}; diff --git a/x-pack/plugins/security/public/session/session_lifespan_warning.tsx b/x-pack/plugins/security/public/session/session_lifespan_warning.tsx deleted file mode 100644 index e611c9aa2ac5f..0000000000000 --- a/x-pack/plugins/security/public/session/session_lifespan_warning.tsx +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EuiProgress } from '@elastic/eui'; -import React from 'react'; - -import { i18n } from '@kbn/i18n'; -import { FormattedMessage, FormattedRelative } from '@kbn/i18n/react'; -import type { ToastInput } from 'src/core/public'; - -import { toMountPoint } from '../../../../../src/plugins/kibana_react/public'; - -interface Props { - timeout: number; -} - -export const SessionLifespanWarning = (props: Props) => { - return ( - <> - -

- - ), - }} - /> -

- - ); -}; - -export const createToast = (toastLifeTimeMs: number): ToastInput => { - const timeout = toastLifeTimeMs + Date.now(); - return { - color: 'danger', - text: toMountPoint(), - title: i18n.translate('xpack.security.components.sessionLifespanWarning.title', { - defaultMessage: 'Warning', - }), - iconType: 'alert', - toastLifeTimeMs, - }; -}; diff --git a/x-pack/plugins/security/public/session/session_timeout.mock.ts b/x-pack/plugins/security/public/session/session_timeout.mock.ts index 47e16810f2c2b..15071b08ded8f 100644 --- a/x-pack/plugins/security/public/session/session_timeout.mock.ts +++ b/x-pack/plugins/security/public/session/session_timeout.mock.ts @@ -5,12 +5,13 @@ * 2.0. */ -import type { ISessionTimeout } from './session_timeout'; +import type { PublicMethodsOf } from '@kbn/utility-types'; + +import type { SessionTimeout } from './session_timeout'; export function createSessionTimeoutMock() { return { start: jest.fn(), stop: jest.fn(), - extend: jest.fn(), - } as jest.Mocked; + } as jest.Mocked>; } diff --git a/x-pack/plugins/security/public/session/session_timeout.test.ts b/x-pack/plugins/security/public/session/session_timeout.test.ts new file mode 100644 index 0000000000000..b6cd758354dfa --- /dev/null +++ b/x-pack/plugins/security/public/session/session_timeout.test.ts @@ -0,0 +1,403 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { BroadcastChannel } from 'broadcast-channel'; + +import type { ToastInputFields } from 'src/core/public'; +import { coreMock } from 'src/core/public/mocks'; + +import { + SESSION_CHECK_MS, + SESSION_EXPIRATION_WARNING_MS, + SESSION_EXTENSION_THROTTLE_MS, + SESSION_GRACE_PERIOD_MS, + SESSION_ROUTE, +} from '../../common/constants'; +import type { SessionInfo } from '../../common/types'; +import { createSessionExpiredMock } from './session_expired.mock'; +import type { SessionState } from './session_timeout'; +import { SessionTimeout, startTimer } from './session_timeout'; + +jest.mock('broadcast-channel'); + +jest.useFakeTimers(); + +jest.spyOn(window, 'addEventListener'); +jest.spyOn(window, 'removeEventListener'); + +jest.spyOn(document, 'addEventListener'); +jest.spyOn(document, 'removeEventListener'); + +const nowMock = jest.spyOn(Date, 'now'); +const visibilityStateMock = jest.spyOn(document, 'visibilityState', 'get'); + +function createSessionTimeout(expiresInMs: number | null = 60 * 60 * 1000, canBeExtended = true) { + const { notifications, http } = coreMock.createSetup(); + const toast = Symbol(); + notifications.toasts.add.mockReturnValue(toast as any); + const sessionExpired = createSessionExpiredMock(); + const tenant = 'test'; + const sessionTimeout = new SessionTimeout(notifications, sessionExpired, http, tenant); + + http.fetch.mockResolvedValue({ + expiresInMs, + canBeExtended, + provider: { type: 'basic', name: 'basic1' }, + } as SessionInfo); + + return { sessionTimeout, sessionExpired, notifications, http }; +} + +describe('SessionTimeout', () => { + afterEach(async () => { + jest.clearAllMocks(); + jest.clearAllTimers(); + }); + + test(`does not initialize when starting an anonymous path`, async () => { + const { sessionTimeout, http } = createSessionTimeout(); + http.anonymousPaths.isAnonymous.mockReturnValue(true); + await sessionTimeout.start(); + expect(http.fetch).not.toHaveBeenCalled(); + }); + + it('fetches session info when starting', async () => { + const { sessionTimeout, http } = createSessionTimeout(); + await sessionTimeout.start(); + expect(http.fetch).toHaveBeenCalledTimes(1); + expect(http.fetch).toHaveBeenCalledWith(SESSION_ROUTE, { + asSystemRequest: true, + method: 'GET', + }); + }); + + it('adds event handlers when starting', async () => { + const { sessionTimeout } = createSessionTimeout(); + + expect(window.addEventListener).not.toHaveBeenCalled(); + expect(document.addEventListener).not.toHaveBeenCalled(); + + await sessionTimeout.start(); + + expect(window.addEventListener).toHaveBeenCalledWith('mousemove', expect.any(Function)); + expect(window.addEventListener).toHaveBeenCalledWith('mousedown', expect.any(Function)); + expect(window.addEventListener).toHaveBeenCalledWith('wheel', expect.any(Function)); + expect(window.addEventListener).toHaveBeenCalledWith('touchstart', expect.any(Function)); + expect(window.addEventListener).toHaveBeenCalledWith('keydown', expect.any(Function)); + expect(document.addEventListener).toHaveBeenCalledWith( + 'visibilitychange', + expect.any(Function) + ); + }); + + it('removes event handlers when stopping', async () => { + const { sessionTimeout } = createSessionTimeout(); + + await sessionTimeout.start(); + + expect(window.removeEventListener).not.toHaveBeenCalled(); + expect(document.removeEventListener).not.toHaveBeenCalled(); + + sessionTimeout.stop(); + + expect(window.removeEventListener).toHaveBeenCalledWith('mousemove', expect.any(Function)); + expect(window.removeEventListener).toHaveBeenCalledWith('mousedown', expect.any(Function)); + expect(window.removeEventListener).toHaveBeenCalledWith('wheel', expect.any(Function)); + expect(window.removeEventListener).toHaveBeenCalledWith('touchstart', expect.any(Function)); + expect(window.removeEventListener).toHaveBeenCalledWith('keydown', expect.any(Function)); + expect(document.removeEventListener).toHaveBeenCalledWith( + 'visibilitychange', + expect.any(Function) + ); + }); + + it('clears timers when stopping', async () => { + const { sessionTimeout, sessionExpired } = createSessionTimeout(100); + await sessionTimeout.start(); + sessionTimeout.stop(); + + jest.advanceTimersByTime(200); + + expect(sessionExpired.logout).not.toHaveBeenCalled(); + }); + + it('extends session when detecting user activity', async () => { + const { sessionTimeout, http } = createSessionTimeout(); + await sessionTimeout.start(); + expect(http.fetch).toHaveBeenCalledTimes(1); + + // Increment system time enough so that session extension gets triggered + nowMock.mockReturnValue(Date.now() + SESSION_EXTENSION_THROTTLE_MS + 10); + window.dispatchEvent(new Event('mousemove')); + + expect(http.fetch).toHaveBeenCalledTimes(2); + expect(http.fetch).toHaveBeenLastCalledWith( + SESSION_ROUTE, + expect.objectContaining({ asSystemRequest: false }) + ); + }); + + it('refetches session info before warning is displayed', async () => { + const { sessionTimeout, http } = createSessionTimeout(60 * 60 * 1000); + await sessionTimeout.start(); + expect(http.fetch).toHaveBeenCalledTimes(1); + + jest.advanceTimersByTime( + 60 * 60 * 1000 - SESSION_GRACE_PERIOD_MS - SESSION_EXPIRATION_WARNING_MS - SESSION_CHECK_MS + ); + + expect(http.fetch).toHaveBeenCalledTimes(2); + }); + + it('does not extend session if recently extended', async () => { + const { sessionTimeout, http } = createSessionTimeout(); + await sessionTimeout.start(); + + expect(http.fetch).toHaveBeenCalledTimes(1); + + window.dispatchEvent(new Event('mousemove')); + + expect(http.fetch).toHaveBeenCalledTimes(1); + }); + + it('marks HTTP requests as system requests when tab is not visible', async () => { + const { sessionTimeout, http } = createSessionTimeout(); + await sessionTimeout.start(); + + visibilityStateMock.mockReturnValue('hidden'); + document.dispatchEvent(new Event('visibilitychange')); + + const [{ request: handleHttpRequest }] = http.intercept.mock.calls[0]; + const fetchOptions = handleHttpRequest!({ path: '/test' }, {} as any); + expect(fetchOptions).toEqual({ path: '/test', asSystemRequest: true }); + }); + + it('does not mark HTTP requests to external URLs as system requests', async () => { + const { sessionTimeout, http } = createSessionTimeout(); + await sessionTimeout.start(); + + visibilityStateMock.mockReturnValue('hidden'); + document.dispatchEvent(new Event('visibilitychange')); + + const [{ request: handleHttpRequest }] = http.intercept.mock.calls[0]; + const fetchOptions = handleHttpRequest!({ path: 'http://elastic.co/' }, {} as any); + expect(fetchOptions).toBe(undefined); + }); + + it('marks HTTP requests as user requests when tab is visible', async () => { + const { sessionTimeout, http } = createSessionTimeout(); + await sessionTimeout.start(); + + visibilityStateMock.mockReturnValue('visible'); + document.dispatchEvent(new Event('visibilitychange')); + + const [{ request: handleHttpRequest }] = http.intercept.mock.calls[0]; + const fetchOptions = handleHttpRequest!({ path: '/test' }, {} as any); + expect(fetchOptions).toEqual({ path: '/test', asSystemRequest: false }); + }); + + it('resets timers when receiving message from other tabs', async () => { + const { sessionTimeout, sessionExpired } = createSessionTimeout(60 * 1000); + await sessionTimeout.start(); + + jest.advanceTimersByTime(30 * 1000); + + const [broadcastChannelMock] = jest.requireMock('broadcast-channel').BroadcastChannel.mock + .instances as [BroadcastChannel]; + + broadcastChannelMock.onmessage!({ + lastExtensionTime: Date.now(), + expiresInMs: 60 * 1000, + canBeExtended: true, + }); + + jest.advanceTimersByTime(30 * 1000); + + expect(sessionExpired.logout).not.toHaveBeenCalled(); + }); + + it('shows warning before session expires', async () => { + const { sessionTimeout, notifications } = createSessionTimeout(60 * 60 * 1000); + await sessionTimeout.start(); + + jest.advanceTimersByTime( + 60 * 60 * 1000 - SESSION_GRACE_PERIOD_MS - SESSION_EXPIRATION_WARNING_MS + ); + + expect(notifications.toasts.add).toHaveBeenCalledWith( + expect.objectContaining({ color: 'warning', iconType: 'clock' }) + ); + }); + + it('extends session when closing expiration warning', async () => { + const { sessionTimeout, notifications, http } = createSessionTimeout(60 * 60 * 1000); + await sessionTimeout.start(); + + expect(http.fetch).toHaveBeenCalledTimes(1); + expect(http.fetch).toHaveBeenLastCalledWith( + SESSION_ROUTE, + expect.objectContaining({ asSystemRequest: true }) + ); + + jest.runOnlyPendingTimers(); + + expect(http.fetch).toHaveBeenCalledTimes(2); + expect(http.fetch).toHaveBeenLastCalledWith( + SESSION_ROUTE, + expect.objectContaining({ asSystemRequest: true }) + ); + + const [toast] = notifications.toasts.add.mock.calls[0] as [ToastInputFields]; + + await toast.onClose!(); + + expect(http.fetch).toHaveBeenCalledTimes(3); + expect(http.fetch).toHaveBeenLastCalledWith( + SESSION_ROUTE, + expect.objectContaining({ asSystemRequest: false }) + ); + }); + + it('show warning 5 minutes before expiration if not previously dismissed', async () => { + const { sessionTimeout, notifications } = createSessionTimeout(null); + await sessionTimeout.start(); + + const expiresInMs = 10 * 60 * 1000; + const showWarningInMs = expiresInMs - SESSION_GRACE_PERIOD_MS - SESSION_EXPIRATION_WARNING_MS; + + // eslint-disable-next-line dot-notation + sessionTimeout['resetTimers']({ + lastExtensionTime: Date.now() + 1 * 60 * 1000, + expiresInMs, + canBeExtended: false, + }); + + jest.advanceTimersByTime(showWarningInMs); + + expect(notifications.toasts.add).toHaveBeenCalled(); + }); + + it('do not show warning again if previously dismissed', async () => { + const { sessionTimeout, notifications } = createSessionTimeout(null); + await sessionTimeout.start(); + + const expiresInMs = 10 * 60 * 1000; + const showWarningInMs = expiresInMs - SESSION_GRACE_PERIOD_MS - SESSION_EXPIRATION_WARNING_MS; + + // eslint-disable-next-line dot-notation + sessionTimeout['snoozedWarningState'] = { + lastExtensionTime: Date.now(), + expiresInMs, + canBeExtended: false, + }; + + // eslint-disable-next-line dot-notation + sessionTimeout['resetTimers']({ + lastExtensionTime: Date.now() + 1 * 60 * 1000, + expiresInMs, + canBeExtended: false, + }); + + // We would normally show the warning at this point in time. However, since the warning has been + // dismissed for 10 minutes we will only show it after 10 minutes have elapsed + jest.advanceTimersByTime(showWarningInMs); + expect(notifications.toasts.add).not.toHaveBeenCalled(); + + // Advance the timer further so that a total have 10 minutes would have passed. This is the + // expiration time of the warning that was dismissed. + jest.advanceTimersByTime(9 * 60 * 1000 - showWarningInMs); + expect(notifications.toasts.add).toHaveBeenCalled(); + }); + + it('hides warning if session gets extended', async () => { + const { sessionTimeout, notifications } = createSessionTimeout(60 * 60 * 1000); + await sessionTimeout.start(); + + jest.advanceTimersByTime( + 60 * 60 * 1000 - SESSION_GRACE_PERIOD_MS - SESSION_EXPIRATION_WARNING_MS + ); + + expect(notifications.toasts.add).toHaveBeenCalled(); + + // eslint-disable-next-line dot-notation + await sessionTimeout['fetchSessionInfo'](true); + + expect(notifications.toasts.remove).toHaveBeenCalled(); + }); + + it('logs user out slightly before session expires', async () => { + const { sessionTimeout, sessionExpired } = createSessionTimeout(60 * 60 * 1000); + await sessionTimeout.start(); + + jest.advanceTimersByTime(60 * 60 * 1000 - SESSION_GRACE_PERIOD_MS); + + expect(sessionExpired.logout).toHaveBeenCalled(); + }); + + it('logs user out immediately if session expiration is imminent', async () => { + const { sessionTimeout, sessionExpired } = createSessionTimeout(SESSION_GRACE_PERIOD_MS / 2); + await sessionTimeout.start(); + + jest.advanceTimersByTime(0); + + expect(sessionExpired.logout).toHaveBeenCalled(); + }); + + it('does not log user out if session does not expire', async () => { + const { sessionTimeout, sessionExpired } = createSessionTimeout(null); + await sessionTimeout.start(); + + jest.runOnlyPendingTimers(); + + expect(sessionExpired.logout).not.toHaveBeenCalled(); + }); + + it('does not log user out if session does not exist', async () => { + const { sessionTimeout, sessionExpired, http } = createSessionTimeout(); + + http.fetch.mockResolvedValue(''); // Session endpoint return 204 No content when session does not exist + await sessionTimeout.start(); + + jest.runOnlyPendingTimers(); + + expect(sessionExpired.logout).not.toHaveBeenCalled(); + }); +}); + +describe('startTimer', () => { + it('executes callback after time elapses', () => { + const callback = jest.fn(); + startTimer(callback, 100); + jest.advanceTimersByTime(100); + expect(callback).toHaveBeenCalledTimes(1); + }); + + it('executes callback after a very long time elapses', () => { + const callback = jest.fn(); + startTimer(callback, 0x7fffffff + 100); + jest.advanceTimersByTime(0x7fffffff + 100); + expect(callback).toHaveBeenCalledTimes(1); + }); + + it('does not executes callback if stopped before time elapses', () => { + const callback = jest.fn(); + const stop = startTimer(callback, 100); + jest.advanceTimersByTime(50); + stop(); + jest.advanceTimersByTime(50); + expect(callback).toHaveBeenCalledTimes(0); + }); + + it('does not executes callback if stopped before a very long time elapses', () => { + const callback = jest.fn(); + const stop = startTimer(callback, 0x7fffffff + 100); + jest.advanceTimersByTime(0x7fffffff + 50); + stop(); + jest.advanceTimersByTime(50); + expect(callback).toHaveBeenCalledTimes(0); + }); +}); diff --git a/x-pack/plugins/security/public/session/session_timeout.test.tsx b/x-pack/plugins/security/public/session/session_timeout.test.tsx deleted file mode 100644 index 1faa105691259..0000000000000 --- a/x-pack/plugins/security/public/session/session_timeout.test.tsx +++ /dev/null @@ -1,495 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import BroadcastChannel from 'broadcast-channel'; - -import { mountWithIntl, nextTick } from '@kbn/test/jest'; -import { coreMock } from 'src/core/public/mocks'; - -import { createSessionExpiredMock } from './session_expired.mock'; -import { SessionTimeout } from './session_timeout'; - -jest.useFakeTimers(); - -const expectNoWarningToast = ( - notifications: ReturnType['notifications'] -) => { - expect(notifications.toasts.add).not.toHaveBeenCalled(); -}; - -const expectIdleTimeoutWarningToast = ( - notifications: ReturnType['notifications'], - toastLifeTimeMs: number = 60000 -) => { - expect(notifications.toasts.add).toHaveBeenCalledTimes(1); - expect(notifications.toasts.add.mock.calls[0][0]).toMatchInlineSnapshot( - { - text: expect.any(Function), - }, - ` - Object { - "color": "warning", - "iconType": "clock", - "text": Any, - "title": "Warning", - "toastLifeTimeMs": ${toastLifeTimeMs}, - } - ` - ); -}; - -const expectLifespanWarningToast = ( - notifications: ReturnType['notifications'], - toastLifeTimeMs: number = 60000 -) => { - expect(notifications.toasts.add).toHaveBeenCalledTimes(1); - expect(notifications.toasts.add.mock.calls[0][0]).toMatchInlineSnapshot( - { - text: expect.any(Function), - }, - ` - Object { - "color": "danger", - "iconType": "alert", - "text": Any, - "title": "Warning", - "toastLifeTimeMs": ${toastLifeTimeMs}, - } - ` - ); -}; - -const expectWarningToastHidden = ( - notifications: ReturnType['notifications'], - toast: symbol -) => { - expect(notifications.toasts.remove).toHaveBeenCalledTimes(1); - expect(notifications.toasts.remove).toHaveBeenCalledWith(toast); -}; - -describe('Session Timeout', () => { - const now = new Date().getTime(); - const defaultSessionInfo = { - now, - idleTimeoutExpiration: now + 2 * 60 * 1000, - lifespanExpiration: null, - provider: { type: 'basic', name: 'basic1' }, - }; - let notifications: ReturnType['notifications']; - let http: ReturnType['http']; - let sessionExpired: ReturnType; - let sessionTimeout: SessionTimeout; - const toast = Symbol(); - - beforeAll(() => { - BroadcastChannel.enforceOptions({ - type: 'simulate', - }); - Object.defineProperty(window, 'sessionStorage', { - value: { - setItem: jest.fn(() => null), - }, - writable: true, - }); - }); - - beforeEach(() => { - const setup = coreMock.createSetup(); - notifications = setup.notifications; - http = setup.http; - notifications.toasts.add.mockReturnValue(toast as any); - sessionExpired = createSessionExpiredMock(); - const tenant = ''; - sessionTimeout = new SessionTimeout(notifications, sessionExpired, http, tenant); - - // default mocked response for checking session info - http.fetch.mockResolvedValue(defaultSessionInfo); - }); - - afterEach(async () => { - jest.clearAllMocks(); - jest.unmock('broadcast-channel'); - sessionTimeout.stop(); - }); - - afterAll(() => { - BroadcastChannel.enforceOptions(null); - delete (window as any).sessionStorage; - }); - - describe('Lifecycle', () => { - test(`starts and initializes on a non-anonymous path`, async () => { - sessionTimeout.start(); - await nextTick(); - // eslint-disable-next-line dot-notation - expect(sessionTimeout['channel']).not.toBeUndefined(); - expect(http.fetch).toHaveBeenCalledTimes(1); - }); - - test(`starts and initializes if the broadcast channel fails to load`, async () => { - jest.mock('broadcast-channel', () => { - throw new Error('Unable to load broadcast channel!'); - }); - const consoleSpy = jest.spyOn(console, 'warn'); - - sessionTimeout.start(); - await nextTick(); - // eslint-disable-next-line dot-notation - expect(sessionTimeout['channel']).toBeUndefined(); - expect(http.fetch).toHaveBeenCalledTimes(1); - expect(consoleSpy).toHaveBeenCalledTimes(1); - expect(consoleSpy.mock.calls[0][0]).toMatchInlineSnapshot( - `"Failed to load broadcast channel. Session management will not be kept in sync when multiple tabs are loaded."` - ); - }); - - test(`starts and does not initialize on an anonymous path`, async () => { - http.anonymousPaths.isAnonymous.mockReturnValue(true); - sessionTimeout.start(); - await nextTick(); - // eslint-disable-next-line dot-notation - expect(sessionTimeout['channel']).toBeUndefined(); - expect(http.fetch).not.toHaveBeenCalled(); - }); - - test(`stops`, async () => { - sessionTimeout.start(); - await nextTick(); - // eslint-disable-next-line dot-notation - const close = jest.fn(sessionTimeout['channel']!.close); - // eslint-disable-next-line dot-notation - sessionTimeout['channel']!.close = close; - // eslint-disable-next-line dot-notation - const cleanup = jest.fn(sessionTimeout['cleanup']); - // eslint-disable-next-line dot-notation - sessionTimeout['cleanup'] = cleanup; - - sessionTimeout.stop(); - expect(close).toHaveBeenCalled(); - expect(cleanup).toHaveBeenCalled(); - }); - - test(`stop works properly for large timeouts`, async () => { - http.fetch.mockResolvedValue({ - ...defaultSessionInfo, - idleTimeoutExpiration: now + 5_000_000_000, - }); - sessionTimeout.start(); - await nextTick(); - - // Advance timers far enough to call intermediate `setTimeout` multiple times, but before any - // of the timers is supposed to be triggered. - jest.advanceTimersByTime(5_000_000_000 - (60 + 5 + 2) * 1000); - - sessionTimeout.stop(); - - // Advance timer even further and make sure that timers were properly cleaned up. - jest.runAllTimers(); - - expect(http.fetch).toHaveBeenCalledTimes(1); - expect(sessionExpired.logout).not.toHaveBeenCalled(); - expectNoWarningToast(notifications); - }); - }); - - describe('API calls', () => { - const methodName = 'handleSessionInfoAndResetTimers'; - let method: jest.Mock; - - beforeEach(() => { - method = jest.fn(sessionTimeout[methodName]); - sessionTimeout[methodName] = method; - }); - - test(`handles success`, async () => { - sessionTimeout.start(); - await nextTick(); - - expect(http.fetch).toHaveBeenCalledTimes(1); - // eslint-disable-next-line dot-notation - expect(sessionTimeout['sessionInfo']).toBe(defaultSessionInfo); - expect(method).toHaveBeenCalledTimes(1); - }); - - test(`handles error`, async () => { - const mockErrorResponse = new Error('some-error'); - http.fetch.mockRejectedValue(mockErrorResponse); - sessionTimeout.start(); - await nextTick(); - - expect(http.fetch).toHaveBeenCalledTimes(1); - // eslint-disable-next-line dot-notation - expect(sessionTimeout['sessionInfo']).toBeUndefined(); - expect(method).not.toHaveBeenCalled(); - }); - }); - - describe('warning toast', () => { - test(`shows idle timeout warning toast`, async () => { - sessionTimeout.start(); - await nextTick(); - - // we display the warning a minute before we expire the the session, which is 5 seconds before it actually expires - jest.advanceTimersByTime(55 * 1000); - expectIdleTimeoutWarningToast(notifications); - }); - - test(`shows idle timeout warning toast even for large timeouts`, async () => { - http.fetch.mockResolvedValue({ - ...defaultSessionInfo, - idleTimeoutExpiration: now + 5_000_000_000, - }); - sessionTimeout.start(); - await nextTick(); - - // we display the warning a minute before we expire the the session, which is 5 seconds before it actually expires - jest.advanceTimersByTime(5_000_000_000 - 66 * 1000); - expectNoWarningToast(notifications); - - jest.advanceTimersByTime(1000); - expectIdleTimeoutWarningToast(notifications); - }); - - test(`shows lifespan warning toast`, async () => { - const sessionInfo = { - now, - idleTimeoutExpiration: null, - lifespanExpiration: now + 2 * 60 * 1000, - provider: { type: 'basic', name: 'basic1' }, - }; - http.fetch.mockResolvedValue(sessionInfo); - sessionTimeout.start(); - await nextTick(); - - // we display the warning a minute before we expire the the session, which is 5 seconds before it actually expires - jest.advanceTimersByTime(55 * 1000); - expectLifespanWarningToast(notifications); - }); - - test(`shows lifespan warning toast even for large timeouts`, async () => { - const sessionInfo = { - ...defaultSessionInfo, - idleTimeoutExpiration: null, - lifespanExpiration: now + 5_000_000_000, - }; - http.fetch.mockResolvedValue(sessionInfo); - sessionTimeout.start(); - await nextTick(); - - // we display the warning a minute before we expire the the session, which is 5 seconds before it actually expires - jest.advanceTimersByTime(5_000_000_000 - 66 * 1000); - expectNoWarningToast(notifications); - - jest.advanceTimersByTime(1000); - expectLifespanWarningToast(notifications); - }); - - test(`extend only results in an HTTP call if a warning is shown`, async () => { - sessionTimeout.start(); - await nextTick(); - expect(http.fetch).toHaveBeenCalledTimes(1); - - await sessionTimeout.extend('/foo'); - expect(http.fetch).toHaveBeenCalledTimes(1); - jest.advanceTimersByTime(54 * 1000); - expect(http.fetch).toHaveBeenCalledTimes(2); - expectNoWarningToast(notifications); - - await sessionTimeout.extend('/foo'); - expect(http.fetch).toHaveBeenCalledTimes(2); - jest.advanceTimersByTime(10 * 1000); - expectIdleTimeoutWarningToast(notifications); - - await sessionTimeout.extend('/foo'); - expect(http.fetch).toHaveBeenCalledTimes(3); - }); - - test(`extend does not result in an HTTP call if a lifespan warning is shown`, async () => { - const sessionInfo = { - now, - idleTimeoutExpiration: null, - lifespanExpiration: now + 2 * 60 * 1000, - provider: { type: 'basic', name: 'basic1' }, - }; - http.fetch.mockResolvedValue(sessionInfo); - sessionTimeout.start(); - await nextTick(); - - // we display the warning a minute before we expire the the session, which is 5 seconds before it actually expires - jest.advanceTimersByTime(55 * 1000); - expectLifespanWarningToast(notifications); - - expect(http.fetch).toHaveBeenCalledTimes(1); - await sessionTimeout.extend('/foo'); - expect(http.fetch).toHaveBeenCalledTimes(1); - }); - - test(`extend hides displayed warning toast`, async () => { - sessionTimeout.start(); - await nextTick(); - expect(http.fetch).toHaveBeenCalledTimes(1); - - // we display the warning a minute before we expire the the session, which is 5 seconds before it actually expires - const elapsed = 55 * 1000; - jest.advanceTimersByTime(elapsed); - expectIdleTimeoutWarningToast(notifications); - - http.fetch.mockResolvedValue({ - now: now + elapsed, - idleTimeoutExpiration: now + elapsed + 2 * 60 * 1000, - lifespanExpiration: null, - provider: { type: 'basic', name: 'basic1' }, - }); - await sessionTimeout.extend('/foo'); - expect(http.fetch).toHaveBeenCalledTimes(3); - expectWarningToastHidden(notifications, toast); - }); - - test(`extend does nothing for session-related routes`, async () => { - sessionTimeout.start(); - await nextTick(); - expect(http.fetch).toHaveBeenCalledTimes(1); - - // we display the warning a minute before we expire the the session, which is 5 seconds before it actually expires - const elapsed = 55 * 1000; - jest.advanceTimersByTime(elapsed); - expect(http.fetch).toHaveBeenCalledTimes(2); - expectIdleTimeoutWarningToast(notifications); - - await sessionTimeout.extend('/internal/security/session'); - expect(http.fetch).toHaveBeenCalledTimes(2); - }); - - test(`checks for updated session info before the warning displays`, async () => { - sessionTimeout.start(); - await nextTick(); - expect(http.fetch).toHaveBeenCalledTimes(1); - - // we check for updated session info 1 second before the warning is shown - const elapsed = 54 * 1000; - jest.advanceTimersByTime(elapsed); - expect(http.fetch).toHaveBeenCalledTimes(2); - }); - - test('clicking "extend" causes a new HTTP request (which implicitly extends the session)', async () => { - sessionTimeout.start(); - await nextTick(); - expect(http.fetch).toHaveBeenCalledTimes(1); - - // we display the warning a minute before we expire the the session, which is 5 seconds before it actually expires - jest.advanceTimersByTime(55 * 1000); - expect(http.fetch).toHaveBeenCalledTimes(2); - expectIdleTimeoutWarningToast(notifications); - - const toastInput = notifications.toasts.add.mock.calls[0][0]; - expect(toastInput).toHaveProperty('text'); - const mountPoint = (toastInput as any).text; - const wrapper = mountWithIntl(mountPoint.__reactMount__); - wrapper.find('EuiButton[data-test-subj="refreshSessionButton"]').simulate('click'); - expect(http.fetch).toHaveBeenCalledTimes(3); - }); - - test('when the session timeout is shorter than 65 seconds, display the warning immediately and for a shorter duration', async () => { - http.fetch.mockResolvedValue({ - now, - idleTimeoutExpiration: now + 64 * 1000, - lifespanExpiration: null, - provider: { type: 'basic', name: 'basic1' }, - }); - sessionTimeout.start(); - await nextTick(); - expect(http.fetch).toHaveBeenCalled(); - - jest.advanceTimersByTime(0); - expectIdleTimeoutWarningToast(notifications, 59 * 1000); - }); - }); - - describe('session expiration', () => { - test(`expires the session 5 seconds before it really expires`, async () => { - sessionTimeout.start(); - await nextTick(); - - jest.advanceTimersByTime(114 * 1000); - expect(sessionExpired.logout).not.toHaveBeenCalled(); - - jest.advanceTimersByTime(1 * 1000); - expect(sessionExpired.logout).toHaveBeenCalled(); - }); - - test(`expires the session 5 seconds before it really expires even for large timeouts`, async () => { - http.fetch.mockResolvedValue({ - ...defaultSessionInfo, - idleTimeoutExpiration: now + 5_000_000_000, - }); - - sessionTimeout.start(); - await nextTick(); - - jest.advanceTimersByTime(5_000_000_000 - 6000); - expect(sessionExpired.logout).not.toHaveBeenCalled(); - - jest.advanceTimersByTime(1000); - expect(sessionExpired.logout).toHaveBeenCalled(); - }); - - test(`extend delays the expiration`, async () => { - sessionTimeout.start(); - await nextTick(); - expect(http.fetch).toHaveBeenCalledTimes(1); - - const elapsed = 114 * 1000; - jest.advanceTimersByTime(elapsed); - expect(http.fetch).toHaveBeenCalledTimes(2); - expectIdleTimeoutWarningToast(notifications); - - const sessionInfo = { - now: now + elapsed, - idleTimeoutExpiration: now + elapsed + 2 * 60 * 1000, - lifespanExpiration: null, - provider: { type: 'basic', name: 'basic1' }, - }; - http.fetch.mockResolvedValue(sessionInfo); - await sessionTimeout.extend('/foo'); - expect(http.fetch).toHaveBeenCalledTimes(3); - // eslint-disable-next-line dot-notation - expect(sessionTimeout['sessionInfo']).toEqual(sessionInfo); - - // at this point, the session is good for another 120 seconds - jest.advanceTimersByTime(114 * 1000); - expect(sessionExpired.logout).not.toHaveBeenCalled(); - - // because "extend" results in an async request and HTTP call, there is a slight delay when timers are updated - // so we need an extra 100ms of padding for this test to ensure that logout has been called - jest.advanceTimersByTime(1 * 1000 + 100); - expect(sessionExpired.logout).toHaveBeenCalled(); - }); - - test(`if the session timeout is shorter than 5 seconds, expire session immediately`, async () => { - http.fetch.mockResolvedValue({ - now, - idleTimeoutExpiration: now + 4 * 1000, - lifespanExpiration: null, - provider: { type: 'basic', name: 'basic1' }, - }); - sessionTimeout.start(); - await nextTick(); - - jest.advanceTimersByTime(0); - expect(sessionExpired.logout).toHaveBeenCalled(); - }); - - test(`'null' sessionTimeout never logs you out`, async () => { - http.fetch.mockResolvedValue({ now, idleTimeoutExpiration: null, lifespanExpiration: null }); - sessionTimeout.start(); - await nextTick(); - - jest.advanceTimersByTime(Number.MAX_VALUE); - expect(sessionExpired.logout).not.toHaveBeenCalled(); - }); - }); -}); diff --git a/x-pack/plugins/security/public/session/session_timeout.ts b/x-pack/plugins/security/public/session/session_timeout.ts new file mode 100644 index 0000000000000..7a23f1bb25ad2 --- /dev/null +++ b/x-pack/plugins/security/public/session/session_timeout.ts @@ -0,0 +1,330 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { BroadcastChannel as BroadcastChannelType } from 'broadcast-channel'; +import type { Subscription } from 'rxjs'; +import { BehaviorSubject } from 'rxjs'; +import { skip, tap, throttleTime } from 'rxjs/operators'; + +import type { + HttpFetchOptionsWithPath, + HttpSetup, + NotificationsSetup, + Toast, +} from 'src/core/public'; + +import { + SESSION_CHECK_MS, + SESSION_EXPIRATION_WARNING_MS, + SESSION_EXTENSION_THROTTLE_MS, + SESSION_GRACE_PERIOD_MS, + SESSION_ROUTE, +} from '../../common/constants'; +import type { SessionInfo } from '../../common/types'; +import { createSessionExpirationToast } from './session_expiration_toast'; +import type { ISessionExpired } from './session_expired'; + +export interface SessionState extends Pick { + lastExtensionTime: number; +} + +export class SessionTimeout { + private channel?: BroadcastChannelType; + + private isVisible = document.visibilityState !== 'hidden'; + private isFetchingSessionInfo = false; + private snoozedWarningState?: SessionState; + + private sessionState$ = new BehaviorSubject({ + lastExtensionTime: 0, + expiresInMs: null, + canBeExtended: false, + }); + private subscription?: Subscription; + + private warningToast?: Toast; + + private stopActivityMonitor?: Function; + private stopVisibilityMonitor?: Function; + private removeHttpInterceptor?: Function; + + private stopRefreshTimer?: Function; + private stopWarningTimer?: Function; + private stopLogoutTimer?: Function; + + constructor( + private notifications: NotificationsSetup, + private sessionExpired: ISessionExpired, + private http: HttpSetup, + private tenant: string + ) {} + + public async start() { + if (this.http.anonymousPaths.isAnonymous(window.location.pathname)) { + return; + } + + this.subscription = this.sessionState$ + .pipe(skip(1), throttleTime(1000), tap(this.toggleEventHandlers)) + .subscribe(this.resetTimers); + + // Subscribe to a broadcast channel for session timeout messages. + // This allows us to synchronize the UX across tabs and avoid repetitive API calls. + try { + const { BroadcastChannel } = await import('broadcast-channel'); + this.channel = new BroadcastChannel(`${this.tenant}/session_timeout`, { + webWorkerSupport: false, + }); + this.channel.onmessage = this.handleChannelMessage; + } catch (error) { + // eslint-disable-next-line no-console + console.warn( + `Failed to load broadcast channel. Session management will not be kept in sync when multiple tabs are loaded.`, + error + ); + } + + return this.fetchSessionInfo(); + } + + public stop() { + const nextState = { + lastExtensionTime: 0, + expiresInMs: null, + canBeExtended: false, + }; + this.toggleEventHandlers(nextState); + this.resetTimers(nextState); + this.subscription?.unsubscribe(); + this.channel?.close(); + } + + /** + * Event handler that receives session information from other browser tabs. + */ + private handleChannelMessage = (message: SessionState) => { + this.sessionState$.next(message); + }; + + /** + * HTTP request interceptor which ensures that API calls extend the session only if tab is + * visible. + */ + private handleHttpRequest = (fetchOptions: HttpFetchOptionsWithPath) => { + // Ignore requests to external URLs + if (fetchOptions.path.indexOf('://') !== -1 || fetchOptions.path.startsWith('//')) { + return; + } + + if (!fetchOptions.asSystemRequest) { + return { ...fetchOptions, asSystemRequest: !this.isVisible }; + } + }; + + /** + * Event handler that tracks user activity and extends the session if needed. + */ + private handleUserActivity = () => { + if (this.shouldExtend()) { + this.fetchSessionInfo(true); + } + }; + + /** + * Event handler that tracks page visibility. + */ + private handleVisibilityChange = (isVisible: boolean) => { + this.isVisible = isVisible; + if (isVisible) { + this.handleUserActivity(); + } + }; + + private resetTimers = ({ lastExtensionTime, expiresInMs }: SessionState) => { + this.stopRefreshTimer = this.stopRefreshTimer?.(); + this.stopWarningTimer = this.stopWarningTimer?.(); + this.stopLogoutTimer = this.stopLogoutTimer?.(); + + if (expiresInMs !== null) { + const logoutInMs = Math.max(expiresInMs - SESSION_GRACE_PERIOD_MS, 0); + + // Show warning before session expires. However, do not show warning again if previously + // dismissed. The snooze time is the expiration time that was remaining in the warning. + const showWarningInMs = Math.max( + logoutInMs - SESSION_EXPIRATION_WARNING_MS, + this.snoozedWarningState + ? this.snoozedWarningState.lastExtensionTime + + this.snoozedWarningState.expiresInMs! - + SESSION_GRACE_PERIOD_MS - + lastExtensionTime + : 0, + 0 + ); + + const fetchSessionInMs = showWarningInMs - SESSION_CHECK_MS; + + // Schedule logout when session is about to expire + this.stopLogoutTimer = startTimer(() => this.sessionExpired.logout(), logoutInMs); + + // Hide warning if session has been extended + if (showWarningInMs > 0) { + this.hideWarning(); + } + + // Schedule warning before session expires + if (showWarningInMs < logoutInMs) { + this.stopWarningTimer = startTimer(this.showWarning, showWarningInMs); + } + + // Refresh session info before showing warning + if (fetchSessionInMs > 0 && fetchSessionInMs < logoutInMs) { + this.stopRefreshTimer = startTimer(this.fetchSessionInfo, fetchSessionInMs); + } + } + }; + + private toggleEventHandlers = ({ expiresInMs, canBeExtended }: SessionState) => { + if (expiresInMs !== null) { + // Monitor activity if session can be extended + if (canBeExtended && !this.stopActivityMonitor) { + this.stopActivityMonitor = monitorActivity(this.handleUserActivity); + } + + // Intercept HTTP requests if session can expire + if (!this.removeHttpInterceptor) { + this.removeHttpInterceptor = this.http.intercept({ request: this.handleHttpRequest }); + } + + if (!this.stopVisibilityMonitor) { + this.stopVisibilityMonitor = monitorVisibility(this.handleVisibilityChange); + } + } else { + this.removeHttpInterceptor = this.removeHttpInterceptor?.(); + this.stopActivityMonitor = this.stopActivityMonitor?.(); + this.stopVisibilityMonitor = this.stopVisibilityMonitor?.(); + } + }; + + private shouldExtend() { + const { lastExtensionTime } = this.sessionState$.getValue(); + return ( + !this.isFetchingSessionInfo && + !this.warningToast && + Date.now() > lastExtensionTime + SESSION_EXTENSION_THROTTLE_MS + ); + } + + private fetchSessionInfo = async (extend = false) => { + this.isFetchingSessionInfo = true; + try { + const sessionInfo = await this.http.fetch(SESSION_ROUTE, { + method: extend ? 'POST' : 'GET', + asSystemRequest: !extend, + }); + if (sessionInfo) { + const { expiresInMs, canBeExtended } = sessionInfo; + const nextState: SessionState = { + lastExtensionTime: Date.now(), + expiresInMs, + canBeExtended, + }; + this.sessionState$.next(nextState); + if (this.channel) { + this.channel.postMessage(nextState); + } + return nextState; + } + } catch (error) { + // ignore + } finally { + this.isFetchingSessionInfo = false; + } + }; + + private showWarning = () => { + if (!this.warningToast) { + const onExtend = async () => { + const { canBeExtended } = this.sessionState$.getValue(); + if (canBeExtended) { + await this.fetchSessionInfo(true); + } + }; + const onClose = () => { + this.hideWarning(true); + return onExtend(); + }; + const toast = createSessionExpirationToast(this.sessionState$, onExtend, onClose); + this.warningToast = this.notifications.toasts.add(toast); + } + }; + + private hideWarning = (snooze = false) => { + if (this.warningToast) { + this.notifications.toasts.remove(this.warningToast); + this.warningToast = undefined; + if (snooze) { + this.snoozedWarningState = this.sessionState$.getValue(); + } + } + }; +} + +/** + * Starts a timer that uses a native `setTimeout` under the hood. When `timeout` is larger + * than the maximum supported one then method calls itself recursively as many times as needed. + * @param callback A function to be executed after the timer expires. + * @param timeout The time, in milliseconds the timer should wait before the specified function is + * executed. + * @returns Function to stop the timer. + */ +export function startTimer(callback: () => void, timeout: number, updater?: (id: number) => void) { + // Max timeout is the largest possible 32-bit signed integer or 2,147,483,647 or 0x7fffffff. + const maxTimeout = 0x7fffffff; + let timeoutID: number; + updater = updater ?? ((id: number) => (timeoutID = id)); + + updater( + timeout > maxTimeout + ? window.setTimeout(() => startTimer(callback, timeout - maxTimeout, updater), maxTimeout) + : window.setTimeout(callback, timeout) + ); + + return () => clearTimeout(timeoutID); +} + +/** + * Adds event handlers to the window object that track user activity. + * @param callback Function to be executed when user activity is detected. + * @returns Function to remove all event handlers from window. + */ +function monitorActivity(callback: () => void) { + const eventTypes = ['mousemove', 'mousedown', 'wheel', 'touchstart', 'keydown']; + for (const eventType of eventTypes) { + window.addEventListener(eventType, callback); + } + + return () => { + for (const eventType of eventTypes) { + window.removeEventListener(eventType, callback); + } + }; +} + +/** + * Adds event handlers to the document object that track page visibility. + * @param callback Function to be executed when page visibility changes. + * @returns Function to remove all event handlers from document. + */ +function monitorVisibility(callback: (isVisible: boolean) => void) { + const handleVisibilityChange = () => callback(document.visibilityState !== 'hidden'); + + document.addEventListener('visibilitychange', handleVisibilityChange); + + return () => { + document.removeEventListener('visibilitychange', handleVisibilityChange); + }; +} diff --git a/x-pack/plugins/security/public/session/session_timeout.tsx b/x-pack/plugins/security/public/session/session_timeout.tsx deleted file mode 100644 index 2288fce8d30af..0000000000000 --- a/x-pack/plugins/security/public/session/session_timeout.tsx +++ /dev/null @@ -1,255 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { BroadcastChannel as BroadcastChannelType } from 'broadcast-channel'; - -import type { HttpSetup, NotificationsSetup, Toast, ToastInput } from 'src/core/public'; - -import type { SessionInfo } from '../../common/types'; -import type { ISessionExpired } from './session_expired'; -import { createToast as createIdleTimeoutToast } from './session_idle_timeout_warning'; -import { createToast as createLifespanToast } from './session_lifespan_warning'; - -/** - * Client session timeout is decreased by this number so that Kibana server - * can still access session content during logout request to properly clean - * user session up (invalidate access tokens, redirect to logout portal etc.). - */ -const GRACE_PERIOD_MS = 5 * 1000; - -/** - * Duration we'll normally display the warning toast - */ -const WARNING_MS = 60 * 1000; - -/** - * Current session info is checked this number of milliseconds before the - * warning toast shows. This will prevent the toast from being shown if the - * session has already been extended. - */ -const SESSION_CHECK_MS = 1000; - -/** - * Route to get session info and extend session expiration - */ -const SESSION_ROUTE = '/internal/security/session'; - -export interface ISessionTimeout { - start(): void; - stop(): void; - extend(url: string): void; -} - -export class SessionTimeout implements ISessionTimeout { - private channel?: BroadcastChannelType; - private sessionInfo?: SessionInfo; - private fetchTimer?: number; - private warningTimer?: number; - private expirationTimer?: number; - private warningToast?: Toast; - - constructor( - private notifications: NotificationsSetup, - private sessionExpired: ISessionExpired, - private http: HttpSetup, - private tenant: string - ) {} - - start() { - if (this.http.anonymousPaths.isAnonymous(window.location.pathname)) { - return; - } - - import('broadcast-channel') - .then(({ BroadcastChannel }) => { - // subscribe to a broadcast channel for session timeout messages - // this allows us to synchronize the UX across tabs and avoid repetitive API calls - const name = `${this.tenant}/session_timeout`; - this.channel = new BroadcastChannel(name, { webWorkerSupport: false }); - this.channel.onmessage = this.handleSessionInfoAndResetTimers; - }) - .catch((e) => { - // eslint-disable-next-line no-console - console.warn( - `Failed to load broadcast channel. Session management will not be kept in sync when multiple tabs are loaded.`, - e - ); - }) - .finally(() => { - // Triggers an initial call to the endpoint to get session info; - // when that returns, it will set the timeout - return this.fetchSessionInfoAndResetTimers(); - }); - } - - stop() { - if (this.channel) { - this.channel.close(); - } - this.cleanup(); - } - - /** - * When the user makes an authenticated, non-system API call, this function is used to check - * and see if the session has been extended. - * @param url The URL that was called - */ - extend(url: string) { - // avoid an additional API calls when the user clicks the button on the session idle timeout - if (url.endsWith(SESSION_ROUTE)) { - return; - } - - const { isLifespanTimeout } = this.getTimeout(); - if (this.warningToast && !isLifespanTimeout) { - // the idle timeout warning is currently showing and the user has clicked elsewhere on the page; - // make a new call to get the latest session info - return this.fetchSessionInfoAndResetTimers(); - } - } - - /** - * Fetch latest session information from the server, and optionally attempt to extend - * the session expiration. - */ - private fetchSessionInfoAndResetTimers = async (extend = false) => { - const method = extend ? 'POST' : 'GET'; - try { - const result = await this.http.fetch(SESSION_ROUTE, { method, asSystemRequest: !extend }); - - this.handleSessionInfoAndResetTimers(result); - - // share this updated session info with any other tabs to sync the UX - if (this.channel) { - this.channel.postMessage(result); - } - } catch (err) { - // do nothing; 401 errors will be caught by the http interceptor - } - }; - - /** - * Processes latest session information, and resets timers based on it. These timers are - * used to trigger an HTTP call for updated session information, to show a timeout - * warning, and to log the user out when their session is expired. - */ - private handleSessionInfoAndResetTimers = (sessionInfo: SessionInfo) => { - this.sessionInfo = sessionInfo; - // save the provider name in session storage, we will need it when we log out - const key = `${this.tenant}/session_provider`; - sessionStorage.setItem(key, sessionInfo.provider.name); - - const { timeout, isLifespanTimeout } = this.getTimeout(); - if (timeout == null) { - return; - } - - this.cleanup(); - - // set timers - const timeoutVal = timeout - WARNING_MS - GRACE_PERIOD_MS - SESSION_CHECK_MS; - if (timeoutVal > 0 && !isLifespanTimeout) { - // we should check for the latest session info before the warning displays - this.startTimer( - (timeoutID) => (this.fetchTimer = timeoutID), - this.fetchSessionInfoAndResetTimers, - timeoutVal - ); - } - - this.startTimer( - (timeoutID) => (this.warningTimer = timeoutID), - this.showWarning, - Math.max(timeout - WARNING_MS - GRACE_PERIOD_MS, 0) - ); - - this.startTimer( - (timeoutID) => (this.expirationTimer = timeoutID), - () => this.sessionExpired.logout(), - Math.max(timeout - GRACE_PERIOD_MS, 0) - ); - }; - - private cleanup = () => { - if (this.fetchTimer) { - window.clearTimeout(this.fetchTimer); - } - if (this.warningTimer) { - window.clearTimeout(this.warningTimer); - } - if (this.expirationTimer) { - window.clearTimeout(this.expirationTimer); - } - if (this.warningToast) { - this.notifications.toasts.remove(this.warningToast); - this.warningToast = undefined; - } - }; - - /** - * Get the amount of time until the session times out, and whether or not the - * session has reached it maximum lifespan. - */ - private getTimeout = (): { timeout: number | null; isLifespanTimeout: boolean } => { - let timeout = null; - let isLifespanTimeout = false; - if (this.sessionInfo) { - const { now, idleTimeoutExpiration, lifespanExpiration } = this.sessionInfo; - if (idleTimeoutExpiration) { - timeout = idleTimeoutExpiration - now; - } - if ( - lifespanExpiration && - (idleTimeoutExpiration === null || lifespanExpiration <= idleTimeoutExpiration) - ) { - timeout = lifespanExpiration - now; - isLifespanTimeout = true; - } - } - return { timeout, isLifespanTimeout }; - }; - - /** - * Show a warning toast depending on the session state. - */ - private showWarning = () => { - const { timeout, isLifespanTimeout } = this.getTimeout(); - const toastLifeTimeMs = Math.min(timeout! - GRACE_PERIOD_MS, WARNING_MS); - let toast: ToastInput; - if (!isLifespanTimeout) { - const refresh = () => this.fetchSessionInfoAndResetTimers(true); - toast = createIdleTimeoutToast(toastLifeTimeMs, refresh); - } else { - toast = createLifespanToast(toastLifeTimeMs); - } - this.warningToast = this.notifications.toasts.add(toast); - }; - - /** - * Starts a timer that uses a native `setTimeout` under the hood. When `timeout` is larger - * than the maximum supported one then method calls itself recursively as many times as needed. - * @param updater Method that is supposed to update a reference to a native timer ID that can be - * used with native `clearTimeout`. It's essential for the larger timeouts when `setTimeout` is - * called multiple times and timer ID changes. - * when timer ID changes - * @param callback A function to be executed after the timer expires. - * @param timeout The time, in milliseconds the timer should wait before the specified function is - * executed. - */ - private startTimer(updater: (timeoutID: number) => void, callback: () => void, timeout: number) { - // Max timeout is the largest possible 32-bit signed integer or 2,147,483,647 or 0x7fffffff. - const maxTimeout = 0x7fffffff; - updater( - timeout > maxTimeout - ? window.setTimeout( - () => this.startTimer(updater, callback, timeout - maxTimeout), - maxTimeout - ) - : window.setTimeout(callback, timeout) - ); - } -} diff --git a/x-pack/plugins/security/public/session/session_timeout_http_interceptor.test.ts b/x-pack/plugins/security/public/session/session_timeout_http_interceptor.test.ts deleted file mode 100644 index 4f216d331c3f2..0000000000000 --- a/x-pack/plugins/security/public/session/session_timeout_http_interceptor.test.ts +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -// @ts-ignore -import fetchMock from 'fetch-mock/es5/client'; - -import { setup } from 'src/core/test_helpers/http_test_setup'; - -import { createSessionTimeoutMock } from './session_timeout.mock'; -import { SessionTimeoutHttpInterceptor } from './session_timeout_http_interceptor'; - -const mockCurrentUrl = (url: string) => window.history.pushState({}, '', url); - -const setupHttp = (basePath: string) => { - const { http } = setup((injectedMetadata) => { - injectedMetadata.getBasePath.mockReturnValue(basePath); - }); - return http; -}; - -afterEach(() => { - fetchMock.restore(); -}); - -describe('response', () => { - test('extends session timeouts', async () => { - const http = setupHttp('/foo'); - const sessionTimeoutMock = createSessionTimeoutMock(); - const interceptor = new SessionTimeoutHttpInterceptor(sessionTimeoutMock, http.anonymousPaths); - http.intercept(interceptor); - fetchMock.mock('*', 200); - - await http.fetch('/foo-api'); - - expect(sessionTimeoutMock.extend).toHaveBeenCalled(); - }); - - test(`doesn't extend session timeouts for anonymous paths`, async () => { - mockCurrentUrl('/foo/bar'); - const http = setupHttp('/foo'); - const sessionTimeoutMock = createSessionTimeoutMock(); - const { anonymousPaths } = http; - anonymousPaths.register('/bar'); - const interceptor = new SessionTimeoutHttpInterceptor(sessionTimeoutMock, anonymousPaths); - http.intercept(interceptor); - fetchMock.mock('*', 200); - - await http.fetch('/foo-api'); - - expect(sessionTimeoutMock.extend).not.toHaveBeenCalled(); - }); - - test(`doesn't extend session timeouts for system api requests`, async () => { - const http = setupHttp('/foo'); - const sessionTimeoutMock = createSessionTimeoutMock(); - const interceptor = new SessionTimeoutHttpInterceptor(sessionTimeoutMock, http.anonymousPaths); - http.intercept(interceptor); - fetchMock.mock('*', 200); - - await http.fetch('/foo-api', { asSystemRequest: true }); - - expect(sessionTimeoutMock.extend).not.toHaveBeenCalled(); - }); -}); - -describe('responseError', () => { - test('extends session timeouts', async () => { - const http = setupHttp('/foo'); - const sessionTimeoutMock = createSessionTimeoutMock(); - const interceptor = new SessionTimeoutHttpInterceptor(sessionTimeoutMock, http.anonymousPaths); - http.intercept(interceptor); - fetchMock.mock('*', 401); - - await expect(http.fetch('/foo-api')).rejects.toMatchInlineSnapshot(`[Error: Unauthorized]`); - - expect(sessionTimeoutMock.extend).toHaveBeenCalled(); - }); - - test(`doesn't extend session timeouts for anonymous paths`, async () => { - mockCurrentUrl('/foo/bar'); - const http = setupHttp('/foo'); - const sessionTimeoutMock = createSessionTimeoutMock(); - const { anonymousPaths } = http; - anonymousPaths.register('/bar'); - const interceptor = new SessionTimeoutHttpInterceptor(sessionTimeoutMock, anonymousPaths); - http.intercept(interceptor); - fetchMock.mock('*', 401); - - await expect(http.fetch('/foo-api')).rejects.toMatchInlineSnapshot(`[Error: Unauthorized]`); - - expect(sessionTimeoutMock.extend).not.toHaveBeenCalled(); - }); - - test(`doesn't extend session timeouts for system api requests`, async () => { - const http = setupHttp('/foo'); - const sessionTimeoutMock = createSessionTimeoutMock(); - const interceptor = new SessionTimeoutHttpInterceptor(sessionTimeoutMock, http.anonymousPaths); - http.intercept(interceptor); - fetchMock.mock('*', 401); - - await expect(http.fetch('/foo-api', { asSystemRequest: true })).rejects.toMatchInlineSnapshot( - `[Error: Unauthorized]` - ); - - expect(sessionTimeoutMock.extend).not.toHaveBeenCalled(); - }); - - test(`doesn't extend session timeouts when there is no response`, async () => { - const http = setupHttp('/foo'); - const sessionTimeoutMock = createSessionTimeoutMock(); - const interceptor = new SessionTimeoutHttpInterceptor(sessionTimeoutMock, http.anonymousPaths); - http.intercept(interceptor); - fetchMock.mock('*', new Promise((resolve, reject) => reject(new Error('Network is down')))); - - await expect(http.fetch('/foo-api')).rejects.toMatchInlineSnapshot(`[Error: Network is down]`); - - expect(sessionTimeoutMock.extend).not.toHaveBeenCalled(); - }); -}); diff --git a/x-pack/plugins/security/public/session/session_timeout_http_interceptor.ts b/x-pack/plugins/security/public/session/session_timeout_http_interceptor.ts deleted file mode 100644 index 6fc7fb88578ff..0000000000000 --- a/x-pack/plugins/security/public/session/session_timeout_http_interceptor.ts +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { - HttpInterceptor, - HttpInterceptorResponseError, - HttpResponse, - IAnonymousPaths, -} from 'src/core/public'; - -import type { ISessionTimeout } from './session_timeout'; - -export class SessionTimeoutHttpInterceptor implements HttpInterceptor { - constructor(private sessionTimeout: ISessionTimeout, private anonymousPaths: IAnonymousPaths) {} - - response(httpResponse: HttpResponse) { - if (this.anonymousPaths.isAnonymous(window.location.pathname)) { - return; - } - - if (httpResponse.fetchOptions.asSystemRequest) { - return; - } - - this.sessionTimeout.extend(httpResponse.request.url); - } - - responseError(httpErrorResponse: HttpInterceptorResponseError) { - if (this.anonymousPaths.isAnonymous(window.location.pathname)) { - return; - } - - if (httpErrorResponse.fetchOptions.asSystemRequest) { - return; - } - - // if we happen to not have a response, for example if there is no - // network connectivity, we won't extend the session because there - // won't be a response with a set-cookie header, which is required - // to extend the session - const { response } = httpErrorResponse; - if (!response) { - return; - } - - this.sessionTimeout.extend(httpErrorResponse.request.url); - } -} diff --git a/x-pack/plugins/security/server/routes/session_management/info.test.ts b/x-pack/plugins/security/server/routes/session_management/info.test.ts index 29c0865b8c561..4c420f3b14346 100644 --- a/x-pack/plugins/security/server/routes/session_management/info.test.ts +++ b/x-pack/plugins/security/server/routes/session_management/info.test.ts @@ -10,6 +10,7 @@ import type { RequestHandler, RouteConfig } from 'src/core/server'; import { kibanaResponseFactory } from 'src/core/server'; import { httpServerMock } from 'src/core/server/mocks'; +import { SESSION_EXPIRATION_WARNING_MS } from '../../../common/constants'; import type { Session } from '../../session_management'; import { sessionMock } from '../../session_management/session.mock'; import type { SecurityRequestHandlerContext, SecurityRouter } from '../../types'; @@ -29,7 +30,7 @@ describe('Info session routes', () => { defineSessionInfoRoutes(routeParamsMock); }); - describe('extend session', () => { + describe('session info', () => { let routeHandler: RequestHandler; let routeConfig: RouteConfig; beforeEach(() => { @@ -63,30 +64,94 @@ describe('Info session routes', () => { }); it('returns session info.', async () => { - session.get.mockResolvedValue( - sessionMock.createValue({ idleTimeoutExpiration: 100, lifespanExpiration: 200 }) - ); - + const now = 1000; const dateSpy = jest.spyOn(Date, 'now'); - dateSpy.mockReturnValue(1234); - - const expectedBody = { - now: 1234, - provider: { type: 'basic', name: 'basic1' }, - idleTimeoutExpiration: 100, - lifespanExpiration: 200, - }; - await expect( - routeHandler( - ({} as unknown) as SecurityRequestHandlerContext, - httpServerMock.createKibanaRequest(), - kibanaResponseFactory - ) - ).resolves.toEqual({ - status: 200, - payload: expectedBody, - options: { body: expectedBody }, - }); + dateSpy.mockReturnValue(now); + + const assertions = [ + [ + { + idleTimeoutExpiration: 100 + now, + lifespanExpiration: 200 + SESSION_EXPIRATION_WARNING_MS + now, + }, + { + canBeExtended: true, + expiresInMs: 100, + }, + ], + [ + { + idleTimeoutExpiration: 100 + now, + lifespanExpiration: 200 + now, + }, + { + canBeExtended: false, + expiresInMs: 100, + }, + ], + [ + { + idleTimeoutExpiration: 200 + now, + lifespanExpiration: 100 + now, + }, + { + canBeExtended: false, + expiresInMs: 100, + }, + ], + [ + { + idleTimeoutExpiration: null, + lifespanExpiration: 100 + now, + }, + { + canBeExtended: false, + expiresInMs: 100, + }, + ], + [ + { + idleTimeoutExpiration: 100 + now, + lifespanExpiration: null, + }, + { + canBeExtended: true, + expiresInMs: 100, + }, + ], + [ + { + idleTimeoutExpiration: null, + lifespanExpiration: null, + }, + { + canBeExtended: false, + expiresInMs: null, + }, + ], + ]; + + for (const [sessionInfo, expected] of assertions) { + session.get.mockResolvedValue(sessionMock.createValue(sessionInfo)); + + const expectedBody = { + canBeExtended: expected.canBeExtended, + expiresInMs: expected.expiresInMs, + provider: { type: 'basic', name: 'basic1' }, + }; + + await expect( + routeHandler( + ({} as unknown) as SecurityRequestHandlerContext, + httpServerMock.createKibanaRequest(), + kibanaResponseFactory + ) + ).resolves.toEqual({ + status: 200, + payload: expectedBody, + options: { body: expectedBody }, + }); + } }); it('returns empty response if session is not available.', async () => { diff --git a/x-pack/plugins/security/server/routes/session_management/info.ts b/x-pack/plugins/security/server/routes/session_management/info.ts index cfd76bb8abbc0..b141f42c4b072 100644 --- a/x-pack/plugins/security/server/routes/session_management/info.ts +++ b/x-pack/plugins/security/server/routes/session_management/info.ts @@ -6,6 +6,7 @@ */ import type { RouteDefinitionParams } from '../'; +import { SESSION_EXPIRATION_WARNING_MS } from '../../../common/constants'; import type { SessionInfo } from '../../../common/types'; /** @@ -17,13 +18,19 @@ export function defineSessionInfoRoutes({ router, getSession }: RouteDefinitionP async (_context, request, response) => { const sessionValue = await getSession().get(request); if (sessionValue) { + const expirationTime = + sessionValue.idleTimeoutExpiration && sessionValue.lifespanExpiration + ? Math.min(sessionValue.idleTimeoutExpiration, sessionValue.lifespanExpiration) + : sessionValue.idleTimeoutExpiration || sessionValue.lifespanExpiration; + return response.ok({ body: { - // We can't rely on the client's system clock, so in addition to returning expiration timestamps, we also return - // the current server time -- that way the client can calculate the relative time to expiration. - now: Date.now(), - idleTimeoutExpiration: sessionValue.idleTimeoutExpiration, - lifespanExpiration: sessionValue.lifespanExpiration, + expiresInMs: expirationTime ? expirationTime - Date.now() : null, + canBeExtended: + sessionValue.idleTimeoutExpiration !== null && + expirationTime !== null && + (sessionValue.lifespanExpiration === null || + expirationTime + SESSION_EXPIRATION_WARNING_MS < sessionValue.lifespanExpiration), provider: sessionValue.provider, } as SessionInfo, }); diff --git a/x-pack/plugins/security_solution/common/endpoint/data_generators/fleet_agent_generator.ts b/x-pack/plugins/security_solution/common/endpoint/data_generators/fleet_agent_generator.ts new file mode 100644 index 0000000000000..6e508a099003a --- /dev/null +++ b/x-pack/plugins/security_solution/common/endpoint/data_generators/fleet_agent_generator.ts @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { estypes } from '@elastic/elasticsearch'; +import { DeepPartial } from 'utility-types'; +import { merge } from 'lodash'; +import { BaseDataGenerator } from './base_data_generator'; +import { Agent, AGENTS_INDEX, FleetServerAgent } from '../../../../fleet/common'; + +export class FleetAgentGenerator extends BaseDataGenerator { + /** + * @param [overrides] any partial value to the full Agent record + * + * @example + * + * fleetAgentGenerator.generate({ + * local_metadata: { + * elastic: { + * agent: { + * log_level: `debug` + * } + * } + * } + * }); + */ + generate(overrides: DeepPartial = {}): Agent { + const hit = this.generateEsHit(); + + // The mapping below is identical to `searchHitToAgent()` located in + // `x-pack/plugins/fleet/server/services/agents/helpers.ts:19` + return merge( + { + // Casting here is needed because several of the attributes in `FleetServerAgent` are + // defined as optional, but required in `Agent` type. + ...(hit._source as Agent), + id: hit._id, + policy_revision: hit._source?.policy_revision_idx, + access_api_key: undefined, + status: undefined, + packages: hit._source?.packages ?? [], + }, + overrides + ); + } + + /** + * @param [overrides] any partial value to the full document + */ + generateEsHit( + overrides: DeepPartial> = {} + ): estypes.Hit { + const hostname = this.randomHostname(); + const now = new Date().toISOString(); + const osFamily = this.randomOSFamily(); + + return merge, DeepPartial>>( + { + _index: AGENTS_INDEX, + _id: this.randomUUID(), + _score: 1.0, + _source: { + access_api_key_id: 'jY3dWnkBj1tiuAw9pAmq', + action_seq_no: -1, + active: true, + enrolled_at: now, + local_metadata: { + elastic: { + agent: { + 'build.original': `8.0.0-SNAPSHOT (build: ${this.randomString( + 5 + )} at 2021-05-07 18:42:49 +0000 UTC)`, + id: this.randomUUID(), + log_level: 'info', + snapshot: true, + upgradeable: true, + version: '8.0.0', + }, + }, + host: { + architecture: 'x86_64', + hostname, + id: this.randomUUID(), + ip: [this.randomIP()], + mac: [this.randomMac()], + name: hostname, + }, + os: { + family: osFamily, + full: `${osFamily} 2019 Datacenter`, + kernel: '10.0.17763.1879 (Build.160101.0800)', + name: `${osFamily} Server 2019 Datacenter`, + platform: osFamily, + version: this.randomVersion(), + }, + }, + user_provided_metadata: {}, + policy_id: this.randomUUID(), + type: 'PERMANENT', + default_api_key: 'so3dWnkBj1tiuAw9yAm3:t7jNlnPnR6azEI_YpXuBXQ', + // policy_output_permissions_hash: + // '81b3d070dddec145fafcbdfb6f22888495a12edc31881f6b0511fa10de66daa7', + default_api_key_id: 'so3dWnkBj1tiuAw9yAm3', + updated_at: now, + last_checkin: now, + policy_revision_idx: 2, + policy_coordinator_idx: 1, + }, + }, + overrides + ); + } +} diff --git a/x-pack/plugins/security_solution/common/endpoint/index_data.ts b/x-pack/plugins/security_solution/common/endpoint/index_data.ts index fd26a2d95c9b4..0dc7891560c2d 100644 --- a/x-pack/plugins/security_solution/common/endpoint/index_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/index_data.ts @@ -5,31 +5,31 @@ * 2.0. */ -import { Client } from '@elastic/elasticsearch'; +import { Client, estypes } from '@elastic/elasticsearch'; import seedrandom from 'seedrandom'; // eslint-disable-next-line import/no-extraneous-dependencies import { KbnClient } from '@kbn/test'; import { AxiosResponse } from 'axios'; -import { EndpointDocGenerator, TreeOptions, Event } from './generate_data'; +import { EndpointDocGenerator, Event, TreeOptions } from './generate_data'; import { firstNonNullValue } from './models/ecs_safety_helpers'; import { + AGENT_POLICY_API_ROUTES, CreateAgentPolicyRequest, CreateAgentPolicyResponse, CreatePackagePolicyRequest, CreatePackagePolicyResponse, - GetPackagesResponse, - AGENT_API_ROUTES, - AGENT_POLICY_API_ROUTES, EPM_API_ROUTES, + FLEET_SERVER_SERVERS_INDEX, + FleetServerAgent, + GetPackagesResponse, PACKAGE_POLICY_API_ROUTES, - ENROLLMENT_API_KEY_ROUTES, - GetEnrollmentAPIKeysResponse, - GetOneEnrollmentAPIKeyResponse, - Agent, } from '../../../fleet/common'; import { policyFactory as policyConfigFactory } from './models/policy_config'; import { HostMetadata } from './types'; import { KbnClientWithApiKeySupport } from '../../scripts/endpoint/kbn_client_with_api_key_support'; +import { FleetAgentGenerator } from './data_generators/fleet_agent_generator'; + +const fleetAgentGenerator = new FleetAgentGenerator(); export async function indexHostsAndAlerts( client: Client, @@ -47,8 +47,15 @@ export async function indexHostsAndAlerts( ) { const random = seedrandom(seed); const epmEndpointPackage = await getEndpointPackageInfo(kbnClient); + + // If `fleet` integration is true, then ensure a (fake) fleet-server is connected + if (fleet) { + await enableFleetServerIfNecessary(client); + } + // Keep a map of host applied policy ids (fake) to real ingest package configs (policy record) const realPolicies: Record = {}; + for (let i = 0; i < numHosts; i++) { const generator = new EndpointDocGenerator(random); await indexHostDocs({ @@ -71,9 +78,11 @@ export async function indexHostsAndAlerts( options, }); } + await client.indices.refresh({ index: eventIndex, }); + // TODO: Unclear why the documents are not showing up after the call to refresh. // Waiting 5 seconds allows the indices to refresh automatically and // the documents become available in API/integration tests. @@ -107,9 +116,10 @@ async function indexHostDocs({ }) { const timeBetweenDocs = 6 * 3600 * 1000; // 6 hours between metadata documents const timestamp = new Date().getTime(); + const kibanaVersion = await fetchKibanaVersion(kbnClient); let hostMetadata: HostMetadata; let wasAgentEnrolled = false; - let enrolledAgent: undefined | Agent; + let enrolledAgent: undefined | estypes.Hit; for (let j = 0; j < numDocs; j++) { generator.updateHostData(); @@ -136,10 +146,12 @@ async function indexHostDocs({ // If we did not yet enroll an agent for this Host, do it now that we have good policy id if (!wasAgentEnrolled) { wasAgentEnrolled = true; - enrolledAgent = await fleetEnrollAgentForHost( - kbnClient, + + enrolledAgent = await indexFleetAgentForHost( + client, hostMetadata!, - realPolicies[appliedPolicyId].policy_id + realPolicies[appliedPolicyId].policy_id, + kibanaVersion ); } // Update the Host metadata record with the ID of the "real" policy along with the enrolled agent id @@ -149,7 +161,7 @@ async function indexHostDocs({ ...hostMetadata.elastic, agent: { ...hostMetadata.elastic.agent, - id: enrolledAgent?.id ?? hostMetadata.elastic.agent.id, + id: enrolledAgent?._id ?? hostMetadata.elastic.agent.id, }, }, Endpoint: { @@ -295,208 +307,93 @@ const getEndpointPackageInfo = async ( return endpointPackage; }; -const fleetEnrollAgentForHost = async ( - kbnClient: KbnClientWithApiKeySupport, - endpointHost: HostMetadata, - agentPolicyId: string -): Promise => { - // Get Enrollement key for host's applied policy - const enrollmentApiKey = await kbnClient - .request({ - path: ENROLLMENT_API_KEY_ROUTES.LIST_PATTERN, - method: 'GET', - query: { - kuery: `policy_id:"${agentPolicyId}"`, - }, - }) - .then((apiKeysResponse) => { - const apiKey = apiKeysResponse.data.list[0]; +const fetchKibanaVersion = async (kbnClient: KbnClientWithApiKeySupport) => { + const version = ((await kbnClient.request({ + path: '/api/status', + method: 'GET', + })) as AxiosResponse).data.version.number; - if (!apiKey) { - return Promise.reject( - new Error(`no API enrollment key found for agent policy id ${agentPolicyId}`) - ); - } + if (!version) { + // eslint-disable-next-line no-console + console.log('failed to retrieve kibana version'); + return '8.0.0'; + } - return kbnClient - .request({ - path: ENROLLMENT_API_KEY_ROUTES.INFO_PATTERN.replace('{keyId}', apiKey.id), - method: 'GET', - }) - .catch((error) => { - // eslint-disable-next-line no-console - console.log('unable to retrieve enrollment api key for policy'); - return Promise.reject(error); - }); - }) - .then((apiKeyDetailsResponse) => { - return apiKeyDetailsResponse.data.item.api_key; - }) - .catch((error) => { - // eslint-disable-next-line no-console - console.error(error); - return ''; - }); + return version; +}; + +/** + * Will ensure that at least one fleet server is present in the `.fleet-servers` index. This will + * enable the `Agent` section of kibana Fleet to be displayed + * + * @param esClient + * @param version + */ +const enableFleetServerIfNecessary = async (esClient: Client, version: string = '8.0.0') => { + const res = await esClient.search<{}, {}>({ + index: FLEET_SERVER_SERVERS_INDEX, + ignore_unavailable: true, + }); - if (enrollmentApiKey.length === 0) { + // @ts-expect-error value is number | TotalHits + if (res.body.hits.total.value > 0) { return; } - const fetchKibanaVersion = async () => { - const version = ((await kbnClient.request({ - path: '/api/status', - method: 'GET', - })) as AxiosResponse).data.version.number; - if (!version) { - // eslint-disable-next-line no-console - console.log('failed to retrieve kibana version'); - } - return version; - }; + // Create a Fake fleet-server in this kibana instance + await esClient.index({ + index: FLEET_SERVER_SERVERS_INDEX, + body: { + agent: { + id: '12988155-475c-430d-ac89-84dc84b67cd1', + version: '', + }, + host: { + architecture: 'linux', + id: 'c3e5f4f690b4a3ff23e54900701a9513', + ip: ['127.0.0.1', '::1', '10.201.0.213', 'fe80::4001:aff:fec9:d5'], + name: 'endpoint-data-generator', + }, + server: { + id: '12988155-475c-430d-ac89-84dc84b67cd1', + version: '8.0.0-SNAPSHOT', + }, + '@timestamp': '2021-05-12T18:42:52.009482058Z', + }, + }); +}; - // Enroll an agent for the Host - const body = { - type: 'PERMANENT', - metadata: { - local: { +const indexFleetAgentForHost = async ( + esClient: Client, + endpointHost: HostMetadata, + agentPolicyId: string, + kibanaVersion: string = '8.0.0' +): Promise> => { + const agentDoc = fleetAgentGenerator.generateEsHit({ + _source: { + local_metadata: { elastic: { agent: { - version: await fetchKibanaVersion(), + version: kibanaVersion, }, }, host: { ...endpointHost.host, }, os: { - family: 'windows', - kernel: '10.0.19041.388 (WinBuild.160101.0800)', - platform: 'windows', - version: '10.0', - name: 'Windows 10 Pro', - full: 'Windows 10 Pro(10.0)', + ...endpointHost.host.os, }, }, - user_provided: { - dev_agent_version: '0.0.1', - region: 'us-east', - }, + policy_id: agentPolicyId, }, - }; - - try { - // First enroll the agent - const res = await kbnClient.requestWithApiKey(AGENT_API_ROUTES.ENROLL_PATTERN, { - method: 'POST', - body: JSON.stringify(body), - headers: { - 'kbn-xsrf': 'xxx', - Authorization: `ApiKey ${enrollmentApiKey}`, - 'Content-Type': 'application/json', - }, - }); - - if (res) { - const enrollObj = await res.json(); - if (!res.ok) { - // eslint-disable-next-line no-console - console.error('unable to enroll agent', enrollObj); - return; - } - // ------------------------------------------------ - // now check the agent in so that it can complete enrollment - const checkinBody = { - events: [ - { - type: 'STATE', - subtype: 'RUNNING', - message: 'state changed from STOPPED to RUNNING', - timestamp: new Date().toISOString(), - payload: { - random: 'data', - state: 'RUNNING', - previous_state: 'STOPPED', - }, - agent_id: enrollObj.item.id, - }, - ], - }; - const checkinRes = await kbnClient - .requestWithApiKey( - AGENT_API_ROUTES.CHECKIN_PATTERN.replace('{agentId}', enrollObj.item.id), - { - method: 'POST', - body: JSON.stringify(checkinBody), - headers: { - 'kbn-xsrf': 'xxx', - Authorization: `ApiKey ${enrollObj.item.access_api_key}`, - 'Content-Type': 'application/json', - }, - } - ) - .catch((error) => { - return Promise.reject(error); - }); - - // Agent unenrolling? - if (checkinRes.status === 403) { - return; - } - - const checkinObj = await checkinRes.json(); - if (!checkinRes.ok) { - // eslint-disable-next-line no-console - console.error( - `failed to checkin agent [${enrollObj.item.id}] for endpoint [${endpointHost.host.id}]` - ); - return enrollObj.item; - } - - // ------------------------------------------------ - // If we have an action to ack(), then do it now - if (checkinObj.actions.length) { - const ackActionBody = { - // @ts-ignore - events: checkinObj.actions.map((action) => { - return { - action_id: action.id, - type: 'ACTION_RESULT', - subtype: 'CONFIG', - timestamp: new Date().toISOString(), - agent_id: action.agent_id, - policy_id: agentPolicyId, - message: `endpoint generator: Endpoint Started`, - }; - }), - }; - const ackActionResp = await kbnClient.requestWithApiKey( - AGENT_API_ROUTES.ACKS_PATTERN.replace('{agentId}', enrollObj.item.id), - { - method: 'POST', - body: JSON.stringify(ackActionBody), - headers: { - 'kbn-xsrf': 'xxx', - Authorization: `ApiKey ${enrollObj.item.access_api_key}`, - 'Content-Type': 'application/json', - }, - } - ); + }); - const ackActionObj = await ackActionResp.json(); - if (!ackActionResp.ok) { - // eslint-disable-next-line no-console - console.error( - `failed to ACK Actions provided to agent [${enrollObj.item.id}] for endpoint [${endpointHost.host.id}]` - ); - // eslint-disable-next-line no-console - console.error(JSON.stringify(ackActionObj, null, 2)); - return enrollObj.item; - } - } + await esClient.index({ + index: agentDoc._index, + id: agentDoc._id, + body: agentDoc._source!, + op_type: 'create', + }); - return enrollObj.item; - } - } catch (error) { - // eslint-disable-next-line no-console - console.error(error); - } + return agentDoc; }; diff --git a/x-pack/plugins/security_solution/common/endpoint/types/index.ts b/x-pack/plugins/security_solution/common/endpoint/types/index.ts index c58e67b5d4fd4..b9e72bcd625ec 100644 --- a/x-pack/plugins/security_solution/common/endpoint/types/index.ts +++ b/x-pack/plugins/security_solution/common/endpoint/types/index.ts @@ -414,6 +414,11 @@ export type PolicyInfo = Immutable<{ id: string; }>; +export interface HostMetaDataInfo { + metadata: HostMetadata; + query_strategy_version: MetadataQueryStrategyVersions; +} + export type HostInfo = Immutable<{ metadata: HostMetadata; host_status: HostStatus; diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/hosts/common/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/hosts/common/index.ts index a579d8f8d8ef3..3175876a8299c 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/hosts/common/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/hosts/common/index.ts @@ -25,10 +25,16 @@ export interface EndpointFields { endpointPolicy?: Maybe; sensorVersion?: Maybe; policyStatus?: Maybe; + id?: Maybe; +} + +interface AgentFields { + id?: Maybe; } export interface HostItem { _id?: Maybe; + agent?: Maybe; cloud?: Maybe; endpoint?: Maybe; host?: Maybe; @@ -70,6 +76,9 @@ export interface HostAggEsItem { cloud_machine_type?: HostBuckets; cloud_provider?: HostBuckets; cloud_region?: HostBuckets; + endpoint?: { + id: HostBuckets; + }; host_architecture?: HostBuckets; host_id?: HostBuckets; host_ip?: HostBuckets; diff --git a/x-pack/plugins/security_solution/cypress/integration/detection_rules/indicator_match_rule.spec.ts b/x-pack/plugins/security_solution/cypress/integration/detection_rules/indicator_match_rule.spec.ts index 098c9aca571eb..d0f2cd9f45743 100644 --- a/x-pack/plugins/security_solution/cypress/integration/detection_rules/indicator_match_rule.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/detection_rules/indicator_match_rule.spec.ts @@ -16,7 +16,16 @@ import { ALERT_RULE_VERSION, NUMBER_OF_ALERTS, } from '../../screens/alerts'; -import { JSON_LINES } from '../../screens/alerts_details'; +import { + JSON_LINES, + TABLE_CELL, + TABLE_ROWS, + THREAT_CONTENT, + THREAT_DETAILS_VIEW, + THREAT_INTEL_TAB, + THREAT_SUMMARY_VIEW, + TITLE, +} from '../../screens/alerts_details'; import { CUSTOM_RULES_BTN, RISK_SCORE, @@ -63,7 +72,11 @@ import { waitForAlertsIndexToBeCreated, waitForAlertsPanelToBeLoaded, } from '../../tasks/alerts'; -import { openJsonView, scrollJsonViewToBottom } from '../../tasks/alerts_details'; +import { + openJsonView, + openThreatIndicatorDetails, + scrollJsonViewToBottom, +} from '../../tasks/alerts_details'; import { changeRowsPerPageTo300, duplicateFirstRule, @@ -585,6 +598,90 @@ describe('indicator match', () => { }); }); }); + + it('Displays threat summary data on alerts details', () => { + const expectedThreatSummary = [ + { field: 'matched.field', value: 'myhash.mysha256' }, + { field: 'matched.type', value: 'file' }, + { field: 'first_seen', value: '2021-03-10T08:02:14.000Z' }, + ]; + + expandFirstAlert(); + + cy.get(THREAT_SUMMARY_VIEW).within(() => { + cy.get(TABLE_ROWS).should('have.length', expectedThreatSummary.length); + expectedThreatSummary.forEach((row, index) => { + cy.get(TABLE_ROWS) + .eq(index) + .within(() => { + cy.get(TITLE).should('have.text', row.field); + cy.get(THREAT_CONTENT).should('have.text', row.value); + }); + }); + }); + }); + + it('Displays threat indicator data on the threat intel tab', () => { + const expectedThreatIndicatorData = [ + { field: 'first_seen', value: '2021-03-10T08:02:14.000Z' }, + { field: 'file.size', value: '80280' }, + { field: 'file.type', value: 'elf' }, + { + field: 'file.hash.sha256', + value: 'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3', + }, + { + field: 'file.hash.tlsh', + value: '6D7312E017B517CC1371A8353BED205E9128223972AE35302E97528DF957703BAB2DBE', + }, + { + field: 'file.hash.ssdeep', + value: + '1536:87vbq1lGAXSEYQjbChaAU2yU23M51DjZgSQAvcYkFtZTjzBht5:8D+CAXFYQChaAUk5ljnQssL', + }, + { field: 'file.hash.md5', value: '9b6c3518a91d23ed77504b5416bfb5b3' }, + { field: 'type', value: 'file' }, + { + field: 'event.reference', + value: + 'https://urlhaus-api.abuse.ch/v1/download/a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3/(opens in a new tab or window)', + }, + { field: 'event.ingested', value: '2021-03-10T14:51:09.809069Z' }, + { field: 'event.created', value: '2021-03-10T14:51:07.663Z' }, + { field: 'event.kind', value: 'enrichment' }, + { field: 'event.module', value: 'threatintel' }, + { field: 'event.category', value: 'threat' }, + { field: 'event.type', value: 'indicator' }, + { field: 'event.dataset', value: 'threatintel.abusemalware' }, + { + field: 'matched.atomic', + value: 'a04ac6d98ad989312783d4fe3456c53730b212c79a426fb215708b6c6daa3de3', + }, + { field: 'matched.field', value: 'myhash.mysha256' }, + { + field: 'matched.id', + value: '84cf452c1e0375c3d4412cb550bd1783358468a3b3b777da4829d72c7d6fb74f', + }, + { field: 'matched.index', value: 'filebeat-7.12.0-2021.03.10-000001' }, + { field: 'matched.type', value: 'file' }, + ]; + + expandFirstAlert(); + openThreatIndicatorDetails(); + + cy.get(THREAT_INTEL_TAB).should('have.text', 'Threat Intel (1)'); + cy.get(THREAT_DETAILS_VIEW).within(() => { + cy.get(TABLE_ROWS).should('have.length', expectedThreatIndicatorData.length); + expectedThreatIndicatorData.forEach((row, index) => { + cy.get(TABLE_ROWS) + .eq(index) + .within(() => { + cy.get(TABLE_CELL).eq(0).should('have.text', row.field); + cy.get(TABLE_CELL).eq(1).should('have.text', row.value); + }); + }); + }); + }); }); describe('Duplicates the indicator rule', () => { diff --git a/x-pack/plugins/security_solution/cypress/integration/detection_rules/threshold_rule.spec.ts b/x-pack/plugins/security_solution/cypress/integration/detection_rules/threshold_rule.spec.ts index f9971dfc0f791..7c09b311807be 100644 --- a/x-pack/plugins/security_solution/cypress/integration/detection_rules/threshold_rule.spec.ts +++ b/x-pack/plugins/security_solution/cypress/integration/detection_rules/threshold_rule.spec.ts @@ -6,7 +6,7 @@ */ import { formatMitreAttackDescription } from '../../helpers/rules'; -import { indexPatterns, newThresholdRule } from '../../objects/rule'; +import { indexPatterns, newRule, newThresholdRule } from '../../objects/rule'; import { ALERT_RULE_METHOD, @@ -26,6 +26,7 @@ import { RULES_TABLE, SEVERITY, } from '../../screens/alerts_detection_rules'; +import { PREVIEW_HEADER_SUBTITLE } from '../../screens/create_new_rule'; import { ABOUT_DETAILS, ABOUT_INVESTIGATION_NOTES, @@ -64,13 +65,16 @@ import { goToRuleDetails, waitForRulesTableToBeLoaded, } from '../../tasks/alerts_detection_rules'; +import { createCustomRuleActivated } from '../../tasks/api_calls/rules'; import { createTimeline } from '../../tasks/api_calls/timelines'; import { cleanKibana } from '../../tasks/common'; import { createAndActivateRule, fillAboutRuleAndContinue, fillDefineThresholdRuleAndContinue, + fillDefineThresholdRule, fillScheduleRuleAndContinue, + previewResults, selectThresholdRuleType, waitForAlertsToPopulate, waitForTheRuleToBeExecuted, @@ -92,12 +96,12 @@ describe('Detection rules, threshold', () => { createTimeline(newThresholdRule.timeline).then((response) => { rule.timeline.id = response.body.data.persistTimeline.timeline.savedObjectId; }); - }); - - it('Creates and activates a new threshold rule', () => { loginAndWaitForPageWithoutDateRange(DETECTIONS_URL); waitForAlertsPanelToBeLoaded(); waitForAlertsIndexToBeCreated(); + }); + + it('Creates and activates a new threshold rule', () => { goToManageAlertsDetectionRules(); waitForRulesTableToBeLoaded(); goToCreateNewRule(); @@ -175,4 +179,19 @@ describe('Detection rules, threshold', () => { cy.get(ALERT_RULE_SEVERITY).first().should('have.text', rule.severity.toLowerCase()); cy.get(ALERT_RULE_RISK_SCORE).first().should('have.text', rule.riskScore); }); + + it('Preview results', () => { + const previewRule = { ...newThresholdRule }; + previewRule.index!.push('.siem-signals*'); + + createCustomRuleActivated(newRule); + goToManageAlertsDetectionRules(); + waitForRulesTableToBeLoaded(); + goToCreateNewRule(); + selectThresholdRuleType(); + fillDefineThresholdRule(previewRule); + previewResults(); + + cy.get(PREVIEW_HEADER_SUBTITLE).should('have.text', '3 unique hits'); + }); }); diff --git a/x-pack/plugins/security_solution/cypress/screens/alerts_details.ts b/x-pack/plugins/security_solution/cypress/screens/alerts_details.ts index c18949224d4c0..12bef09b8356d 100644 --- a/x-pack/plugins/security_solution/cypress/screens/alerts_details.ts +++ b/x-pack/plugins/security_solution/cypress/screens/alerts_details.ts @@ -13,6 +13,18 @@ export const JSON_LINES = '.ace_line'; export const JSON_VIEW_TAB = '[data-test-subj="jsonViewTab"]'; +export const TABLE_CELL = '.euiTableRowCell'; + export const TABLE_TAB = '[data-test-subj="tableTab"]'; export const TABLE_ROWS = '.euiTableRow'; + +export const THREAT_CONTENT = '[data-test-subj^=draggable-content-threat]'; + +export const THREAT_DETAILS_VIEW = '[data-test-subj="threat-details-view-0"]'; + +export const THREAT_INTEL_TAB = '[data-test-subj="threatIntelTab"]'; + +export const THREAT_SUMMARY_VIEW = '[data-test-subj="threat-summary-view"]'; + +export const TITLE = '.euiTitle'; diff --git a/x-pack/plugins/security_solution/cypress/screens/create_new_rule.ts b/x-pack/plugins/security_solution/cypress/screens/create_new_rule.ts index db8d93dfbbef9..a580068b636e4 100644 --- a/x-pack/plugins/security_solution/cypress/screens/create_new_rule.ts +++ b/x-pack/plugins/security_solution/cypress/screens/create_new_rule.ts @@ -127,6 +127,8 @@ export const MITRE_ATTACK_ADD_TECHNIQUE_BUTTON = '[data-test-subj="addMitreAttac export const MITRE_ATTACK_ADD_SUBTECHNIQUE_BUTTON = '[data-test-subj="addMitreAttackSubtechnique"]'; +export const PREVIEW_HEADER_SUBTITLE = '[data-test-subj="header-panel-subtitle"]'; + export const QUERY_PREVIEW_BUTTON = '[data-test-subj="queryPreviewButton"]'; export const REFERENCE_URLS_INPUT = diff --git a/x-pack/plugins/security_solution/cypress/tasks/alerts_details.ts b/x-pack/plugins/security_solution/cypress/tasks/alerts_details.ts index 16c30afe117ef..44b30a7c89365 100644 --- a/x-pack/plugins/security_solution/cypress/tasks/alerts_details.ts +++ b/x-pack/plugins/security_solution/cypress/tasks/alerts_details.ts @@ -5,7 +5,12 @@ * 2.0. */ -import { JSON_CONTENT, JSON_VIEW_TAB, TABLE_TAB } from '../screens/alerts_details'; +import { + JSON_CONTENT, + JSON_VIEW_TAB, + TABLE_TAB, + THREAT_INTEL_TAB, +} from '../screens/alerts_details'; export const openJsonView = () => { cy.get(JSON_VIEW_TAB).click(); @@ -15,6 +20,10 @@ export const openTable = () => { cy.get(TABLE_TAB).click(); }; +export const openThreatIndicatorDetails = () => { + cy.get(THREAT_INTEL_TAB).click(); +}; + export const scrollJsonViewToBottom = () => { cy.get(JSON_CONTENT).click({ force: true }); cy.get(JSON_CONTENT).type('{pagedown}{pagedown}{pagedown}'); diff --git a/x-pack/plugins/security_solution/cypress/tasks/create_new_rule.ts b/x-pack/plugins/security_solution/cypress/tasks/create_new_rule.ts index cd342e9456906..9c15b1f03932d 100644 --- a/x-pack/plugins/security_solution/cypress/tasks/create_new_rule.ts +++ b/x-pack/plugins/security_solution/cypress/tasks/create_new_rule.ts @@ -260,12 +260,18 @@ export const fillScheduleRuleAndContinue = (rule: CustomRule | MachineLearningRu cy.get(LOOK_BACK_TIME_TYPE).select(rule.lookBack.timeType); }; -export const fillDefineThresholdRuleAndContinue = (rule: ThresholdRule) => { +export const fillDefineThresholdRule = (rule: ThresholdRule) => { const thresholdField = 0; const threshold = 1; cy.get(IMPORT_QUERY_FROM_SAVED_TIMELINE_LINK).click(); cy.get(TIMELINE(rule.timeline.id!)).click(); + cy.get(COMBO_BOX_CLEAR_BTN).click(); + + rule.index!.forEach((index) => { + cy.get(COMBO_BOX_INPUT).first().type(`${index}{enter}`); + }); + cy.get(CUSTOM_QUERY_INPUT).should('have.value', rule.customQuery); cy.get(THRESHOLD_INPUT_AREA) .find(INPUT) @@ -274,6 +280,24 @@ export const fillDefineThresholdRuleAndContinue = (rule: ThresholdRule) => { cy.get(THRESHOLD_FIELD_SELECTION).click({ force: true }); cy.wrap(inputs[threshold]).clear().type(rule.threshold); }); +}; + +export const fillDefineThresholdRuleAndContinue = (rule: ThresholdRule) => { + const thresholdField = 0; + const threshold = 1; + + const typeThresholdField = ($el: Cypress.ObjectLike) => cy.wrap($el).type(rule.thresholdField); + + cy.get(IMPORT_QUERY_FROM_SAVED_TIMELINE_LINK).click(); + cy.get(TIMELINE(rule.timeline.id!)).click(); + cy.get(CUSTOM_QUERY_INPUT).should('have.value', rule.customQuery); + cy.get(THRESHOLD_INPUT_AREA) + .find(INPUT) + .then((inputs) => { + cy.wrap(inputs[thresholdField]).pipe(typeThresholdField); + cy.get(THRESHOLD_FIELD_SELECTION).click({ force: true }); + cy.wrap(inputs[threshold]).clear().type(rule.threshold); + }); cy.get(DEFINE_CONTINUE_BUTTON).should('exist').click({ force: true }); cy.get(CUSTOM_QUERY_INPUT).should('not.exist'); @@ -478,6 +502,10 @@ export const selectThresholdRuleType = () => { cy.get(THRESHOLD_TYPE).click({ force: true }); }; +export const previewResults = () => { + cy.get(QUERY_PREVIEW_BUTTON).click(); +}; + export const waitForAlertsToPopulate = async (alertCountThreshold = 1) => { cy.waitUntil( () => { diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/event_details.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/event_details.tsx index 4e924402783c2..c4092214633e5 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/event_details.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/event_details.tsx @@ -136,6 +136,7 @@ const EventDetailsComponent: React.FC = ({ isAlert ? { id: EventsViewType.threatIntelView, + 'data-test-subj': 'threatIntelTab', name: `${i18n.THREAT_INTEL} (${threatCount})`, content: , } diff --git a/x-pack/plugins/security_solution/public/common/components/exceptions/viewer/exception_item/index.tsx b/x-pack/plugins/security_solution/public/common/components/exceptions/viewer/exception_item/index.tsx index 4b6ae29de3bbf..caff2be29ab2d 100644 --- a/x-pack/plugins/security_solution/public/common/components/exceptions/viewer/exception_item/index.tsx +++ b/x-pack/plugins/security_solution/public/common/components/exceptions/viewer/exception_item/index.tsx @@ -38,6 +38,7 @@ export interface ExceptionItemProps { onEditException: (item: ExceptionListItemSchema) => void; showName?: boolean; showModified?: boolean; + 'data-test-subj'?: string; } const ExceptionItemComponent = ({ @@ -48,6 +49,7 @@ const ExceptionItemComponent = ({ onEditException, showModified = false, showName = false, + 'data-test-subj': dataTestSubj, }: ExceptionItemProps): JSX.Element => { const [entryItems, setEntryItems] = useState([]); const [showComments, setShowComments] = useState(false); @@ -82,7 +84,7 @@ const ExceptionItemComponent = ({ }, [loadingItemIds, exceptionItem.id]); return ( - + diff --git a/x-pack/plugins/security_solution/public/common/components/ml/criteria/host_to_criteria.ts b/x-pack/plugins/security_solution/public/common/components/ml/criteria/host_to_criteria.ts index 19eae99757849..ff454da7b1fcd 100644 --- a/x-pack/plugins/security_solution/public/common/components/ml/criteria/host_to_criteria.ts +++ b/x-pack/plugins/security_solution/public/common/components/ml/criteria/host_to_criteria.ts @@ -9,6 +9,9 @@ import { HostItem } from '../../../../../common/search_strategy/security_solutio import { CriteriaFields } from '../types'; export const hostToCriteria = (hostItem: HostItem): CriteriaFields[] => { + if (hostItem == null) { + return []; + } if (hostItem.host != null && hostItem.host.name != null) { const criteria: CriteriaFields[] = [ { diff --git a/x-pack/plugins/security_solution/public/common/mock/endpoint/app_context_render.tsx b/x-pack/plugins/security_solution/public/common/mock/endpoint/app_context_render.tsx index 2d3a01f820b44..37e60775a7195 100644 --- a/x-pack/plugins/security_solution/public/common/mock/endpoint/app_context_render.tsx +++ b/x-pack/plugins/security_solution/public/common/mock/endpoint/app_context_render.tsx @@ -8,8 +8,7 @@ import React from 'react'; import { createMemoryHistory } from 'history'; import { render as reactRender, RenderOptions, RenderResult } from '@testing-library/react'; -import { Store } from 'redux'; - +import { Action, Reducer, Store } from 'redux'; import { coreMock } from '../../../../../../../src/core/public/mocks'; import { StartPlugins } from '../../../types'; import { depsStartMock } from './dependencies_start_mock'; @@ -20,6 +19,7 @@ import { AppRootProvider } from './app_root_provider'; import { managementMiddlewareFactory } from '../../../management/store/middleware'; import { createKibanaContextProviderMock } from '../../lib/kibana/kibana_react.mock'; import { SUB_PLUGINS_REDUCER, mockGlobalState, createSecuritySolutionStorageMock } from '..'; +import { ExperimentalFeatures } from '../../../../common/experimental_features'; type UiRender = (ui: React.ReactElement, options?: RenderOptions) => RenderResult; @@ -43,8 +43,42 @@ export interface AppContextTestRender { * endpoint runtime context environment */ render: UiRender; + + /** + * Set experimental features on/off. Calling this method updates the Store with the new values + * for the given feature flags + * @param flags + */ + setExperimentalFlag: (flags: Partial) => void; } +// Defined a private custom reducer that reacts to an action that enables us to updat the +// store with new values for experimental features/flags. Because the `action.type` is a `Symbol`, +// and its not exported the action can only be `dispatch`'d from this module +const UpdateExperimentalFeaturesTestActionType = Symbol('updateExperimentalFeaturesTestAction'); + +type UpdateExperimentalFeaturesTestAction = Action< + typeof UpdateExperimentalFeaturesTestActionType +> & { + payload: Partial; +}; + +const experimentalFeaturesReducer: Reducer = ( + state = mockGlobalState.app, + action +) => { + if (action.type === UpdateExperimentalFeaturesTestActionType) { + return { + ...state, + enableExperimental: { + ...state.enableExperimental!, + ...action.payload, + }, + }; + } + return state; +}; + /** * Creates a mocked endpoint app context custom renderer that can be used to render * component that depend upon the application's surrounding context providers. @@ -58,7 +92,14 @@ export const createAppRootMockRenderer = (): AppContextTestRender => { const middlewareSpy = createSpyMiddleware(); const { storage } = createSecuritySolutionStorageMock(); - const store = createStore(mockGlobalState, SUB_PLUGINS_REDUCER, kibanaObservable, storage, [ + const storeReducer = { + ...SUB_PLUGINS_REDUCER, + // This is ok here because the store created by this testing utility (see below) does + // not pull in the non-sub-plugin reducers + app: experimentalFeaturesReducer, + }; + + const store = createStore(mockGlobalState, storeReducer, kibanaObservable, storage, [ ...managementMiddlewareFactory(coreStart, depsStart), middlewareSpy.actionSpyMiddleware, ]); @@ -79,6 +120,13 @@ export const createAppRootMockRenderer = (): AppContextTestRender => { }); }; + const setExperimentalFlag: AppContextTestRender['setExperimentalFlag'] = (flags) => { + store.dispatch({ + type: UpdateExperimentalFeaturesTestActionType, + payload: flags, + }); + }; + return { store, history, @@ -87,5 +135,6 @@ export const createAppRootMockRenderer = (): AppContextTestRender => { middlewareSpy, AppWrapper, render, + setExperimentalFlag, }; }; diff --git a/x-pack/plugins/security_solution/public/common/mock/endpoint/http_handler_mock_factory.ts b/x-pack/plugins/security_solution/public/common/mock/endpoint/http_handler_mock_factory.ts new file mode 100644 index 0000000000000..9d12efca19aed --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/mock/endpoint/http_handler_mock_factory.ts @@ -0,0 +1,275 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/* eslint-disable @typescript-eslint/no-explicit-any */ + +import type { + HttpFetchOptions, + HttpFetchOptionsWithPath, + HttpHandler, + HttpStart, +} from 'kibana/public'; +import { extend } from 'lodash'; +import { act } from '@testing-library/react'; + +class ApiRouteNotMocked extends Error {} + +// Source: https://stackoverflow.com/a/43001581 +type Writeable = { -readonly [P in keyof T]: T[P] }; + +/** The callback that will be executed to retrieve an HTTP API response */ +export type ResponseProviderCallback any = (...args: any) => any> = F; + +/** + * Generic interface to facilitate defining the map of HTTP response provider callbacks + * that will be exposed when the mock is applied + * + * @example + * type FleetSetupResponseProvidersMock = ResponseProvidersInterface<{ + * fleetSetup: () => PostIngestSetupResponse; + * }>; + */ +export type ResponseProvidersInterface< + I extends Record = Record +> = I; + +type SingleResponseProvider< + F extends ResponseProviderCallback = ResponseProviderCallback +> = jest.MockedFunction & { + /** + * Delay responding to the HTTP call until this promise is resolved. Use it to introduce + * elongated delays in order to test intermediate UI states. + * + * @example + * apiMocks.responseProvider.someProvider.mockDelay + * // Delay this response by 1/2 second + * .mockImplementation( + * () => new Promise(r => setTimeout(r, 500)) + * ) + */ + mockDelay: jest.MockedFunction<() => Promise>; +}; + +/** + * The interface for a `core.http` set of mocked API responses. + */ +interface MockedApi { + /** + * Will return a promise that resolves when triggered APIs are all complete. This method uses + * React testing `act()` to await for the API calls, thus ensuring that the UI components are + * updated with any state that was affected by the returned API responses. + */ + waitForApi: () => Promise; + /** + * A object containing the list of API response provider functions that are used by the mocked API. + * These API response methods are wrapped in `jest.MockedFunction`, thus their implementation or + * returned values can be manipulated by each test case using the normal `jest.mock` interface. + * In addition to the normal `jest.Mock` properties and functions, an additional method is also + * available - `mockDelay()` - which can be used to delay the given response being returned by the + * associated HTTP method. + */ + responseProvider: Readonly< + { + [K in keyof R]: SingleResponseProvider; + } + >; +} + +type HttpMethods = keyof Pick< + HttpStart, + 'delete' | 'fetch' | 'get' | 'post' | 'put' | 'head' | 'patch' +>; + +const HTTP_METHODS: HttpMethods[] = ['delete', 'fetch', 'get', 'post', 'put', 'head', 'patch']; + +export type ApiHandlerMock = ( + http: jest.Mocked +) => MockedApi; + +interface RouteMock { + /** + * A legible identifier for the Mock. Will be used as the name in the `MockedApi` + * that is returned when the HTTP mock is applied + */ + id: keyof R; + method: HttpMethods; + path: string; + /** + * The handler for providing a response to for this API call. + * It should return the "raw" value, __NOT__ a `Promise` + */ + handler: (...args: Parameters) => any; + /** + * A function that returns a promise. The API response will be delayed until this promise is + * resolved. This can be helpful when wanting to test an intermediate UI state while the API + * call is inflight. + */ + delay?: () => Promise; +} + +export type ApiHandlerMockFactoryProps< + R extends ResponseProvidersInterface = ResponseProvidersInterface +> = Array>; +/** + * Returns a function that can be used to apply mocked responses to calls made via `core.http` + * methods during testing. + * + * @example + * + * const mockEpmApi = httpHandlerMockFactory<{ epmGetInfo: () => GetInfoResponse }>([ + * { + * id: 'epmGetInfo', + * method: 'get', + * path: '/api/fleet/epm/something', + * handler: () => 'returnValueHere' + * } + * ]); + * // In a test - usually in a `beforeEach()` + * let mockedApi; + * beforeEach(() => { + * mockedApi = mockEpmApi(core.http); + * }); + */ +export const httpHandlerMockFactory = ( + mocks: ApiHandlerMockFactoryProps +): ApiHandlerMock => { + return (http) => { + let inflightApiCalls = 0; + const apiDoneListeners: Array<() => void> = []; + const markApiCallAsHandled = async (delay?: RouteMock['delay']) => { + inflightApiCalls++; + + // If a delay was defined, then await that first + if (delay) { + await delay(); + } + + // We always wait at least 1ms + await new Promise((r) => setTimeout(r, 1)); + + inflightApiCalls--; + + // If no more pending API calls, then notify listeners + if (inflightApiCalls === 0 && apiDoneListeners.length > 0) { + apiDoneListeners.splice(0).forEach((listener) => listener()); + } + }; + + const responseProvider: MockedApi['responseProvider'] = mocks.reduce( + (providers, routeMock) => { + // FIXME: find a way to remove the ignore below. May need to limit the calling signature of `RouteMock['handler']` + // @ts-ignore + const routeResponseCallbackMock: SingleResponseProvider = jest.fn( + routeMock.handler + ); + routeResponseCallbackMock.mockDelay = jest.fn(routeMock.delay || (async () => {})); + + providers[routeMock.id] = routeResponseCallbackMock; + return providers; + }, + {} as Writeable['responseProvider']> + ); + + const mockedApiInterface: MockedApi = { + async waitForApi() { + await act(async () => { + await new Promise((resolve) => { + if (inflightApiCalls > 0) { + apiDoneListeners.push(resolve); + } else { + resolve(); + } + }); + }); + }, + responseProvider, + }; + + HTTP_METHODS.forEach((method) => { + const priorMockedFunction = http[method].getMockImplementation(); + const methodMocks = mocks.filter((mock) => mock.method === method); + + http[method].mockImplementation(async (...args) => { + const path = isHttpFetchOptionsWithPath(args[0]) ? args[0].path : args[0]; + const routeMock = methodMocks.find((handler) => handler.path === path); + + if (routeMock) { + markApiCallAsHandled(responseProvider[routeMock.id].mockDelay); + + await responseProvider[routeMock.id].mockDelay(); + + // Use the handler defined for the HTTP Mocked interface (not the one passed on input to + // the factory) for retrieving the response value because that one could have had its + // response value manipulated by the individual test case. + return responseProvider[routeMock.id](...args); + } else if (priorMockedFunction) { + return priorMockedFunction(...args); + } + + const err = new ApiRouteNotMocked(`API [${method.toUpperCase()} ${path}] is not MOCKED!`); + // eslint-disable-next-line no-console + console.error(err); + throw err; + }); + }); + + return mockedApiInterface; + }; +}; + +const isHttpFetchOptionsWithPath = ( + opt: string | HttpFetchOptions | HttpFetchOptionsWithPath +): opt is HttpFetchOptionsWithPath => { + return 'object' === typeof opt && 'path' in opt; +}; + +/** + * Compose a new API Handler mock based upon a list of one or more Api Handlers. + * Returns a new function (`ApiHandlerMock`) that applies all provided handler mocks to the `core.http` + * service while at the same time supporting a `waitForApi()` method that will wait all handlers. + * + * @example + * import { composeApiHandlerMocks } from './http_handler_mock_factory'; + * import { + * fleetSetupApiMock, + * agentsSetupApiMock, + * } from './setup'; + * + * // Create the new interface as an intersection of all other Api Handler Mocks + * type ComposedApiHandlerMocks = ReturnType & ReturnType + * + * const newComposedHandlerMock = composeApiHandlerMocks< + * ComposedApiHandlerMocks + * >([fleetSetupApiMock, agentsSetupApiMock]); + */ +export const composeHttpHandlerMocks = < + R extends ResponseProvidersInterface = ResponseProvidersInterface +>( + handlerMocks: ApiHandlerMock[] +): ApiHandlerMock => { + return (http) => { + const waitForApiHandlers: Array = []; + const mockedApiInterfaces: MockedApi = { + async waitForApi() { + await act(async () => + Promise.all(waitForApiHandlers.map((handlerWaitFor) => handlerWaitFor())).then(() => {}) + ); + }, + // Ignore here because we populate this object with the entries provided + // via the input argument `handlerMocks` + // @ts-ignore + responseProvider: {}, + }; + + handlerMocks.forEach((handlerMock) => { + const { waitForApi, ...otherInterfaceProps } = handlerMock(http); + extend(mockedApiInterfaces, otherInterfaceProps); + }); + + return mockedApiInterfaces; + }; +}; diff --git a/x-pack/plugins/security_solution/public/detections/components/host_isolation/index.tsx b/x-pack/plugins/security_solution/public/detections/components/host_isolation/index.tsx index 30ee7e77f3a7d..3897458e8459c 100644 --- a/x-pack/plugins/security_solution/public/detections/components/host_isolation/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/host_isolation/index.tsx @@ -21,7 +21,6 @@ import { import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; import { useHostIsolation } from '../../containers/detection_engine/alerts/use_host_isolation'; -import { TimelineEventsDetailsItem } from '../../../../common/search_strategy/timeline'; import { CANCEL, CASES_ASSOCIATED_WITH_ALERT, @@ -31,6 +30,9 @@ import { RETURN_TO_ALERT_DETAILS, } from './translations'; import { Maybe } from '../../../../../observability/common/typings'; +import { useCasesFromAlerts } from '../../containers/detection_engine/alerts/use_cases_from_alerts'; +import { CaseDetailsLink } from '../../../common/components/links'; +import { TimelineEventsDetailsItem } from '../../../../common/search_strategy'; export const HostIsolationPanel = React.memo( ({ @@ -59,7 +61,13 @@ export const HostIsolationPanel = React.memo( return findAlertRule ? findAlertRule[0] : ''; }, [details]); - const { loading, isolateHost } = useHostIsolation({ agentId, comment }); + const alertId = useMemo(() => { + const findAlertId = find({ category: '_id', field: '_id' }, details)?.values; + return findAlertId ? findAlertId[0] : ''; + }, [details]); + + const { caseIds } = useCasesFromAlerts({ alertId }); + const { loading, isolateHost } = useHostIsolation({ agentId, comment, caseIds }); const confirmHostIsolation = useCallback(async () => { const hostIsolated = await isolateHost(); @@ -68,8 +76,25 @@ export const HostIsolationPanel = React.memo( const backToAlertDetails = useCallback(() => cancelCallback(), [cancelCallback]); - // a placeholder until we get the case count returned from a new case route in a future pr - const caseCount: number = 0; + const casesList = useMemo( + () => + caseIds.map((id, index) => { + return ( +
  • + + + +
    • + ); + }), + [caseIds] + ); + + const caseCount: number = useMemo(() => caseIds.length, [caseIds]); const hostIsolated = useMemo(() => { return ( @@ -92,20 +117,13 @@ export const HostIsolationPanel = React.memo(

    -
      -
    • - -
      • -
    +
      {casesList}
     )} @@ -121,7 +139,7 @@ export const HostIsolationPanel = React.memo(     ); - }, [backToAlertDetails, hostName]); + }, [backToAlertDetails, hostName, caseCount, casesList]); const hostNotIsolated = useMemo(() => { return ( @@ -137,7 +155,7 @@ export const HostIsolationPanel = React.memo( cases: ( {caseCount} - {CASES_ASSOCIATED_WITH_ALERT} + {CASES_ASSOCIATED_WITH_ALERT(caseCount)} {alertRule} ), @@ -171,7 +189,15 @@ export const HostIsolationPanel = React.memo(     ); - }, [alertRule, backToAlertDetails, comment, confirmHostIsolation, hostName, loading]); + }, [ + alertRule, + backToAlertDetails, + comment, + confirmHostIsolation, + hostName, + loading, + caseCount, + ]); return isIsolated ? hostIsolated : hostNotIsolated; } diff --git a/x-pack/plugins/security_solution/public/detections/components/host_isolation/translations.ts b/x-pack/plugins/security_solution/public/detections/components/host_isolation/translations.ts index 97a1a278952a6..8d6334f6c340d 100644 --- a/x-pack/plugins/security_solution/public/detections/components/host_isolation/translations.ts +++ b/x-pack/plugins/security_solution/public/detections/components/host_isolation/translations.ts @@ -31,12 +31,14 @@ export const CONFIRM = i18n.translate('xpack.securitySolution.endpoint.hostIsola defaultMessage: 'Confirm', }); -export const CASES_ASSOCIATED_WITH_ALERT = i18n.translate( - 'xpack.securitySolution.endpoint.hostIsolation.isolateHost.casesAssociatedWihtAlert', - { - defaultMessage: ' cases associated with the rule ', - } -); +export const CASES_ASSOCIATED_WITH_ALERT = (caseCount: number): string => + i18n.translate( + 'xpack.securitySolution.endpoint.hostIsolation.isolateHost.casesAssociatedWithAlert', + { + defaultMessage: ' {caseCount, plural, one {case} other {cases}} associated with the rule ', + values: { caseCount }, + } + ); export const RETURN_TO_ALERT_DETAILS = i18n.translate( 'xpack.securitySolution.endpoint.hostIsolation.returnToAlertDetails', diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/__mocks__/api.ts b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/__mocks__/api.ts index e1f5b53e2f4c3..ea64f39226cd2 100644 --- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/__mocks__/api.ts +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/__mocks__/api.ts @@ -5,8 +5,15 @@ * 2.0. */ -import { QueryAlerts, AlertSearchResponse, BasicSignals, AlertsIndex, Privilege } from '../types'; -import { alertsMock, mockSignalIndex, mockUserPrivilege } from '../mock'; +import { + QueryAlerts, + AlertSearchResponse, + BasicSignals, + AlertsIndex, + Privilege, + CasesFromAlertsResponse, +} from '../types'; +import { alertsMock, mockSignalIndex, mockUserPrivilege, mockCaseIdsFromAlertId } from '../mock'; export const fetchQueryAlerts = async ({ query, @@ -22,3 +29,9 @@ export const getUserPrivilege = async ({ signal }: BasicSignals): Promise => Promise.resolve(mockSignalIndex); + +export const getCaseIdsFromAlertId = async ({ + alertId, +}: { + alertId: string; +}): Promise => Promise.resolve(mockCaseIdsFromAlertId); diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/api.test.ts b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/api.test.ts index 82f275f7dc9ba..9aa5cfd229292 100644 --- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/api.test.ts +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/api.test.ts @@ -12,6 +12,7 @@ import { mockStatusAlertQuery, mockSignalIndex, mockUserPrivilege, + mockHostIsolation, } from './mock'; import { fetchQueryAlerts, @@ -19,6 +20,7 @@ import { getSignalIndex, getUserPrivilege, createSignalIndex, + createHostIsolation, } from './api'; const abortCtrl = new AbortController(); @@ -163,4 +165,33 @@ describe('Detections Alerts API', () => { expect(alertsResp).toEqual(mockSignalIndex); }); }); + + describe('createHostIsolation', () => { + beforeEach(() => { + fetchMock.mockClear(); + fetchMock.mockResolvedValue(mockHostIsolation); + }); + + test('check parameter url', async () => { + await createHostIsolation({ + agentId: 'fd8a122b-4c54-4c05-b295-e5f8381fc59d', + comment: 'commento', + caseIds: ['88c04a90-b19c-11eb-b838-bf3c7840b969'], + }); + expect(fetchMock).toHaveBeenCalledWith('/api/endpoint/isolate', { + method: 'POST', + body: + '{"agent_ids":["fd8a122b-4c54-4c05-b295-e5f8381fc59d"],"comment":"commento","case_ids":["88c04a90-b19c-11eb-b838-bf3c7840b969"]}', + }); + }); + + test('happy path', async () => { + const hostIsolationResponse = await createHostIsolation({ + agentId: 'fd8a122b-4c54-4c05-b295-e5f8381fc59d', + comment: 'commento', + caseIds: ['88c04a90-b19c-11eb-b838-bf3c7840b969'], + }); + expect(hostIsolationResponse).toEqual(mockHostIsolation); + }); + }); }); diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/api.ts b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/api.ts index dbcb11383432f..300005b23caaa 100644 --- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/api.ts +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/api.ts @@ -6,6 +6,7 @@ */ import { UpdateDocumentByQueryResponse } from 'elasticsearch'; +import { getCasesFromAlertsUrl } from '../../../../../../cases/common'; import { HostIsolationResponse } from '../../../../../common/endpoint/types'; import { DETECTION_ENGINE_QUERY_SIGNALS_URL, @@ -22,6 +23,7 @@ import { AlertSearchResponse, AlertsIndex, UpdateAlertStatusProps, + CasesFromAlertsResponse, } from './types'; /** @@ -109,20 +111,38 @@ export const createSignalIndex = async ({ signal }: BasicSignals): Promise => KibanaServices.get().http.fetch(ISOLATE_HOST_ROUTE, { method: 'POST', body: JSON.stringify({ agent_ids: [agentId], comment, + case_ids: caseIds, }), }); + +/** + * Get list of associated case ids from alert id + * + * @param alert id + */ +export const getCaseIdsFromAlertId = async ({ + alertId, +}: { + alertId: string; +}): Promise => + KibanaServices.get().http.fetch(getCasesFromAlertsUrl(alertId), { + method: 'get', + }); diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/mock.ts b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/mock.ts index 18651063df8ca..69358958a395c 100644 --- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/mock.ts +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/mock.ts @@ -5,7 +5,8 @@ * 2.0. */ -import { AlertSearchResponse, AlertsIndex, Privilege } from './types'; +import { HostIsolationResponse } from '../../../../../common/endpoint/types/actions'; +import { AlertSearchResponse, AlertsIndex, Privilege, CasesFromAlertsResponse } from './types'; export const alertsMock: AlertSearchResponse = { took: 7, @@ -1039,3 +1040,12 @@ export const mockUserPrivilege: Privilege = { is_authenticated: true, has_encryption_key: true, }; + +export const mockHostIsolation: HostIsolationResponse = { + action: '713085d6-ab45-4e9e-b41d-96563cafdd97', +}; + +export const mockCaseIdsFromAlertId: CasesFromAlertsResponse = [ + '818601a0-b26b-11eb-8759-6b318e8cf4bc', + '8a774850-b26b-11eb-8759-6b318e8cf4bc', +]; diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/translations.ts b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/translations.ts index 2998c97376c26..ed6a22375a776 100644 --- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/translations.ts +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/translations.ts @@ -32,3 +32,8 @@ export const HOST_ISOLATION_FAILURE = i18n.translate( 'xpack.securitySolution.endpoint.hostIsolation.failedToIsolate.title', { defaultMessage: 'Failed to isolate host' } ); + +export const CASES_FROM_ALERTS_FAILURE = i18n.translate( + 'xpack.securitySolution.endpoint.hostIsolation.casesFromAlerts.title', + { defaultMessage: 'Failed to find associated cases' } +); diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/types.ts b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/types.ts index 26108ca939a57..52b477d95076b 100644 --- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/types.ts +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/types.ts @@ -48,6 +48,8 @@ export interface AlertsIndex { index_mapping_outdated: boolean; } +export type CasesFromAlertsResponse = string[]; + export interface Privilege { username: string; has_all_requested: boolean; diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_cases_from_alerts.test.tsx b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_cases_from_alerts.test.tsx new file mode 100644 index 0000000000000..0867fb001051a --- /dev/null +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_cases_from_alerts.test.tsx @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { renderHook } from '@testing-library/react-hooks'; +import { useCasesFromAlerts } from './use_cases_from_alerts'; +import * as api from './api'; +import { useAppToasts } from '../../../../common/hooks/use_app_toasts'; +import { useAppToastsMock } from '../../../../common/hooks/use_app_toasts.mock'; +import { mockCaseIdsFromAlertId } from './mock'; + +jest.mock('./api'); +jest.mock('../../../../common/hooks/use_app_toasts'); + +describe('useCasesFromAlerts hook', () => { + let appToastsMock: jest.Mocked>; + beforeEach(() => { + jest.resetAllMocks(); + appToastsMock = useAppToastsMock.create(); + (useAppToasts as jest.Mock).mockReturnValue(appToastsMock); + }); + afterEach(() => { + jest.restoreAllMocks(); + }); + + it('returns an array of caseIds', async () => { + const spyOnCases = jest.spyOn(api, 'getCaseIdsFromAlertId'); + const { result, waitForNextUpdate } = renderHook(() => + useCasesFromAlerts({ alertId: 'anAlertId' }) + ); + await waitForNextUpdate(); + expect(spyOnCases).toHaveBeenCalledTimes(1); + expect(result.current).toEqual({ + loading: false, + caseIds: mockCaseIdsFromAlertId, + }); + }); +}); diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_cases_from_alerts.tsx b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_cases_from_alerts.tsx new file mode 100644 index 0000000000000..fb130eb744700 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_cases_from_alerts.tsx @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isEmpty } from 'lodash'; +import { useEffect, useState } from 'react'; +import { useAppToasts } from '../../../../common/hooks/use_app_toasts'; +import { getCaseIdsFromAlertId } from './api'; +import { CASES_FROM_ALERTS_FAILURE } from './translations'; +import { CasesFromAlertsResponse } from './types'; + +interface CasesFromAlertsStatus { + loading: boolean; + caseIds: CasesFromAlertsResponse; +} + +export const useCasesFromAlerts = ({ alertId }: { alertId: string }): CasesFromAlertsStatus => { + const [loading, setLoading] = useState(false); + const [cases, setCases] = useState([]); + const { addError } = useAppToasts(); + + useEffect(() => { + // isMounted tracks if a component is mounted before changing state + let isMounted = true; + setLoading(true); + const fetchData = async () => { + try { + const casesResponse = await getCaseIdsFromAlertId({ alertId }); + if (isMounted) { + setCases(casesResponse); + } + } catch (error) { + addError(error.message, { title: CASES_FROM_ALERTS_FAILURE }); + } + if (isMounted) { + setLoading(false); + } + }; + if (!isEmpty(alertId)) { + fetchData(); + } + return () => { + // updates to show component is unmounted + isMounted = false; + }; + }, [alertId, addError]); + return { loading, caseIds: cases }; +}; diff --git a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_host_isolation.tsx b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_host_isolation.tsx index 684bc6af5d2c7..ad3c6e91c03fe 100644 --- a/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_host_isolation.tsx +++ b/x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_host_isolation.tsx @@ -18,11 +18,13 @@ interface HostIsolationStatus { interface UseHostIsolationProps { agentId: string; comment: string; + caseIds?: string[]; } export const useHostIsolation = ({ agentId, comment, + caseIds, }: UseHostIsolationProps): HostIsolationStatus => { const [loading, setLoading] = useState(false); const { addError } = useAppToasts(); @@ -30,7 +32,7 @@ export const useHostIsolation = ({ const isolateHost = useCallback(async () => { try { setLoading(true); - const isolationStatus = await createHostIsolation({ agentId, comment }); + const isolationStatus = await createHostIsolation({ agentId, comment, caseIds }); setLoading(false); return isolationStatus.action ? true : false; } catch (error) { @@ -38,6 +40,6 @@ export const useHostIsolation = ({ addError(error.message, { title: HOST_ISOLATION_FAILURE }); return false; } - }, [agentId, comment, addError]); + }, [agentId, comment, caseIds, addError]); return { loading, isolateHost }; }; diff --git a/x-pack/plugins/security_solution/public/hosts/containers/hosts/details/index.tsx b/x-pack/plugins/security_solution/public/hosts/containers/hosts/details/index.tsx index dd55bdb4c6948..a0f4386be59a4 100644 --- a/x-pack/plugins/security_solution/public/hosts/containers/hosts/details/index.tsx +++ b/x-pack/plugins/security_solution/public/hosts/containers/hosts/details/index.tsx @@ -145,14 +145,14 @@ export const useHostDetails = ({ } return prevRequest; }); - return () => { - searchSubscription$.current.unsubscribe(); - abortCtrl.current.abort(); - }; }, [endDate, hostName, indexNames, startDate]); useEffect(() => { hostDetailsSearch(hostDetailsRequest); + return () => { + searchSubscription$.current.unsubscribe(); + abortCtrl.current.abort(); + }; }, [hostDetailsRequest, hostDetailsSearch]); return [loading, hostDetailsResponse]; diff --git a/x-pack/plugins/security_solution/public/hosts/pages/details/index.tsx b/x-pack/plugins/security_solution/public/hosts/pages/details/index.tsx index 1ff4abb78b210..d88e4f048f917 100644 --- a/x-pack/plugins/security_solution/public/hosts/pages/details/index.tsx +++ b/x-pack/plugins/security_solution/public/hosts/pages/details/index.tsx @@ -50,6 +50,9 @@ import { timelineDefaults } from '../../../timelines/store/timeline/defaults'; import { useSourcererScope } from '../../../common/containers/sourcerer'; import { useDeepEqualSelector, useShallowEqualSelector } from '../../../common/hooks/use_selector'; import { useHostDetails } from '../../containers/hosts/details'; +import { manageQuery } from '../../../common/components/page/manage_query'; + +const HostOverviewManage = manageQuery(HostOverview); const HostDetailsComponent: React.FC = ({ detailName, hostDetailsPagePath }) => { const dispatch = useDispatch(); @@ -93,11 +96,12 @@ const HostDetailsComponent: React.FC = ({ detailName, hostDeta ); const { docValueFields, indicesExist, indexPattern, selectedPatterns } = useSourcererScope(); - const [loading, { hostDetails: hostOverview, id }] = useHostDetails({ + const [loading, { hostDetails: hostOverview, id, refetch }] = useHostDetails({ endDate: to, startDate: from, hostName: detailName, indexNames: selectedPatterns, + skip: selectedPatterns.length === 0, }); const filterQuery = convertToBuildEsQuery({ config: esQuery.getEsQueryConfig(kibana.services.uiSettings), @@ -141,7 +145,7 @@ const HostDetailsComponent: React.FC = ({ detailName, hostDeta skip={isInitializing} > {({ isLoadingAnomaliesData, anomaliesData }) => ( - = ({ detailName, hostDeta to: fromTo.to, }); }} + setQuery={setQuery} + refetch={refetch} /> )} diff --git a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/store/utils.test.ts b/x-pack/plugins/security_solution/public/management/common/utils.test.ts similarity index 79% rename from x-pack/plugins/security_solution/public/management/pages/trusted_apps/store/utils.test.ts rename to x-pack/plugins/security_solution/public/management/common/utils.test.ts index 4193d9c37c8f2..59455ccd6bb04 100644 --- a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/store/utils.test.ts +++ b/x-pack/plugins/security_solution/public/management/common/utils.test.ts @@ -8,27 +8,33 @@ import { parseQueryFilterToKQL } from './utils'; describe('utils', () => { + const searchableFields = [`name`, `description`, `entries.value`, `entries.entries.value`]; describe('parseQueryFilterToKQL', () => { it('should parse simple query without term', () => { - expect(parseQueryFilterToKQL('')).toBe(''); + expect(parseQueryFilterToKQL('', searchableFields)).toBe(''); }); it('should parse simple query with term', () => { - expect(parseQueryFilterToKQL('simpleQuery')).toBe( + expect(parseQueryFilterToKQL('simpleQuery', searchableFields)).toBe( 'exception-list-agnostic.attributes.name:(*simpleQuery*) OR exception-list-agnostic.attributes.description:(*simpleQuery*) OR exception-list-agnostic.attributes.entries.value:(*simpleQuery*) OR exception-list-agnostic.attributes.entries.entries.value:(*simpleQuery*)' ); }); it('should parse complex query with term', () => { - expect(parseQueryFilterToKQL('complex query')).toBe( + expect(parseQueryFilterToKQL('complex query', searchableFields)).toBe( 'exception-list-agnostic.attributes.name:(*complex*query*) OR exception-list-agnostic.attributes.description:(*complex*query*) OR exception-list-agnostic.attributes.entries.value:(*complex*query*) OR exception-list-agnostic.attributes.entries.entries.value:(*complex*query*)' ); }); it('should parse complex query with colon and backslash chars term', () => { - expect(parseQueryFilterToKQL('C:\\tmpes')).toBe( + expect(parseQueryFilterToKQL('C:\\tmpes', searchableFields)).toBe( 'exception-list-agnostic.attributes.name:(*C\\:\\\\tmpes*) OR exception-list-agnostic.attributes.description:(*C\\:\\\\tmpes*) OR exception-list-agnostic.attributes.entries.value:(*C\\:\\\\tmpes*) OR exception-list-agnostic.attributes.entries.entries.value:(*C\\:\\\\tmpes*)' ); }); it('should parse complex query with special chars term', () => { - expect(parseQueryFilterToKQL("this'is%&query{}[]!¿?with.,-+`´special<>ºª@#|·chars")).toBe( + expect( + parseQueryFilterToKQL( + "this'is%&query{}[]!¿?with.,-+`´special<>ºª@#|·chars", + searchableFields + ) + ).toBe( "exception-list-agnostic.attributes.name:(*this'is%&query\\{\\}[]!¿?with.,-+`´special\\<\\>ºª@#|·chars*) OR exception-list-agnostic.attributes.description:(*this'is%&query\\{\\}[]!¿?with.,-+`´special\\<\\>ºª@#|·chars*) OR exception-list-agnostic.attributes.entries.value:(*this'is%&query\\{\\}[]!¿?with.,-+`´special\\<\\>ºª@#|·chars*) OR exception-list-agnostic.attributes.entries.entries.value:(*this'is%&query\\{\\}[]!¿?with.,-+`´special\\<\\>ºª@#|·chars*)" ); }); diff --git a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/store/utils.ts b/x-pack/plugins/security_solution/public/management/common/utils.ts similarity index 78% rename from x-pack/plugins/security_solution/public/management/pages/trusted_apps/store/utils.ts rename to x-pack/plugins/security_solution/public/management/common/utils.ts index b14942e626d1b..c8cf761ccaf86 100644 --- a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/store/utils.ts +++ b/x-pack/plugins/security_solution/public/management/common/utils.ts @@ -5,9 +5,9 @@ * 2.0. */ -export const parseQueryFilterToKQL = (filter: string): string => { +export const parseQueryFilterToKQL = (filter: string, fields: Readonly): string => { if (!filter) return ''; - const kuery = [`name`, `description`, `entries.value`, `entries.entries.value`] + const kuery = fields .map( (field) => `exception-list-agnostic.attributes.${field}:(*${filter diff --git a/x-pack/plugins/security_solution/public/management/components/paginated_content/paginated_content.tsx b/x-pack/plugins/security_solution/public/management/components/paginated_content/paginated_content.tsx index ea97ce5e30706..b3d1f21649d33 100644 --- a/x-pack/plugins/security_solution/public/management/components/paginated_content/paginated_content.tsx +++ b/x-pack/plugins/security_solution/public/management/components/paginated_content/paginated_content.tsx @@ -106,15 +106,17 @@ const DefaultNoItemsFound = memo(() => { DefaultNoItemsFound.displayName = 'DefaultNoItemsFound'; -const ErrorMessage = memo<{ message: string }>(({ message }) => { - return ( - - - {message} - - - ); -}); +const ErrorMessage = memo<{ message: string; 'data-test-subj'?: string }>( + ({ message, 'data-test-subj': dataTestSubj }) => { + return ( + + + {message} + + + ); + } +); ErrorMessage.displayName = 'ErrorMessage'; @@ -166,7 +168,11 @@ export const PaginatedContent = memo( const generatedBodyItemContent = useMemo(() => { if (error) { - return 'string' === typeof error ? : error; + return 'string' === typeof error ? ( + + ) : ( + error + ); } // This casting here is needed in order to avoid the following a TS error (TS2322) @@ -199,11 +205,27 @@ export const PaginatedContent = memo( } return noItemsMessage || ; - }, [ItemComponent, error, itemComponentProps, itemId, itemKeys, items, noItemsMessage]); + }, [ + ItemComponent, + error, + getTestId, + itemComponentProps, + itemId, + itemKeys, + items, + noItemsMessage, + ]); return ( - {loading && } + {loading && ( + + )}
    diff --git a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/view/components/search_bar/index.test.tsx b/x-pack/plugins/security_solution/public/management/components/search_bar/index.test.tsx similarity index 88% rename from x-pack/plugins/security_solution/public/management/pages/trusted_apps/view/components/search_bar/index.test.tsx rename to x-pack/plugins/security_solution/public/management/components/search_bar/index.test.tsx index f12c979c2afe4..6daea8e53282d 100644 --- a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/view/components/search_bar/index.test.tsx +++ b/x-pack/plugins/security_solution/public/management/components/search_bar/index.test.tsx @@ -28,7 +28,7 @@ describe('Search bar', () => { it('should have a default value', () => { const expectedDefaultValue = 'this is a default value'; const element = mount(getElement(expectedDefaultValue)); - const defaultValue = element.find('[data-test-subj="trustedAppSearchField"]').first().props() + const defaultValue = element.find('[data-test-subj="searchField"]').first().props() .defaultValue; expect(defaultValue).toBe(expectedDefaultValue); }); @@ -38,7 +38,7 @@ describe('Search bar', () => { const element = mount(getElement()); expect(onSearchMock).toHaveBeenCalledTimes(0); const searchFieldProps = element - .find('[data-test-subj="trustedAppSearchField"]') + .find('[data-test-subj="searchField"]') .first() .props() as EuiFieldSearchPropsFake; @@ -53,7 +53,7 @@ describe('Search bar', () => { const element = mount(getElement(expectedDefaultValue)); expect(onSearchMock).toHaveBeenCalledTimes(0); - element.find('[data-test-subj="trustedAppSearchButton"]').first().simulate('click'); + element.find('[data-test-subj="searchButton"]').first().simulate('click'); expect(onSearchMock).toHaveBeenCalledTimes(1); expect(onSearchMock).toHaveBeenCalledWith(expectedDefaultValue); }); diff --git a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/view/components/search_bar/index.tsx b/x-pack/plugins/security_solution/public/management/components/search_bar/index.tsx similarity index 78% rename from x-pack/plugins/security_solution/public/management/pages/trusted_apps/view/components/search_bar/index.tsx rename to x-pack/plugins/security_solution/public/management/components/search_bar/index.tsx index 6fc6b2af4d585..0d4fcf8fec87b 100644 --- a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/view/components/search_bar/index.tsx +++ b/x-pack/plugins/security_solution/public/management/components/search_bar/index.tsx @@ -27,23 +27,20 @@ export const SearchBar = memo(({ defaultValue = '', onSearch }) - + - {i18n.translate('xpack.securitySolution.trustedapps.list.search.button', { + {i18n.translate('xpack.securitySolution.management.search.button', { defaultMessage: 'Refresh', })} diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.tsx b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.tsx index d28bf6b38fd31..f654efdd89ce1 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.tsx @@ -321,7 +321,7 @@ export const EndpointList = () => { render: (hostStatus: HostInfo['host_status']) => { return ( diff --git a/x-pack/plugins/security_solution/public/management/pages/event_filters/constants.ts b/x-pack/plugins/security_solution/public/management/pages/event_filters/constants.ts index 882b964d54bb5..5d600f471994b 100644 --- a/x-pack/plugins/security_solution/public/management/pages/event_filters/constants.ts +++ b/x-pack/plugins/security_solution/public/management/pages/event_filters/constants.ts @@ -24,4 +24,11 @@ export const EVENT_FILTER_LIST = { type: EVENT_FILTER_LIST_TYPE, }; +export const SEARCHABLE_FIELDS: Readonly = [ + `name`, + `entries.value`, + `entries.entries.value`, + `comments.comment`, +]; + export { ENDPOINT_EVENT_FILTERS_LIST_ID, EXCEPTION_LIST_URL, EXCEPTION_LIST_ITEM_URL }; diff --git a/x-pack/plugins/security_solution/public/management/pages/event_filters/index.tsx b/x-pack/plugins/security_solution/public/management/pages/event_filters/index.tsx index 7d2cb526367c5..86c2f2364961d 100644 --- a/x-pack/plugins/security_solution/public/management/pages/event_filters/index.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/event_filters/index.tsx @@ -19,4 +19,3 @@ export const EventFiltersContainer = () => { ); }; -export { EventFiltersService } from './types'; diff --git a/x-pack/plugins/security_solution/public/management/pages/event_filters/service/index.ts b/x-pack/plugins/security_solution/public/management/pages/event_filters/service/index.ts index 0c01cbfc6a24f..6a95ac5c15e83 100644 --- a/x-pack/plugins/security_solution/public/management/pages/event_filters/service/index.ts +++ b/x-pack/plugins/security_solution/public/management/pages/event_filters/service/index.ts @@ -48,11 +48,13 @@ export class EventFiltersHttpService implements EventFiltersService { page, sortField, sortOrder, + filter, }: Partial<{ page: number; perPage: number; sortField: string; sortOrder: string; + filter: string; }> = {}): Promise { const http = await this.httpWrapper(); return http.get(`${EXCEPTION_LIST_ITEM_URL}/_find`, { @@ -63,6 +65,7 @@ export class EventFiltersHttpService implements EventFiltersService { sort_order: sortOrder, list_id: [ENDPOINT_EVENT_FILTERS_LIST_ID], namespace_type: ['agnostic'], + filter, }, }); } @@ -89,4 +92,13 @@ export class EventFiltersHttpService implements EventFiltersService { body: JSON.stringify(exception), }); } + + async deleteOne(id: string): Promise { + return (await this.httpWrapper()).delete(EXCEPTION_LIST_ITEM_URL, { + query: { + id, + namespace_type: 'agnostic', + }, + }); + } } diff --git a/x-pack/plugins/security_solution/public/management/pages/event_filters/state/index.ts b/x-pack/plugins/security_solution/public/management/pages/event_filters/state/index.ts deleted file mode 100644 index 4a7788e0b9225..0000000000000 --- a/x-pack/plugins/security_solution/public/management/pages/event_filters/state/index.ts +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ExceptionListItemSchema } from '../../../../shared_imports'; -import { AsyncResourceState } from '../../../state/async_resource_state'; -import { FoundExceptionListItemSchema } from '../../../../../../lists/common/schemas'; -import { - EventFiltersForm, - EventFiltersPageLocation, - EventFiltersServiceGetListOptions, -} from '../types'; - -export interface EventFiltersListPageState { - entries: ExceptionListItemSchema[]; - form: EventFiltersForm; - location: EventFiltersPageLocation; - /** State for the Event Filters List page */ - listPage: { - active: boolean; - forceRefresh: boolean; - data: AsyncResourceState<{ - /** The query that was used to retrieve the data */ - query: EventFiltersServiceGetListOptions; - /** The data retrieved from the API */ - content: FoundExceptionListItemSchema; - }>; - /** tracks if the overall list (not filtered or with invalid page numbers) contains data */ - dataExist: AsyncResourceState; - }; -} diff --git a/x-pack/plugins/security_solution/public/management/pages/event_filters/store/action.ts b/x-pack/plugins/security_solution/public/management/pages/event_filters/store/action.ts index 76cb6c570221c..4ae90e7abba90 100644 --- a/x-pack/plugins/security_solution/public/management/pages/event_filters/store/action.ts +++ b/x-pack/plugins/security_solution/public/management/pages/event_filters/store/action.ts @@ -12,11 +12,7 @@ import { UpdateExceptionListItemSchema, } from '../../../../shared_imports'; import { AsyncResourceState } from '../../../state/async_resource_state'; -import { EventFiltersListPageState } from '../state'; - -export type EventFiltersListPageStateChanged = Action<'eventFiltersListPageStateChanged'> & { - payload: EventFiltersListPageState['listPage']; -}; +import { EventFiltersListPageState } from '../types'; export type EventFiltersListPageDataChanged = Action<'eventFiltersListPageDataChanged'> & { payload: EventFiltersListPageState['listPage']['data']; @@ -26,6 +22,18 @@ export type EventFiltersListPageDataExistsChanged = Action<'eventFiltersListPage payload: EventFiltersListPageState['listPage']['dataExist']; }; +export type EventFilterForDeletion = Action<'eventFilterForDeletion'> & { + payload: ExceptionListItemSchema; +}; + +export type EventFilterDeletionReset = Action<'eventFilterDeletionReset'>; + +export type EventFilterDeleteSubmit = Action<'eventFilterDeleteSubmit'>; + +export type EventFilterDeleteStatusChanged = Action<'eventFilterDeleteStatusChanged'> & { + payload: EventFiltersListPageState['listPage']['deletion']['status']; +}; + export type EventFiltersInitForm = Action<'eventFiltersInitForm'> & { payload: { entry: UpdateExceptionListItemSchema | CreateExceptionListItemSchema; @@ -59,7 +67,6 @@ export type EventFiltersFormStateChanged = Action<'eventFiltersFormStateChanged' }; export type EventFiltersPageAction = - | EventFiltersListPageStateChanged | EventFiltersListPageDataChanged | EventFiltersListPageDataExistsChanged | EventFiltersInitForm @@ -70,4 +77,8 @@ export type EventFiltersPageAction = | EventFiltersCreateStart | EventFiltersCreateSuccess | EventFiltersCreateError - | EventFiltersFormStateChanged; + | EventFiltersFormStateChanged + | EventFilterForDeletion + | EventFilterDeletionReset + | EventFilterDeleteSubmit + | EventFilterDeleteStatusChanged; diff --git a/x-pack/plugins/security_solution/public/management/pages/event_filters/store/builders.ts b/x-pack/plugins/security_solution/public/management/pages/event_filters/store/builders.ts index 460d5c2ce7398..30722a33270ee 100644 --- a/x-pack/plugins/security_solution/public/management/pages/event_filters/store/builders.ts +++ b/x-pack/plugins/security_solution/public/management/pages/event_filters/store/builders.ts @@ -5,8 +5,9 @@ * 2.0. */ -import { EventFiltersListPageState } from '../state'; import { MANAGEMENT_DEFAULT_PAGE, MANAGEMENT_DEFAULT_PAGE_SIZE } from '../../../common/constants'; +import { EventFiltersListPageState } from '../types'; +import { createLoadedResourceState, createUninitialisedResourceState } from '../../../state'; export const initialEventFiltersPageState = (): EventFiltersListPageState => ({ entries: [], @@ -16,7 +17,7 @@ export const initialEventFiltersPageState = (): EventFiltersListPageState => ({ hasItemsError: false, hasOSError: false, newComment: '', - submissionResourceState: { type: 'UninitialisedResourceState' }, + submissionResourceState: createUninitialisedResourceState(), }, location: { page_index: MANAGEMENT_DEFAULT_PAGE, @@ -26,8 +27,12 @@ export const initialEventFiltersPageState = (): EventFiltersListPageState => ({ listPage: { active: false, forceRefresh: false, - data: { type: 'UninitialisedResourceState' }, - /** We started off assuming data exists, until we can confirm othewise */ - dataExist: { type: 'LoadedResourceState', data: true }, + data: createUninitialisedResourceState(), + /** We started off assuming data exists, until we can confirm otherwise */ + dataExist: createLoadedResourceState(true), + deletion: { + item: undefined, + status: createUninitialisedResourceState(), + }, }, }); diff --git a/x-pack/plugins/security_solution/public/management/pages/event_filters/store/middleware.test.ts b/x-pack/plugins/security_solution/public/management/pages/event_filters/store/middleware.test.ts index 2151762b04d03..b55a32a937c45 100644 --- a/x-pack/plugins/security_solution/public/management/pages/event_filters/store/middleware.test.ts +++ b/x-pack/plugins/security_solution/public/management/pages/event_filters/store/middleware.test.ts @@ -15,19 +15,20 @@ import { AppAction } from '../../../../common/store/actions'; import { createEventFiltersPageMiddleware } from './middleware'; import { eventFiltersPageReducer } from './reducer'; -import { EventFiltersListPageState } from '../state'; import { initialEventFiltersPageState } from './builders'; import { getInitialExceptionFromEvent } from './utils'; import { createdEventFilterEntryMock, ecsEventMock } from '../test_utils'; -import { EventFiltersService } from '../types'; - -const initialState: EventFiltersListPageState = initialEventFiltersPageState(); +import { EventFiltersListPageState, EventFiltersService } from '../types'; +import { getFoundExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/found_exception_list_item_schema.mock'; +import { isFailedResourceState, isLoadedResourceState } from '../../../state'; +import { getListFetchError } from './selector'; const createEventFiltersServiceMock = (): jest.Mocked => ({ addEventFilters: jest.fn(), getList: jest.fn(), getOne: jest.fn(), updateOne: jest.fn(), + deleteOne: jest.fn(), }); const createStoreSetup = (eventFiltersService: EventFiltersService) => { @@ -45,26 +46,132 @@ const createStoreSetup = (eventFiltersService: EventFiltersService) => { }; }; -describe('middleware', () => { - describe('initial state', () => { - it('sets initial state properly', async () => { - expect(createStoreSetup(createEventFiltersServiceMock()).store.getState()).toStrictEqual( - initialState - ); - }); - }); - +describe('Event filters middleware', () => { let service: jest.Mocked; let store: Store; let spyMiddleware: MiddlewareActionSpyHelper; + let initialState: EventFiltersListPageState; beforeEach(() => { + initialState = initialEventFiltersPageState(); service = createEventFiltersServiceMock(); + const storeSetup = createStoreSetup(service); + store = storeSetup.store as Store; spyMiddleware = storeSetup.spyMiddleware; }); + describe('initial state', () => { + it('sets initial state properly', async () => { + expect(createStoreSetup(createEventFiltersServiceMock()).store.getState()).toStrictEqual( + initialState + ); + }); + }); + + describe('when on the List page', () => { + const changeUrl = (searchParams: string = '') => { + store.dispatch({ + type: 'userChangedUrl', + payload: { + pathname: '/event_filters', + search: searchParams, + hash: '', + key: 'ylsd7h', + }, + }); + }; + + beforeEach(() => { + service.getList.mockResolvedValue(getFoundExceptionListItemSchemaMock()); + }); + + it.each([ + [undefined, undefined], + [3, 50], + ])( + 'should trigger api call to retrieve event filters with url params page_index[%s] page_size[%s]', + async (pageIndex, perPage) => { + const dataLoaded = spyMiddleware.waitForAction('eventFiltersListPageDataChanged', { + validate({ payload }) { + return isLoadedResourceState(payload); + }, + }); + + changeUrl((pageIndex && perPage && `?page_index=${pageIndex}&page_size=${perPage}`) || ''); + await dataLoaded; + + expect(service.getList).toHaveBeenCalledWith({ + page: (pageIndex ?? 0) + 1, + perPage: perPage ?? 10, + sortField: 'created_at', + sortOrder: 'desc', + }); + } + ); + + it('should not refresh the list if nothing in the query has changed', async () => { + const dataLoaded = spyMiddleware.waitForAction('eventFiltersListPageDataChanged', { + validate({ payload }) { + return isLoadedResourceState(payload); + }, + }); + + changeUrl(); + await dataLoaded; + const getListCallCount = service.getList.mock.calls.length; + changeUrl('&show=create'); + + expect(service.getList.mock.calls.length).toBe(getListCallCount); + }); + + it('should trigger second api call to check if data exists if first returned no records', async () => { + const dataLoaded = spyMiddleware.waitForAction('eventFiltersListPageDataExistsChanged', { + validate({ payload }) { + return isLoadedResourceState(payload); + }, + }); + + service.getList.mockResolvedValue({ + data: [], + total: 0, + page: 1, + per_page: 10, + }); + + changeUrl(); + await dataLoaded; + + expect(service.getList).toHaveBeenCalledTimes(2); + expect(service.getList).toHaveBeenNthCalledWith(2, { + page: 1, + perPage: 1, + }); + }); + + it('should dispatch a Failure if an API error was encountered', async () => { + const dataLoaded = spyMiddleware.waitForAction('eventFiltersListPageDataChanged', { + validate({ payload }) { + return isFailedResourceState(payload); + }, + }); + + service.getList.mockRejectedValue({ + body: { message: 'error message', statusCode: 500, error: 'Internal Server Error' }, + }); + + changeUrl(); + await dataLoaded; + + expect(getListFetchError(store.getState())).toEqual({ + message: 'error message', + statusCode: 500, + error: 'Internal Server Error', + }); + }); + }); + describe('submit creation event filter', () => { it('does not submit when entry is undefined', async () => { store.dispatch({ type: 'eventFiltersCreateStart' }); diff --git a/x-pack/plugins/security_solution/public/management/pages/event_filters/store/middleware.ts b/x-pack/plugins/security_solution/public/management/pages/event_filters/store/middleware.ts index f1cf3e49e4382..b32b39fb9293c 100644 --- a/x-pack/plugins/security_solution/public/management/pages/event_filters/store/middleware.ts +++ b/x-pack/plugins/security_solution/public/management/pages/event_filters/store/middleware.ts @@ -14,11 +14,9 @@ import { import { EventFiltersHttpService } from '../service'; -import { EventFiltersListPageState } from '../state'; -import { getLastLoadedResourceState } from '../../../state/async_resource_state'; - import { CreateExceptionListItemSchema, + ExceptionListItemSchema, transformNewItemOutput, transformOutput, UpdateExceptionListItemSchema, @@ -33,8 +31,26 @@ import { getFormEntry, getSubmissionResource, getNewComment, + isDeletionInProgress, + getItemToDelete, + getDeletionState, } from './selector'; -import { EventFiltersService, EventFiltersServiceGetListOptions } from '../types'; + +import { parseQueryFilterToKQL } from '../../../common/utils'; +import { SEARCHABLE_FIELDS } from '../constants'; +import { + EventFiltersListPageData, + EventFiltersListPageState, + EventFiltersService, + EventFiltersServiceGetListOptions, +} from '../types'; +import { + createFailedResourceState, + createLoadedResourceState, + createLoadingResourceState, + getLastLoadedResourceState, +} from '../../../state'; +import { ServerApiError } from '../../../../common/types'; const addNewComments = ( entry: UpdateExceptionListItemSchema | CreateExceptionListItemSchema, @@ -59,10 +75,9 @@ const eventFiltersCreate: MiddlewareActionHandler = async (store, eventFiltersSe if (!formEntry) return; store.dispatch({ type: 'eventFiltersFormStateChanged', - payload: { - type: 'LoadingResourceState', - previousState: { type: 'UninitialisedResourceState' }, - }, + payload: createLoadingResourceState({ + type: 'UninitialisedResourceState', + }), }); const sanitizedEntry = transformNewItemOutput(formEntry as CreateExceptionListItemSchema); @@ -145,19 +160,15 @@ const eventFiltersUpdate = async ( }); store.dispatch({ type: 'eventFiltersFormStateChanged', - payload: { - type: 'LoadedResourceState', - data: exception, - }, + payload: createLoadedResourceState(exception), }); } catch (error) { store.dispatch({ type: 'eventFiltersFormStateChanged', - payload: { - type: 'FailedResourceState', - error: error.body || error, - lastLoadedState: getLastLoadedResourceState(submissionResourceState), - }, + payload: createFailedResourceState( + error.body ?? error, + getLastLoadedResourceState(submissionResourceState) + ), }); } }; @@ -192,12 +203,9 @@ const checkIfEventFilterDataExist: MiddlewareActionHandler = async ( ) => { dispatch({ type: 'eventFiltersListPageDataExistsChanged', - payload: { - type: 'LoadingResourceState', - // Ignore will be fixed with when AsyncResourceState is refactored (#830) - // @ts-ignore - previousState: getListPageDataExistsState(getState()), - }, + // Ignore will be fixed with when AsyncResourceState is refactored (#830) + // @ts-ignore + payload: createLoadingResourceState(getListPageDataExistsState(getState())), }); try { @@ -205,18 +213,12 @@ const checkIfEventFilterDataExist: MiddlewareActionHandler = async ( dispatch({ type: 'eventFiltersListPageDataExistsChanged', - payload: { - type: 'LoadedResourceState', - data: Boolean(anythingInListResults.total), - }, + payload: createLoadedResourceState(Boolean(anythingInListResults.total)), }); } catch (error) { dispatch({ type: 'eventFiltersListPageDataExistsChanged', - payload: { - type: 'FailedResourceState', - error: error.body || error, - }, + payload: createFailedResourceState(error.body ?? error), }); } }; @@ -237,12 +239,13 @@ const refreshListDataIfNeeded: MiddlewareActionHandler = async (store, eventFilt }, }); - const { page_size: pageSize, page_index: pageIndex } = getCurrentLocation(state); + const { page_size: pageSize, page_index: pageIndex, filter } = getCurrentLocation(state); const query: EventFiltersServiceGetListOptions = { page: pageIndex + 1, perPage: pageSize, sortField: 'created_at', sortOrder: 'desc', + filter: parseQueryFilterToKQL(filter, SEARCHABLE_FIELDS) || undefined, }; try { @@ -250,13 +253,10 @@ const refreshListDataIfNeeded: MiddlewareActionHandler = async (store, eventFilt dispatch({ type: 'eventFiltersListPageDataChanged', - payload: { - type: 'LoadedResourceState', - data: { - query, - content: results, - }, - }, + payload: createLoadedResourceState({ + query, + content: results, + }), }); dispatch({ @@ -276,15 +276,49 @@ const refreshListDataIfNeeded: MiddlewareActionHandler = async (store, eventFilt } catch (error) { dispatch({ type: 'eventFiltersListPageDataChanged', - payload: { - type: 'FailedResourceState', - error: error.body || error, - }, + payload: createFailedResourceState(error.body ?? error), }); } } }; +const eventFilterDeleteEntry: MiddlewareActionHandler = async ( + { getState, dispatch }, + eventFiltersService +) => { + const state = getState(); + + if (isDeletionInProgress(state)) { + return; + } + + const itemId = getItemToDelete(state)?.id; + + if (!itemId) { + return; + } + + dispatch({ + type: 'eventFilterDeleteStatusChanged', + // Ignore will be fixed with when AsyncResourceState is refactored (#830) + // @ts-ignore + payload: createLoadingResourceState(getDeletionState(state).status), + }); + + try { + const response = await eventFiltersService.deleteOne(itemId); + dispatch({ + type: 'eventFilterDeleteStatusChanged', + payload: createLoadedResourceState(response), + }); + } catch (e) { + dispatch({ + type: 'eventFilterDeleteStatusChanged', + payload: createFailedResourceState(e.body ?? e), + }); + } +}; + export const createEventFiltersPageMiddleware = ( eventFiltersService: EventFiltersService ): ImmutableMiddleware => { @@ -304,9 +338,12 @@ export const createEventFiltersPageMiddleware = ( if ( action.type === 'userChangedUrl' || action.type === 'eventFiltersCreateSuccess' || - action.type === 'eventFiltersUpdateSuccess' + action.type === 'eventFiltersUpdateSuccess' || + action.type === 'eventFilterDeleteStatusChanged' ) { refreshListDataIfNeeded(store, eventFiltersService); + } else if (action.type === 'eventFilterDeleteSubmit') { + eventFilterDeleteEntry(store, eventFiltersService); } } }; diff --git a/x-pack/plugins/security_solution/public/management/pages/event_filters/store/reducer.test.ts b/x-pack/plugins/security_solution/public/management/pages/event_filters/store/reducer.test.ts index 3196ac0c19b28..ed665135317af 100644 --- a/x-pack/plugins/security_solution/public/management/pages/event_filters/store/reducer.test.ts +++ b/x-pack/plugins/security_solution/public/management/pages/event_filters/store/reducer.test.ts @@ -9,10 +9,17 @@ import { initialEventFiltersPageState } from './builders'; import { eventFiltersPageReducer } from './reducer'; import { getInitialExceptionFromEvent } from './utils'; import { createdEventFilterEntryMock, ecsEventMock } from '../test_utils'; +import { UserChangedUrl } from '../../../../common/store/routing/action'; +import { getListPageIsActive } from './selector'; +import { EventFiltersListPageState } from '../types'; -const initialState = initialEventFiltersPageState(); +describe('event filters reducer', () => { + let initialState: EventFiltersListPageState; + + beforeEach(() => { + initialState = initialEventFiltersPageState(); + }); -describe('reducer', () => { describe('EventFiltersForm', () => { it('sets the initial form values', () => { const entry = getInitialExceptionFromEvent(ecsEventMock()); @@ -100,23 +107,45 @@ describe('reducer', () => { }); }); describe('UserChangedUrl', () => { - it('receives a url change with show=create', () => { - const result = eventFiltersPageReducer(initialState, { - type: 'userChangedUrl', - payload: { search: '?show=create', pathname: '/event_filters', hash: '' }, + const userChangedUrlAction = ( + search: string = '', + pathname = '/event_filters' + ): UserChangedUrl => ({ + type: 'userChangedUrl', + payload: { search, pathname, hash: '' }, + }); + + describe('When url is the Event List page', () => { + it('should mark page active when on the list url', () => { + const result = eventFiltersPageReducer(initialState, userChangedUrlAction()); + expect(getListPageIsActive(result)).toBe(true); }); - expect(result).toStrictEqual({ - ...initialState, - location: { - ...initialState.location, - id: undefined, - show: 'create', - }, - listPage: { - ...initialState.listPage, - active: true, - }, + it('should mark page not active when not on the list url', () => { + const result = eventFiltersPageReducer( + initialState, + userChangedUrlAction('', '/some-other-page') + ); + expect(getListPageIsActive(result)).toBe(false); + }); + }); + + describe('When `show=create`', () => { + it('receives a url change with show=create', () => { + const result = eventFiltersPageReducer(initialState, userChangedUrlAction('?show=create')); + + expect(result).toStrictEqual({ + ...initialState, + location: { + ...initialState.location, + id: undefined, + show: 'create', + }, + listPage: { + ...initialState.listPage, + active: true, + }, + }); }); }); }); diff --git a/x-pack/plugins/security_solution/public/management/pages/event_filters/store/reducer.ts b/x-pack/plugins/security_solution/public/management/pages/event_filters/store/reducer.ts index 858d62ae67744..d69efb689c877 100644 --- a/x-pack/plugins/security_solution/public/management/pages/event_filters/store/reducer.ts +++ b/x-pack/plugins/security_solution/public/management/pages/event_filters/store/reducer.ts @@ -14,7 +14,10 @@ import { AppLocation, Immutable } from '../../../../../common/endpoint/types'; import { UserChangedUrl } from '../../../../common/store/routing/action'; import { MANAGEMENT_ROUTING_EVENT_FILTERS_PATH } from '../../../common/constants'; import { extractEventFiltetrsPageLocation } from '../../../common/routing'; -import { isUninitialisedResourceState } from '../../../state/async_resource_state'; +import { + isLoadedResourceState, + isUninitialisedResourceState, +} from '../../../state/async_resource_state'; import { EventFiltersInitForm, @@ -22,14 +25,16 @@ import { EventFiltersFormStateChanged, EventFiltersCreateSuccess, EventFiltersUpdateSuccess, - EventFiltersListPageStateChanged, EventFiltersListPageDataChanged, EventFiltersListPageDataExistsChanged, + EventFilterForDeletion, + EventFilterDeletionReset, + EventFilterDeleteStatusChanged, } from './action'; -import { EventFiltersListPageState } from '../state'; import { initialEventFiltersPageState } from './builders'; import { getListPageIsActive } from './selector'; +import { EventFiltersListPageState } from '../types'; type StateReducer = ImmutableReducer; type CaseReducer = ( @@ -46,17 +51,6 @@ const isEventFiltersPageLocation = (location: Immutable) => { ); }; -// FIXME:PT might not need this. maybe delete -const handleEventFiltersListPageChanges: CaseReducer = ( - state, - action -) => { - return { - ...state, - listPage: action.payload, - }; -}; - const handleEventFiltersListPageDataChanges: CaseReducer = ( state, action @@ -185,6 +179,46 @@ const userChangedUrl: CaseReducer = (state, action) => { } }; +const handleEventFilterForDeletion: CaseReducer = (state, action) => { + return { + ...state, + listPage: { + ...state.listPage, + deletion: { + ...state.listPage.deletion, + item: action.payload, + }, + }, + }; +}; + +const handleEventFilterDeletionReset: CaseReducer = (state) => { + return { + ...state, + listPage: { + ...state.listPage, + deletion: initialEventFiltersPageState().listPage.deletion, + }, + }; +}; + +const handleEventFilterDeleteStatusChanges: CaseReducer = ( + state, + action +) => { + return { + ...state, + listPage: { + ...state.listPage, + forceRefresh: isLoadedResourceState(action.payload) ? true : state.listPage.forceRefresh, + deletion: { + ...state.listPage.deletion, + status: action.payload, + }, + }, + }; +}; + export const eventFiltersPageReducer: StateReducer = ( state = initialEventFiltersPageState(), action @@ -207,12 +241,16 @@ export const eventFiltersPageReducer: StateReducer = ( // actions only handled if we're on the List Page if (getListPageIsActive(state)) { switch (action.type) { - case 'eventFiltersListPageStateChanged': - return handleEventFiltersListPageChanges(state, action); case 'eventFiltersListPageDataChanged': return handleEventFiltersListPageDataChanges(state, action); case 'eventFiltersListPageDataExistsChanged': return handleEventFiltersListPageDataExistChanges(state, action); + case 'eventFilterForDeletion': + return handleEventFilterForDeletion(state, action); + case 'eventFilterDeletionReset': + return handleEventFilterDeletionReset(state, action); + case 'eventFilterDeleteStatusChanged': + return handleEventFilterDeleteStatusChanges(state, action); } } diff --git a/x-pack/plugins/security_solution/public/management/pages/event_filters/store/selector.ts b/x-pack/plugins/security_solution/public/management/pages/event_filters/store/selector.ts index b33383d683e90..e6f67f7329e6a 100644 --- a/x-pack/plugins/security_solution/public/management/pages/event_filters/store/selector.ts +++ b/x-pack/plugins/security_solution/public/management/pages/event_filters/store/selector.ts @@ -7,9 +7,8 @@ import { createSelector } from 'reselect'; import { Pagination } from '@elastic/eui'; -import { EventFiltersServiceGetListOptions } from '../types'; +import { EventFiltersListPageState, EventFiltersServiceGetListOptions } from '../types'; -import { EventFiltersListPageState } from '../state'; import { ExceptionListItemSchema } from '../../../../shared_imports'; import { ServerApiError } from '../../../../common/types'; import { @@ -17,6 +16,7 @@ import { isLoadedResourceState, isFailedResourceState, isUninitialisedResourceState, + getLastLoadedResourceState, } from '../../../state/async_resource_state'; import { FoundExceptionListItemSchema } from '../../../../../../lists/common/schemas'; import { @@ -48,15 +48,7 @@ export const getCurrentListPageDataState: EventFiltersSelector | undefined > = createSelector(getCurrentListPageDataState, (listPageData) => { - if (isLoadedResourceState(listPageData)) { - return listPageData.data.content; - } else if ( - isLoadingResourceState(listPageData) && - isLoadedResourceState(listPageData.previousState) - ) { - return listPageData.previousState.data.content; - } - return undefined; + return getLastLoadedResourceState(listPageData)?.data.content; }); export const getListItems: EventFiltersSelector< @@ -73,13 +65,7 @@ export const getListItems: EventFiltersSelector< export const getCurrentListItemsQuery: EventFiltersSelector = createSelector( getCurrentListPageDataState, (pageDataState) => { - return ( - (isLoadedResourceState(pageDataState) && pageDataState.data.query) || - (isLoadingResourceState(pageDataState) && - isLoadedResourceState(pageDataState.previousState) && - pageDataState.previousState.data.query) || - {} - ); + return getLastLoadedResourceState(pageDataState)?.data.query ?? {}; } ); @@ -192,7 +178,47 @@ export const listDataNeedsRefresh: EventFiltersSelector = createSelecto return ( forceRefresh || location.page_index + 1 !== currentQuery.page || - location.page_size !== currentQuery.perPage + location.page_size !== currentQuery.perPage || + (!!location.filter && location.filter !== currentQuery.filter) ); } ); + +export const getDeletionState = createSelector( + getCurrentListPageState, + (listState) => listState.deletion +); + +export const showDeleteModal: EventFiltersSelector = createSelector( + getDeletionState, + ({ item }) => { + return Boolean(item); + } +); + +export const getItemToDelete: EventFiltersSelector< + StoreState['listPage']['deletion']['item'] +> = createSelector(getDeletionState, ({ item }) => item); + +export const isDeletionInProgress: EventFiltersSelector = createSelector( + getDeletionState, + ({ status }) => { + return isLoadingResourceState(status); + } +); + +export const wasDeletionSuccessful: EventFiltersSelector = createSelector( + getDeletionState, + ({ status }) => { + return isLoadedResourceState(status); + } +); + +export const getDeleteError: EventFiltersSelector = createSelector( + getDeletionState, + ({ status }) => { + if (isFailedResourceState(status)) { + return status.error; + } + } +); diff --git a/x-pack/plugins/security_solution/public/management/pages/event_filters/store/selectors.test.ts b/x-pack/plugins/security_solution/public/management/pages/event_filters/store/selectors.test.ts index 3c15bc57e6c06..cba069775a90f 100644 --- a/x-pack/plugins/security_solution/public/management/pages/event_filters/store/selectors.test.ts +++ b/x-pack/plugins/security_solution/public/management/pages/event_filters/store/selectors.test.ts @@ -26,8 +26,7 @@ import { } from './selector'; import { ecsEventMock } from '../test_utils'; import { getInitialExceptionFromEvent } from './utils'; -import { EventFiltersPageLocation } from '../types'; -import { EventFiltersListPageState } from '../state'; +import { EventFiltersListPageState, EventFiltersPageLocation } from '../types'; import { MANAGEMENT_DEFAULT_PAGE, MANAGEMENT_DEFAULT_PAGE_SIZE } from '../../../common/constants'; import { getFoundExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/found_exception_list_item_schema.mock'; import { @@ -47,14 +46,14 @@ describe('event filters selectors', () => { const setToLoadedState = () => { initialState.listPage.data = createLoadedResourceState({ - query: { page: 2, perPage: 10 }, + query: { page: 2, perPage: 10, filter: '' }, content: getFoundExceptionListItemSchemaMock(), }); }; const setToLoadingState = ( previousState: EventFiltersListPageState['listPage']['data'] = createLoadedResourceState({ - query: { page: 5, perPage: 50 }, + query: { page: 5, perPage: 50, filter: '' }, content: getFoundExceptionListItemSchemaMock(), }) ) => { @@ -128,12 +127,12 @@ describe('event filters selectors', () => { it('should return query from current loaded state', () => { setToLoadedState(); - expect(getCurrentListItemsQuery(initialState)).toEqual({ page: 2, perPage: 10 }); + expect(getCurrentListItemsQuery(initialState)).toEqual({ page: 2, perPage: 10, filter: '' }); }); it('should return query from previous state while Loading new page', () => { setToLoadingState(); - expect(getCurrentListItemsQuery(initialState)).toEqual({ page: 5, perPage: 50 }); + expect(getCurrentListItemsQuery(initialState)).toEqual({ page: 5, perPage: 50, filter: '' }); }); }); @@ -237,6 +236,11 @@ describe('event filters selectors', () => { initialState.location.page_index = 10; expect(listDataNeedsRefresh(initialState)).toBe(true); }); + + it('should should return true if filter param differ from last api call', () => { + initialState.location.filter = 'query'; + expect(listDataNeedsRefresh(initialState)).toBe(true); + }); }); describe('getFormEntry()', () => { diff --git a/x-pack/plugins/security_solution/public/management/pages/event_filters/test_utils/index.ts b/x-pack/plugins/security_solution/public/management/pages/event_filters/test_utils/index.ts index 62f73ddfc57e1..701fb8d77b2e6 100644 --- a/x-pack/plugins/security_solution/public/management/pages/event_filters/test_utils/index.ts +++ b/x-pack/plugins/security_solution/public/management/pages/event_filters/test_utils/index.ts @@ -12,9 +12,20 @@ import { MANAGEMENT_STORE_GLOBAL_NAMESPACE, MANAGEMENT_STORE_EVENT_FILTERS_NAMESPACE, } from '../../../common/constants'; -import { ExceptionListItemSchema } from '../../../../shared_imports'; +import { + EXCEPTION_LIST_ITEM_URL, + EXCEPTION_LIST_URL, + ExceptionListItemSchema, +} from '../../../../shared_imports'; import { eventFiltersPageReducer } from '../store/reducer'; +import { + httpHandlerMockFactory, + ResponseProvidersInterface, +} from '../../../../common/mock/endpoint/http_handler_mock_factory'; +import { FoundExceptionListItemSchema } from '../../../../../../lists/common/schemas'; +import { getFoundExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/found_exception_list_item_schema.mock'; +import { getExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; export const createGlobalNoMiddlewareStore = () => { return createStore( @@ -89,3 +100,32 @@ export const createdEventFilterEntryMock = (): ExceptionListItemSchema => ({ updated_at: '2021-04-19T10:30:36.428Z', updated_by: 'elastic', }); + +export type EventFiltersListQueryHttpMockProviders = ResponseProvidersInterface<{ + eventFiltersList: () => FoundExceptionListItemSchema; + eventFiltersCreateList: () => ExceptionListItemSchema; +}>; + +/** + * Mock `core.http` methods used by Event Filters List page + */ +export const eventFiltersListQueryHttpMock = httpHandlerMockFactory( + [ + { + id: 'eventFiltersCreateList', + method: 'post', + path: EXCEPTION_LIST_URL, + handler: () => { + return getExceptionListItemSchemaMock(); + }, + }, + { + id: 'eventFiltersList', + method: 'get', + path: `${EXCEPTION_LIST_ITEM_URL}/_find`, + handler: (): FoundExceptionListItemSchema => { + return getFoundExceptionListItemSchemaMock(); + }, + }, + ] +); diff --git a/x-pack/plugins/security_solution/public/management/pages/event_filters/types.ts b/x-pack/plugins/security_solution/public/management/pages/event_filters/types.ts index b23d353c21e36..cc70a2037a5af 100644 --- a/x-pack/plugins/security_solution/public/management/pages/event_filters/types.ts +++ b/x-pack/plugins/security_solution/public/management/pages/event_filters/types.ts @@ -6,9 +6,9 @@ */ import { - UpdateExceptionListItemSchema, CreateExceptionListItemSchema, ExceptionListItemSchema, + UpdateExceptionListItemSchema, } from '../../../shared_imports'; import { AsyncResourceState } from '../../state/async_resource_state'; import { Immutable } from '../../../../common/endpoint/types'; @@ -37,6 +37,7 @@ export type EventFiltersServiceGetListOptions = Partial<{ perPage: number; sortField: keyof ExceptionListItemSchema; sortOrder: 'asc' | 'desc'; + filter: string; }>; export interface EventFiltersService { @@ -47,4 +48,31 @@ export interface EventFiltersService { getList(options?: EventFiltersServiceGetListOptions): Promise; getOne(id: string): Promise; updateOne(exception: Immutable): Promise; + deleteOne(id: string): Promise; +} + +export interface EventFiltersListPageData { + /** The query that was used to retrieve the data */ + query: EventFiltersServiceGetListOptions; + /** The data retrieved from the API */ + content: FoundExceptionListItemSchema; +} + +export interface EventFiltersListPageState { + entries: ExceptionListItemSchema[]; + form: EventFiltersForm; + location: EventFiltersPageLocation; + /** State for the Event Filters List page */ + listPage: { + active: boolean; + forceRefresh: boolean; + data: AsyncResourceState; + /** tracks if the overall list (not filtered or with invalid page numbers) contains data */ + dataExist: AsyncResourceState; + /** state for deletion of items from the list */ + deletion: { + item: ExceptionListItemSchema | undefined; + status: AsyncResourceState; + }; + }; } diff --git a/x-pack/plugins/security_solution/public/management/pages/event_filters/view/components/empty/index.tsx b/x-pack/plugins/security_solution/public/management/pages/event_filters/view/components/empty/index.tsx index e3c9532708871..d448b7644cc24 100644 --- a/x-pack/plugins/security_solution/public/management/pages/event_filters/view/components/empty/index.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/event_filters/view/components/empty/index.tsx @@ -44,7 +44,7 @@ export const EventFiltersListEmptyState = memo<{ fill isDisabled={isAddDisabled} onClick={onAdd} - data-test-subj="eventFiltersListAddButton" + data-test-subj="eventFiltersListEmptyStateAddButton" > { + let renderAndSetup: () => Promise>; + let renderResult: ReturnType; + let coreStart: AppContextTestRender['coreStart']; + let history: AppContextTestRender['history']; + let waitForAction: AppContextTestRender['middlewareSpy']['waitForAction']; + let store: AppContextTestRender['store']; + + const getBody = () => + renderResult.baseElement.querySelector('[data-test-subj="eventFilterDeleteModalBody"]')!; + + const getConfirmButton = () => + renderResult.baseElement.querySelector( + '[data-test-subj="eventFilterDeleteModalConfirmButton"]' + )! as HTMLButtonElement; + + const getCancelButton = () => + renderResult.baseElement.querySelector( + '[data-test-subj="eventFilterDeleteModalCancelButton"]' + )! as HTMLButtonElement; + + const getCurrentState = () => store.getState().management.eventFilters; + + beforeEach(() => { + const mockedContext = createAppRootMockRenderer(); + + ({ history, store, coreStart } = mockedContext); + renderAndSetup = async () => { + renderResult = mockedContext.render(); + + await act(async () => { + history.push('/event_filters'); + + await waitForAction('userChangedUrl'); + + mockedContext.store.dispatch({ + type: 'eventFilterForDeletion', + payload: { + id: '123', + name: 'tic-tac-toe', + }, + }); + }); + + return renderResult; + }; + + waitForAction = mockedContext.middlewareSpy.waitForAction; + mockedContext.setExperimentalFlag({ eventFilteringEnabled: true }); + }); + + it('should display name of event filter in body message', async () => { + await renderAndSetup(); + expect(getBody().textContent).toMatch(/You are removing event filter "tic-tac-toe"/); + }); + + it('should close dialog if cancel button is clicked', async () => { + await renderAndSetup(); + act(() => { + fireEvent.click(getCancelButton()); + }); + + expect(showDeleteModal(getCurrentState())).toBe(false); + }); + + it('should close dialog if the close X button is clicked', async () => { + await renderAndSetup(); + const dialogCloseButton = renderResult.baseElement.querySelector( + '[aria-label="Closes this modal window"]' + )!; + act(() => { + fireEvent.click(dialogCloseButton); + }); + + expect(showDeleteModal(getCurrentState())).toBe(false); + }); + + it('should disable action buttons when confirmed', async () => { + await renderAndSetup(); + act(() => { + fireEvent.click(getConfirmButton()); + }); + + expect(getCancelButton().disabled).toBe(true); + expect(getConfirmButton().disabled).toBe(true); + }); + + it('should set confirm button to loading', async () => { + await renderAndSetup(); + act(() => { + fireEvent.click(getConfirmButton()); + }); + + expect(getConfirmButton().querySelector('.euiLoadingSpinner')).not.toBeNull(); + }); + + it('should show success toast', async () => { + await renderAndSetup(); + const updateCompleted = waitForAction('eventFilterDeleteStatusChanged', { + validate(action) { + return isLoadedResourceState(action.payload); + }, + }); + + await act(async () => { + fireEvent.click(getConfirmButton()); + await updateCompleted; + }); + + expect(coreStart.notifications.toasts.addSuccess).toHaveBeenCalledWith( + '"tic-tac-toe" has been removed from the Event Filters list.' + ); + }); + + it('should show error toast if error is countered', async () => { + coreStart.http.delete.mockRejectedValue(new Error('oh oh')); + await renderAndSetup(); + const updateFailure = waitForAction('eventFilterDeleteStatusChanged', { + validate(action) { + return isFailedResourceState(action.payload); + }, + }); + + await act(async () => { + fireEvent.click(getConfirmButton()); + await updateFailure; + }); + + expect(coreStart.notifications.toasts.addDanger).toHaveBeenCalledWith( + 'Unable to remove "tic-tac-toe" from the Event Filters list. Reason: oh oh' + ); + expect(showDeleteModal(getCurrentState())).toBe(true); + }); +}); diff --git a/x-pack/plugins/security_solution/public/management/pages/event_filters/view/components/event_filter_delete_modal.tsx b/x-pack/plugins/security_solution/public/management/pages/event_filters/view/components/event_filter_delete_modal.tsx new file mode 100644 index 0000000000000..74a023965a57d --- /dev/null +++ b/x-pack/plugins/security_solution/public/management/pages/event_filters/view/components/event_filter_delete_modal.tsx @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { memo, useCallback, useEffect } from 'react'; +import { + EuiButton, + EuiButtonEmpty, + EuiModal, + EuiModalBody, + EuiModalFooter, + EuiModalHeader, + EuiModalHeaderTitle, + EuiText, +} from '@elastic/eui'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { useDispatch } from 'react-redux'; +import { Dispatch } from 'redux'; +import { i18n } from '@kbn/i18n'; +import { useEventFiltersSelector } from '../hooks'; +import { + getDeleteError, + getItemToDelete, + isDeletionInProgress, + wasDeletionSuccessful, +} from '../../store/selector'; +import { AppAction } from '../../../../../common/store/actions'; +import { useToasts } from '../../../../../common/lib/kibana'; + +export const EventFilterDeleteModal = memo<{}>(() => { + const dispatch = useDispatch>(); + const toasts = useToasts(); + + const isDeleting = useEventFiltersSelector(isDeletionInProgress); + const eventFilter = useEventFiltersSelector(getItemToDelete); + const wasDeleted = useEventFiltersSelector(wasDeletionSuccessful); + const deleteError = useEventFiltersSelector(getDeleteError); + + const onCancel = useCallback(() => { + dispatch({ type: 'eventFilterDeletionReset' }); + }, [dispatch]); + + const onConfirm = useCallback(() => { + dispatch({ type: 'eventFilterDeleteSubmit' }); + }, [dispatch]); + + // Show toast for success + useEffect(() => { + if (wasDeleted) { + toasts.addSuccess( + i18n.translate('xpack.securitySolution.eventFilters.deletionDialog.deleteSuccess', { + defaultMessage: '"{name}" has been removed from the Event Filters list.', + values: { name: eventFilter?.name }, + }) + ); + + dispatch({ type: 'eventFilterDeletionReset' }); + } + }, [dispatch, eventFilter?.name, toasts, wasDeleted]); + + // show toast for failures + useEffect(() => { + if (deleteError) { + toasts.addDanger( + i18n.translate('xpack.securitySolution.eventFilters.deletionDialog.deleteFailure', { + defaultMessage: + 'Unable to remove "{name}" from the Event Filters list. Reason: {message}', + values: { name: eventFilter?.name, message: deleteError.message }, + }) + ); + } + }, [deleteError, eventFilter?.name, toasts]); + + return ( + + + + + + + + + +

    + {eventFilter?.name} }} + /> +

    +

    + +

    +     +     + + + + + + + + + + +     + ); +}); + +EventFilterDeleteModal.displayName = 'EventFilterDeleteModal'; diff --git a/x-pack/plugins/security_solution/public/management/pages/event_filters/view/components/flyout/index.test.tsx b/x-pack/plugins/security_solution/public/management/pages/event_filters/view/components/flyout/index.test.tsx index 48e7f1a6f22d5..722eb57bf872c 100644 --- a/x-pack/plugins/security_solution/public/management/pages/event_filters/view/components/flyout/index.test.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/event_filters/view/components/flyout/index.test.tsx @@ -21,7 +21,7 @@ import { import { EventFiltersHttpService } from '../../../service'; import { createdEventFilterEntryMock } from '../../../test_utils'; import { getFormEntryState, isUninitialisedForm } from '../../../store/selector'; -import { EventFiltersListPageState } from '../../../state'; +import { EventFiltersListPageState } from '../../../types'; jest.mock('../form'); jest.mock('../../../service'); diff --git a/x-pack/plugins/security_solution/public/management/pages/event_filters/view/components/flyout/index.tsx b/x-pack/plugins/security_solution/public/management/pages/event_filters/view/components/flyout/index.tsx index c8443ee171d50..c36e711879b8e 100644 --- a/x-pack/plugins/security_solution/public/management/pages/event_filters/view/components/flyout/index.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/event_filters/view/components/flyout/index.tsx @@ -108,7 +108,7 @@ export const EventFiltersFlyout: React.FC = memo( ); return ( - +

    diff --git a/x-pack/plugins/security_solution/public/management/pages/event_filters/view/event_filters_list_page.test.tsx b/x-pack/plugins/security_solution/public/management/pages/event_filters/view/event_filters_list_page.test.tsx new file mode 100644 index 0000000000000..2fbabad746cad --- /dev/null +++ b/x-pack/plugins/security_solution/public/management/pages/event_filters/view/event_filters_list_page.test.tsx @@ -0,0 +1,179 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { AppContextTestRender, createAppRootMockRenderer } from '../../../../common/mock/endpoint'; +import React from 'react'; +import { fireEvent, act } from '@testing-library/react'; +import { EventFiltersListPage } from './event_filters_list_page'; +import { eventFiltersListQueryHttpMock } from '../test_utils'; +import { isFailedResourceState, isLoadedResourceState } from '../../../state'; + +// Needed to mock the data services used by the ExceptionItem component +jest.mock('../../../../common/lib/kibana'); + +describe('When on the Event Filters List Page', () => { + let render: () => ReturnType; + let renderResult: ReturnType; + let history: AppContextTestRender['history']; + let coreStart: AppContextTestRender['coreStart']; + let waitForAction: AppContextTestRender['middlewareSpy']['waitForAction']; + let mockedApi: ReturnType; + + const dataReceived = () => + act(async () => { + await waitForAction('eventFiltersListPageDataChanged', { + validate(action) { + return isLoadedResourceState(action.payload); + }, + }); + }); + + beforeEach(() => { + const mockedContext = createAppRootMockRenderer(); + + ({ history, coreStart } = mockedContext); + render = () => (renderResult = mockedContext.render()); + mockedApi = eventFiltersListQueryHttpMock(coreStart.http); + waitForAction = mockedContext.middlewareSpy.waitForAction; + + act(() => { + mockedContext.setExperimentalFlag({ eventFilteringEnabled: true }); + history.push('/event_filters'); + }); + }); + + describe('And no data exists', () => { + beforeEach(async () => { + mockedApi.responseProvider.eventFiltersList.mockReturnValue({ + data: [], + page: 1, + per_page: 10, + total: 0, + }); + + render(); + + await act(async () => { + await waitForAction('eventFiltersListPageDataExistsChanged', { + validate(action) { + return isLoadedResourceState(action.payload); + }, + }); + }); + }); + + it('should show the Empty message', () => { + expect(renderResult.getByTestId('eventFiltersEmpty')).toBeTruthy(); + expect(renderResult.getByTestId('eventFiltersListEmptyStateAddButton')).toBeTruthy(); + }); + + it('should open create flyout when add button in empty state is clicked', async () => { + act(() => { + fireEvent.click(renderResult.getByTestId('eventFiltersListEmptyStateAddButton')); + }); + + expect(renderResult.getByTestId('eventFiltersCreateEditFlyout')).toBeTruthy(); + expect(history.location.search).toEqual('?show=create'); + }); + }); + + describe('And data exists', () => { + it('should show loading indicator while retrieving data', async () => { + let releaseApiResponse: () => void; + + mockedApi.responseProvider.eventFiltersList.mockDelay.mockReturnValue( + new Promise((r) => (releaseApiResponse = r)) + ); + render(); + + expect(renderResult.getByTestId('eventFiltersContent-loader')).toBeTruthy(); + + const wasReceived = dataReceived(); + releaseApiResponse!(); + await wasReceived; + + expect(renderResult.container.querySelector('.euiProgress')).toBeNull(); + }); + + it('should show items on the list', async () => { + render(); + await dataReceived(); + + expect(renderResult.getByTestId('eventFilterCard')).toBeTruthy(); + }); + + it('should render expected fields on card', async () => { + render(); + await dataReceived(); + const eventMeta = ([] as HTMLElement[]).map.call( + renderResult.getByTestId('exceptionsViewerItemDetails').querySelectorAll('dd'), + (ele) => ele.textContent + ); + + expect(eventMeta).toEqual([ + 'some name', + 'Linux', + 'April 20th 2020 @ 11:25:31', + 'some user', + 'April 20th 2020 @ 11:25:31', + 'some user', + 'some description', + ]); + }); + + it('should show API error if one is encountered', async () => { + mockedApi.responseProvider.eventFiltersList.mockImplementation(() => { + throw new Error('oh no'); + }); + render(); + await act(async () => { + await waitForAction('eventFiltersListPageDataChanged', { + validate(action) { + return isFailedResourceState(action.payload); + }, + }); + }); + + expect(renderResult.getByTestId('eventFiltersContent-error').textContent).toEqual(' oh no'); + }); + + it('should show modal when delete is clicked on a card', async () => { + render(); + await dataReceived(); + act(() => { + fireEvent.click(renderResult.getByTestId('exceptionsViewerDeleteBtn')); + }); + + expect( + renderResult.baseElement.querySelector('[data-test-subj="eventFilterDeleteModalHeader"]') + ).not.toBeNull(); + }); + }); + + describe('And search is dispatched', () => { + beforeEach(async () => { + act(() => { + history.push('/event_filters?filter=test'); + }); + renderResult = render(); + await act(async () => { + await waitForAction('eventFiltersListPageDataChanged'); + }); + }); + + it('search bar is filled with query params', () => { + expect(renderResult.getByDisplayValue('test')).not.toBeNull(); + }); + + it('search action is dispatched', async () => { + await act(async () => { + fireEvent.click(renderResult.getByTestId('searchButton')); + expect(await waitForAction('userChangedUrl')).not.toBeNull(); + }); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/public/management/pages/event_filters/view/event_filters_list_page.tsx b/x-pack/plugins/security_solution/public/management/pages/event_filters/view/event_filters_list_page.tsx index 04a42370cbfa6..0898f93cf6f9b 100644 --- a/x-pack/plugins/security_solution/public/management/pages/event_filters/view/event_filters_list_page.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/event_filters/view/event_filters_list_page.tsx @@ -11,7 +11,7 @@ import { Dispatch } from 'redux'; import { useHistory } from 'react-router-dom'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import { EuiButton } from '@elastic/eui'; +import { EuiButton, EuiSpacer, EuiHorizontalRule, EuiText } from '@elastic/eui'; import styled from 'styled-components'; import { AppAction } from '../../../../common/store/actions'; @@ -30,6 +30,7 @@ import { getListPageDoesDataExist, getActionError, getFormEntry, + showDeleteModal, } from '../store/selector'; import { PaginatedContent, PaginatedContentProps } from '../../../components/paginated_content'; import { ExceptionListItemSchema } from '../../../../../../lists/common'; @@ -38,6 +39,9 @@ import { ExceptionItem, ExceptionItemProps, } from '../../../../common/components/exceptions/viewer/exception_item'; +import { EventFilterDeleteModal } from './components/event_filter_delete_modal'; + +import { SearchBar } from '../../../components/search_bar'; type EventListPaginatedContent = PaginatedContentProps< Immutable, @@ -65,6 +69,7 @@ export const EventFiltersListPage = memo(() => { const fetchError = useEventFiltersSelector(getListFetchError); const location = useEventFiltersSelector(getCurrentLocation); const doesDataExist = useEventFiltersSelector(getListPageDoesDataExist); + const showDelete = useEventFiltersSelector(showDeleteModal); const navigateCallback = useEventFiltersNavigateCallback(); const showFlyout = !!location.show; @@ -126,9 +131,16 @@ export const EventFiltersListPage = memo(() => { [navigateCallback] ); - const handleItemDelete: ExceptionItemProps['onDeleteException'] = useCallback((args) => { - // TODO: implement delete item - }, []); + const handleItemDelete: ExceptionItemProps['onDeleteException'] = useCallback( + ({ id }) => { + dispatch({ + type: 'eventFilterForDeletion', + // Casting below needed due to error around the comments array needing to be mutable + payload: listItems.find((item) => item.id === id)! as ExceptionListItemSchema, + }); + }, + [dispatch, listItems] + ); const handleItemComponentProps: EventListPaginatedContent['itemComponentProps'] = useCallback( (exceptionItem) => ({ @@ -139,6 +151,7 @@ export const EventFiltersListPage = memo(() => { onDeleteException: handleItemDelete, showModified: true, showName: true, + 'data-test-subj': `eventFilterCard`, }), [handleItemDelete, handleItemEdit] ); @@ -153,6 +166,10 @@ export const EventFiltersListPage = memo(() => { [navigateCallback] ); + const handleOnSearch = useCallback((query: string) => navigateCallback({ filter: query }), [ + navigateCallback, + ]); + return ( { /> )} + {showDelete && } + + {doesDataExist && ( + <> + + + + + + + + )} + , typeof ExceptionItem> items={listItems} ItemComponent={ExceptionItem} @@ -201,6 +235,7 @@ export const EventFiltersListPage = memo(() => { loading={isLoading} pagination={pagination} contentClassName="event-filter-container" + data-test-subj="eventFiltersContent" noItemsMessage={ !doesDataExist && ( diff --git a/x-pack/plugins/security_solution/public/management/pages/event_filters/view/hooks.ts b/x-pack/plugins/security_solution/public/management/pages/event_filters/view/hooks.ts index 65be130160601..e48f11c7f8bae 100644 --- a/x-pack/plugins/security_solution/public/management/pages/event_filters/view/hooks.ts +++ b/x-pack/plugins/security_solution/public/management/pages/event_filters/view/hooks.ts @@ -26,8 +26,7 @@ import { } from './translations'; import { State } from '../../../../common/store'; -import { EventFiltersListPageState } from '../state'; -import { EventFiltersPageLocation } from '../types'; +import { EventFiltersListPageState, EventFiltersPageLocation } from '../types'; import { getEventFiltersListPath } from '../../../common/routing'; import { diff --git a/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/constants.ts b/x-pack/plugins/security_solution/public/management/pages/trusted_apps/constants.ts similarity index 58% rename from x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/constants.ts rename to x-pack/plugins/security_solution/public/management/pages/trusted_apps/constants.ts index 58bd2aa7e5cec..0602ae18c1408 100644 --- a/x-pack/plugins/enterprise_search/public/applications/app_search/components/search_ui/constants.ts +++ b/x-pack/plugins/security_solution/public/management/pages/trusted_apps/constants.ts @@ -5,9 +5,9 @@ * 2.0. */ -import { i18n } from '@kbn/i18n'; - -export const SEARCH_UI_TITLE = i18n.translate( - 'xpack.enterpriseSearch.appSearch.engine.searchUI.title', - { defaultMessage: 'Search UI' } -); +export const SEARCHABLE_FIELDS: Readonly = [ + `name`, + `description`, + `entries.value`, + `entries.entries.value`, +]; diff --git a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/store/middleware.ts b/x-pack/plugins/security_solution/public/management/pages/trusted_apps/store/middleware.ts index 71a49caf66fd6..878938aa20e1b 100644 --- a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/store/middleware.ts +++ b/x-pack/plugins/security_solution/public/management/pages/trusted_apps/store/middleware.ts @@ -64,8 +64,9 @@ import { getListItems, editItemState, } from './selectors'; -import { parseQueryFilterToKQL } from './utils'; +import { parseQueryFilterToKQL } from '../../../common/utils'; import { toUpdateTrustedApp } from '../../../../../common/endpoint/service/trusted_apps/to_update_trusted_app'; +import { SEARCHABLE_FIELDS } from '../constants'; const createTrustedAppsListResourceStateChangedAction = ( newState: Immutable> @@ -97,7 +98,7 @@ const refreshListIfNeeded = async ( const response = await trustedAppsService.getTrustedAppsList({ page: pageIndex + 1, per_page: pageSize, - kuery: parseQueryFilterToKQL(filter) || undefined, + kuery: parseQueryFilterToKQL(filter, SEARCHABLE_FIELDS) || undefined, }); store.dispatch( diff --git a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/view/trusted_apps_page.test.tsx b/x-pack/plugins/security_solution/public/management/pages/trusted_apps/view/trusted_apps_page.test.tsx index 5efe8cfc16185..874d1a4a969c4 100644 --- a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/view/trusted_apps_page.test.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/trusted_apps/view/trusted_apps_page.test.tsx @@ -882,7 +882,7 @@ describe('When on the Trusted Apps Page', () => { it('search action is dispatched', async () => { await act(async () => { - fireEvent.click(renderResult.getByTestId('trustedAppSearchButton')); + fireEvent.click(renderResult.getByTestId('searchButton')); expect(await waitForAction('userChangedUrl')).not.toBeNull(); }); }); diff --git a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/view/trusted_apps_page.tsx b/x-pack/plugins/security_solution/public/management/pages/trusted_apps/view/trusted_apps_page.tsx index 1bf44769c15b4..39432f2d06505 100644 --- a/x-pack/plugins/security_solution/public/management/pages/trusted_apps/view/trusted_apps_page.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/trusted_apps/view/trusted_apps_page.tsx @@ -39,7 +39,7 @@ import { TrustedAppsListPageRouteState } from '../../../../../common/endpoint/ty import { useNavigateToAppEventHandler } from '../../../../common/hooks/endpoint/use_navigate_to_app_event_handler'; import { ABOUT_TRUSTED_APPS } from './translations'; import { EmptyState } from './components/empty_state'; -import { SearchBar } from './components/search_bar'; +import { SearchBar } from '../../../components/search_bar'; export const TrustedAppsPage = memo(() => { const { state: routeState } = useLocation(); diff --git a/x-pack/plugins/security_solution/public/management/state/async_resource_builders.ts b/x-pack/plugins/security_solution/public/management/state/async_resource_builders.ts index f7de2f68aecd9..0c18e121d25b1 100644 --- a/x-pack/plugins/security_solution/public/management/state/async_resource_builders.ts +++ b/x-pack/plugins/security_solution/public/management/state/async_resource_builders.ts @@ -12,12 +12,13 @@ import { StaleResourceState, UninitialisedResourceState, } from './async_resource_state'; +import { ServerApiError } from '../../common/types'; export const createUninitialisedResourceState = (): UninitialisedResourceState => { return { type: 'UninitialisedResourceState' }; }; -export const createLoadingResourceState = ( +export const createLoadingResourceState = ( previousState: StaleResourceState ): LoadingResourceState => { return { @@ -33,7 +34,7 @@ export const createLoadedResourceState = (data: Data): LoadedResourceState }; }; -export const createFailedResourceState = ( +export const createFailedResourceState = ( error: Error, lastLoadedState?: LoadedResourceState ): FailedResourceState => { diff --git a/x-pack/plugins/security_solution/public/management/types.ts b/x-pack/plugins/security_solution/public/management/types.ts index 460a30eaca783..cadb3b91f66a6 100644 --- a/x-pack/plugins/security_solution/public/management/types.ts +++ b/x-pack/plugins/security_solution/public/management/types.ts @@ -10,7 +10,7 @@ import { SecurityPageName } from '../app/types'; import { PolicyDetailsState } from './pages/policy/types'; import { EndpointState } from './pages/endpoint_hosts/types'; import { TrustedAppsListPageState } from './pages/trusted_apps/state'; -import { EventFiltersListPageState } from './pages/event_filters/state'; +import { EventFiltersListPageState } from './pages/event_filters/types'; /** * The type for the management store global namespace. Used mostly internally to reference diff --git a/x-pack/plugins/security_solution/public/overview/components/host_overview/index.tsx b/x-pack/plugins/security_solution/public/overview/components/host_overview/index.tsx index c5d51a9466235..fa644d1cbcdac 100644 --- a/x-pack/plugins/security_solution/public/overview/components/host_overview/index.tsx +++ b/x-pack/plugins/security_solution/public/overview/components/host_overview/index.tsx @@ -86,14 +86,15 @@ export const HostOverview = React.memo( () => [ { title: i18n.HOST_ID, - description: data.host - ? hostIdRenderer({ host: data.host, noLink: true }) - : getEmptyTagValue(), + description: + data && data.host + ? hostIdRenderer({ host: data.host, noLink: true }) + : getEmptyTagValue(), }, { title: i18n.FIRST_SEEN, description: - data.host != null && data.host.name && data.host.name.length ? ( + data && data.host != null && data.host.name && data.host.name.length ? ( ( { title: i18n.LAST_SEEN, description: - data.host != null && data.host.name && data.host.name.length ? ( + data && data.host != null && data.host.name && data.host.name.length ? ( ( )} - {data.endpoint != null ? ( + {data && data.endpoint != null ? ( <> diff --git a/x-pack/plugins/security_solution/server/endpoint/mocks.ts b/x-pack/plugins/security_solution/server/endpoint/mocks.ts index 40d4b1a877b2b..23ea6cc29c3d2 100644 --- a/x-pack/plugins/security_solution/server/endpoint/mocks.ts +++ b/x-pack/plugins/security_solution/server/endpoint/mocks.ts @@ -5,8 +5,8 @@ * 2.0. */ -import { ILegacyScopedClusterClient, SavedObjectsClientContract } from 'kibana/server'; -import { loggingSystemMock, savedObjectsServiceMock } from 'src/core/server/mocks'; +import { loggingSystemMock, savedObjectsServiceMock } from '../../../../../src/core/server/mocks'; +import { IScopedClusterClient, SavedObjectsClientContract } from '../../../../../src/core/server'; import { listMock } from '../../../lists/server/mocks'; import { securityMock } from '../../../security/server/mocks'; import { alertsMock } from '../../../alerting/server/mocks'; @@ -131,11 +131,11 @@ export const createMockMetadataRequestContext = (): jest.Mocked, + dataClient: jest.Mocked, savedObjectsClient: jest.Mocked ) { - const context = xpackMocks.createRequestHandlerContext(); - context.core.elasticsearch.legacy.client = dataClient; + const context = (xpackMocks.createRequestHandlerContext() as unknown) as jest.Mocked; + context.core.elasticsearch.client = dataClient; context.core.savedObjects.client = savedObjectsClient; return context; } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/actions/isolation.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/actions/isolation.test.ts index 306f37796c3fd..3340ef38d73cb 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/actions/isolation.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/actions/isolation.test.ts @@ -6,12 +6,7 @@ */ /* eslint-disable @typescript-eslint/no-explicit-any */ -import { - ILegacyClusterClient, - KibanaResponseFactory, - RequestHandler, - RouteConfig, -} from 'kibana/server'; +import { KibanaResponseFactory, RequestHandler, RouteConfig } from 'kibana/server'; import { elasticsearchServiceMock, httpServerMock, @@ -78,8 +73,8 @@ describe('Host Isolation', () => { beforeEach(() => { // instantiate... everything - const mockScopedClient = elasticsearchServiceMock.createLegacyScopedClusterClient(); - const mockClusterClient = elasticsearchServiceMock.createLegacyClusterClient() as jest.Mocked; + const mockScopedClient = elasticsearchServiceMock.createScopedClusterClient(); + const mockClusterClient = elasticsearchServiceMock.createClusterClient(); mockClusterClient.asScoped.mockReturnValue(mockScopedClient); const routerMock = httpServiceMock.createRouter(); mockResponse = httpServerMock.createResponseFactory(); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/handlers.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/handlers.ts index 0d59ff2f4ed7b..104383f398646 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/handlers.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/handlers.ts @@ -6,11 +6,18 @@ */ import Boom from '@hapi/boom'; -import type { Logger, RequestHandler } from 'kibana/server'; + import { TypeOf } from '@kbn/config-schema'; +import { + IScopedClusterClient, + Logger, + RequestHandler, + SavedObjectsClientContract, +} from '../../../../../../../src/core/server'; import { HostInfo, HostMetadata, + HostMetaDataInfo, HostResultList, HostStatus, MetadataQueryStrategyVersions, @@ -27,9 +34,11 @@ import { findAgentIDsByStatus } from './support/agent_status'; import { EndpointAppContextService } from '../../endpoint_app_context_services'; export interface MetadataRequestContext { + esClient?: IScopedClusterClient; endpointAppContextService: EndpointAppContextService; logger: Logger; - requestHandlerContext: SecuritySolutionRequestHandlerContext; + requestHandlerContext?: SecuritySolutionRequestHandlerContext; + savedObjectsClient?: SavedObjectsClientContract; } const HOST_STATUS_MAPPING = new Map([ @@ -75,9 +84,11 @@ export const getMetadataListRequestHandler = function ( } const metadataRequestContext: MetadataRequestContext = { + esClient: context.core.elasticsearch.client, endpointAppContextService: endpointAppContext.service, logger, requestHandlerContext: context, + savedObjectsClient: context.core.savedObjects.client, }; const unenrolledAgentIds = await findAllUnenrolledAgentIds( @@ -110,9 +121,10 @@ export const getMetadataListRequestHandler = function ( } ); - const hostListQueryResult = queryStrategy!.queryResponseToHostListResult( - await context.core.elasticsearch.legacy.client.callAsCurrentUser('search', queryParams) + const result = await context.core.elasticsearch.client.asCurrentUser.search( + queryParams ); + const hostListQueryResult = queryStrategy!.queryResponseToHostListResult(result.body); return response.ok({ body: await mapToHostResultList(queryParams, hostListQueryResult, metadataRequestContext), }); @@ -136,9 +148,11 @@ export const getMetadataRequestHandler = function ( } const metadataRequestContext: MetadataRequestContext = { + esClient: context.core.elasticsearch.client, endpointAppContextService: endpointAppContext.service, logger, requestHandlerContext: context, + savedObjectsClient: context.core.savedObjects.client, }; try { @@ -164,42 +178,86 @@ export const getMetadataRequestHandler = function ( }; }; -export async function getHostData( +export async function getHostMetaData( metadataRequestContext: MetadataRequestContext, id: string, queryStrategyVersion?: MetadataQueryStrategyVersions -): Promise { +): Promise { + if ( + !metadataRequestContext.esClient && + !metadataRequestContext.requestHandlerContext?.core.elasticsearch.client + ) { + throw Boom.badRequest('esClient not found'); + } + + if ( + !metadataRequestContext.savedObjectsClient && + !metadataRequestContext.requestHandlerContext?.core.savedObjects + ) { + throw Boom.badRequest('savedObjectsClient not found'); + } + + const esClient = (metadataRequestContext?.esClient ?? + metadataRequestContext.requestHandlerContext?.core.elasticsearch + .client) as IScopedClusterClient; + + const esSavedObjectClient = + metadataRequestContext?.savedObjectsClient ?? + (metadataRequestContext.requestHandlerContext?.core.savedObjects + .client as SavedObjectsClientContract); + const queryStrategy = await metadataRequestContext.endpointAppContextService ?.getMetadataService() - ?.queryStrategy( - metadataRequestContext.requestHandlerContext.core.savedObjects.client, - queryStrategyVersion - ); - + ?.queryStrategy(esSavedObjectClient, queryStrategyVersion); const query = getESQueryHostMetadataByID(id, queryStrategy!); - const hostResult = queryStrategy!.queryResponseToHostResult( - await metadataRequestContext.requestHandlerContext.core.elasticsearch.legacy.client.callAsCurrentUser( - 'search', - query - ) - ); + + const response = await esClient.asCurrentUser.search(query); + + const hostResult = queryStrategy!.queryResponseToHostResult(response.body); + const hostMetadata = hostResult.result; if (!hostMetadata) { return undefined; } - const agent = await findAgent(metadataRequestContext, hostMetadata); + return { metadata: hostMetadata, query_strategy_version: hostResult.queryStrategyVersion }; +} + +export async function getHostData( + metadataRequestContext: MetadataRequestContext, + id: string, + queryStrategyVersion?: MetadataQueryStrategyVersions +): Promise { + if (!metadataRequestContext.savedObjectsClient) { + throw Boom.badRequest('savedObjectsClient not found'); + } + + if ( + !metadataRequestContext.esClient && + !metadataRequestContext.requestHandlerContext?.core.elasticsearch.client + ) { + throw Boom.badRequest('esClient not found'); + } + + const hostResult = await getHostMetaData(metadataRequestContext, id, queryStrategyVersion); + + if (!hostResult) { + return undefined; + } + + const agent = await findAgent(metadataRequestContext, hostResult.metadata); if (agent && !agent.active) { throw Boom.badRequest('the requested endpoint is unenrolled'); } const metadata = await enrichHostMetadata( - hostMetadata, + hostResult.metadata, metadataRequestContext, - hostResult.queryStrategyVersion + hostResult.query_strategy_version ); - return { ...metadata, query_strategy_version: hostResult.queryStrategyVersion }; + + return { ...metadata, query_strategy_version: hostResult.query_strategy_version }; } async function findAgent( @@ -207,12 +265,20 @@ async function findAgent( hostMetadata: HostMetadata ): Promise { try { + if ( + !metadataRequestContext.esClient && + !metadataRequestContext.requestHandlerContext?.core.elasticsearch.client + ) { + throw new Error('esClient not found'); + } + + const esClient = (metadataRequestContext?.esClient ?? + metadataRequestContext.requestHandlerContext?.core.elasticsearch + .client) as IScopedClusterClient; + return await metadataRequestContext.endpointAppContextService ?.getAgentService() - ?.getAgent( - metadataRequestContext.requestHandlerContext.core.elasticsearch.client.asCurrentUser, - hostMetadata.elastic.agent.id - ); + ?.getAgent(esClient.asCurrentUser, hostMetadata.elastic.agent.id); } catch (e) { if (e instanceof AgentNotFoundError) { metadataRequestContext.logger.warn( @@ -232,7 +298,7 @@ export async function mapToHostResultList( metadataRequestContext: MetadataRequestContext ): Promise { const totalNumberOfHosts = hostListQueryResult.resultLength; - if (hostListQueryResult.resultList.length > 0) { + if ((hostListQueryResult.resultList?.length ?? 0) > 0) { return { request_page_size: queryParams.size, request_page_index: queryParams.from, @@ -267,6 +333,35 @@ export async function enrichHostMetadata( let hostStatus = HostStatus.UNHEALTHY; let elasticAgentId = hostMetadata?.elastic?.agent?.id; const log = metadataRequestContext.logger; + + try { + if ( + !metadataRequestContext.esClient && + !metadataRequestContext.requestHandlerContext?.core.elasticsearch.client + ) { + throw new Error('esClient not found'); + } + + if ( + !metadataRequestContext.savedObjectsClient && + !metadataRequestContext.requestHandlerContext?.core.savedObjects + ) { + throw new Error('esSavedObjectClient not found'); + } + } catch (e) { + log.error(e); + throw e; + } + + const esClient = (metadataRequestContext?.esClient ?? + metadataRequestContext.requestHandlerContext?.core.elasticsearch + .client) as IScopedClusterClient; + + const esSavedObjectClient = + metadataRequestContext?.savedObjectsClient ?? + (metadataRequestContext.requestHandlerContext?.core.savedObjects + .client as SavedObjectsClientContract); + try { /** * Get agent status by elastic agent id if available or use the endpoint-agent id. @@ -279,10 +374,7 @@ export async function enrichHostMetadata( const status = await metadataRequestContext.endpointAppContextService ?.getAgentService() - ?.getAgentStatusById( - metadataRequestContext.requestHandlerContext.core.elasticsearch.client.asCurrentUser, - elasticAgentId - ); + ?.getAgentStatusById(esClient.asCurrentUser, elasticAgentId); hostStatus = HOST_STATUS_MAPPING.get(status!) || HostStatus.UNHEALTHY; } catch (e) { if (e instanceof AgentNotFoundError) { @@ -297,17 +389,10 @@ export async function enrichHostMetadata( try { const agent = await metadataRequestContext.endpointAppContextService ?.getAgentService() - ?.getAgent( - metadataRequestContext.requestHandlerContext.core.elasticsearch.client.asCurrentUser, - elasticAgentId - ); + ?.getAgent(esClient.asCurrentUser, elasticAgentId); const agentPolicy = await metadataRequestContext.endpointAppContextService .getAgentPolicyService() - ?.get( - metadataRequestContext.requestHandlerContext.core.savedObjects.client, - agent?.policy_id!, - true - ); + ?.get(esSavedObjectClient, agent?.policy_id!, true); const endpointPolicy = ((agentPolicy?.package_policies || []) as PackagePolicy[]).find( (policy: PackagePolicy) => policy.package?.name === 'endpoint' ); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts index f4698cbed6203..b916ec19da17f 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts @@ -6,8 +6,6 @@ */ import { - ILegacyClusterClient, - ILegacyScopedClusterClient, KibanaResponseFactory, RequestHandler, RouteConfig, @@ -50,12 +48,17 @@ import { PackageService } from '../../../../../fleet/server/services'; import { metadataTransformPrefix } from '../../../../common/endpoint/constants'; import type { SecuritySolutionPluginRouter } from '../../../types'; import { PackagePolicyServiceInterface } from '../../../../../fleet/server'; +import { + ClusterClientMock, + ScopedClusterClientMock, + // eslint-disable-next-line @kbn/eslint/no-restricted-paths +} from '../../../../../../../src/core/server/elasticsearch/client/mocks'; describe('test endpoint route', () => { let routerMock: jest.Mocked; let mockResponse: jest.Mocked; - let mockClusterClient: jest.Mocked; - let mockScopedClient: jest.Mocked; + let mockClusterClient: ClusterClientMock; + let mockScopedClient: ScopedClusterClientMock; let mockSavedObjectClient: jest.Mocked; let mockPackageService: jest.Mocked; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -76,8 +79,8 @@ describe('test endpoint route', () => { }; beforeEach(() => { - mockScopedClient = elasticsearchServiceMock.createLegacyScopedClusterClient(); - mockClusterClient = elasticsearchServiceMock.createLegacyClusterClient() as jest.Mocked; + mockScopedClient = elasticsearchServiceMock.createScopedClusterClient(); + mockClusterClient = elasticsearchServiceMock.createClusterClient(); mockSavedObjectClient = savedObjectsClientMock.create(); mockClusterClient.asScoped.mockReturnValue(mockScopedClient); routerMock = httpServiceMock.createRouter(); @@ -119,7 +122,9 @@ describe('test endpoint route', () => { it('test find the latest of all endpoints', async () => { const mockRequest = httpServerMock.createKibanaRequest({}); const response = createV1SearchResponse(new EndpointDocGenerator().generateHostMetadata()); - mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response)); + (mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ body: response }) + ); [routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) => path.startsWith(`${METADATA_REQUEST_ROUTE}`) )!; @@ -131,7 +136,7 @@ describe('test endpoint route', () => { mockResponse ); - expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1); + expect(mockScopedClient.asCurrentUser.search).toHaveBeenCalledTimes(1); expect(routeConfig.options).toEqual({ authRequired: true, tags: ['access:securitySolution'], @@ -157,7 +162,9 @@ describe('test endpoint route', () => { mockAgentService.getAgent = jest.fn().mockReturnValue(({ active: true, } as unknown) as Agent); - mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response)); + (mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ body: response }) + ); [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => path.startsWith(`${METADATA_REQUEST_ROUTE}`) @@ -169,7 +176,7 @@ describe('test endpoint route', () => { mockResponse ); - expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1); + expect(mockScopedClient.asCurrentUser.search).toHaveBeenCalledTimes(1); expect(routeConfig.options).toEqual({ authRequired: true, tags: ['access:securitySolution'], @@ -214,7 +221,9 @@ describe('test endpoint route', () => { it('test find the latest of all endpoints', async () => { const mockRequest = httpServerMock.createKibanaRequest({}); const response = createV2SearchResponse(new EndpointDocGenerator().generateHostMetadata()); - mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response)); + (mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ body: response }) + ); [routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) => path.startsWith(`${METADATA_REQUEST_ROUTE}`) )!; @@ -226,7 +235,7 @@ describe('test endpoint route', () => { mockResponse ); - expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1); + expect(mockScopedClient.asCurrentUser.search).toHaveBeenCalledTimes(1); expect(routeConfig.options).toEqual({ authRequired: true, tags: ['access:securitySolution'], @@ -258,8 +267,10 @@ describe('test endpoint route', () => { mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error'); mockAgentService.listAgents = jest.fn().mockReturnValue(noUnenrolledAgent); - mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => - Promise.resolve(createV2SearchResponse(new EndpointDocGenerator().generateHostMetadata())) + (mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ + body: createV2SearchResponse(new EndpointDocGenerator().generateHostMetadata()), + }) ); [routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) => path.startsWith(`${METADATA_REQUEST_ROUTE}`) @@ -270,10 +281,10 @@ describe('test endpoint route', () => { mockRequest, mockResponse ); - - expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1); + expect(mockScopedClient.asCurrentUser.search).toHaveBeenCalledTimes(1); expect( - mockScopedClient.callAsCurrentUser.mock.calls[0][1]?.body?.query.bool.must_not + (mockScopedClient.asCurrentUser.search as jest.Mock).mock.calls[0][0]?.body?.query.bool + .must_not ).toContainEqual({ terms: { 'elastic.agent.id': [ @@ -315,8 +326,10 @@ describe('test endpoint route', () => { mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error'); mockAgentService.listAgents = jest.fn().mockReturnValue(noUnenrolledAgent); - mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => - Promise.resolve(createV2SearchResponse(new EndpointDocGenerator().generateHostMetadata())) + (mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ + body: createV2SearchResponse(new EndpointDocGenerator().generateHostMetadata()), + }) ); [routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) => path.startsWith(`${METADATA_REQUEST_ROUTE}`) @@ -328,10 +341,10 @@ describe('test endpoint route', () => { mockResponse ); - expect(mockScopedClient.callAsCurrentUser).toBeCalled(); + expect(mockScopedClient.asCurrentUser.search).toBeCalled(); expect( // KQL filter to be passed through - mockScopedClient.callAsCurrentUser.mock.calls[0][1]?.body?.query.bool.must + (mockScopedClient.asCurrentUser.search as jest.Mock).mock.calls[0][0]?.body?.query.bool.must ).toContainEqual({ bool: { must_not: { @@ -349,7 +362,7 @@ describe('test endpoint route', () => { }, }); expect( - mockScopedClient.callAsCurrentUser.mock.calls[0][1]?.body?.query.bool.must + (mockScopedClient.asCurrentUser.search as jest.Mock).mock.calls[0][0]?.body?.query.bool.must ).toContainEqual({ bool: { must_not: [ @@ -393,8 +406,8 @@ describe('test endpoint route', () => { it('should return 404 on no results', async () => { const mockRequest = httpServerMock.createKibanaRequest({ params: { id: 'BADID' } }); - mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => - Promise.resolve(createV2SearchResponse()) + (mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ body: createV2SearchResponse() }) ); mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error'); @@ -411,7 +424,7 @@ describe('test endpoint route', () => { mockResponse ); - expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1); + expect(mockScopedClient.asCurrentUser.search).toHaveBeenCalledTimes(1); expect(routeConfig.options).toEqual({ authRequired: true, tags: ['access:securitySolution'], @@ -431,7 +444,9 @@ describe('test endpoint route', () => { mockAgentService.getAgent = jest.fn().mockReturnValue(({ active: true, } as unknown) as Agent); - mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response)); + (mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ body: response }) + ); [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => path.startsWith(`${METADATA_REQUEST_ROUTE}`) @@ -443,7 +458,7 @@ describe('test endpoint route', () => { mockResponse ); - expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1); + expect(mockScopedClient.asCurrentUser.search).toHaveBeenCalledTimes(1); expect(routeConfig.options).toEqual({ authRequired: true, tags: ['access:securitySolution'], @@ -470,7 +485,9 @@ describe('test endpoint route', () => { SavedObjectsErrorHelpers.createGenericNotFoundError(); }); - mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response)); + (mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ body: response }) + ); [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => path.startsWith(`${METADATA_REQUEST_ROUTE}`) @@ -482,7 +499,7 @@ describe('test endpoint route', () => { mockResponse ); - expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1); + expect(mockScopedClient.asCurrentUser.search).toHaveBeenCalledTimes(1); expect(routeConfig.options).toEqual({ authRequired: true, tags: ['access:securitySolution'], @@ -503,7 +520,9 @@ describe('test endpoint route', () => { mockAgentService.getAgent = jest.fn().mockReturnValue(({ active: true, } as unknown) as Agent); - mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response)); + (mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ body: response }) + ); [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => path.startsWith(`${METADATA_REQUEST_ROUTE}`) @@ -515,7 +534,7 @@ describe('test endpoint route', () => { mockResponse ); - expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1); + expect(mockScopedClient.asCurrentUser.search).toHaveBeenCalledTimes(1); expect(routeConfig.options).toEqual({ authRequired: true, tags: ['access:securitySolution'], @@ -531,7 +550,9 @@ describe('test endpoint route', () => { const mockRequest = httpServerMock.createKibanaRequest({ params: { id: response.hits.hits[0]._id }, }); - mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response)); + (mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ body: response }) + ); mockAgentService.getAgent = jest.fn().mockReturnValue(({ active: false, } as unknown) as Agent); @@ -546,7 +567,7 @@ describe('test endpoint route', () => { mockResponse ); - expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1); + expect(mockScopedClient.asCurrentUser.search).toHaveBeenCalledTimes(1); expect(mockResponse.customError).toBeCalled(); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata_v1.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata_v1.test.ts index e3f859c26601e..0d56514e7d395 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata_v1.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata_v1.test.ts @@ -6,14 +6,17 @@ */ import { - ILegacyClusterClient, - ILegacyScopedClusterClient, KibanaResponseFactory, RequestHandler, RouteConfig, SavedObjectsClientContract, -} from 'kibana/server'; -import { SavedObjectsErrorHelpers } from '../../../../../../../src/core/server/'; + SavedObjectsErrorHelpers, +} from '../../../../../../../src/core/server'; +import { + ClusterClientMock, + ScopedClusterClientMock, + // eslint-disable-next-line @kbn/eslint/no-restricted-paths +} from '../../../../../../../src/core/server/elasticsearch/client/mocks'; import { elasticsearchServiceMock, httpServerMock, @@ -49,8 +52,8 @@ import { PackagePolicyServiceInterface } from '../../../../../fleet/server'; describe('test endpoint route v1', () => { let routerMock: jest.Mocked; let mockResponse: jest.Mocked; - let mockClusterClient: jest.Mocked; - let mockScopedClient: jest.Mocked; + let mockClusterClient: ClusterClientMock; + let mockScopedClient: ScopedClusterClientMock; let mockSavedObjectClient: jest.Mocked; let mockPackageService: jest.Mocked; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -71,8 +74,8 @@ describe('test endpoint route v1', () => { }; beforeEach(() => { - mockClusterClient = elasticsearchServiceMock.createLegacyClusterClient() as jest.Mocked; - mockScopedClient = elasticsearchServiceMock.createLegacyScopedClusterClient(); + mockClusterClient = elasticsearchServiceMock.createClusterClient(); + mockScopedClient = elasticsearchServiceMock.createScopedClusterClient(); mockSavedObjectClient = savedObjectsClientMock.create(); mockClusterClient.asScoped.mockReturnValue(mockScopedClient); routerMock = httpServiceMock.createRouter(); @@ -110,7 +113,9 @@ describe('test endpoint route v1', () => { it('test find the latest of all endpoints', async () => { const mockRequest = httpServerMock.createKibanaRequest({}); const response = createV1SearchResponse(new EndpointDocGenerator().generateHostMetadata()); - mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response)); + (mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ body: response }) + ); [routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) => path.startsWith(`${METADATA_REQUEST_V1_ROUTE}`) )!; @@ -122,7 +127,7 @@ describe('test endpoint route v1', () => { mockResponse ); - expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1); + expect(mockScopedClient.asCurrentUser.search).toHaveBeenCalledTimes(1); expect(routeConfig.options).toEqual({ authRequired: true, tags: ['access:securitySolution'] }); expect(mockResponse.ok).toBeCalled(); const endpointResultList = mockResponse.ok.mock.calls[0][0]?.body as HostResultList; @@ -151,8 +156,10 @@ describe('test endpoint route v1', () => { mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error'); mockAgentService.listAgents = jest.fn().mockReturnValue(noUnenrolledAgent); - mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => - Promise.resolve(createV1SearchResponse(new EndpointDocGenerator().generateHostMetadata())) + (mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ + body: createV1SearchResponse(new EndpointDocGenerator().generateHostMetadata()), + }) ); [routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) => path.startsWith(`${METADATA_REQUEST_V1_ROUTE}`) @@ -164,9 +171,10 @@ describe('test endpoint route v1', () => { mockResponse ); - expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1); + expect(mockScopedClient.asCurrentUser.search).toHaveBeenCalledTimes(1); expect( - mockScopedClient.callAsCurrentUser.mock.calls[0][1]?.body?.query.bool.must_not + (mockScopedClient.asCurrentUser.search as jest.Mock).mock.calls[0][0]?.body?.query.bool + .must_not ).toContainEqual({ terms: { 'elastic.agent.id': [ @@ -205,8 +213,10 @@ describe('test endpoint route v1', () => { mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error'); mockAgentService.listAgents = jest.fn().mockReturnValue(noUnenrolledAgent); - mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => - Promise.resolve(createV1SearchResponse(new EndpointDocGenerator().generateHostMetadata())) + (mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ + body: createV1SearchResponse(new EndpointDocGenerator().generateHostMetadata()), + }) ); [routeConfig, routeHandler] = routerMock.post.mock.calls.find(([{ path }]) => path.startsWith(`${METADATA_REQUEST_V1_ROUTE}`) @@ -218,10 +228,10 @@ describe('test endpoint route v1', () => { mockResponse ); - expect(mockScopedClient.callAsCurrentUser).toBeCalled(); + expect(mockScopedClient.asCurrentUser.search).toBeCalled(); // needs to have the KQL filter passed through expect( - mockScopedClient.callAsCurrentUser.mock.calls[0][1]?.body?.query.bool.must + (mockScopedClient.asCurrentUser.search as jest.Mock).mock.calls[0][0]?.body?.query.bool.must ).toContainEqual({ bool: { must_not: { @@ -240,7 +250,7 @@ describe('test endpoint route v1', () => { }); // and unenrolled should be filtered out. expect( - mockScopedClient.callAsCurrentUser.mock.calls[0][1]?.body?.query.bool.must + (mockScopedClient.asCurrentUser.search as jest.Mock).mock.calls[0][0]?.body?.query.bool.must ).toContainEqual({ bool: { must_not: [ @@ -281,8 +291,8 @@ describe('test endpoint route v1', () => { it('should return 404 on no results', async () => { const mockRequest = httpServerMock.createKibanaRequest({ params: { id: 'BADID' } }); - mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => - Promise.resolve(createV1SearchResponse()) + (mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ body: createV1SearchResponse() }) ); mockAgentService.getAgentStatusById = jest.fn().mockReturnValue('error'); @@ -299,7 +309,7 @@ describe('test endpoint route v1', () => { mockResponse ); - expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1); + expect(mockScopedClient.asCurrentUser.search).toHaveBeenCalledTimes(1); expect(routeConfig.options).toEqual({ authRequired: true, tags: ['access:securitySolution'], @@ -319,7 +329,9 @@ describe('test endpoint route v1', () => { mockAgentService.getAgent = jest.fn().mockReturnValue(({ active: true, } as unknown) as Agent); - mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response)); + (mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ body: response }) + ); [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => path.startsWith(`${METADATA_REQUEST_V1_ROUTE}`) @@ -331,7 +343,7 @@ describe('test endpoint route v1', () => { mockResponse ); - expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1); + expect(mockScopedClient.asCurrentUser.search).toHaveBeenCalledTimes(1); expect(routeConfig.options).toEqual({ authRequired: true, tags: ['access:securitySolution'], @@ -357,7 +369,9 @@ describe('test endpoint route v1', () => { SavedObjectsErrorHelpers.createGenericNotFoundError(); }); - mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response)); + (mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ body: response }) + ); [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => path.startsWith(`${METADATA_REQUEST_V1_ROUTE}`) @@ -369,7 +383,7 @@ describe('test endpoint route v1', () => { mockResponse ); - expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1); + expect(mockScopedClient.asCurrentUser.search).toHaveBeenCalledTimes(1); expect(routeConfig.options).toEqual({ authRequired: true, tags: ['access:securitySolution'], @@ -390,7 +404,9 @@ describe('test endpoint route v1', () => { mockAgentService.getAgent = jest.fn().mockReturnValue(({ active: true, } as unknown) as Agent); - mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response)); + (mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ body: response }) + ); [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => path.startsWith(`${METADATA_REQUEST_V1_ROUTE}`) @@ -402,7 +418,7 @@ describe('test endpoint route v1', () => { mockResponse ); - expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1); + expect(mockScopedClient.asCurrentUser.search).toHaveBeenCalledTimes(1); expect(routeConfig.options).toEqual({ authRequired: true, tags: ['access:securitySolution'], @@ -418,7 +434,9 @@ describe('test endpoint route v1', () => { const mockRequest = httpServerMock.createKibanaRequest({ params: { id: response.hits.hits[0]._id }, }); - mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response)); + (mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ body: response }) + ); mockAgentService.getAgent = jest.fn().mockReturnValue(({ active: false, } as unknown) as Agent); @@ -433,7 +451,7 @@ describe('test endpoint route v1', () => { mockResponse ); - expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1); + expect(mockScopedClient.asCurrentUser.search).toHaveBeenCalledTimes(1); expect(mockResponse.customError).toBeCalled(); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.test.ts index 5c09fd5ce05e4..e790c1de1a5b8 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.test.ts @@ -12,6 +12,7 @@ import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__ import { metadataCurrentIndexPattern } from '../../../../common/endpoint/constants'; import { parseExperimentalConfigValue } from '../../../../common/experimental_features'; import { metadataQueryStrategyV2 } from './support/query_strategies'; +import { get } from 'lodash'; describe('query builder', () => { describe('MetadataListESQuery', () => { @@ -204,7 +205,7 @@ describe('query builder', () => { const mockID = 'AABBCCDD-0011-2233-AA44-DEADBEEF8899'; const query = getESQueryHostMetadataByID(mockID, metadataQueryStrategyV2()); - expect(query.body.query.bool.filter[0].bool.should).toContainEqual({ + expect(get(query, 'body.query.bool.filter.0.bool.should')).toContainEqual({ term: { 'agent.id': mockID }, }); }); @@ -213,7 +214,7 @@ describe('query builder', () => { const mockID = 'AABBCCDD-0011-2233-AA44-DEADBEEF8899'; const query = getESQueryHostMetadataByID(mockID, metadataQueryStrategyV2()); - expect(query.body.query.bool.filter[0].bool.should).toContainEqual({ + expect(get(query, 'body.query.bool.filter.0.bool.should')).toContainEqual({ term: { 'HostDetails.agent.id': mockID }, }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.ts index a5259dd44cf2b..51e3495938606 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.ts @@ -5,7 +5,8 @@ * 2.0. */ -import { KibanaRequest } from 'kibana/server'; +import { SearchRequest, SortContainer } from '@elastic/elasticsearch/api/types'; +import { KibanaRequest } from '../../../../../../../src/core/server'; import { esKuery } from '../../../../../../../src/plugins/data/server'; import { EndpointAppContext, MetadataQueryStrategy } from '../../types'; @@ -19,7 +20,7 @@ export interface QueryBuilderOptions { // using unmapped_type avoids errors when the given field doesn't exist, and sets to the 0-value for that type // effectively ignoring it // https://www.elastic.co/guide/en/elasticsearch/reference/current/sort-search-results.html#_ignoring_unmapped_fields -const MetadataSortMethod = [ +const MetadataSortMethod: SortContainer[] = [ { 'event.created': { order: 'desc', @@ -146,7 +147,7 @@ function buildQueryBody( export function getESQueryHostMetadataByID( agentID: string, metadataQueryStrategy: MetadataQueryStrategy -) { +): SearchRequest { return { body: { query: { diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders_v1.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders_v1.test.ts index 9ce6130ff7dd3..c18c585cd3d34 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders_v1.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders_v1.test.ts @@ -12,6 +12,7 @@ import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__ import { metadataIndexPattern } from '../../../../common/endpoint/constants'; import { parseExperimentalConfigValue } from '../../../../common/experimental_features'; import { metadataQueryStrategyV1 } from './support/query_strategies'; +import { get } from 'lodash'; describe('query builder v1', () => { describe('MetadataListESQuery', () => { @@ -179,7 +180,7 @@ describe('query builder v1', () => { const mockID = 'AABBCCDD-0011-2233-AA44-DEADBEEF8899'; const query = getESQueryHostMetadataByID(mockID, metadataQueryStrategyV1()); - expect(query.body.query.bool.filter[0].bool.should).toContainEqual({ + expect(get(query, 'body.query.bool.filter.0.bool.should')).toContainEqual({ term: { 'agent.id': mockID }, }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/query_strategies.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/query_strategies.ts index 2f875ec2754a4..506c02fc2f1ec 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/query_strategies.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/support/query_strategies.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { SearchResponse } from 'elasticsearch'; +import { SearchResponse } from '@elastic/elasticsearch/api/types'; import { metadataCurrentIndexPattern, metadataIndexPattern, @@ -13,10 +13,6 @@ import { import { HostMetadata, MetadataQueryStrategyVersions } from '../../../../../common/endpoint/types'; import { HostListQueryResult, HostQueryResult, MetadataQueryStrategy } from '../../../types'; -interface HitSource { - _source: HostMetadata; -} - export function metadataQueryStrategyV1(): MetadataQueryStrategy { return { index: metadataIndexPattern, @@ -42,11 +38,13 @@ export function metadataQueryStrategyV1(): MetadataQueryStrategy { ): HostListQueryResult => { const response = searchResponse as SearchResponse; return { - resultLength: response?.aggregations?.total?.value || 0, + resultLength: + ((response?.aggregations?.total as unknown) as { value?: number; relation: string }) + ?.value || 0, resultList: response.hits.hits - .map((hit) => hit.inner_hits.most_recent.hits.hits) - .flatMap((data) => data as HitSource) - .map((entry) => entry._source), + .map((hit) => hit.inner_hits?.most_recent.hits.hits) + .flatMap((data) => data) + .map((entry) => (entry?._source ?? {}) as HostMetadata), queryStrategyVersion: MetadataQueryStrategyVersions.VERSION_1, }; }, @@ -75,7 +73,7 @@ export function metadataQueryStrategyV2(): MetadataQueryStrategy { >; const list = response.hits.hits.length > 0 - ? response.hits.hits.map((entry) => stripHostDetails(entry._source)) + ? response.hits.hits.map((entry) => stripHostDetails(entry?._source as HostMetadata)) : []; return { @@ -95,7 +93,7 @@ export function metadataQueryStrategyV2(): MetadataQueryStrategy { resultLength: response.hits.hits.length, result: response.hits.hits.length > 0 - ? stripHostDetails(response.hits.hits[0]._source) + ? stripHostDetails(response.hits.hits[0]._source as HostMetadata) : undefined, queryStrategyVersion: MetadataQueryStrategyVersions.VERSION_2, }; diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/policy/handlers.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/policy/handlers.test.ts index ca9b8832bebd0..c8b36a22b359a 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/policy/handlers.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/policy/handlers.test.ts @@ -13,10 +13,9 @@ import { import { createMockAgentService } from '../../../../../fleet/server/mocks'; import { getHostPolicyResponseHandler, getAgentPolicySummaryHandler } from './handlers'; import { - ILegacyScopedClusterClient, KibanaResponseFactory, SavedObjectsClientContract, -} from 'kibana/server'; +} from '../../../../../../../src/core/server'; import { elasticsearchServiceMock, httpServerMock, @@ -30,16 +29,19 @@ import { parseExperimentalConfigValue } from '../../../../common/experimental_fe import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__'; import { Agent } from '../../../../../fleet/common/types/models'; import { AgentService } from '../../../../../fleet/server/services'; +import { get } from 'lodash'; +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { ScopedClusterClientMock } from '../../../../../../../src/core/server/elasticsearch/client/mocks'; describe('test policy response handler', () => { let endpointAppContextService: EndpointAppContextService; - let mockScopedClient: jest.Mocked; + let mockScopedClient: ScopedClusterClientMock; let mockSavedObjectClient: jest.Mocked; let mockResponse: jest.Mocked; describe('test policy response handler', () => { beforeEach(() => { - mockScopedClient = elasticsearchServiceMock.createLegacyScopedClusterClient(); + mockScopedClient = elasticsearchServiceMock.createScopedClusterClient(); mockSavedObjectClient = savedObjectsClientMock.create(); mockResponse = httpServerMock.createResponseFactory(); endpointAppContextService = new EndpointAppContextService(); @@ -52,7 +54,9 @@ describe('test policy response handler', () => { const response = createSearchResponse(new EndpointDocGenerator().generatePolicyResponse()); const hostPolicyResponseHandler = getHostPolicyResponseHandler(); - mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => Promise.resolve(response)); + (mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ body: response }) + ); const mockRequest = httpServerMock.createKibanaRequest({ params: { agentId: 'id' }, }); @@ -65,14 +69,16 @@ describe('test policy response handler', () => { expect(mockResponse.ok).toBeCalled(); const result = mockResponse.ok.mock.calls[0][0]?.body as GetHostPolicyResponse; - expect(result.policy_response.agent.id).toEqual(response.hits.hits[0]._source.agent.id); + expect(result.policy_response.agent.id).toEqual( + get(response, 'hits.hits.0._source.agent.id') + ); }); it('should return not found when there is no response policy for host', async () => { const hostPolicyResponseHandler = getHostPolicyResponseHandler(); - mockScopedClient.callAsCurrentUser.mockImplementationOnce(() => - Promise.resolve(createSearchResponse()) + (mockScopedClient.asCurrentUser.search as jest.Mock).mockImplementationOnce(() => + Promise.resolve({ body: createSearchResponse() }) ); const mockRequest = httpServerMock.createKibanaRequest({ @@ -109,7 +115,7 @@ describe('test policy response handler', () => { }; beforeEach(() => { - mockScopedClient = elasticsearchServiceMock.createLegacyScopedClusterClient(); + mockScopedClient = elasticsearchServiceMock.createScopedClusterClient(); mockSavedObjectClient = savedObjectsClientMock.create(); mockResponse = httpServerMock.createResponseFactory(); endpointAppContextService = new EndpointAppContextService(); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/policy/handlers.ts b/x-pack/plugins/security_solution/server/endpoint/routes/policy/handlers.ts index ec1fad80701b6..45b6201c47773 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/policy/handlers.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/policy/handlers.ts @@ -25,7 +25,7 @@ export const getHostPolicyResponseHandler = function (): RequestHandler< const doc = await getPolicyResponseByAgentId( policyIndexPattern, request.query.agentId, - context.core.elasticsearch.legacy.client + context.core.elasticsearch.client ); if (doc) { diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/policy/service.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/policy/service.test.ts index 8043eae20b30e..8646a05900f80 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/policy/service.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/policy/service.test.ts @@ -24,7 +24,7 @@ describe('test policy query', () => { it('queries for the correct host', async () => { const agentId = 'f757d3c0-e874-11ea-9ad9-015510b487f4'; const query = getESQueryPolicyResponseByAgentID(agentId, 'anyindex'); - expect(query.body.query.bool.filter.term).toEqual({ 'agent.id': agentId }); + expect(query.body?.query?.bool?.filter).toEqual({ term: { 'agent.id': agentId } }); }); it('filters out initial policy by ID', async () => { @@ -32,8 +32,10 @@ describe('test policy query', () => { 'f757d3c0-e874-11ea-9ad9-015510b487f4', 'anyindex' ); - expect(query.body.query.bool.must_not.term).toEqual({ - 'Endpoint.policy.applied.id': '00000000-0000-0000-0000-000000000000', + expect(query.body?.query?.bool?.must_not).toEqual({ + term: { + 'Endpoint.policy.applied.id': '00000000-0000-0000-0000-000000000000', + }, }); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/policy/service.ts b/x-pack/plugins/security_solution/server/endpoint/routes/policy/service.ts index af5a885b78040..987bef15afe98 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/policy/service.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/policy/service.ts @@ -5,18 +5,21 @@ * 2.0. */ -import { SearchResponse } from 'elasticsearch'; import { ElasticsearchClient, - ILegacyScopedClusterClient, + IScopedClusterClient, SavedObjectsClientContract, -} from 'kibana/server'; +} from '../../../../../../../src/core/server'; import { GetHostPolicyResponse, HostPolicyResponse } from '../../../../common/endpoint/types'; import { INITIAL_POLICY_ID } from './index'; import { Agent } from '../../../../../fleet/common/types/models'; import { EndpointAppContext } from '../../types'; +import { ISearchRequestParams } from '../../../../../../../src/plugins/data/common'; -export function getESQueryPolicyResponseByAgentID(agentID: string, index: string) { +export const getESQueryPolicyResponseByAgentID = ( + agentID: string, + index: string +): ISearchRequestParams => { return { body: { query: { @@ -44,26 +47,23 @@ export function getESQueryPolicyResponseByAgentID(agentID: string, index: string }, index, }; -} +}; export async function getPolicyResponseByAgentId( index: string, agentID: string, - dataClient: ILegacyScopedClusterClient + dataClient: IScopedClusterClient ): Promise { const query = getESQueryPolicyResponseByAgentID(agentID, index); - const response = (await dataClient.callAsCurrentUser( - 'search', - query - )) as SearchResponse; + const response = await dataClient.asCurrentUser.search(query); - if (response.hits.hits.length === 0) { - return undefined; + if (response.body.hits.hits.length > 0 && response.body.hits.hits[0]._source != null) { + return { + policy_response: response.body.hits.hits[0]._source, + }; } - return { - policy_response: response.hits.hits[0]._source, - }; + return undefined; } const transformAgentVersionMap = (versionMap: Map): { [key: string]: number } => { diff --git a/x-pack/plugins/security_solution/server/endpoint/types.ts b/x-pack/plugins/security_solution/server/endpoint/types.ts index 8006bf20d4517..b3c7e58afe991 100644 --- a/x-pack/plugins/security_solution/server/endpoint/types.ts +++ b/x-pack/plugins/security_solution/server/endpoint/types.ts @@ -6,7 +6,8 @@ */ import { LoggerFactory } from 'kibana/server'; -import { SearchResponse } from 'elasticsearch'; + +import { SearchResponse } from '@elastic/elasticsearch/api/types'; import { ConfigType } from '../config'; import { EndpointAppContextService } from './endpoint_app_context_services'; import { JsonObject } from '../../../../../src/plugins/kibana_utils/common'; diff --git a/x-pack/plugins/security_solution/server/index.ts b/x-pack/plugins/security_solution/server/index.ts index a4b9dddec812e..5b95ddf13e033 100644 --- a/x-pack/plugins/security_solution/server/index.ts +++ b/x-pack/plugins/security_solution/server/index.ts @@ -48,17 +48,4 @@ export const config: PluginConfigDescriptor = { export { ConfigType, Plugin, PluginSetup, PluginStart }; export { AppClient }; -// Exports to be shared with plugins such as x-pack/lists plugin -export { deleteTemplate } from './lib/detection_engine/index/delete_template'; -export { deletePolicy } from './lib/detection_engine/index/delete_policy'; -export { deleteAllIndex } from './lib/detection_engine/index/delete_all_index'; -export { setPolicy } from './lib/detection_engine/index/set_policy'; -export { setTemplate } from './lib/detection_engine/index/set_template'; -export { getTemplateExists } from './lib/detection_engine/index/get_template_exists'; -export { getPolicyExists } from './lib/detection_engine/index/get_policy_exists'; -export { createBootstrapIndex } from './lib/detection_engine/index/create_bootstrap_index'; -export { getIndexExists } from './lib/detection_engine/index/get_index_exists'; -export { buildRouteValidation } from './utils/build_validation/route_validation'; -export { transformError, buildSiemResponse } from './lib/detection_engine/routes/utils'; -export { readPrivileges } from './lib/detection_engine/privileges/read_privileges'; export type { AppRequestContext } from './types'; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/index/create_bootstrap_index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/index/create_bootstrap_index.ts index fd9b63152ddd3..02f8f3f7b36ab 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/index/create_bootstrap_index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/index/create_bootstrap_index.ts @@ -10,6 +10,10 @@ import { ElasticsearchClient } from 'kibana/server'; // See the reference(s) below on explanations about why -000001 was chosen and // why the is_write_index is true as well as the bootstrapping step which is needed. // Ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/applying-policy-to-template.html + +/** + * @deprecated Use the one from kbn-securitysolution-es-utils + */ export const createBootstrapIndex = async ( esClient: ElasticsearchClient, index: string diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/index/delete_all_index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/index/delete_all_index.ts index 48bbbdcbf3a48..d76290921fac8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/index/delete_all_index.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/index/delete_all_index.ts @@ -7,6 +7,9 @@ import { ElasticsearchClient } from 'kibana/server'; +/** + * @deprecated Use the one from kbn-securitysolution-es-utils + */ export const deleteAllIndex = async ( esClient: ElasticsearchClient, pattern: string, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/index/delete_policy.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/index/delete_policy.ts index d671d256f56aa..924970d304c88 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/index/delete_policy.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/index/delete_policy.ts @@ -7,6 +7,9 @@ import { ElasticsearchClient } from 'kibana/server'; +/** + * @deprecated Use the one from kbn-securitysolution-es-utils + */ export const deletePolicy = async ( esClient: ElasticsearchClient, policy: string diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/index/delete_template.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/index/delete_template.ts index e57bbd77120f2..5466fd03f534c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/index/delete_template.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/index/delete_template.ts @@ -6,6 +6,9 @@ */ import { ElasticsearchClient } from 'kibana/server'; +/** + * @deprecated Use the one from kbn-securitysolution-es-utils + */ export const deleteTemplate = async ( esClient: ElasticsearchClient, name: string diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/index/get_index_exists.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/index/get_index_exists.ts index cc7f22064572c..7ca7f9818ba0b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/index/get_index_exists.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/index/get_index_exists.ts @@ -7,6 +7,9 @@ import { ElasticsearchClient } from 'kibana/server'; +/** + * @deprecated Use the one from kbn-securitysolution-es-utils + */ export const getIndexExists = async ( esClient: ElasticsearchClient, index: string diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/index/get_policy_exists.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/index/get_policy_exists.ts index c0d7c38a4bb02..6ebdac0d244cb 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/index/get_policy_exists.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/index/get_policy_exists.ts @@ -7,6 +7,9 @@ import { ElasticsearchClient } from 'kibana/server'; +/** + * @deprecated Use the one from kbn-securitysolution-es-utils + */ export const getPolicyExists = async ( esClient: ElasticsearchClient, policy: string diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/index/get_template_exists.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/index/get_template_exists.ts index 50ec3bfc670d5..af5f874a05688 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/index/get_template_exists.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/index/get_template_exists.ts @@ -7,6 +7,9 @@ import { ElasticsearchClient } from 'kibana/server'; +/** + * @deprecated Use the one from kbn-securitysolution-es-utils + */ export const getTemplateExists = async ( esClient: ElasticsearchClient, template: string diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/index/set_policy.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/index/set_policy.ts index 9dbcdd795ac71..113b9d368e0d9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/index/set_policy.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/index/set_policy.ts @@ -7,6 +7,9 @@ import { ElasticsearchClient } from 'kibana/server'; +/** + * @deprecated Use the one from kbn-securitysolution-es-utils + */ export const setPolicy = async ( esClient: ElasticsearchClient, policy: string, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/index/set_template.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/index/set_template.ts index e63dbbd6c3e8f..288377c306325 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/index/set_template.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/index/set_template.ts @@ -7,6 +7,9 @@ import { ElasticsearchClient } from 'kibana/server'; +/** + * @deprecated Use the one from kbn-securitysolution-es-utils + */ export const setTemplate = async ( esClient: ElasticsearchClient, name: string, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.ts index cedf1744ab06e..c2acbf9c5cc0a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.ts @@ -25,6 +25,9 @@ export interface OutputError { statusCode: number; } +/** + * @deprecated Use kbn-securitysolution-es-utils version + */ export const transformError = (err: Error & Partial): OutputError => { if (Boom.isBoom(err)) { return { diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 46467a21ca7ab..158c2e94b2d7a 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -305,7 +305,10 @@ export class Plugin implements IPlugin { - const securitySolutionSearchStrategy = securitySolutionSearchStrategyProvider(depsStart.data); + const securitySolutionSearchStrategy = securitySolutionSearchStrategyProvider( + depsStart.data, + endpointContext + ); const securitySolutionTimelineSearchStrategy = securitySolutionTimelineSearchStrategyProvider( depsStart.data ); diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/index.tsx b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/index.tsx index 9e85eefe21e8a..fa78a8d59803d 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/index.tsx +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/authentications/index.tsx @@ -58,7 +58,6 @@ export const authentications: SecuritySolutionFactory fakeTotalCount; - return { ...response, inspect, diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/__mocks__/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/__mocks__/index.ts index 7561682e070fc..9dfff5e11715d 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/__mocks__/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/__mocks__/index.ts @@ -1370,6 +1370,20 @@ export const formattedSearchStrategyResponse = { terms: { field: 'cloud.region', size: 10, order: { timestamp: 'desc' } }, aggs: { timestamp: { max: { field: '@timestamp' } } }, }, + endpoint_id: { + filter: { + term: { + 'agent.type': 'endpoint', + }, + }, + aggs: { + value: { + terms: { + field: 'agent.id', + }, + }, + }, + }, }, query: { bool: { @@ -1413,6 +1427,20 @@ export const expectedDsl = { track_total_hits: false, body: { aggregations: { + endpoint_id: { + filter: { + term: { + 'agent.type': 'endpoint', + }, + }, + aggs: { + value: { + terms: { + field: 'agent.id', + }, + }, + }, + }, host_architecture: { terms: { field: 'host.architecture', diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/helpers.ts index a581370cb5720..1b6e927f33638 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/helpers.ts @@ -7,16 +7,23 @@ import { set } from '@elastic/safer-lodash-set/fp'; import { get, has, head } from 'lodash/fp'; +import { + IScopedClusterClient, + SavedObjectsClientContract, +} from '../../../../../../../../../src/core/server'; import { hostFieldsMap } from '../../../../../../common/ecs/ecs_fields'; -import { toObjectArrayOfStrings } from '../../../../../../common/utils/to_array'; import { Direction } from '../../../../../../common/search_strategy/common'; import { AggregationRequest, + EndpointFields, HostAggEsItem, HostBuckets, HostItem, HostValue, } from '../../../../../../common/search_strategy/security_solution/hosts'; +import { toObjectArrayOfStrings } from '../../../../../../common/utils/to_array'; +import { getHostMetaData } from '../../../../../endpoint/routes/metadata/handlers'; +import { EndpointAppContext } from '../../../../../endpoint/types'; export const HOST_FIELDS = [ '_id', @@ -38,6 +45,8 @@ export const HOST_FIELDS = [ 'endpoint.endpointPolicy', 'endpoint.policyStatus', 'endpoint.sensorVersion', + 'agent.type', + 'endpoint.id', ]; export const buildFieldsTermAggregation = (esFields: readonly string[]): AggregationRequest => @@ -99,8 +108,8 @@ const getTermsAggregationTypeFromField = (field: string): AggregationRequest => }; }; -export const formatHostItem = (bucket: HostAggEsItem): HostItem => - HOST_FIELDS.reduce((flattenedFields, fieldName) => { +export const formatHostItem = (bucket: HostAggEsItem): HostItem => { + return HOST_FIELDS.reduce((flattenedFields, fieldName) => { const fieldValue = getHostFieldValue(fieldName, bucket); if (fieldValue != null) { if (fieldName === '_id') { @@ -114,11 +123,13 @@ export const formatHostItem = (bucket: HostAggEsItem): HostItem => } return flattenedFields; }, {}); +}; const getHostFieldValue = (fieldName: string, bucket: HostAggEsItem): string | string[] | null => { const aggField = hostFieldsMap[fieldName] ? hostFieldsMap[fieldName].replace(/\./g, '_') : fieldName.replace(/\./g, '_'); + if ( [ 'host.ip', @@ -134,10 +145,7 @@ const getHostFieldValue = (fieldName: string, bucket: HostAggEsItem): string | s return data.buckets.map((obj) => obj.key); } else if (has(`${aggField}.buckets`, bucket)) { return getFirstItem(get(`${aggField}`, bucket)); - } else if (has(aggField, bucket)) { - const valueObj: HostValue = get(aggField, bucket); - return valueObj.value_as_string; - } else if (['host.name', 'host.os.name', 'host.os.version'].includes(fieldName)) { + } else if (['host.name', 'host.os.name', 'host.os.version', 'endpoint.id'].includes(fieldName)) { switch (fieldName) { case 'host.name': return get('key', bucket) || null; @@ -145,7 +153,12 @@ const getHostFieldValue = (fieldName: string, bucket: HostAggEsItem): string | s return get('os.hits.hits[0]._source.host.os.name', bucket) || null; case 'host.os.version': return get('os.hits.hits[0]._source.host.os.version', bucket) || null; + case 'endpoint.id': + return get('endpoint_id.value.buckets[0].key', bucket) || null; } + } else if (has(aggField, bucket)) { + const valueObj: HostValue = get(aggField, bucket); + return valueObj.value_as_string; } else if (aggField === '_id') { const hostName = get(`host_name`, bucket); return hostName ? getFirstItem(hostName) : null; @@ -160,3 +173,42 @@ const getFirstItem = (data: HostBuckets): string | null => { } return firstItem.key; }; + +export const getHostEndpoint = async ( + id: string | null, + deps: { + esClient: IScopedClusterClient; + savedObjectsClient: SavedObjectsClientContract; + endpointContext: EndpointAppContext; + } +): Promise => { + const { esClient, endpointContext, savedObjectsClient } = deps; + const logger = endpointContext.logFactory.get('metadata'); + try { + const agentService = endpointContext.service.getAgentService(); + if (agentService === undefined) { + throw new Error('agentService not available'); + } + const metadataRequestContext = { + esClient, + endpointAppContextService: endpointContext.service, + logger, + savedObjectsClient, + }; + const endpointData = + id != null && metadataRequestContext.endpointAppContextService.getAgentService() != null + ? await getHostMetaData(metadataRequestContext, id, undefined) + : null; + + return endpointData != null && endpointData.metadata + ? { + endpointPolicy: endpointData.metadata.Endpoint.policy.applied.name, + policyStatus: endpointData.metadata.Endpoint.policy.applied.status, + sensorVersion: endpointData.metadata.agent.version, + } + : null; + } catch (err) { + logger.warn(JSON.stringify(err, null, 2)); + return null; + } +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/index.test.tsx b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/index.test.tsx index 244b826c7caeb..4474b9f288570 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/index.test.tsx +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/index.test.tsx @@ -12,6 +12,32 @@ import { mockSearchStrategyResponse, formattedSearchStrategyResponse, } from './__mocks__'; +import { + IScopedClusterClient, + SavedObjectsClientContract, +} from '../../../../../../../../../src/core/server'; +import { EndpointAppContext } from '../../../../../endpoint/types'; +import { EndpointAppContextService } from '../../../../../endpoint/endpoint_app_context_services'; + +const mockDeps = { + esClient: {} as IScopedClusterClient, + savedObjectsClient: {} as SavedObjectsClientContract, + endpointContext: { + logFactory: { + get: jest.fn().mockReturnValue({ + warn: jest.fn(), + }), + }, + config: jest.fn().mockResolvedValue({}), + experimentalFeatures: { + trustedAppsByPolicyEnabled: false, + metricsEntitiesEnabled: false, + eventFilteringEnabled: false, + hostIsolationEnabled: false, + }, + service: {} as EndpointAppContextService, + } as EndpointAppContext, +}; describe('hostDetails search strategy', () => { const buildHostDetailsQuery = jest.spyOn(buildQuery, 'buildHostDetailsQuery'); @@ -29,7 +55,7 @@ describe('hostDetails search strategy', () => { describe('parse', () => { test('should parse data correctly', async () => { - const result = await hostDetails.parse(mockOptions, mockSearchStrategyResponse); + const result = await hostDetails.parse(mockOptions, mockSearchStrategyResponse, mockDeps); expect(result).toMatchObject(formattedSearchStrategyResponse); }); }); diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/index.ts index 5da64cc8f7a90..562b7e4fbc167 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/index.ts @@ -10,28 +10,58 @@ import { get } from 'lodash/fp'; import { IEsSearchResponse } from '../../../../../../../../../src/plugins/data/common'; import { HostAggEsData, - HostAggEsItem, HostDetailsStrategyResponse, HostsQueries, HostDetailsRequestOptions, + EndpointFields, } from '../../../../../../common/search_strategy/security_solution/hosts'; import { inspectStringifyObject } from '../../../../../utils/build_query'; import { SecuritySolutionFactory } from '../../types'; import { buildHostDetailsQuery } from './query.host_details.dsl'; -import { formatHostItem } from './helpers'; +import { formatHostItem, getHostEndpoint } from './helpers'; +import { EndpointAppContext } from '../../../../../endpoint/types'; +import { + IScopedClusterClient, + SavedObjectsClientContract, +} from '../../../../../../../../../src/core/server'; export const hostDetails: SecuritySolutionFactory = { buildDsl: (options: HostDetailsRequestOptions) => buildHostDetailsQuery(options), parse: async ( options: HostDetailsRequestOptions, - response: IEsSearchResponse + response: IEsSearchResponse, + deps?: { + esClient: IScopedClusterClient; + savedObjectsClient: SavedObjectsClientContract; + endpointContext: EndpointAppContext; + } ): Promise => { - const aggregations: HostAggEsItem = get('aggregations', response.rawResponse) || {}; + const aggregations = get('aggregations', response.rawResponse); + const inspect = { dsl: [inspectStringifyObject(buildHostDetailsQuery(options))], }; + + if (aggregations == null) { + return { ...response, inspect, hostDetails: {} }; + } + const formattedHostItem = formatHostItem(aggregations); - return { ...response, inspect, hostDetails: formattedHostItem }; + const ident = // endpoint-generated ID, NOT elastic-agent-id + formattedHostItem.endpoint && formattedHostItem.endpoint.id + ? Array.isArray(formattedHostItem.endpoint.id) + ? formattedHostItem.endpoint.id[0] + : formattedHostItem.endpoint.id + : null; + if (deps == null) { + return { ...response, inspect, hostDetails: { ...formattedHostItem } }; + } + const endpoint: EndpointFields | null = await getHostEndpoint(ident, deps); + return { + ...response, + inspect, + hostDetails: endpoint != null ? { ...formattedHostItem, endpoint } : formattedHostItem, + }; }, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/query.host_details.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/query.host_details.dsl.ts index fb8296d6593b0..45afed2526aa3 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/query.host_details.dsl.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/hosts/details/query.host_details.dsl.ts @@ -16,7 +16,10 @@ export const buildHostDetailsQuery = ({ defaultIndex, timerange: { from, to }, }: HostDetailsRequestOptions): ISearchRequestParams => { - const esFields = reduceFields(HOST_FIELDS, { ...hostFieldsMap, ...cloudFieldsMap }); + const esFields = reduceFields(HOST_FIELDS, { + ...hostFieldsMap, + ...cloudFieldsMap, + }); const filter = [ { term: { 'host.name': hostName } }, @@ -39,6 +42,20 @@ export const buildHostDetailsQuery = ({ body: { aggregations: { ...buildFieldsTermAggregation(esFields.filter((field) => !['@timestamp'].includes(field))), + endpoint_id: { + filter: { + term: { + 'agent.type': 'endpoint', + }, + }, + aggs: { + value: { + terms: { + field: 'agent.id', + }, + }, + }, + }, }, query: { bool: { filter } }, size: 0, diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/types.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/types.ts index 3455b627144bf..4bdf97b489805 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/types.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/types.ts @@ -5,6 +5,10 @@ * 2.0. */ +import { + IScopedClusterClient, + SavedObjectsClientContract, +} from '../../../../../../../src/core/server'; import { IEsSearchResponse, ISearchRequestParams, @@ -14,11 +18,17 @@ import { StrategyRequestType, StrategyResponseType, } from '../../../../common/search_strategy/security_solution'; +import { EndpointAppContext } from '../../../endpoint/types'; export interface SecuritySolutionFactory { buildDsl: (options: StrategyRequestType) => ISearchRequestParams; parse: ( options: StrategyRequestType, - response: IEsSearchResponse + response: IEsSearchResponse, + deps?: { + esClient: IScopedClusterClient; + savedObjectsClient: SavedObjectsClientContract; + endpointContext: EndpointAppContext; + } ) => Promise>; } diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/index.ts index 2980f63df8a67..0883a144615bc 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/index.ts @@ -19,9 +19,11 @@ import { } from '../../../common/search_strategy/security_solution'; import { securitySolutionFactory } from './factory'; import { SecuritySolutionFactory } from './factory/types'; +import { EndpointAppContext } from '../../endpoint/types'; export const securitySolutionSearchStrategyProvider = ( - data: PluginStart + data: PluginStart, + endpointContext: EndpointAppContext ): ISearchStrategy, StrategyResponseType> => { const es = data.search.getSearchStrategy(ENHANCED_ES_SEARCH_STRATEGY); @@ -42,7 +44,13 @@ export const securitySolutionSearchStrategyProvider = queryFactory.parse(request, esSearchRes)) + mergeMap((esSearchRes) => + queryFactory.parse(request, esSearchRes, { + esClient: deps.esClient, + savedObjectsClient: deps.savedObjectsClient, + endpointContext, + }) + ) ); }, cancel: async (id, options, deps) => { diff --git a/x-pack/plugins/security_solution/server/usage/detections/detections_metrics_helpers.ts b/x-pack/plugins/security_solution/server/usage/detections/detections_metrics_helpers.ts index 97fc8c2884289..8697fa138d4b0 100644 --- a/x-pack/plugins/security_solution/server/usage/detections/detections_metrics_helpers.ts +++ b/x-pack/plugins/security_solution/server/usage/detections/detections_metrics_helpers.ts @@ -193,7 +193,7 @@ export const getDetectionRuleMetrics = async ( filterPath: [], ignoreUnavailable: true, index: kibanaIndex, - size: 1, + size: 10_000, // elasticsearch index.max_result_window default value }; try { diff --git a/x-pack/plugins/spaces/public/plugin.tsx b/x-pack/plugins/spaces/public/plugin.tsx index 8892671551896..1ba1cd1a1f3d4 100644 --- a/x-pack/plugins/spaces/public/plugin.tsx +++ b/x-pack/plugins/spaces/public/plugin.tsx @@ -59,7 +59,7 @@ export class SpacesPlugin implements Plugin this.spacesManager.onActiveSpaceChange$, getActiveSpace: () => this.spacesManager.getActiveSpace(), }; diff --git a/x-pack/plugins/translations/translations/ja-JP.json b/x-pack/plugins/translations/translations/ja-JP.json index a2eb51e929fd4..244b294baffe5 100644 --- a/x-pack/plugins/translations/translations/ja-JP.json +++ b/x-pack/plugins/translations/translations/ja-JP.json @@ -8156,6 +8156,7 @@ "xpack.enterpriseSearch.appSearch.engine.sampleEngineBadge": "サンプルエンジン", "xpack.enterpriseSearch.appSearch.engine.schema.conflicts": "スキーマ競合", "xpack.enterpriseSearch.appSearch.engine.schema.title": "スキーマ", + "xpack.enterpriseSearch.appSearch.engine.schema.unconfirmedFieldLabel": "最近追加された項目", "xpack.enterpriseSearch.appSearch.engine.schema.unconfirmedFields": "新しい未確認のフィールド", "xpack.enterpriseSearch.appSearch.engine.searchUI.title": "Search UI", "xpack.enterpriseSearch.appSearch.engine.synonyms.title": "同義語", @@ -18751,11 +18752,6 @@ "xpack.security.checkup.insecureClusterMessage": "1 ビットを失わないでください。無料のセキュリティ機能を有効にしてください。", "xpack.security.checkup.insecureClusterTitle": "データが保護されていません", "xpack.security.common.extendedRoleDeprecationNotice": "{roleName} ロールは非推奨です。{reason}", - "xpack.security.components.sessionIdleTimeoutWarning.message": "操作がないため間もなくログアウト{timeout}します。再開するには [OK] をクリックしてください。", - "xpack.security.components.sessionIdleTimeoutWarning.okButtonText": "OK", - "xpack.security.components.sessionIdleTimeoutWarning.title": "警告", - "xpack.security.components.sessionLifespanWarning.message": "セッションは最大時間制限{timeout}に達しました。もう一度ログインする必要があります。", - "xpack.security.components.sessionLifespanWarning.title": "警告", "xpack.security.conflictingSessionError": "申し訳ありません。すでに有効なKibanaセッションがあります。新しいセッションを開始する場合は、先に既存のセッションからログアウトしてください。", "xpack.security.copyTokenField.copyButton": "クリップボードにコピー", "xpack.security.copyTokenField.tokenLabel": "トークン", @@ -21847,8 +21843,6 @@ "xpack.securitySolution.trustedapps.list.backButton": "戻る", "xpack.securitySolution.trustedapps.list.columns.actions": "アクション", "xpack.securitySolution.trustedapps.list.pageTitle": "信頼できるアプリケーション", - "xpack.securitySolution.trustedapps.list.search.button": "更新", - "xpack.securitySolution.trustedapps.list.search.placeholder": "検索", "xpack.securitySolution.trustedapps.listEmptyState.message": "現在、エンドポイントには信頼できるアプリケーションがありません。", "xpack.securitySolution.trustedapps.listEmptyState.title": "最初の信頼できるアプリケーションを追加", "xpack.securitySolution.trustedapps.logicalConditionBuilder.entry.field.description.hash": "md5、sha1、sha256", diff --git a/x-pack/plugins/translations/translations/zh-CN.json b/x-pack/plugins/translations/translations/zh-CN.json index b07661eaf5e26..8bdffce98d4ab 100644 --- a/x-pack/plugins/translations/translations/zh-CN.json +++ b/x-pack/plugins/translations/translations/zh-CN.json @@ -8225,6 +8225,7 @@ "xpack.enterpriseSearch.appSearch.engine.sampleEngineBadge": "样本引擎", "xpack.enterpriseSearch.appSearch.engine.schema.conflicts": "架构冲突", "xpack.enterpriseSearch.appSearch.engine.schema.title": "架构", + "xpack.enterpriseSearch.appSearch.engine.schema.unconfirmedFieldLabel": "最近添加", "xpack.enterpriseSearch.appSearch.engine.schema.unconfirmedFields": "新的未确认字段", "xpack.enterpriseSearch.appSearch.engine.searchUI.title": "搜索 UI", "xpack.enterpriseSearch.appSearch.engine.synonyms.title": "同义词", @@ -19008,11 +19009,6 @@ "xpack.security.checkup.insecureClusterMessage": "切勿丢下任何一位。启用我们的免费安全功能。", "xpack.security.checkup.insecureClusterTitle": "您的数据并非安全无忧", "xpack.security.common.extendedRoleDeprecationNotice": "{roleName} 角色已弃用。 {reason}", - "xpack.security.components.sessionIdleTimeoutWarning.message": "由于不活动，您会在 {timeout} 后自动注销。单击“确定”可以恢复。", - "xpack.security.components.sessionIdleTimeoutWarning.okButtonText": "确定", - "xpack.security.components.sessionIdleTimeoutWarning.title": "警告", - "xpack.security.components.sessionLifespanWarning.message": "您的会话将达到最大时间限制 {timeout}。您将需要重新登录。", - "xpack.security.components.sessionLifespanWarning.title": "警告", "xpack.security.conflictingSessionError": "抱歉，您已有活动的 Kibana 会话。如果希望开始新的会话，请首先从现有会话注销。", "xpack.security.copyTokenField.copyButton": "复制到剪贴板", "xpack.security.copyTokenField.tokenLabel": "令牌", @@ -22187,8 +22183,6 @@ "xpack.securitySolution.trustedapps.list.backButton": "返回", "xpack.securitySolution.trustedapps.list.columns.actions": "操作", "xpack.securitySolution.trustedapps.list.pageTitle": "受信任的应用程序", - "xpack.securitySolution.trustedapps.list.search.button": "刷新", - "xpack.securitySolution.trustedapps.list.search.placeholder": "搜索", "xpack.securitySolution.trustedapps.list.totalCount": "{totalItemCount, plural, other {# 个受信任的应用程序}}", "xpack.securitySolution.trustedapps.listEmptyState.message": "当前在您的终端上没有受信任应用程序。", "xpack.securitySolution.trustedapps.listEmptyState.title": "添加您的首个受信任应用程序", diff --git a/x-pack/plugins/uptime/public/components/fleet_package/custom_fields.tsx b/x-pack/plugins/uptime/public/components/fleet_package/custom_fields.tsx index 1dbd37dc00803..0bc2fc8823cec 100644 --- a/x-pack/plugins/uptime/public/components/fleet_package/custom_fields.tsx +++ b/x-pack/plugins/uptime/public/components/fleet_package/custom_fields.tsx @@ -327,7 +327,6 @@ export const CustomFields = memo( defaultMessage="Timeout must be 0 or greater and less than schedule interval" /> } - labelAppend={} helpText={ - - docs -   - - , - } - } - /> - -`; diff --git a/x-pack/plugins/uptime/public/components/monitor/ping_list/__snapshots__/expanded_row.test.tsx.snap b/x-pack/plugins/uptime/public/components/monitor/ping_list/__snapshots__/expanded_row.test.tsx.snap index 8ace0445d0eb7..377a2c9389bbd 100644 --- a/x-pack/plugins/uptime/public/components/monitor/ping_list/__snapshots__/expanded_row.test.tsx.snap +++ b/x-pack/plugins/uptime/public/components/monitor/ping_list/__snapshots__/expanded_row.test.tsx.snap @@ -174,10 +174,7 @@ exports[`PingListExpandedRow renders link to docs if body is not recorded but it rel="noopener" target="_blank" > - docs  - + docs { it('renders expected elements for valid props', () => { - expect(shallowWithIntl()).toMatchSnapshot(); + render(); + + expect(screen.getByText(/Body not recorded. Read our/)); + expect( + screen.getByRole('link', { name: 'docs External link (opens in a new tab or window)' }) + ).toBeInTheDocument(); + expect(screen.getByText(/for more information on recording response bodies./)); }); }); diff --git a/x-pack/plugins/uptime/public/components/monitor/ping_list/doc_link_body.tsx b/x-pack/plugins/uptime/public/components/monitor/ping_list/doc_link_body.tsx index 32d0b1e4122d8..67561022bce93 100644 --- a/x-pack/plugins/uptime/public/components/monitor/ping_list/doc_link_body.tsx +++ b/x-pack/plugins/uptime/public/components/monitor/ping_list/doc_link_body.tsx @@ -8,7 +8,7 @@ import React from 'react'; import { FormattedMessage } from '@kbn/i18n/react'; import { i18n } from '@kbn/i18n'; -import { EuiIcon, EuiLink, EuiText } from '@elastic/eui'; +import { EuiLink, EuiText } from '@elastic/eui'; const bodyDocsLink = 'https://www.elastic.co/guide/en/beats/heartbeat/current/configuration-heartbeat-options.html#monitor-http-response'; @@ -20,8 +20,6 @@ export const DocLinkForBody = () => { defaultMessage: 'docs', description: 'Docs link to set response body', })} -   - ); diff --git a/x-pack/plugins/uptime/public/state/reducers/selected_filters.test.ts b/x-pack/plugins/uptime/public/state/reducers/selected_filters.test.ts new file mode 100644 index 0000000000000..f69af7bd1b5a3 --- /dev/null +++ b/x-pack/plugins/uptime/public/state/reducers/selected_filters.test.ts @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { selectedFiltersReducer } from './selected_filters'; +import { + getSelectedFilters, + setSelectedFilters, + SelectedFilters, +} from '../actions/selected_filters'; +import { createAction } from 'redux-actions'; + +describe('selectedFiltersReducer', () => { + let state: SelectedFilters; + + beforeEach(() => { + state = { + locations: [], + ports: [], + schemes: ['http'], + tags: [], + }; + }); + + it('returns state by default', () => { + expect(selectedFiltersReducer(state, createAction('fake action')())).toEqual(state); + }); + + describe('setSelectedFilters', () => { + it('returns null for null payload', () => { + expect(selectedFiltersReducer(state, setSelectedFilters(null))).toBeNull(); + }); + + it('sets state to the action payload if state is null', () => { + expect( + selectedFiltersReducer( + null, + setSelectedFilters({ + locations: [], + ports: [5601], + schemes: [], + tags: [], + }) + ) + ).toEqual({ locations: [], ports: [5601], schemes: [], tags: [] }); + }); + + it('merges state and action payload', () => { + expect( + selectedFiltersReducer( + { + locations: [], + ports: [], + schemes: [], + tags: [], + }, + setSelectedFilters({ + locations: [], + ports: [5601], + schemes: [], + tags: ['prod'], + }) + ) + ).toEqual({ locations: [], ports: [5601], schemes: [], tags: ['prod'] }); + }); + }); + + describe('getSelectedFilters', () => { + it('returns the state', () => { + expect(selectedFiltersReducer(state, getSelectedFilters())).toEqual(state); + }); + }); +}); diff --git a/x-pack/plugins/uptime/public/state/reducers/selected_filters.ts b/x-pack/plugins/uptime/public/state/reducers/selected_filters.ts index fc24241331169..b5e3544b66156 100644 --- a/x-pack/plugins/uptime/public/state/reducers/selected_filters.ts +++ b/x-pack/plugins/uptime/public/state/reducers/selected_filters.ts @@ -19,14 +19,12 @@ export function selectedFiltersReducer( action: Action ): SelectedFilters | null { switch (action.type) { - case String(getSelectedFilters): - return state; case String(setSelectedFilters): - if (state === null) return { ...action.payload }; + if (action.payload === null) return null; return { - ...(state || {}), ...action.payload, }; + case String(getSelectedFilters): default: return state; } diff --git a/x-pack/plugins/uptime/server/lib/requests/get_journey_screenshot.ts b/x-pack/plugins/uptime/server/lib/requests/get_journey_screenshot.ts index 6a533d558c721..b5ff1de44000c 100644 --- a/x-pack/plugins/uptime/server/lib/requests/get_journey_screenshot.ts +++ b/x-pack/plugins/uptime/server/lib/requests/get_journey_screenshot.ts @@ -14,10 +14,21 @@ export interface GetJourneyScreenshotParams { stepIndex: number; } +export interface GetJourneyScreenshotResults { + blob: string | null; + mimeType: string | null; + stepName: string; + totalSteps: number; +} + export const getJourneyScreenshot: UMElasticsearchQueryFn< GetJourneyScreenshotParams, any -> = async ({ uptimeEsClient, checkGroup, stepIndex }) => { +> = async ({ + uptimeEsClient, + checkGroup, + stepIndex, +}): Promise => { const params = { track_total_hits: true, size: 0, @@ -48,7 +59,7 @@ export const getJourneyScreenshot: UMElasticsearchQueryFn< image: { top_hits: { size: 1, - _source: ['synthetics.blob', 'synthetics.step.name'], + _source: ['synthetics.blob', 'synthetics.blob_mime', 'synthetics.step.name'], }, }, }, @@ -65,6 +76,7 @@ export const getJourneyScreenshot: UMElasticsearchQueryFn< return { blob: stepHit?.synthetics?.blob ?? null, + mimeType: stepHit?.synthetics?.blob_mime ?? null, stepName: stepHit?.synthetics?.step?.name ?? '', totalSteps: result?.hits?.total.value, }; diff --git a/x-pack/plugins/uptime/server/rest_api/pings/journey_screenshots.ts b/x-pack/plugins/uptime/server/rest_api/pings/journey_screenshots.ts index ab8a01cfb9c3f..4d8c8f86ddf2d 100644 --- a/x-pack/plugins/uptime/server/rest_api/pings/journey_screenshots.ts +++ b/x-pack/plugins/uptime/server/rest_api/pings/journey_screenshots.ts @@ -37,7 +37,7 @@ export const createJourneyScreenshotRoute: UMRestApiRouteFactory = (libs: UMServ return response.ok({ body: Buffer.from(result.blob, 'base64'), headers: { - 'content-type': 'image/png', + 'content-type': result.mimeType || 'image/png', // falls back to 'image/png' for earlier versions of synthetics 'cache-control': 'max-age=600', 'caption-name': result.stepName, 'max-steps': result.totalSteps, diff --git a/x-pack/test/api_integration/apis/management/index_management/indices.js b/x-pack/test/api_integration/apis/management/index_management/indices.js index 3653d9916466d..cef1bdbba754b 100644 --- a/x-pack/test/api_integration/apis/management/index_management/indices.js +++ b/x-pack/test/api_integration/apis/management/index_management/indices.js @@ -34,8 +34,7 @@ export default function ({ getService }) { clearCache, } = registerHelpers({ supertest }); - // Failing: See https://github.com/elastic/kibana/issues/64473 - describe.skip('indices', () => { + describe('indices', () => { after(() => Promise.all([cleanUpEsResources()])); describe('clear cache', () => { diff --git a/x-pack/test/api_integration/apis/ml/index.ts b/x-pack/test/api_integration/apis/ml/index.ts index 41e94d69d2e9b..2206355003a75 100644 --- a/x-pack/test/api_integration/apis/ml/index.ts +++ b/x-pack/test/api_integration/apis/ml/index.ts @@ -36,6 +36,11 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { await ml.testResources.deleteIndexPatternByTitle('ft_module_siem_winlogbeat'); await ml.testResources.deleteIndexPatternByTitle('ft_farequote'); await ml.testResources.deleteIndexPatternByTitle('ft_logs-endpoint.events.*'); + await ml.testResources.deleteIndexPatternByTitle('ft_module_metricbeat'); + await ml.testResources.deleteIndexPatternByTitle('ft_module_siem_cloudtrail'); + await ml.testResources.deleteIndexPatternByTitle('ft_module_metrics_ui'); + await ml.testResources.deleteIndexPatternByTitle('ft_module_apache_data_stream'); + await ml.testResources.deleteIndexPatternByTitle('ft_module_nginx_data_stream'); await esArchiver.unload('ml/ecommerce'); await esArchiver.unload('ml/categorization'); @@ -54,6 +59,11 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { await esArchiver.unload('ml/farequote'); await esArchiver.unload('ml/bm_classification'); await esArchiver.unload('ml/ihp_outlier'); + await esArchiver.unload('ml/module_metricbeat'); + await esArchiver.unload('ml/module_siem_cloudtrail'); + await esArchiver.unload('ml/module_metrics_ui'); + await esArchiver.unload('ml/module_apache_data_stream'); + await esArchiver.unload('ml/module_nginx_data_stream'); await ml.testResources.resetKibanaTimeZone(); }); diff --git a/x-pack/test/api_integration/apis/ml/modules/get_module.ts b/x-pack/test/api_integration/apis/ml/modules/get_module.ts index 59aa6102b54e2..4fa79b915cc5d 100644 --- a/x-pack/test/api_integration/apis/ml/modules/get_module.ts +++ b/x-pack/test/api_integration/apis/ml/modules/get_module.ts @@ -14,6 +14,7 @@ import { COMMON_REQUEST_HEADERS } from '../../../../functional/services/ml/commo import { isPopulatedObject } from '../../../../../plugins/ml/common/util/object_utils'; const moduleIds = [ + 'apache_data_stream', 'apache_ecs', 'apm_jsbase', 'apm_nodejs', @@ -25,6 +26,7 @@ const moduleIds = [ 'metricbeat_system_ecs', 'metrics_ui_hosts', 'metrics_ui_k8s', + 'nginx_data_stream', 'nginx_ecs', 'sample_data_ecommerce', 'sample_data_weblogs', diff --git a/x-pack/test/api_integration/apis/ml/modules/index.ts b/x-pack/test/api_integration/apis/ml/modules/index.ts index c5a22921b92c5..ab46c4f0333c8 100644 --- a/x-pack/test/api_integration/apis/ml/modules/index.ts +++ b/x-pack/test/api_integration/apis/ml/modules/index.ts @@ -7,8 +7,24 @@ import { FtrProviderContext } from '../../../ftr_provider_context'; -export default function ({ loadTestFile }: FtrProviderContext) { +export default function ({ getService, loadTestFile }: FtrProviderContext) { + const ml = getService('ml'); + + const fleetPackages = ['apache-0.5.0', 'nginx-0.5.0']; + describe('modules', function () { + before(async () => { + for (const fleetPackage of fleetPackages) { + await ml.testResources.installFleetPackage(fleetPackage); + } + }); + + after(async () => { + for (const fleetPackage of fleetPackages) { + await ml.testResources.removeFleetPackage(fleetPackage); + } + }); + loadTestFile(require.resolve('./get_module')); loadTestFile(require.resolve('./recognize_module')); loadTestFile(require.resolve('./setup_module')); diff --git a/x-pack/test/api_integration/apis/ml/modules/recognize_module.ts b/x-pack/test/api_integration/apis/ml/modules/recognize_module.ts index d6020e17801fd..8cdc7168ac62b 100644 --- a/x-pack/test/api_integration/apis/ml/modules/recognize_module.ts +++ b/x-pack/test/api_integration/apis/ml/modules/recognize_module.ts @@ -104,7 +104,12 @@ export default ({ getService }: FtrProviderContext) => { user: USER.ML_POWERUSER, expected: { responseCode: 200, - moduleIds: ['siem_winlogbeat'], + moduleIds: [ + 'security_network', + 'security_windows', + 'siem_winlogbeat', + 'siem_winlogbeat_auth', + ], }, }, { @@ -146,6 +151,56 @@ export default ({ getService }: FtrProviderContext) => { moduleIds: ['security_linux', 'security_network', 'security_windows'], }, }, + { + testTitleSuffix: 'for metricbeat dataset', + sourceDataArchive: 'ml/module_metricbeat', + indexPattern: 'ft_module_metricbeat', + user: USER.ML_POWERUSER, + expected: { + responseCode: 200, + moduleIds: ['metricbeat_system_ecs', 'security_linux'], + }, + }, + { + testTitleSuffix: 'for siem clodutrail dataset', + sourceDataArchive: 'ml/module_siem_cloudtrail', + indexPattern: 'ft_module_siem_cloudtrail', + user: USER.ML_POWERUSER, + expected: { + responseCode: 200, + moduleIds: ['siem_cloudtrail'], + }, + }, + { + testTitleSuffix: 'for metrics ui dataset', + sourceDataArchive: 'ml/module_metrics_ui', + indexPattern: 'ft_module_metrics_ui', + user: USER.ML_POWERUSER, + expected: { + responseCode: 200, + moduleIds: ['security_linux'], // the metrics ui modules don't define a query and can't be recognized + }, + }, + { + testTitleSuffix: 'for apache data stream dataset', + sourceDataArchive: 'ml/module_apache_data_stream', + indexPattern: 'ft_module_apache_data_stream', + user: USER.ML_POWERUSER, + expected: { + responseCode: 200, + moduleIds: ['apache_data_stream'], + }, + }, + { + testTitleSuffix: 'for nginx data stream dataset', + sourceDataArchive: 'ml/module_nginx_data_stream', + indexPattern: 'ft_module_nginx_data_stream', + user: USER.ML_POWERUSER, + expected: { + responseCode: 200, + moduleIds: ['nginx_data_stream'], + }, + }, ]; async function executeRecognizeModuleRequest(indexPattern: string, user: USER, rspCode: number) { diff --git a/x-pack/test/api_integration/apis/ml/modules/setup_module.ts b/x-pack/test/api_integration/apis/ml/modules/setup_module.ts index 8e5da7c56bb64..186a87e547382 100644 --- a/x-pack/test/api_integration/apis/ml/modules/setup_module.ts +++ b/x-pack/test/api_integration/apis/ml/modules/setup_module.ts @@ -41,19 +41,16 @@ export default ({ getService }: FtrProviderContext) => { jobId: 'pf1_low_request_rate', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '10mb', }, { jobId: 'pf1_response_code_rates', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '10mb', }, { jobId: 'pf1_url_scanning', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '10mb', }, ], searches: [] as string[], @@ -81,19 +78,16 @@ export default ({ getService }: FtrProviderContext) => { jobId: 'pf2_low_request_rate', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, { jobId: 'pf2_response_code_rates', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, { jobId: 'pf2_url_scanning', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '16mb', }, ], searches: [] as string[], @@ -121,31 +115,26 @@ export default ({ getService }: FtrProviderContext) => { jobId: 'pf3_low_request_rate_ecs', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, { jobId: 'pf3_source_ip_request_rate_ecs', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, { jobId: 'pf3_source_ip_url_count_ecs', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '16mb', }, { jobId: 'pf3_status_code_rate_ecs', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, { jobId: 'pf3_visitor_rate_ecs', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, ], searches: ['ml_http_access_filebeat_ecs'] as string[], @@ -181,19 +170,16 @@ export default ({ getService }: FtrProviderContext) => { jobId: 'pf4_abnormal_span_durations_nodejs', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, { jobId: 'pf4_abnormal_trace_durations_nodejs', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, { jobId: 'pf4_decreased_throughput_nodejs', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, ], searches: [] as string[], @@ -221,7 +207,6 @@ export default ({ getService }: FtrProviderContext) => { jobId: 'pf5_high_mean_transaction_duration', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, ], searches: [] as string[], @@ -249,7 +234,6 @@ export default ({ getService }: FtrProviderContext) => { jobId: 'pf6_log-entry-rate', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, ], searches: [] as string[], @@ -277,7 +261,6 @@ export default ({ getService }: FtrProviderContext) => { jobId: 'pf7_log-entry-categories-count', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '41mb', }, ], searches: [] as string[], @@ -304,31 +287,26 @@ export default ({ getService }: FtrProviderContext) => { jobId: 'pf8_visitor_rate_ecs', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, { jobId: 'pf8_status_code_rate_ecs', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, { jobId: 'pf8_source_ip_url_count_ecs', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '16mb', }, { jobId: 'pf8_source_ip_request_rate_ecs', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, { jobId: 'pf8_low_request_rate_ecs', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, ], searches: ['ml_http_access_filebeat_ecs'] as string[], @@ -364,7 +342,6 @@ export default ({ getService }: FtrProviderContext) => { jobId: 'pf9_high_sum_total_sales', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, ], searches: [] as string[], @@ -392,7 +369,6 @@ export default ({ getService }: FtrProviderContext) => { jobId: 'pf11_suspicious_login_activity_ecs', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, ], searches: [] as string[], @@ -420,31 +396,26 @@ export default ({ getService }: FtrProviderContext) => { jobId: 'pf12_packetbeat_dns_tunneling', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '16mb', }, { jobId: 'pf12_packetbeat_rare_dns_question', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, { jobId: 'pf12_packetbeat_rare_server_domain', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, { jobId: 'pf12_packetbeat_rare_urls', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, { jobId: 'pf12_packetbeat_rare_user_agent', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, ], searches: [] as string[], @@ -472,7 +443,6 @@ export default ({ getService }: FtrProviderContext) => { jobId: 'pf13_high_latency_by_geo', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, ], searches: [] as string[], @@ -500,13 +470,11 @@ export default ({ getService }: FtrProviderContext) => { jobId: 'pf14_hosts_high_count_process_events_ecs', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, { jobId: 'pf14_hosts_rare_process_activity_ecs', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, ], searches: ['ml_auditbeat_hosts_process_events_ecs'] as string[], @@ -541,37 +509,31 @@ export default ({ getService }: FtrProviderContext) => { jobId: 'pf15_v2_rare_process_by_host_linux_ecs', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, { jobId: 'pf15_v2_linux_rare_metadata_user', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, { jobId: 'pf15_v2_linux_rare_metadata_process', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, { jobId: 'pf15_v2_linux_anomalous_user_name_ecs', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, { jobId: 'pf15_v2_linux_anomalous_process_all_hosts_ecs', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, { jobId: 'pf15_v2_linux_anomalous_network_port_activity_ecs', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, ], searches: [] as string[], @@ -599,49 +561,397 @@ export default ({ getService }: FtrProviderContext) => { jobId: 'pf16_v2_rare_process_by_host_windows_ecs', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, { jobId: 'pf16_v2_windows_anomalous_network_activity_ecs', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, { jobId: 'pf16_v2_windows_anomalous_path_activity_ecs', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '10mb', }, { jobId: 'pf16_v2_windows_anomalous_process_all_hosts_ecs', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, { jobId: 'pf16_v2_windows_anomalous_process_creation', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, { jobId: 'pf16_v2_windows_anomalous_user_name_ecs', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, { jobId: 'pf16_v2_windows_rare_metadata_process', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', }, { jobId: 'pf16_v2_windows_rare_metadata_user', jobState: JOB_STATE.CLOSED, datafeedState: DATAFEED_STATE.STOPPED, - modelMemoryLimit: '11mb', + }, + ], + searches: [] as string[], + visualizations: [] as string[], + dashboards: [] as string[], + }, + }, + { + testTitleSuffix: + 'for metricbeat_system_ecs with prefix, startDatafeed true and estimateModelMemory true', + sourceDataArchive: 'ml/module_metricbeat', + indexPattern: { name: 'ft_module_metricbeat', timeField: '@timestamp' }, + module: 'metricbeat_system_ecs', + user: USER.ML_POWERUSER, + requestBody: { + prefix: 'pf17_', + indexPatternName: 'ft_module_metricbeat', + startDatafeed: true, + end: Date.now(), + }, + expected: { + responseCode: 200, + jobs: [ + { + jobId: 'pf17_max_disk_utilization_ecs', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf17_metricbeat_outages_ecs', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf17_high_mean_cpu_iowait_ecs', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + ], + searches: [] as string[], + visualizations: [] as string[], + dashboards: [] as string[], + }, + }, + { + testTitleSuffix: + 'for metrics_ui_hosts with prefix, startDatafeed true and estimateModelMemory true', + sourceDataArchive: 'ml/module_metrics_ui', + indexPattern: { name: 'ft_module_metrics_ui', timeField: '@timestamp' }, + module: 'metrics_ui_hosts', + user: USER.ML_POWERUSER, + requestBody: { + prefix: 'pf18_', + indexPatternName: 'ft_module_metrics_ui', + startDatafeed: true, + end: Date.now(), + }, + expected: { + responseCode: 200, + jobs: [ + { + jobId: 'pf18_hosts_memory_usage', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf18_hosts_network_in', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf18_hosts_network_out', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + ], + searches: [] as string[], + visualizations: [] as string[], + dashboards: [] as string[], + }, + }, + { + testTitleSuffix: + 'for metrics_ui_k8s with prefix, startDatafeed true and estimateModelMemory true', + sourceDataArchive: 'ml/module_metrics_ui', + indexPattern: { name: 'ft_module_metrics_ui', timeField: '@timestamp' }, + module: 'metrics_ui_k8s', + user: USER.ML_POWERUSER, + requestBody: { + prefix: 'pf19_', + indexPatternName: 'ft_module_metrics_ui', + startDatafeed: true, + end: Date.now(), + }, + expected: { + responseCode: 200, + jobs: [ + { + jobId: 'pf19_k8s_memory_usage', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf19_k8s_network_in', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf19_k8s_network_out', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + ], + searches: [] as string[], + visualizations: [] as string[], + dashboards: [] as string[], + }, + }, + { + testTitleSuffix: + 'for siem_cloudtrail with prefix, startDatafeed true and estimateModelMemory true', + sourceDataArchive: 'ml/module_siem_cloudtrail', + indexPattern: { name: 'ft_module_siem_cloudtrail', timeField: '@timestamp' }, + module: 'siem_cloudtrail', + user: USER.ML_POWERUSER, + requestBody: { + prefix: 'pf20_', + indexPatternName: 'ft_module_siem_cloudtrail', + startDatafeed: true, + end: Date.now(), + }, + expected: { + responseCode: 200, + jobs: [ + { + jobId: 'pf20_rare_method_for_a_city', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf20_rare_method_for_a_country', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf20_rare_method_for_a_username', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf20_high_distinct_count_error_message', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf20_rare_error_code', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + ], + searches: [] as string[], + visualizations: [] as string[], + dashboards: [] as string[], + }, + }, + { + testTitleSuffix: + 'for siem_winlogbeat with prefix, startDatafeed true and estimateModelMemory true', + sourceDataArchive: 'ml/module_siem_winlogbeat', + indexPattern: { name: 'ft_module_siem_winlogbeat', timeField: '@timestamp' }, + module: 'siem_winlogbeat', + user: USER.ML_POWERUSER, + requestBody: { + prefix: 'pf21_', + indexPatternName: 'ft_module_siem_winlogbeat', + startDatafeed: true, + end: Date.now(), + }, + expected: { + responseCode: 200, + jobs: [ + { + jobId: 'pf21_rare_process_by_host_windows_ecs', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf21_windows_anomalous_network_activity_ecs', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf21_windows_anomalous_path_activity_ecs', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf21_windows_anomalous_process_all_hosts_ecs', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf21_windows_anomalous_process_creation', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf21_windows_anomalous_script', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf21_windows_anomalous_service', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf21_windows_anomalous_user_name_ecs', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf21_windows_rare_user_runas_event', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf21_windows_rare_metadata_process', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf21_windows_rare_metadata_user', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + ], + searches: [] as string[], + visualizations: [] as string[], + dashboards: [] as string[], + }, + }, + { + testTitleSuffix: + 'for siem_winlogbeat_auth with prefix, startDatafeed true and estimateModelMemory true', + sourceDataArchive: 'ml/module_siem_winlogbeat', + indexPattern: { name: 'ft_module_siem_winlogbeat', timeField: '@timestamp' }, + module: 'siem_winlogbeat_auth', + user: USER.ML_POWERUSER, + requestBody: { + prefix: 'pf22_', + indexPatternName: 'ft_module_siem_winlogbeat', + startDatafeed: true, + end: Date.now(), + }, + expected: { + responseCode: 200, + jobs: [ + { + jobId: 'pf22_windows_rare_user_type10_remote_login', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + ], + searches: [] as string[], + visualizations: [] as string[], + dashboards: [] as string[], + }, + }, + { + testTitleSuffix: + 'for apache_data_stream with prefix, startDatafeed true and estimateModelMemory true', + sourceDataArchive: 'ml/module_apache_data_stream', + indexPattern: { name: 'ft_module_apache_data_stream', timeField: '@timestamp' }, + module: 'apache_data_stream', + user: USER.ML_POWERUSER, + requestBody: { + prefix: 'pf23_', + indexPatternName: 'ft_module_apache_data_stream', + startDatafeed: true, + end: Date.now(), + }, + expected: { + responseCode: 200, + jobs: [ + { + jobId: 'pf23_visitor_rate_apache', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf23_status_code_rate_apache', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf23_source_ip_url_count_apache', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf23_source_ip_request_rate_apache', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf23_low_request_rate_apache', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + ], + searches: [] as string[], + visualizations: [] as string[], + dashboards: [] as string[], + }, + }, + { + testTitleSuffix: + 'for nginx_data_stream with prefix, startDatafeed true and estimateModelMemory true', + sourceDataArchive: 'ml/module_nginx_data_stream', + indexPattern: { name: 'ft_module_nginx_data_stream', timeField: '@timestamp' }, + module: 'nginx_data_stream', + user: USER.ML_POWERUSER, + requestBody: { + prefix: 'pf24_', + indexPatternName: 'ft_module_nginx_data_stream', + startDatafeed: true, + end: Date.now(), + }, + expected: { + responseCode: 200, + jobs: [ + { + jobId: 'pf24_visitor_rate_nginx', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf24_status_code_rate_nginx', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf24_source_ip_url_count_nginx', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf24_source_ip_request_rate_nginx', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, + }, + { + jobId: 'pf24_low_request_rate_nginx', + jobState: JOB_STATE.CLOSED, + datafeedState: DATAFEED_STATE.STOPPED, }, ], searches: [] as string[], @@ -828,7 +1138,7 @@ export default ({ getService }: FtrProviderContext) => { ); } - // verify job and datafeed creation + states + // verify job + datafeed creation + states and model memory limit for (const job of testData.expected.jobs) { const datafeedId = `datafeed-${job.jobId}`; await ml.api.waitForAnomalyDetectionJobToExist(job.jobId); @@ -838,41 +1148,18 @@ export default ({ getService }: FtrProviderContext) => { } await ml.api.waitForJobState(job.jobId, job.jobState); await ml.api.waitForDatafeedState(datafeedId, job.datafeedState); - } - // compare model memory limits for created jobs - const expectedModelMemoryLimits = sortBy( - testData.expected.jobs.map((j) => ({ - id: j.jobId, - modelMemoryLimit: j.modelMemoryLimit, - })), - 'id' - ); - - const { - body: { jobs }, - }: { - body: { - jobs: Job[]; - }; - } = await ml.api.getAnomalyDetectionJob( - testData.expected.jobs.map((j) => j.jobId).join() - ); - - const actualModelMemoryLimits = sortBy( - jobs.map((j) => ({ - id: j.job_id, - modelMemoryLimit: j.analysis_limits!.model_memory_limit, - })), - 'id' - ); - - expect(actualModelMemoryLimits).to.eql( - expectedModelMemoryLimits, - `Expected job model memory limits '${JSON.stringify( - expectedModelMemoryLimits - )}' (got '${JSON.stringify(actualModelMemoryLimits)}')` - ); + // model memory limit should be <= 99mb + const { + body: jobsDetails, + }: { + body: { + jobs: Job[]; + }; + } = await ml.api.getAnomalyDetectionJob(job.jobId); + const actualModelMemoryLimit = jobsDetails.jobs[0].analysis_limits?.model_memory_limit; + expect(actualModelMemoryLimit).to.match(/\d{1,2}mb/); + } // verify saved objects creation for (const search of testData.expected.searches) { diff --git a/x-pack/test/api_integration/apis/monitoring/apm/fixtures/cluster.json b/x-pack/test/api_integration/apis/monitoring/apm/fixtures/cluster.json index 1b89349785f26..42ad837963ef4 100644 --- a/x-pack/test/api_integration/apis/monitoring/apm/fixtures/cluster.json +++ b/x-pack/test/api_integration/apis/monitoring/apm/fixtures/cluster.json @@ -2,7 +2,6 @@ "stats": { "totalEvents": 18, "memRss": 3821568, - "memTotal": 2404475016, "apms": { "total": 2 }, diff --git a/x-pack/test/api_integration/apis/monitoring/apm/instances.js b/x-pack/test/api_integration/apis/monitoring/apm/instances.js index a18569711cd63..9d0be53cff97f 100644 --- a/x-pack/test/api_integration/apis/monitoring/apm/instances.js +++ b/x-pack/test/api_integration/apis/monitoring/apm/instances.js @@ -67,6 +67,7 @@ export default function ({ getService }) { time_of_last_event: '2018-08-31T13:59:21.163Z', }, ], + cgroup: false, }; expect(body).to.eql(expected); diff --git a/x-pack/test/api_integration/apis/monitoring/apm/instances_mb.js b/x-pack/test/api_integration/apis/monitoring/apm/instances_mb.js index 4ebbd50691d12..879549659936b 100644 --- a/x-pack/test/api_integration/apis/monitoring/apm/instances_mb.js +++ b/x-pack/test/api_integration/apis/monitoring/apm/instances_mb.js @@ -67,6 +67,7 @@ export default function ({ getService }) { time_of_last_event: '2018-08-31T13:59:21.163Z', }, ], + cgroup: false, }; expect(body).to.eql(expected); diff --git a/x-pack/test/api_integration/apis/monitoring/cluster/fixtures/multicluster.json b/x-pack/test/api_integration/apis/monitoring/cluster/fixtures/multicluster.json index 6abd3a8ecff9d..080517f46f9c9 100644 --- a/x-pack/test/api_integration/apis/monitoring/cluster/fixtures/multicluster.json +++ b/x-pack/test/api_integration/apis/monitoring/cluster/fixtures/multicluster.json @@ -76,7 +76,6 @@ "apm": { "totalEvents": 0, "memRss": 0, - "memTotal": 0, "apms": { "total": 0 }, @@ -173,7 +172,6 @@ "apm": { "totalEvents": 0, "memRss": 0, - "memTotal": 0, "apms": { "total": 0 }, @@ -270,7 +268,6 @@ "apm": { "totalEvents": 0, "memRss": 0, - "memTotal": 0, "apms": { "total": 0 }, diff --git a/x-pack/test/api_integration/apis/monitoring/cluster/fixtures/overview.json b/x-pack/test/api_integration/apis/monitoring/cluster/fixtures/overview.json index 4f1024f2c94b0..a779546624ee1 100644 --- a/x-pack/test/api_integration/apis/monitoring/cluster/fixtures/overview.json +++ b/x-pack/test/api_integration/apis/monitoring/cluster/fixtures/overview.json @@ -90,7 +90,6 @@ "apm": { "totalEvents": 0, "memRss": 0, - "memTotal": 0, "apms": { "total": 0 }, diff --git a/x-pack/test/api_integration/apis/monitoring/standalone_cluster/fixtures/cluster.json b/x-pack/test/api_integration/apis/monitoring/standalone_cluster/fixtures/cluster.json index 3e590656753f1..13535347f437c 100644 --- a/x-pack/test/api_integration/apis/monitoring/standalone_cluster/fixtures/cluster.json +++ b/x-pack/test/api_integration/apis/monitoring/standalone_cluster/fixtures/cluster.json @@ -49,7 +49,6 @@ "apm": { "totalEvents": 0, "memRss": 0, - "memTotal": 0, "apms": { "total": 0 }, diff --git a/x-pack/test/api_integration/apis/monitoring/standalone_cluster/fixtures/clusters.json b/x-pack/test/api_integration/apis/monitoring/standalone_cluster/fixtures/clusters.json index 8f20dce44ee8a..87c269068ed33 100644 --- a/x-pack/test/api_integration/apis/monitoring/standalone_cluster/fixtures/clusters.json +++ b/x-pack/test/api_integration/apis/monitoring/standalone_cluster/fixtures/clusters.json @@ -76,7 +76,6 @@ "apm": { "totalEvents": 0, "memRss": 0, - "memTotal": 0, "apms": { "total": 0 }, @@ -153,7 +152,6 @@ "apm": { "totalEvents": 0, "memRss": 0, - "memTotal": 0, "apms": { "total": 0 }, diff --git a/x-pack/test/api_integration/apis/uptime/rest/certs.ts b/x-pack/test/api_integration/apis/uptime/rest/certs.ts index 76be0af03e908..d7c0a4eed4a72 100644 --- a/x-pack/test/api_integration/apis/uptime/rest/certs.ts +++ b/x-pack/test/api_integration/apis/uptime/rest/certs.ts @@ -15,7 +15,7 @@ import { makeChecksWithStatus } from './helper/make_checks'; export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); - const legacyEsService = getService('legacyEs'); + const esService = getService('es'); const esArchiver = getService('esArchiver'); describe('certs api', () => { @@ -33,7 +33,7 @@ export default function ({ getService }: FtrProviderContext) { const monitorId = 'monitor1'; before(async () => { makeChecksWithStatus( - legacyEsService, + esService, monitorId, 3, 1, diff --git a/x-pack/test/api_integration/apis/uptime/rest/helper/make_checks.ts b/x-pack/test/api_integration/apis/uptime/rest/helper/make_checks.ts index 829ae07eccf20..51338d2bb6e03 100644 --- a/x-pack/test/api_integration/apis/uptime/rest/helper/make_checks.ts +++ b/x-pack/test/api_integration/apis/uptime/rest/helper/make_checks.ts @@ -7,11 +7,12 @@ import uuid from 'uuid'; import { merge, flattenDeep } from 'lodash'; +import { KibanaClient } from '@elastic/elasticsearch/api/kibana'; import { makePing } from './make_ping'; import { TlsProps } from './make_tls'; interface CheckProps { - es: any; + es: KibanaClient; monitorId?: string; numIps?: number; fields?: { [key: string]: any }; @@ -76,7 +77,7 @@ export const makeCheck = async ({ }; export const makeChecks = async ( - es: any, + es: KibanaClient, monitorId: string, numChecks: number = 1, numIps: number = 1, @@ -121,7 +122,7 @@ export const makeChecks = async ( }; export const makeChecksWithStatus = async ( - es: any, + es: KibanaClient, monitorId: string, numChecks: number, numIps: number, diff --git a/x-pack/test/api_integration/apis/uptime/rest/monitor_states_generated.ts b/x-pack/test/api_integration/apis/uptime/rest/monitor_states_generated.ts index abd3c5d51928b..5268a2146d123 100644 --- a/x-pack/test/api_integration/apis/uptime/rest/monitor_states_generated.ts +++ b/x-pack/test/api_integration/apis/uptime/rest/monitor_states_generated.ts @@ -36,7 +36,7 @@ export default function ({ getService }: FtrProviderContext) { describe('checks with no summaries', async () => { const testMonitorId = 'scope-test-id'; before(async () => { - const es = getService('legacyEs'); + const es = getService('es'); dateRangeStart = new Date().toISOString(); await makeChecksWithStatus(es, testMonitorId, 1, numIps, 1, {}, 'up', (d) => { delete d.summary; @@ -64,7 +64,7 @@ export default function ({ getService }: FtrProviderContext) { }; before(async () => { - const es = getService('legacyEs'); + const es = getService('es'); dateRangeStart = new Date().toISOString(); checks = await makeChecksWithStatus(es, testMonitorId, 1, numIps, 1, {}, 'up', (d) => { // turn an all up status into having at least one down @@ -141,7 +141,7 @@ export default function ({ getService }: FtrProviderContext) { before('generate three monitors with up, down, mix state', async () => { await getService('esArchiver').load('uptime/blank'); - const es = getService('legacyEs'); + const es = getService('es'); const observer = { geo: { diff --git a/x-pack/test/api_integration/apis/uptime/rest/snapshot.ts b/x-pack/test/api_integration/apis/uptime/rest/snapshot.ts index 7a5487c611612..358e667bcb05b 100644 --- a/x-pack/test/api_integration/apis/uptime/rest/snapshot.ts +++ b/x-pack/test/api_integration/apis/uptime/rest/snapshot.ts @@ -52,7 +52,7 @@ export default function ({ getService }: FtrProviderContext) { const makeMonitorChecks = async (monitorId: string, status: 'up' | 'down') => { return makeChecksWithStatus( - getService('legacyEs'), + getService('es'), monitorId, checksPerMonitor, numIps, diff --git a/x-pack/test/api_integration/apis/uptime/rest/telemetry_collectors.ts b/x-pack/test/api_integration/apis/uptime/rest/telemetry_collectors.ts index f0f7a520d098a..fd8894d7e9164 100644 --- a/x-pack/test/api_integration/apis/uptime/rest/telemetry_collectors.ts +++ b/x-pack/test/api_integration/apis/uptime/rest/telemetry_collectors.ts @@ -12,7 +12,7 @@ import { makeChecksWithStatus } from './helper/make_checks'; export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); - const es = getService('legacyEs'); + const es = getService('es'); describe('telemetry collectors heartbeat', () => { before('generating data', async () => { diff --git a/x-pack/test/apm_api_integration/configs/index.ts b/x-pack/test/apm_api_integration/configs/index.ts index 97d18c2419840..3393580153215 100644 --- a/x-pack/test/apm_api_integration/configs/index.ts +++ b/x-pack/test/apm_api_integration/configs/index.ts @@ -18,7 +18,7 @@ const apmFtrConfigs = { rules: { license: 'trial' as const, kibanaConfig: { - 'xpack.ruleRegistry.unsafe.write.enabled': 'true', + 'xpack.ruleRegistry.index': '.kibana-alerts', }, }, }; diff --git a/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts b/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts index 8d0b87782ff7c..e0a3e4d3a3f8b 100644 --- a/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts +++ b/x-pack/test/apm_api_integration/tests/alerts/rule_registry.ts @@ -6,7 +6,7 @@ */ import expect from '@kbn/expect'; -import { get, merge, omit } from 'lodash'; +import { merge, omit } from 'lodash'; import { format } from 'url'; import { FtrProviderContext } from '../../common/ftr_provider_context'; import { registry } from '../../common/registry'; @@ -30,7 +30,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { const supertest = getService('supertestAsApmWriteUser'); const es = getService('es'); - const MAX_POLLS = 5; + const MAX_POLLS = 10; const BULK_INDEX_DELAY = 1000; const INDEXING_DELAY = 5000; @@ -108,11 +108,11 @@ export default function ApiTest({ getService }: FtrProviderContext) { } registry.when('Rule registry with write enabled', { config: 'rules', archives: [] }, () => { - it('bootstraps the apm alert indices', async () => { + it('does not bootstrap indices on plugin startup', async () => { const { body } = await es.indices.get({ index: ALERTS_INDEX_TARGET, expand_wildcards: 'open', - allow_no_indices: false, + allow_no_indices: true, }); const indices = Object.entries(body).map(([indexName, index]) => { @@ -122,23 +122,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { }; }); - const indexNames = indices.map((index) => index.indexName); - - const apmIndex = indices[0]; - - // make sure it only creates one index - expect(indices.length).to.be(1); - - const apmIndexName = apmIndex.indexName; - - expect(apmIndexName.split('-').includes('observability')).to.be(true); - expect(apmIndexName.split('-').includes('apm')).to.be(true); - - expect(indexNames[0].startsWith('.kibana-alerts-observability-apm')).to.be(true); - - expect(get(apmIndex, 'index.mappings.properties.service.properties.environment.type')).to.be( - 'keyword' - ); + expect(indices.length).to.be(0); }); describe('when creating a rule', () => { @@ -335,12 +319,14 @@ export default function ApiTest({ getService }: FtrProviderContext) { sort: { '@timestamp': 'desc', }, + _source: false, + fields: [{ field: '*', include_unmapped: true }], }, }); expect(afterViolatingDataResponse.body.hits.hits.length).to.be(1); - const alertEvent = afterViolatingDataResponse.body.hits.hits[0]._source as Record< + const alertEvent = afterViolatingDataResponse.body.hits.hits[0].fields as Record< string, any >; @@ -354,23 +340,56 @@ export default function ApiTest({ getService }: FtrProviderContext) { const toCompare = omit(alertEvent, exclude); - expect(toCompare).to.eql({ - 'event.action': 'open', - 'event.kind': 'state', - 'kibana.rac.alert.duration.us': 0, - 'kibana.rac.alert.id': 'apm.transaction_error_rate_opbeans-go_request', - 'kibana.rac.alert.status': 'open', - 'kibana.rac.producer': 'apm', - 'kibana.observability.evaluation.threshold': 30, - 'kibana.observability.evaluation.value': 50, - 'processor.event': 'transaction', - 'rule.category': 'Transaction error rate threshold', - 'rule.id': 'apm.transaction_error_rate', - 'rule.name': 'Transaction error rate threshold | opbeans-go', - 'service.name': 'opbeans-go', - tags: ['apm', 'service.name:opbeans-go'], - 'transaction.type': 'request', - }); + expectSnapshot(toCompare).toMatchInline(` + Object { + "event.action": Array [ + "open", + ], + "event.kind": Array [ + "state", + ], + "kibana.rac.alert.duration.us": Array [ + 0, + ], + "kibana.rac.alert.evaluation.threshold": Array [ + 30, + ], + "kibana.rac.alert.evaluation.value": Array [ + 50, + ], + "kibana.rac.alert.id": Array [ + "apm.transaction_error_rate_opbeans-go_request", + ], + "kibana.rac.alert.producer": Array [ + "apm", + ], + "kibana.rac.alert.status": Array [ + "open", + ], + "processor.event": Array [ + "transaction", + ], + "rule.category": Array [ + "Transaction error rate threshold", + ], + "rule.id": Array [ + "apm.transaction_error_rate", + ], + "rule.name": Array [ + "Transaction error rate threshold | opbeans-go", + ], + "service.name": Array [ + "opbeans-go", + ], + "tags": Array [ + "apm", + "service.name:opbeans-go", + ], + "transaction.type": Array [ + "request", + ], + } + `); const now = new Date().getTime(); @@ -390,7 +409,56 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(topAlerts.length).to.be.greaterThan(0); - expect(omit(topAlerts[0], exclude)).to.eql(toCompare); + expectSnapshot(omit(topAlerts[0], exclude)).toMatchInline(` + Object { + "event.action": Array [ + "open", + ], + "event.kind": Array [ + "state", + ], + "kibana.rac.alert.duration.us": Array [ + 0, + ], + "kibana.rac.alert.evaluation.threshold": Array [ + 30, + ], + "kibana.rac.alert.evaluation.value": Array [ + 50, + ], + "kibana.rac.alert.id": Array [ + "apm.transaction_error_rate_opbeans-go_request", + ], + "kibana.rac.alert.producer": Array [ + "apm", + ], + "kibana.rac.alert.status": Array [ + "open", + ], + "processor.event": Array [ + "transaction", + ], + "rule.category": Array [ + "Transaction error rate threshold", + ], + "rule.id": Array [ + "apm.transaction_error_rate", + ], + "rule.name": Array [ + "Transaction error rate threshold | opbeans-go", + ], + "service.name": Array [ + "opbeans-go", + ], + "tags": Array [ + "apm", + "service.name:opbeans-go", + ], + "transaction.type": Array [ + "request", + ], + } + `); await es.bulk({ index: APM_TRANSACTION_INDEX_NAME, @@ -423,43 +491,76 @@ export default function ApiTest({ getService }: FtrProviderContext) { sort: { '@timestamp': 'desc', }, + _source: false, + fields: [{ field: '*', include_unmapped: true }], }, }); expect(afterRecoveryResponse.body.hits.hits.length).to.be(1); - const recoveredAlertEvent = afterRecoveryResponse.body.hits.hits[0]._source as Record< + const recoveredAlertEvent = afterRecoveryResponse.body.hits.hits[0].fields as Record< string, any >; - expect(recoveredAlertEvent['kibana.rac.alert.status']).to.eql('closed'); - expect(recoveredAlertEvent['kibana.rac.alert.duration.us']).to.be.greaterThan(0); - expect(new Date(recoveredAlertEvent['kibana.rac.alert.end']).getTime()).to.be.greaterThan( - 0 - ); - + expect(recoveredAlertEvent['kibana.rac.alert.status']?.[0]).to.eql('closed'); + expect(recoveredAlertEvent['kibana.rac.alert.duration.us']?.[0]).to.be.greaterThan(0); expect( + new Date(recoveredAlertEvent['kibana.rac.alert.end']?.[0]).getTime() + ).to.be.greaterThan(0); + + expectSnapshot( omit( recoveredAlertEvent, exclude.concat(['kibana.rac.alert.duration.us', 'kibana.rac.alert.end']) ) - ).to.eql({ - 'event.action': 'close', - 'event.kind': 'state', - 'kibana.rac.alert.id': 'apm.transaction_error_rate_opbeans-go_request', - 'kibana.rac.alert.status': 'closed', - 'kibana.rac.producer': 'apm', - 'kibana.observability.evaluation.threshold': 30, - 'kibana.observability.evaluation.value': 50, - 'processor.event': 'transaction', - 'rule.category': 'Transaction error rate threshold', - 'rule.id': 'apm.transaction_error_rate', - 'rule.name': 'Transaction error rate threshold | opbeans-go', - 'service.name': 'opbeans-go', - tags: ['apm', 'service.name:opbeans-go'], - 'transaction.type': 'request', - }); + ).toMatchInline(` + Object { + "event.action": Array [ + "close", + ], + "event.kind": Array [ + "state", + ], + "kibana.rac.alert.evaluation.threshold": Array [ + 30, + ], + "kibana.rac.alert.evaluation.value": Array [ + 50, + ], + "kibana.rac.alert.id": Array [ + "apm.transaction_error_rate_opbeans-go_request", + ], + "kibana.rac.alert.producer": Array [ + "apm", + ], + "kibana.rac.alert.status": Array [ + "closed", + ], + "processor.event": Array [ + "transaction", + ], + "rule.category": Array [ + "Transaction error rate threshold", + ], + "rule.id": Array [ + "apm.transaction_error_rate", + ], + "rule.name": Array [ + "Transaction error rate threshold | opbeans-go", + ], + "service.name": Array [ + "opbeans-go", + ], + "tags": Array [ + "apm", + "service.name:opbeans-go", + ], + "transaction.type": Array [ + "request", + ], + } + `); const { body: topAlertsAfterRecovery, @@ -480,7 +581,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { expect(topAlertsAfterRecovery.length).to.be(1); - expect(topAlertsAfterRecovery[0]['kibana.rac.alert.status']).to.be('closed'); + expect(topAlertsAfterRecovery[0]['kibana.rac.alert.status']?.[0]).to.be('closed'); }); }); }); diff --git a/x-pack/test/examples/search_examples/index.ts b/x-pack/test/examples/search_examples/index.ts index 65e214cda4cf8..eaaeb22410183 100644 --- a/x-pack/test/examples/search_examples/index.ts +++ b/x-pack/test/examples/search_examples/index.ts @@ -26,5 +26,6 @@ export default function ({ getService, loadTestFile }: PluginFunctionalProviderC loadTestFile(require.resolve('./search_session_example')); loadTestFile(require.resolve('./search_example')); loadTestFile(require.resolve('./search_sessions_cache')); + loadTestFile(require.resolve('./partial_results_example')); }); } diff --git a/x-pack/test/examples/search_examples/partial_results_example.ts b/x-pack/test/examples/search_examples/partial_results_example.ts new file mode 100644 index 0000000000000..269b2e79ab38f --- /dev/null +++ b/x-pack/test/examples/search_examples/partial_results_example.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../functional/ftr_provider_context'; + +// eslint-disable-next-line import/no-default-export +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const testSubjects = getService('testSubjects'); + const PageObjects = getPageObjects(['common']); + const retry = getService('retry'); + + describe('Partial results example', () => { + before(async () => { + await PageObjects.common.navigateToApp('searchExamples'); + await testSubjects.click('/search'); + }); + + it('should update a progress bar', async () => { + await testSubjects.click('responseTab'); + const progressBar = await testSubjects.find('progressBar'); + + const value = await progressBar.getAttribute('value'); + expect(value).to.be('0'); + + await testSubjects.click('requestFibonacci'); + + await retry.waitFor('update progress bar', async () => { + const newValue = await progressBar.getAttribute('value'); + return parseFloat(newValue) > 0; + }); + }); + }); +} diff --git a/x-pack/test/functional/apps/discover/reporting.ts b/x-pack/test/functional/apps/discover/reporting.ts index d7dd961e2f103..25a7674e47e84 100644 --- a/x-pack/test/functional/apps/discover/reporting.ts +++ b/x-pack/test/functional/apps/discover/reporting.ts @@ -41,9 +41,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { describe('Check Available', () => { beforeEach(() => PageObjects.common.navigateToApp('discover')); - it('is not available if new', async () => { + it('is available if new', async () => { await PageObjects.reporting.openCsvReportingPanel(); - expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true'); + expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null); }); it('becomes available when saved', async () => { @@ -52,11 +52,11 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null); }); - it('becomes available/not available when a saved search is created, changed and saved again', async () => { + it('remains available regardless of the saved search state', async () => { // create new search, csv export is not available await PageObjects.discover.clickNewSearchButton(); await PageObjects.reporting.openCsvReportingPanel(); - expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true'); + expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null); // save search, csv export is available await PageObjects.discover.saveSearch('my search - expectEnabledGenerateReportButton 2'); await PageObjects.reporting.openCsvReportingPanel(); @@ -64,7 +64,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { // add filter, csv export is not available await filterBar.addFilter('currency', 'is', 'EUR'); await PageObjects.reporting.openCsvReportingPanel(); - expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true'); + expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null); // save search again, csv export is available await PageObjects.discover.saveSearch('my search - expectEnabledGenerateReportButton 2'); await PageObjects.reporting.openCsvReportingPanel(); diff --git a/x-pack/test/functional/apps/index_lifecycle_management/home_page.ts b/x-pack/test/functional/apps/index_lifecycle_management/home_page.ts index 44cd2cda7e1af..50194552aec0a 100644 --- a/x-pack/test/functional/apps/index_lifecycle_management/home_page.ts +++ b/x-pack/test/functional/apps/index_lifecycle_management/home_page.ts @@ -41,7 +41,10 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { return (await testSubjects.getVisibleText('sectionHeading')) === 'Index Lifecycle Policies'; }); + await pageObjects.indexLifecycleManagement.increasePolicyListPageSize(); + const allPolicies = await pageObjects.indexLifecycleManagement.getPolicyList(); + const filteredPolicies = allPolicies.filter(function (policy) { return policy.name === policyName; }); diff --git a/x-pack/test/functional/apps/lens/geo_field.ts b/x-pack/test/functional/apps/lens/geo_field.ts new file mode 100644 index 0000000000000..2ba833177a135 --- /dev/null +++ b/x-pack/test/functional/apps/lens/geo_field.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../ftr_provider_context'; + +export default function ({ getService, getPageObjects }: FtrProviderContext) { + const PageObjects = getPageObjects(['visualize', 'lens', 'header', 'maps', 'timePicker']); + + describe('lens visualize geo field tests', () => { + it('should visualize geo fields in maps', async () => { + await PageObjects.visualize.navigateToNewVisualization(); + await PageObjects.visualize.clickVisType('lens'); + await PageObjects.timePicker.setAbsoluteRange( + 'Sep 22, 2015 @ 00:00:00.000', + 'Sep 22, 2015 @ 04:00:00.000' + ); + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.lens.dragFieldToGeoFieldWorkspace('geo.coordinates'); + + await PageObjects.header.waitUntilLoadingHasFinished(); + await PageObjects.maps.waitForLayersToLoad(); + const doesLayerExist = await PageObjects.maps.doesLayerExist('logstash-*'); + expect(doesLayerExist).to.equal(true); + const hits = await PageObjects.maps.getHits(); + expect(hits).to.equal('66'); + await PageObjects.maps.refreshAndClearUnsavedChangesWarning(); + }); + }); +} diff --git a/x-pack/test/functional/apps/lens/index.ts b/x-pack/test/functional/apps/lens/index.ts index bfb0aad7177f4..ab7cee13ffebd 100644 --- a/x-pack/test/functional/apps/lens/index.ts +++ b/x-pack/test/functional/apps/lens/index.ts @@ -37,6 +37,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./colors')); loadTestFile(require.resolve('./chart_data')); loadTestFile(require.resolve('./drag_and_drop')); + loadTestFile(require.resolve('./geo_field')); loadTestFile(require.resolve('./lens_reporting')); loadTestFile(require.resolve('./lens_tagging')); diff --git a/x-pack/test/functional/apps/maps/mapbox_styles.js b/x-pack/test/functional/apps/maps/mapbox_styles.js index b483b95e0ca1f..31cb366f826c8 100644 --- a/x-pack/test/functional/apps/maps/mapbox_styles.js +++ b/x-pack/test/functional/apps/maps/mapbox_styles.js @@ -40,13 +40,10 @@ export default function ({ getPageObjects, getService }) { maxzoom: 24, filter: [ 'all', + ['!=', ['get', '__kbn_too_many_features__'], true], + ['!=', ['get', '__kbn_is_centroid_feature__'], true], + ['any', ['==', ['geometry-type'], 'Point'], ['==', ['geometry-type'], 'MultiPoint']], ['==', ['get', '__kbn_isvisibleduetojoin__'], true], - [ - 'all', - ['!=', ['get', '__kbn_too_many_features__'], true], - ['!=', ['get', '__kbn_is_centroid_feature__'], true], - ['any', ['==', ['geometry-type'], 'Point'], ['==', ['geometry-type'], 'MultiPoint']], - ], ], layout: { visibility: 'visible' }, paint: { @@ -124,17 +121,10 @@ export default function ({ getPageObjects, getService }) { maxzoom: 24, filter: [ 'all', + ['!=', ['get', '__kbn_too_many_features__'], true], + ['!=', ['get', '__kbn_is_centroid_feature__'], true], + ['any', ['==', ['geometry-type'], 'Polygon'], ['==', ['geometry-type'], 'MultiPolygon']], ['==', ['get', '__kbn_isvisibleduetojoin__'], true], - [ - 'all', - ['!=', ['get', '__kbn_too_many_features__'], true], - ['!=', ['get', '__kbn_is_centroid_feature__'], true], - [ - 'any', - ['==', ['geometry-type'], 'Polygon'], - ['==', ['geometry-type'], 'MultiPolygon'], - ], - ], ], layout: { visibility: 'visible' }, paint: { @@ -208,19 +198,16 @@ export default function ({ getPageObjects, getService }) { maxzoom: 24, filter: [ 'all', - ['==', ['get', '__kbn_isvisibleduetojoin__'], true], + ['!=', ['get', '__kbn_too_many_features__'], true], + ['!=', ['get', '__kbn_is_centroid_feature__'], true], [ - 'all', - ['!=', ['get', '__kbn_too_many_features__'], true], - ['!=', ['get', '__kbn_is_centroid_feature__'], true], - [ - 'any', - ['==', ['geometry-type'], 'Polygon'], - ['==', ['geometry-type'], 'MultiPolygon'], - ['==', ['geometry-type'], 'LineString'], - ['==', ['geometry-type'], 'MultiLineString'], - ], + 'any', + ['==', ['geometry-type'], 'Polygon'], + ['==', ['geometry-type'], 'MultiPolygon'], + ['==', ['geometry-type'], 'LineString'], + ['==', ['geometry-type'], 'MultiLineString'], ], + ['==', ['get', '__kbn_isvisibleduetojoin__'], true], ], layout: { visibility: 'visible' }, paint: { 'line-color': '#41937c', 'line-opacity': 0.75, 'line-width': 1 }, diff --git a/x-pack/test/functional/apps/uptime/locations.ts b/x-pack/test/functional/apps/uptime/locations.ts index e3f1d04640754..1e71607845ea4 100644 --- a/x-pack/test/functional/apps/uptime/locations.ts +++ b/x-pack/test/functional/apps/uptime/locations.ts @@ -11,7 +11,7 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export default ({ getPageObjects, getService }: FtrProviderContext) => { const { uptime: uptimePage } = getPageObjects(['uptime']); const uptime = getService('uptime'); - const es = getService('legacyEs'); + const es = getService('es'); const monitor = () => uptime.monitor; const MONITOR_ID = 'location-testing-id'; diff --git a/x-pack/test/functional/apps/uptime/ping_redirects.ts b/x-pack/test/functional/apps/uptime/ping_redirects.ts index 9c39ed7017721..39d9292dbbbb8 100644 --- a/x-pack/test/functional/apps/uptime/ping_redirects.ts +++ b/x-pack/test/functional/apps/uptime/ping_redirects.ts @@ -35,7 +35,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { beforeEach(async () => { await makeChecksWithStatus( - getService('legacyEs'), + getService('es'), MONITOR_ID, 5, 2, diff --git a/x-pack/test/functional/es_archives/ml/module_apache_data_stream/data.json.gz b/x-pack/test/functional/es_archives/ml/module_apache_data_stream/data.json.gz new file mode 100644 index 0000000000000..e0ce0d96a6bf6 Binary files /dev/null and b/x-pack/test/functional/es_archives/ml/module_apache_data_stream/data.json.gz differ diff --git a/x-pack/test/functional/es_archives/ml/module_apache_data_stream/mappings.json b/x-pack/test/functional/es_archives/ml/module_apache_data_stream/mappings.json new file mode 100644 index 0000000000000..ef0d508bd9ba3 --- /dev/null +++ b/x-pack/test/functional/es_archives/ml/module_apache_data_stream/mappings.json @@ -0,0 +1,2671 @@ +{ + "type": "index", + "value": { + "aliases": { + }, + "index": "ft_module_apache_data_stream", + "mappings": { + "_meta": { + "beat": "filebeat", + "version": "7.0.0" + }, + "date_detection": false, + "dynamic_templates": [ + { + "container.labels": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "container.labels.*" + } + }, + { + "fields": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "fields.*" + } + }, + { + "docker.container.labels": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "docker.container.labels.*" + } + }, + { + "kibana.log.meta": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "kibana.log.meta.*" + } + }, + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "agent": { + "properties": { + "ephemeral_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "apache": { + "properties": { + "access": { + "properties": { + "agent": { + "ignore_above": 1024, + "type": "keyword" + }, + "ssl": { + "properties": { + "cipher": { + "ignore_above": 1024, + "type": "keyword" + }, + "protocol": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "error": { + "properties": { + "module": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "apache2": { + "properties": { + "access": { + "properties": { + "geoip": { + "type": "object" + }, + "user_agent": { + "type": "object" + } + } + }, + "error": { + "type": "object" + } + } + }, + "auditd": { + "properties": { + "log": { + "properties": { + "a0": { + "ignore_above": 1024, + "type": "keyword" + }, + "addr": { + "type": "ip" + }, + "geoip": { + "type": "object" + }, + "item": { + "ignore_above": 1024, + "type": "keyword" + }, + "items": { + "ignore_above": 1024, + "type": "keyword" + }, + "laddr": { + "type": "ip" + }, + "lport": { + "type": "long" + }, + "new_auid": { + "ignore_above": 1024, + "type": "keyword" + }, + "new_ses": { + "ignore_above": 1024, + "type": "keyword" + }, + "old_auid": { + "ignore_above": 1024, + "type": "keyword" + }, + "old_ses": { + "ignore_above": 1024, + "type": "keyword" + }, + "rport": { + "type": "long" + }, + "sequence": { + "type": "long" + }, + "tty": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "certificate": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "client": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + } + } + }, + "cloud": { + "properties": { + "account": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "instance": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "machine": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "project": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "container": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "image": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "tag": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "labels": { + "type": "object" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "runtime": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "destination": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + } + } + }, + "docker": { + "properties": { + "container": { + "properties": { + "labels": { + "type": "object" + } + } + } + } + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "elasticsearch": { + "properties": { + "audit": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "indices": { + "ignore_above": 1024, + "type": "keyword" + }, + "layer": { + "ignore_above": 1024, + "type": "keyword" + }, + "origin": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "realm": { + "ignore_above": 1024, + "type": "keyword" + }, + "request": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "url": { + "properties": { + "params": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user": { + "properties": { + "realm": { + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "cluster": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "uuid": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "component": { + "ignore_above": 1024, + "type": "keyword" + }, + "deprecation": { + "type": "object" + }, + "gc": { + "properties": { + "heap": { + "properties": { + "size_kb": { + "type": "long" + }, + "used_kb": { + "type": "long" + } + } + }, + "jvm_runtime_sec": { + "type": "float" + }, + "old_gen": { + "properties": { + "size_kb": { + "type": "long" + }, + "used_kb": { + "type": "long" + } + } + }, + "phase": { + "properties": { + "class_unload_time_sec": { + "type": "float" + }, + "cpu_time": { + "properties": { + "real_sec": { + "type": "float" + }, + "sys_sec": { + "type": "float" + }, + "user_sec": { + "type": "float" + } + } + }, + "duration_sec": { + "type": "float" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "parallel_rescan_time_sec": { + "type": "float" + }, + "scrub_string_table_time_sec": { + "type": "float" + }, + "scrub_symbol_table_time_sec": { + "type": "float" + }, + "weak_refs_processing_time_sec": { + "type": "float" + } + } + }, + "stopping_threads_time_sec": { + "type": "float" + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "threads_total_stop_time_sec": { + "type": "float" + }, + "young_gen": { + "properties": { + "size_kb": { + "type": "long" + }, + "used_kb": { + "type": "long" + } + } + } + } + }, + "index": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "node": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "server": { + "properties": { + "component": { + "ignore_above": 1024, + "type": "keyword" + }, + "gc": { + "properties": { + "collection_duration": { + "properties": { + "ms": { + "type": "float" + } + } + }, + "observation_duration": { + "properties": { + "ms": { + "type": "float" + } + } + }, + "overhead_seq": { + "type": "long" + }, + "young": { + "properties": { + "one": { + "type": "long" + }, + "two": { + "type": "long" + } + } + } + } + } + } + }, + "shard": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "slowlog": { + "properties": { + "extra_source": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "logger": { + "ignore_above": 1024, + "type": "keyword" + }, + "routing": { + "ignore_above": 1024, + "type": "keyword" + }, + "search_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "source_query": { + "ignore_above": 1024, + "type": "keyword" + }, + "stats": { + "ignore_above": 1024, + "type": "keyword" + }, + "took": { + "ignore_above": 1024, + "type": "keyword" + }, + "total_hits": { + "ignore_above": 1024, + "type": "keyword" + }, + "total_shards": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "types": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "error": { + "properties": { + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "event": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "created": { + "type": "date" + }, + "dataset": { + "ignore_above": 1024, + "type": "keyword" + }, + "duration": { + "type": "long" + }, + "end": { + "type": "date" + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "doc_values": false, + "ignore_above": 1024, + "index": false, + "type": "keyword" + }, + "outcome": { + "ignore_above": 1024, + "type": "keyword" + }, + "risk_score": { + "type": "float" + }, + "risk_score_norm": { + "type": "float" + }, + "severity": { + "type": "long" + }, + "start": { + "type": "date" + }, + "timezone": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "fields": { + "type": "object" + }, + "file": { + "properties": { + "ctime": { + "type": "date" + }, + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "extension": { + "ignore_above": 1024, + "type": "keyword" + }, + "gid": { + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "ignore_above": 1024, + "type": "keyword" + }, + "inode": { + "ignore_above": 1024, + "type": "keyword" + }, + "mode": { + "ignore_above": 1024, + "type": "keyword" + }, + "mtime": { + "type": "date" + }, + "owner": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "size": { + "type": "long" + }, + "target_path": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "fileset": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "haproxy": { + "properties": { + "backend_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "backend_queue": { + "type": "long" + }, + "bind_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes_read": { + "type": "long" + }, + "client": { + "type": "object" + }, + "connection_wait_time_ms": { + "type": "long" + }, + "connections": { + "properties": { + "active": { + "type": "long" + }, + "backend": { + "type": "long" + }, + "frontend": { + "type": "long" + }, + "retries": { + "type": "long" + }, + "server": { + "type": "long" + } + } + }, + "destination": { + "type": "object" + }, + "error_message": { + "norms": false, + "type": "text" + }, + "frontend_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "geoip": { + "type": "object" + }, + "http": { + "properties": { + "request": { + "properties": { + "captured_cookie": { + "ignore_above": 1024, + "type": "keyword" + }, + "captured_headers": { + "ignore_above": 1024, + "type": "keyword" + }, + "raw_request_line": { + "ignore_above": 1024, + "type": "keyword" + }, + "time_wait_ms": { + "type": "long" + }, + "time_wait_without_data_ms": { + "type": "long" + } + } + }, + "response": { + "properties": { + "captured_cookie": { + "ignore_above": 1024, + "type": "keyword" + }, + "captured_headers": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "mode": { + "ignore_above": 1024, + "type": "keyword" + }, + "server_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "server_queue": { + "type": "long" + }, + "source": { + "norms": false, + "type": "text" + }, + "tcp": { + "properties": { + "connection_waiting_time_ms": { + "type": "long" + } + } + }, + "termination_state": { + "ignore_above": 1024, + "type": "keyword" + }, + "time_backend_connect": { + "type": "long" + }, + "time_queue": { + "type": "long" + }, + "total_waiting_time_ms": { + "type": "long" + } + } + }, + "hash": { + "properties": { + "sha256": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "host": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "build": { + "ignore_above": 1024, + "type": "keyword" + }, + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "http": { + "properties": { + "request": { + "properties": { + "body": { + "properties": { + "bytes": { + "type": "long" + }, + "content": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bytes": { + "type": "long" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "referrer": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "response": { + "properties": { + "body": { + "properties": { + "bytes": { + "type": "long" + }, + "content": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bytes": { + "type": "long" + }, + "status_code": { + "type": "long" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "icinga": { + "properties": { + "debug": { + "properties": { + "facility": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "main": { + "properties": { + "facility": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "startup": { + "properties": { + "facility": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "iis": { + "properties": { + "access": { + "properties": { + "cookie": { + "ignore_above": 1024, + "type": "keyword" + }, + "geoip": { + "type": "object" + }, + "server_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "site_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "sub_status": { + "type": "long" + }, + "user_agent": { + "type": "object" + }, + "win32_status": { + "type": "long" + } + } + }, + "error": { + "properties": { + "geoip": { + "type": "object" + }, + "queue_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "reason_phrase": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "input": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "kafka": { + "properties": { + "log": { + "properties": { + "class": { + "norms": false, + "type": "text" + }, + "component": { + "ignore_above": 1024, + "type": "keyword" + }, + "trace": { + "properties": { + "class": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "norms": false, + "type": "text" + }, + "message": { + "norms": false, + "type": "text" + } + } + } + } + } + } + }, + "kibana": { + "properties": { + "log": { + "properties": { + "meta": { + "type": "object" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "kubernetes": { + "properties": { + "annotations": { + "type": "object" + }, + "container": { + "properties": { + "image": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "labels": { + "type": "object" + }, + "namespace": { + "ignore_above": 1024, + "type": "keyword" + }, + "node": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pod": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "labels": { + "type": "object" + }, + "log": { + "properties": { + "file": { + "properties": { + "path": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "flags": { + "ignore_above": 1024, + "type": "keyword" + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "offset": { + "type": "long" + }, + "original": { + "doc_values": false, + "ignore_above": 1024, + "index": false, + "type": "keyword" + }, + "source": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "logstash": { + "properties": { + "log": { + "properties": { + "log_event": { + "type": "object" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "thread": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "slowlog": { + "properties": { + "event": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "plugin_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "plugin_params": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "plugin_params_object": { + "type": "object" + }, + "plugin_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "thread": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "took_in_millis": { + "type": "long" + } + } + } + } + }, + "message": { + "norms": false, + "type": "text" + }, + "mongodb": { + "properties": { + "log": { + "properties": { + "component": { + "ignore_above": 1024, + "type": "keyword" + }, + "context": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "mysql": { + "properties": { + "error": { + "type": "object" + }, + "slowlog": { + "properties": { + "bytes_sent": { + "type": "long" + }, + "current_user": { + "ignore_above": 1024, + "type": "keyword" + }, + "filesort": { + "type": "boolean" + }, + "filesort_on_disk": { + "type": "boolean" + }, + "full_join": { + "type": "boolean" + }, + "full_scan": { + "type": "boolean" + }, + "innodb": { + "properties": { + "io_r_bytes": { + "type": "long" + }, + "io_r_ops": { + "type": "long" + }, + "io_r_wait": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "pages_distinct": { + "type": "long" + }, + "queue_wait": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "rec_lock_wait": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "trx_id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "killed": { + "ignore_above": 1024, + "type": "keyword" + }, + "last_errno": { + "ignore_above": 1024, + "type": "keyword" + }, + "lock_time": { + "properties": { + "sec": { + "type": "float" + } + } + }, + "log_slow_rate_limit": { + "ignore_above": 1024, + "type": "keyword" + }, + "log_slow_rate_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "merge_passes": { + "type": "long" + }, + "priority_queue": { + "type": "boolean" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "query_cache_hit": { + "type": "boolean" + }, + "rows_affected": { + "type": "long" + }, + "rows_examined": { + "type": "long" + }, + "rows_sent": { + "type": "long" + }, + "schema": { + "ignore_above": 1024, + "type": "keyword" + }, + "tmp_disk_tables": { + "type": "long" + }, + "tmp_table": { + "type": "boolean" + }, + "tmp_table_on_disk": { + "type": "boolean" + }, + "tmp_table_sizes": { + "type": "long" + }, + "tmp_tables": { + "type": "long" + } + } + }, + "thread_id": { + "type": "long" + } + } + }, + "network": { + "properties": { + "application": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "community_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "direction": { + "ignore_above": 1024, + "type": "keyword" + }, + "forwarded_ip": { + "type": "ip" + }, + "iana_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "transport": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "nginx": { + "properties": { + "access": { + "properties": { + "geoip": { + "type": "object" + }, + "user_agent": { + "type": "object" + } + } + }, + "error": { + "properties": { + "connection_id": { + "type": "long" + } + } + } + } + }, + "observer": { + "properties": { + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "vendor": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "organization": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "osquery": { + "properties": { + "result": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "calendar_time": { + "ignore_above": 1024, + "type": "keyword" + }, + "host_identifier": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "unix_time": { + "type": "long" + } + } + } + } + }, + "postgresql": { + "properties": { + "log": { + "properties": { + "core_id": { + "type": "long" + }, + "database": { + "ignore_above": 1024, + "type": "keyword" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "timestamp": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "process": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "executable": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "pid": { + "type": "long" + }, + "ppid": { + "type": "long" + }, + "program": { + "ignore_above": 1024, + "type": "keyword" + }, + "start": { + "type": "date" + }, + "thread": { + "properties": { + "id": { + "type": "long" + } + } + }, + "title": { + "ignore_above": 1024, + "type": "keyword" + }, + "working_directory": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "redis": { + "properties": { + "log": { + "properties": { + "role": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "slowlog": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "cmd": { + "ignore_above": 1024, + "type": "keyword" + }, + "duration": { + "properties": { + "us": { + "type": "long" + } + } + }, + "id": { + "type": "long" + }, + "key": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "related": { + "properties": { + "ip": { + "type": "ip" + } + } + }, + "santa": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "decision": { + "ignore_above": 1024, + "type": "keyword" + }, + "disk": { + "properties": { + "bsdname": { + "ignore_above": 1024, + "type": "keyword" + }, + "bus": { + "ignore_above": 1024, + "type": "keyword" + }, + "fs": { + "ignore_above": 1024, + "type": "keyword" + }, + "model": { + "ignore_above": 1024, + "type": "keyword" + }, + "mount": { + "ignore_above": 1024, + "type": "keyword" + }, + "serial": { + "ignore_above": 1024, + "type": "keyword" + }, + "volume": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "mode": { + "ignore_above": 1024, + "type": "keyword" + }, + "reason": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "server": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + } + } + }, + "service": { + "properties": { + "ephemeral_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "source": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + } + } + }, + "stream": { + "ignore_above": 1024, + "type": "keyword" + }, + "syslog": { + "properties": { + "facility": { + "type": "long" + }, + "facility_label": { + "ignore_above": 1024, + "type": "keyword" + }, + "priority": { + "type": "long" + }, + "severity_label": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "system": { + "properties": { + "auth": { + "properties": { + "groupadd": { + "type": "object" + }, + "ssh": { + "properties": { + "dropped_ip": { + "type": "ip" + }, + "geoip": { + "type": "object" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "signature": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "sudo": { + "properties": { + "command": { + "ignore_above": 1024, + "type": "keyword" + }, + "error": { + "ignore_above": 1024, + "type": "keyword" + }, + "pwd": { + "ignore_above": 1024, + "type": "keyword" + }, + "tty": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "useradd": { + "properties": { + "home": { + "ignore_above": 1024, + "type": "keyword" + }, + "shell": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "syslog": { + "type": "object" + } + } + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "traefik": { + "properties": { + "access": { + "properties": { + "backend_url": { + "ignore_above": 1024, + "type": "keyword" + }, + "frontend_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "geoip": { + "properties": { + "city_name": { + "path": "source.geo.city_name", + "type": "alias" + }, + "continent_name": { + "path": "source.geo.continent_name", + "type": "alias" + }, + "country_iso_code": { + "path": "source.geo.country_iso_code", + "type": "alias" + }, + "location": { + "path": "source.geo.location", + "type": "alias" + }, + "region_iso_code": { + "path": "source.geo.region_iso_code", + "type": "alias" + }, + "region_name": { + "path": "source.geo.region_name", + "type": "alias" + } + } + }, + "request_count": { + "type": "long" + }, + "user_agent": { + "properties": { + "device": { + "path": "user_agent.device.name", + "type": "alias" + }, + "major": { + "path": "user_agent.major", + "type": "alias" + }, + "minor": { + "path": "user_agent.minor", + "type": "alias" + }, + "name": { + "path": "user_agent.name", + "type": "alias" + }, + "original": { + "path": "user_agent.original", + "type": "alias" + }, + "os": { + "path": "user_agent.os.full_name", + "type": "alias" + }, + "os_major": { + "path": "user_agent.os.major", + "type": "alias" + }, + "os_minor": { + "path": "user_agent.os.minor", + "type": "alias" + }, + "os_name": { + "path": "user_agent.os.name", + "type": "alias" + }, + "patch": { + "path": "user_agent.patch", + "type": "alias" + } + } + }, + "user_identifier": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "url": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "fragment": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "ignore_above": 1024, + "type": "keyword" + }, + "password": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "scheme": { + "ignore_above": 1024, + "type": "keyword" + }, + "username": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user": { + "properties": { + "audit": { + "properties": { + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "effective": { + "properties": { + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "filesystem": { + "properties": { + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "full_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "owner": { + "properties": { + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "saved": { + "properties": { + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "terminal": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user_agent": { + "properties": { + "device": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "major": { + "ignore_above": 1024, + "type": "keyword" + }, + "minor": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "major": { + "type": "long" + }, + "minor": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "patch": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} diff --git a/x-pack/test/functional/es_archives/ml/module_metricbeat/data.json.gz b/x-pack/test/functional/es_archives/ml/module_metricbeat/data.json.gz new file mode 100644 index 0000000000000..089499fbea819 Binary files /dev/null and b/x-pack/test/functional/es_archives/ml/module_metricbeat/data.json.gz differ diff --git a/x-pack/test/functional/es_archives/ml/module_metricbeat/mappings.json b/x-pack/test/functional/es_archives/ml/module_metricbeat/mappings.json new file mode 100644 index 0000000000000..d0007464484db --- /dev/null +++ b/x-pack/test/functional/es_archives/ml/module_metricbeat/mappings.json @@ -0,0 +1,441 @@ +{ + "type": "index", + "value": { + "aliases": { + }, + "index": "ft_module_metricbeat", + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "agent": { + "properties": { + "ephemeral_id": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "hostname": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "id": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "type": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "version": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "cloud": { + "properties": { + "availability_zone": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "instance": { + "properties": { + "id": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "name": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "machine": { + "properties": { + "type": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "project": { + "properties": { + "id": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "provider": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "ecs": { + "properties": { + "version": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "event": { + "properties": { + "dataset": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "duration": { + "type": "long" + }, + "module": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "host": { + "properties": { + "architecture": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "containerized": { + "type": "boolean" + }, + "hostname": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "id": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "name": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "os": { + "properties": { + "codename": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "family": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "kernel": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "name": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "platform": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "version": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + } + } + }, + "metricset": { + "properties": { + "name": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "service": { + "properties": { + "type": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "system": { + "properties": { + "cpu": { + "properties": { + "cores": { + "type": "long" + }, + "idle": { + "properties": { + "pct": { + "type": "float" + } + } + }, + "iowait": { + "properties": { + "pct": { + "type": "float" + } + } + }, + "irq": { + "properties": { + "pct": { + "type": "long" + } + } + }, + "nice": { + "properties": { + "pct": { + "type": "long" + } + } + }, + "softirq": { + "properties": { + "pct": { + "type": "float" + } + } + }, + "steal": { + "properties": { + "pct": { + "type": "long" + } + } + }, + "system": { + "properties": { + "pct": { + "type": "float" + } + } + }, + "total": { + "properties": { + "pct": { + "type": "float" + } + } + }, + "user": { + "properties": { + "pct": { + "type": "float" + } + } + } + } + }, + "filesystem": { + "properties": { + "available": { + "type": "long" + }, + "device_name": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "files": { + "type": "long" + }, + "free": { + "type": "long" + }, + "free_files": { + "type": "long" + }, + "mount_point": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "total": { + "type": "long" + }, + "type": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "used": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "settings": { + "index": { + "mapping": { + "total_fields": { + "limit": "5000" + } + }, + "number_of_replicas": "0", + "number_of_shards": "1", + "refresh_interval": "-1" + } + } + } +} \ No newline at end of file diff --git a/x-pack/test/functional/es_archives/ml/module_metrics_ui/data.json.gz b/x-pack/test/functional/es_archives/ml/module_metrics_ui/data.json.gz new file mode 100644 index 0000000000000..b1a9ba3b45cea Binary files /dev/null and b/x-pack/test/functional/es_archives/ml/module_metrics_ui/data.json.gz differ diff --git a/x-pack/test/functional/es_archives/ml/module_metrics_ui/mappings.json b/x-pack/test/functional/es_archives/ml/module_metrics_ui/mappings.json new file mode 100644 index 0000000000000..1b24240597a69 --- /dev/null +++ b/x-pack/test/functional/es_archives/ml/module_metrics_ui/mappings.json @@ -0,0 +1,20668 @@ +{ + "type": "index", + "value": { + "aliases": { + }, + "index": "ft_module_metrics_ui", + "mappings": { + "_meta": { + "beat": "metricbeat", + "version": "8.0.0" + }, + "date_detection": false, + "dynamic_templates": [ + { + "labels": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "labels.*" + } + }, + { + "container.labels": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "container.labels.*" + } + }, + { + "dns.answers": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "dns.answers.*" + } + }, + { + "log.syslog": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "log.syslog.*" + } + }, + { + "network.inner": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "network.inner.*" + } + }, + { + "observer.egress": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "observer.egress.*" + } + }, + { + "observer.ingress": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "observer.ingress.*" + } + }, + { + "fields": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "fields.*" + } + }, + { + "docker.container.labels": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "docker.container.labels.*" + } + }, + { + "kubernetes.labels.*": { + "mapping": { + "type": "keyword" + }, + "path_match": "kubernetes.labels.*" + } + }, + { + "kubernetes.annotations.*": { + "mapping": { + "type": "keyword" + }, + "path_match": "kubernetes.annotations.*" + } + }, + { + "aws.tags.*": { + "mapping": { + "type": "keyword" + }, + "path_match": "aws.tags.*" + } + }, + { + "aws.dimensions.*": { + "mapping": { + "type": "keyword" + }, + "path_match": "aws.dimensions.*" + } + }, + { + "aws.*.metrics.*.*": { + "mapping": { + "type": "double" + }, + "path_match": "aws.*.metrics.*.*" + } + }, + { + "azure.resource.tags.*": { + "mapping": { + "type": "keyword" + }, + "path_match": "azure.resource.tags.*" + } + }, + { + "azure.dimensions.*": { + "mapping": { + "type": "keyword" + }, + "path_match": "azure.dimensions.*" + } + }, + { + "azure.compute_vm.*.*": { + "mapping": { + "type": "float" + }, + "path_match": "azure.compute_vm.*.*" + } + }, + { + "azure.compute_vm_scaleset.*.*": { + "mapping": { + "type": "float" + }, + "path_match": "azure.compute_vm_scaleset.*.*" + } + }, + { + "azure.container_instance.*.*": { + "mapping": { + "type": "float" + }, + "path_match": "azure.container_instance.*.*" + } + }, + { + "azure.container_registry.*.*": { + "mapping": { + "type": "float" + }, + "path_match": "azure.container_registry.*.*" + } + }, + { + "azure.container_service.*.*": { + "mapping": { + "type": "float" + }, + "path_match": "azure.container_service.*.*" + } + }, + { + "azure.database_account.*.*": { + "mapping": { + "type": "float" + }, + "path_match": "azure.database_account.*.*" + } + }, + { + "azure.monitor.metrics.*.*": { + "mapping": { + "type": "float" + }, + "path_match": "azure.monitor.metrics.*.*" + } + }, + { + "azure.storage.*.*": { + "mapping": { + "type": "float" + }, + "path_match": "azure.storage.*.*" + } + }, + { + "coredns.stats.dns.request.duration.ns.bucket.*": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "coredns.stats.dns.request.duration.ns.bucket.*" + } + }, + { + "coredns.stats.dns.request.size.bytes.bucket.*": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "coredns.stats.dns.request.size.bytes.bucket.*" + } + }, + { + "coredns.stats.dns.response.size.bytes.bucket.*": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "coredns.stats.dns.response.size.bytes.bucket.*" + } + }, + { + "docker.cpu.core.*.pct": { + "mapping": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "path_match": "docker.cpu.core.*.pct" + } + }, + { + "docker.cpu.core.*.norm.pct": { + "mapping": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "path_match": "docker.cpu.core.*.norm.pct" + } + }, + { + "docker.cpu.core.*.ticks": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "docker.cpu.core.*.ticks" + } + }, + { + "docker.event.actor.attributes": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "docker.event.actor.attributes.*" + } + }, + { + "docker.image.labels": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "docker.image.labels.*" + } + }, + { + "docker.memory.stats.*": { + "mapping": { + "type": "long" + }, + "path_match": "docker.memory.stats.*" + } + }, + { + "etcd.disk.wal_fsync_duration.ns.bucket.*": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "etcd.disk.wal_fsync_duration.ns.bucket.*" + } + }, + { + "etcd.disk.backend_commit_duration.ns.bucket.*": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "etcd.disk.backend_commit_duration.ns.bucket.*" + } + }, + { + "iis.webserver.*.*": { + "mapping": { + "type": "float" + }, + "path_match": "iis.webserver.*.*" + } + }, + { + "istio.citadel.grpc.server.handling.latency.ms.bucket.*": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "istio.citadel.grpc.server.handling.latency.ms.bucket.*" + } + }, + { + "istio.galley.runtime.processor.event_span.duration.ms.bucket.*": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "istio.galley.runtime.processor.event_span.duration.ms.bucket.*" + } + }, + { + "istio.galley.runtime.processor.snapshot_events.bucket.*": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "istio.galley.runtime.processor.snapshot_events.bucket.*" + } + }, + { + "istio.galley.runtime.processor.snapshot_lifetime.duration.ms.bucket.*": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "istio.galley.runtime.processor.snapshot_lifetime.duration.ms.bucket.*" + } + }, + { + "istio.mesh.request.duration.ms.bucket.*": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "istio.mesh.request.duration.ms.bucket.*" + } + }, + { + "istio.mesh.request.size.bytes.bucket.*": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "istio.mesh.request.size.bytes.bucket.*" + } + }, + { + "istio.mesh.response.size.bytes.bucket.*": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "istio.mesh.response.size.bytes.bucket.*" + } + }, + { + "istio.pilot.proxy.conv.ms.bucket.*": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "istio.pilot.proxy.conv.ms.bucket.*" + } + }, + { + "kubernetes.apiserver.http.request.duration.us.percentile.*": { + "mapping": { + "type": "double" + }, + "match_mapping_type": "double", + "path_match": "kubernetes.apiserver.http.request.duration.us.percentile.*" + } + }, + { + "kubernetes.apiserver.http.request.size.bytes.percentile.*": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "kubernetes.apiserver.http.request.size.bytes.percentile.*" + } + }, + { + "kubernetes.apiserver.http.response.size.bytes.percentile.*": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "kubernetes.apiserver.http.response.size.bytes.percentile.*" + } + }, + { + "kubernetes.apiserver.request.latency.bucket.*": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "kubernetes.apiserver.request.latency.bucket.*" + } + }, + { + "kubernetes.apiserver.request.duration.us.bucket.*": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "kubernetes.apiserver.request.duration.us.bucket.*" + } + }, + { + "kubernetes.controllermanager.http.request.duration.us.percentile.*": { + "mapping": { + "type": "double" + }, + "match_mapping_type": "double", + "path_match": "kubernetes.controllermanager.http.request.duration.us.percentile.*" + } + }, + { + "kubernetes.controllermanager.http.request.size.bytes.percentile.*": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "kubernetes.controllermanager.http.request.size.bytes.percentile.*" + } + }, + { + "kubernetes.controllermanager.http.response.size.bytes.percentile.*": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "kubernetes.controllermanager.http.response.size.bytes.percentile.*" + } + }, + { + "kubernetes.proxy.http.request.duration.us.percentile.*": { + "mapping": { + "type": "double" + }, + "match_mapping_type": "double", + "path_match": "kubernetes.proxy.http.request.duration.us.percentile.*" + } + }, + { + "kubernetes.proxy.http.request.size.bytes.percentile.*": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "kubernetes.proxy.http.request.size.bytes.percentile.*" + } + }, + { + "kubernetes.proxy.http.response.size.bytes.percentile.*": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "kubernetes.proxy.http.response.size.bytes.percentile.*" + } + }, + { + "kubernetes.proxy.sync.rules.duration.us.bucket.*": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "kubernetes.proxy.sync.rules.duration.us.bucket.*" + } + }, + { + "kubernetes.proxy.sync.networkprogramming.duration.us.bucket.*": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "kubernetes.proxy.sync.networkprogramming.duration.us.bucket.*" + } + }, + { + "kubernetes.scheduler.http.request.duration.us.percentile.*": { + "mapping": { + "type": "double" + }, + "match_mapping_type": "double", + "path_match": "kubernetes.scheduler.http.request.duration.us.percentile.*" + } + }, + { + "kubernetes.scheduler.http.request.size.bytes.percentile.*": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "kubernetes.scheduler.http.request.size.bytes.percentile.*" + } + }, + { + "kubernetes.scheduler.http.response.size.bytes.percentile.*": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "kubernetes.scheduler.http.response.size.bytes.percentile.*" + } + }, + { + "kubernetes.scheduler.scheduling.e2e.duration.us.bucket.*": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "kubernetes.scheduler.scheduling.e2e.duration.us.bucket.*" + } + }, + { + "kubernetes.scheduler.scheduling.duration.seconds.percentile.*": { + "mapping": { + "type": "double" + }, + "match_mapping_type": "double", + "path_match": "kubernetes.scheduler.scheduling.duration.seconds.percentile.*" + } + }, + { + "munin.metrics.*": { + "mapping": { + "type": "double" + }, + "path_match": "munin.metrics.*" + } + }, + { + "openmetrics.labels.*": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "openmetrics.labels.*" + } + }, + { + "openmetrics.metrics.*": { + "mapping": { + "type": "double" + }, + "path_match": "openmetrics.metrics.*" + } + }, + { + "prometheus.labels.*": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "prometheus.labels.*" + } + }, + { + "prometheus.metrics.*": { + "mapping": { + "type": "double" + }, + "path_match": "prometheus.metrics.*" + } + }, + { + "prometheus.query.*": { + "mapping": { + "type": "double" + }, + "path_match": "prometheus.query.*" + } + }, + { + "prometheus.*.value": { + "mapping": { + "type": "double" + }, + "path_match": "prometheus.*.value" + } + }, + { + "prometheus.*.counter": { + "mapping": { + "type": "double" + }, + "path_match": "prometheus.*.counter" + } + }, + { + "prometheus.*.rate": { + "mapping": { + "type": "double" + }, + "path_match": "prometheus.*.rate" + } + }, + { + "prometheus.*.histogram": { + "mapping": { + "type": "histogram" + }, + "path_match": "prometheus.*.histogram" + } + }, + { + "sql.metrics.numeric.*": { + "mapping": { + "type": "double" + }, + "match_mapping_type": "double", + "path_match": "sql.metrics.numeric.*" + } + }, + { + "sql.metrics.string.*": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "sql.metrics.string.*" + } + }, + { + "statsd.*.count": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "statsd.*.count" + } + }, + { + "statsd.*.*": { + "mapping": { + "type": "float" + }, + "path_match": "statsd.*.*" + } + }, + { + "system.process.env": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "system.process.env.*" + } + }, + { + "system.process.cgroup.cpuacct.percpu": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "system.process.cgroup.cpuacct.percpu.*" + } + }, + { + "system.raid.disks.states.*": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "system.raid.disks.states.*" + } + }, + { + "traefik.health.response.status_codes.*": { + "mapping": { + "type": "long" + }, + "match_mapping_type": "long", + "path_match": "traefik.health.response.status_codes.*" + } + }, + { + "vsphere.virtualmachine.custom_fields": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "vsphere.virtualmachine.custom_fields.*" + } + }, + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "activemq": { + "properties": { + "broker": { + "properties": { + "connections": { + "properties": { + "count": { + "type": "long" + } + } + }, + "consumers": { + "properties": { + "count": { + "type": "long" + } + } + }, + "mbean": { + "ignore_above": 1024, + "type": "keyword" + }, + "memory": { + "properties": { + "broker": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "store": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "temp": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + }, + "messages": { + "properties": { + "count": { + "type": "long" + }, + "dequeue": { + "properties": { + "count": { + "type": "long" + } + } + }, + "enqueue": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "producers": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "queue": { + "properties": { + "consumers": { + "properties": { + "count": { + "type": "long" + } + } + }, + "mbean": { + "ignore_above": 1024, + "type": "keyword" + }, + "memory": { + "properties": { + "broker": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + }, + "messages": { + "properties": { + "dequeue": { + "properties": { + "count": { + "type": "long" + } + } + }, + "dispatch": { + "properties": { + "count": { + "type": "long" + } + } + }, + "enqueue": { + "properties": { + "count": { + "type": "long" + }, + "time": { + "properties": { + "avg": { + "type": "double" + }, + "max": { + "type": "long" + }, + "min": { + "type": "long" + } + } + } + } + }, + "expired": { + "properties": { + "count": { + "type": "long" + } + } + }, + "inflight": { + "properties": { + "count": { + "type": "long" + } + } + }, + "size": { + "properties": { + "avg": { + "type": "long" + } + } + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "producers": { + "properties": { + "count": { + "type": "long" + } + } + }, + "size": { + "type": "long" + } + } + }, + "topic": { + "properties": { + "consumers": { + "properties": { + "count": { + "type": "long" + } + } + }, + "mbean": { + "ignore_above": 1024, + "type": "keyword" + }, + "memory": { + "properties": { + "broker": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + }, + "messages": { + "properties": { + "dequeue": { + "properties": { + "count": { + "type": "long" + } + } + }, + "dispatch": { + "properties": { + "count": { + "type": "long" + } + } + }, + "enqueue": { + "properties": { + "count": { + "type": "long" + }, + "time": { + "properties": { + "avg": { + "type": "double" + }, + "max": { + "type": "long" + }, + "min": { + "type": "long" + } + } + } + } + }, + "expired": { + "properties": { + "count": { + "type": "long" + } + } + }, + "inflight": { + "properties": { + "count": { + "type": "long" + } + } + }, + "size": { + "properties": { + "avg": { + "type": "long" + } + } + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "producers": { + "properties": { + "count": { + "type": "long" + } + } + } + } + } + } + }, + "aerospike": { + "properties": { + "namespace": { + "properties": { + "client": { + "properties": { + "delete": { + "properties": { + "error": { + "type": "long" + }, + "not_found": { + "type": "long" + }, + "success": { + "type": "long" + }, + "timeout": { + "type": "long" + } + } + }, + "read": { + "properties": { + "error": { + "type": "long" + }, + "not_found": { + "type": "long" + }, + "success": { + "type": "long" + }, + "timeout": { + "type": "long" + } + } + }, + "write": { + "properties": { + "error": { + "type": "long" + }, + "success": { + "type": "long" + }, + "timeout": { + "type": "long" + } + } + } + } + }, + "device": { + "properties": { + "available": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "free": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "total": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "hwm_breached": { + "type": "boolean" + }, + "memory": { + "properties": { + "free": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "used": { + "properties": { + "data": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "index": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "sindex": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "total": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "node": { + "properties": { + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "objects": { + "properties": { + "master": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "stop_writes": { + "type": "boolean" + } + } + } + } + }, + "agent": { + "properties": { + "ephemeral_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "apache": { + "properties": { + "status": { + "properties": { + "bytes_per_request": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "bytes_per_sec": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "connections": { + "properties": { + "async": { + "properties": { + "closing": { + "type": "long" + }, + "keep_alive": { + "type": "long" + }, + "writing": { + "type": "long" + } + } + }, + "total": { + "type": "long" + } + } + }, + "cpu": { + "properties": { + "children_system": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "children_user": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "load": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "system": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "user": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "load": { + "properties": { + "1": { + "scaling_factor": 100, + "type": "scaled_float" + }, + "15": { + "scaling_factor": 100, + "type": "scaled_float" + }, + "5": { + "scaling_factor": 100, + "type": "scaled_float" + } + } + }, + "requests_per_sec": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "scoreboard": { + "properties": { + "closing_connection": { + "type": "long" + }, + "dns_lookup": { + "type": "long" + }, + "gracefully_finishing": { + "type": "long" + }, + "idle_cleanup": { + "type": "long" + }, + "keepalive": { + "type": "long" + }, + "logging": { + "type": "long" + }, + "open_slot": { + "type": "long" + }, + "reading_request": { + "type": "long" + }, + "sending_reply": { + "type": "long" + }, + "starting_up": { + "type": "long" + }, + "total": { + "type": "long" + }, + "waiting_for_connection": { + "type": "long" + } + } + }, + "total_accesses": { + "type": "long" + }, + "total_kbytes": { + "type": "long" + }, + "uptime": { + "properties": { + "server_uptime": { + "type": "long" + }, + "uptime": { + "type": "long" + } + } + }, + "workers": { + "properties": { + "busy": { + "type": "long" + }, + "idle": { + "type": "long" + } + } + } + } + } + } + }, + "appsearch": { + "properties": { + "stats": { + "properties": { + "jvm": { + "properties": { + "memory_usage": { + "properties": { + "heap_committed": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "heap_init": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "heap_max": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "heap_used": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "non_heap_committed": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "non_heap_init": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "queues": { + "properties": { + "analytics_events": { + "properties": { + "count": { + "type": "long" + } + } + }, + "document_destroyer": { + "properties": { + "count": { + "type": "long" + } + } + }, + "engine_destroyer": { + "properties": { + "count": { + "type": "long" + } + } + }, + "failed": { + "properties": { + "count": { + "type": "long" + } + } + }, + "index_adder": { + "properties": { + "count": { + "type": "long" + } + } + }, + "indexed_doc_remover": { + "properties": { + "count": { + "type": "long" + } + } + }, + "mailer": { + "properties": { + "count": { + "type": "long" + } + } + }, + "refresh_document_counts": { + "properties": { + "count": { + "type": "long" + } + } + }, + "reindexer": { + "properties": { + "count": { + "type": "long" + } + } + }, + "schema_updater": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "requests": { + "properties": { + "api": { + "properties": { + "duration": { + "properties": { + "avg": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "max": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + } + } + }, + "count": { + "type": "long" + }, + "web": { + "properties": { + "response_time": { + "properties": { + "avg": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "max": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + } + } + } + } + } + } + } + } + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "aws": { + "properties": { + "*": { + "properties": { + "metrics": { + "properties": { + "*": { + "properties": { + "*": { + "type": "object" + } + } + } + } + } + } + }, + "billing": { + "type": "object" + }, + "cloudwatch": { + "properties": { + "namespace": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "dimensions": { + "properties": { + "*": { + "type": "object" + } + } + }, + "dynamodb": { + "type": "object" + }, + "ebs": { + "type": "object" + }, + "ec2": { + "properties": { + "cpu": { + "properties": { + "credit_balance": { + "type": "long" + }, + "credit_usage": { + "type": "long" + }, + "surplus_credit_balance": { + "type": "long" + }, + "surplus_credits_charged": { + "type": "long" + }, + "total": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + }, + "diskio": { + "properties": { + "read": { + "properties": { + "bytes": { + "type": "long" + }, + "bytes_per_sec": { + "type": "long" + }, + "ops": { + "type": "long" + }, + "ops_per_sec": { + "type": "long" + } + } + }, + "write": { + "properties": { + "bytes": { + "type": "long" + }, + "bytes_per_sec": { + "type": "long" + }, + "ops": { + "type": "long" + }, + "ops_per_sec": { + "type": "long" + } + } + } + } + }, + "instance": { + "properties": { + "core": { + "properties": { + "count": { + "type": "long" + } + } + }, + "image": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "monitoring": { + "properties": { + "state": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "private": { + "properties": { + "dns_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + } + } + }, + "public": { + "properties": { + "dns_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + } + } + }, + "state": { + "properties": { + "code": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "threads_per_core": { + "type": "long" + } + } + }, + "network": { + "properties": { + "in": { + "properties": { + "bytes": { + "type": "long" + }, + "bytes_per_sec": { + "type": "long" + }, + "packets": { + "type": "long" + }, + "packets_per_sec": { + "type": "long" + } + } + }, + "out": { + "properties": { + "bytes": { + "type": "long" + }, + "bytes_per_sec": { + "type": "long" + }, + "packets": { + "type": "long" + }, + "packets_per_sec": { + "type": "long" + } + } + } + } + }, + "status": { + "properties": { + "check_failed": { + "type": "long" + }, + "check_failed_instance": { + "type": "long" + }, + "check_failed_system": { + "type": "long" + } + } + } + } + }, + "elb": { + "type": "object" + }, + "lambda": { + "type": "object" + }, + "natgateway": { + "type": "object" + }, + "rds": { + "properties": { + "aurora_bin_log_replica_lag": { + "type": "long" + }, + "aurora_global_db": { + "properties": { + "data_transfer": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "replicated_write_io": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "replication_lag": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "aurora_replica": { + "properties": { + "lag": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "lag_max": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "lag_min": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "aurora_volume_left_total": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "backtrack_change_records": { + "properties": { + "creation_rate": { + "type": "long" + }, + "stored": { + "type": "long" + } + } + }, + "backtrack_window": { + "properties": { + "actual": { + "type": "long" + }, + "alert": { + "type": "long" + } + } + }, + "backup_storage_billed_total": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "cache_hit_ratio": { + "properties": { + "buffer": { + "type": "long" + }, + "result_set": { + "type": "long" + } + } + }, + "cpu": { + "properties": { + "credit_balance": { + "type": "long" + }, + "credit_usage": { + "type": "long" + }, + "total": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + }, + "database_connections": { + "type": "long" + }, + "db_instance": { + "properties": { + "arn": { + "ignore_above": 1024, + "type": "keyword" + }, + "class": { + "ignore_above": 1024, + "type": "keyword" + }, + "db_cluster_identifier": { + "ignore_above": 1024, + "type": "keyword" + }, + "engine_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "identifier": { + "ignore_above": 1024, + "type": "keyword" + }, + "role": { + "ignore_above": 1024, + "type": "keyword" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "deadlocks": { + "type": "long" + }, + "disk_queue_depth": { + "type": "float" + }, + "disk_usage": { + "properties": { + "bin_log": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "replication_slot": { + "properties": { + "mb": { + "type": "long" + } + } + }, + "transaction_logs": { + "properties": { + "mb": { + "type": "long" + } + } + } + } + }, + "engine_uptime": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "failed_sql_server_agent_jobs": { + "type": "long" + }, + "free_local_storage": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "free_storage": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "freeable_memory": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "latency": { + "properties": { + "commit": { + "type": "float" + }, + "ddl": { + "type": "float" + }, + "delete": { + "type": "float" + }, + "dml": { + "type": "float" + }, + "insert": { + "type": "float" + }, + "read": { + "type": "float" + }, + "select": { + "type": "float" + }, + "update": { + "type": "float" + }, + "write": { + "type": "float" + } + } + }, + "login_failures": { + "type": "long" + }, + "maximum_used_transaction_ids": { + "type": "long" + }, + "oldest_replication_slot_lag": { + "properties": { + "mb": { + "type": "long" + } + } + }, + "queries": { + "type": "long" + }, + "rds_to_aurora_postgresql_replica_lag": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "read_io": { + "properties": { + "ops_per_sec": { + "type": "float" + } + } + }, + "replica_lag": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "storage_used": { + "properties": { + "backup_retention_period": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "snapshot": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "swap_usage": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "throughput": { + "properties": { + "commit": { + "type": "float" + }, + "ddl": { + "type": "float" + }, + "delete": { + "type": "float" + }, + "dml": { + "type": "float" + }, + "insert": { + "type": "float" + }, + "network": { + "type": "float" + }, + "network_receive": { + "type": "float" + }, + "network_transmit": { + "type": "float" + }, + "read": { + "type": "float" + }, + "select": { + "type": "float" + }, + "update": { + "type": "float" + }, + "write": { + "type": "float" + } + } + }, + "transaction_logs_generation": { + "type": "long" + }, + "transactions": { + "properties": { + "active": { + "type": "long" + }, + "blocked": { + "type": "long" + } + } + }, + "volume": { + "properties": { + "read": { + "properties": { + "iops": { + "type": "long" + } + } + }, + "write": { + "properties": { + "iops": { + "type": "long" + } + } + } + } + }, + "volume_used": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "write_io": { + "properties": { + "ops_per_sec": { + "type": "float" + } + } + } + } + }, + "s3": { + "properties": { + "bucket": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "s3_daily_storage": { + "properties": { + "bucket": { + "properties": { + "size": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "number_of_objects": { + "type": "long" + } + } + }, + "s3_request": { + "properties": { + "downloaded": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "errors": { + "properties": { + "4xx": { + "type": "long" + }, + "5xx": { + "type": "long" + } + } + }, + "latency": { + "properties": { + "first_byte": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "total_request": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "requests": { + "properties": { + "delete": { + "type": "long" + }, + "get": { + "type": "long" + }, + "head": { + "type": "long" + }, + "list": { + "type": "long" + }, + "post": { + "type": "long" + }, + "put": { + "type": "long" + }, + "select": { + "type": "long" + }, + "select_returned": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "select_scanned": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "total": { + "type": "long" + } + } + }, + "uploaded": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "sns": { + "type": "object" + }, + "sqs": { + "properties": { + "empty_receives": { + "type": "long" + }, + "messages": { + "properties": { + "delayed": { + "type": "long" + }, + "deleted": { + "type": "long" + }, + "not_visible": { + "type": "long" + }, + "received": { + "type": "long" + }, + "sent": { + "type": "long" + }, + "visible": { + "type": "long" + } + } + }, + "oldest_message_age": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "queue": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "sent_message_size": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "tags": { + "properties": { + "*": { + "type": "object" + } + } + }, + "transitgateway": { + "type": "object" + }, + "usage": { + "type": "object" + }, + "vpn": { + "type": "object" + } + } + }, + "azure": { + "properties": { + "compute_vm": { + "properties": { + "*": { + "properties": { + "*": { + "type": "object" + } + } + } + } + }, + "compute_vm_scaleset": { + "properties": { + "*": { + "properties": { + "*": { + "type": "object" + } + } + } + } + }, + "container_instance": { + "properties": { + "*": { + "properties": { + "*": { + "type": "object" + } + } + } + } + }, + "container_registry": { + "properties": { + "*": { + "properties": { + "*": { + "type": "object" + } + } + } + } + }, + "container_service": { + "properties": { + "*": { + "properties": { + "*": { + "type": "object" + } + } + } + } + }, + "database_account": { + "properties": { + "*": { + "properties": { + "*": { + "type": "object" + } + } + } + } + }, + "dimensions": { + "properties": { + "*": { + "type": "object" + } + } + }, + "monitor": { + "properties": { + "metrics": { + "properties": { + "*": { + "properties": { + "*": { + "type": "object" + } + } + } + } + } + } + }, + "namespace": { + "ignore_above": 1024, + "type": "keyword" + }, + "resource": { + "properties": { + "group": { + "ignore_above": 1024, + "type": "keyword" + }, + "tags": { + "properties": { + "*": { + "type": "object" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "storage": { + "properties": { + "*": { + "properties": { + "*": { + "type": "object" + } + } + } + } + }, + "subscription_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "timegrain": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "beat": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "state": { + "properties": { + "management": { + "properties": { + "enabled": { + "type": "boolean" + } + } + }, + "module": { + "properties": { + "count": { + "type": "long" + } + } + }, + "output": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "queue": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "stats": { + "properties": { + "libbeat": { + "properties": { + "output": { + "properties": { + "events": { + "properties": { + "acked": { + "type": "long" + }, + "active": { + "type": "long" + }, + "batches": { + "type": "long" + }, + "dropped": { + "type": "long" + }, + "duplicates": { + "type": "long" + }, + "failed": { + "type": "long" + }, + "toomany": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "read": { + "properties": { + "bytes": { + "type": "long" + }, + "errors": { + "type": "long" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "write": { + "properties": { + "bytes": { + "type": "long" + }, + "errors": { + "type": "long" + } + } + } + } + } + } + }, + "runtime": { + "properties": { + "goroutines": { + "type": "long" + } + } + }, + "uptime": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ceph": { + "properties": { + "cluster_disk": { + "properties": { + "available": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "total": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "cluster_health": { + "properties": { + "overall_status": { + "ignore_above": 1024, + "type": "keyword" + }, + "timechecks": { + "properties": { + "epoch": { + "type": "long" + }, + "round": { + "properties": { + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "value": { + "type": "long" + } + } + } + } + } + } + }, + "cluster_status": { + "properties": { + "degraded": { + "properties": { + "objects": { + "type": "long" + }, + "ratio": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "total": { + "type": "long" + } + } + }, + "misplace": { + "properties": { + "objects": { + "type": "long" + }, + "ratio": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "total": { + "type": "long" + } + } + }, + "osd": { + "properties": { + "epoch": { + "type": "long" + }, + "full": { + "type": "boolean" + }, + "nearfull": { + "type": "boolean" + }, + "num_in_osds": { + "type": "long" + }, + "num_osds": { + "type": "long" + }, + "num_remapped_pgs": { + "type": "long" + }, + "num_up_osds": { + "type": "long" + } + } + }, + "pg": { + "properties": { + "avail_bytes": { + "type": "long" + }, + "data_bytes": { + "type": "long" + }, + "total_bytes": { + "type": "long" + }, + "used_bytes": { + "type": "long" + } + } + }, + "pg_state": { + "properties": { + "count": { + "type": "long" + }, + "state_name": { + "type": "long" + }, + "version": { + "type": "long" + } + } + }, + "traffic": { + "properties": { + "read_bytes": { + "type": "long" + }, + "read_op_per_sec": { + "type": "long" + }, + "write_bytes": { + "type": "long" + }, + "write_op_per_sec": { + "type": "long" + } + } + }, + "version": { + "type": "long" + } + } + }, + "mgr_cluster_disk": { + "type": "object" + }, + "mgr_cluster_health": { + "type": "object" + }, + "mgr_osd_perf": { + "properties": { + "id": { + "type": "long" + }, + "stats": { + "properties": { + "apply_latency_ms": { + "type": "long" + }, + "apply_latency_ns": { + "type": "long" + }, + "commit_latency_ms": { + "type": "long" + }, + "commit_latency_ns": { + "type": "long" + } + } + } + } + }, + "mgr_osd_pool_stats": { + "properties": { + "client_io_rate": { + "type": "object" + }, + "pool_id": { + "type": "long" + }, + "pool_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "mgr_osd_tree": { + "type": "object" + }, + "mgr_pool_disk": { + "type": "object" + }, + "monitor_health": { + "properties": { + "available": { + "properties": { + "kb": { + "type": "long" + }, + "pct": { + "type": "long" + } + } + }, + "health": { + "ignore_above": 1024, + "type": "keyword" + }, + "last_updated": { + "type": "date" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "store_stats": { + "properties": { + "last_updated": { + "type": "long" + }, + "log": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "misc": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "sst": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "total": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "total": { + "properties": { + "kb": { + "type": "long" + } + } + }, + "used": { + "properties": { + "kb": { + "type": "long" + } + } + } + } + }, + "osd_df": { + "properties": { + "available": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "device_class": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "pg_num": { + "type": "long" + }, + "total": { + "properties": { + "byte": { + "type": "long" + } + } + }, + "used": { + "properties": { + "byte": { + "type": "long" + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + }, + "osd_tree": { + "properties": { + "children": { + "ignore_above": 1024, + "type": "keyword" + }, + "crush_weight": { + "type": "float" + }, + "depth": { + "type": "long" + }, + "device_class": { + "ignore_above": 1024, + "type": "keyword" + }, + "exists": { + "type": "boolean" + }, + "father": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "primary_affinity": { + "type": "float" + }, + "reweight": { + "type": "long" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "type_id": { + "type": "long" + } + } + }, + "pool_disk": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "stats": { + "properties": { + "available": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "objects": { + "type": "long" + }, + "used": { + "properties": { + "bytes": { + "type": "long" + }, + "kb": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "client": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "cloud": { + "properties": { + "account": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "image": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "instance": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "machine": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "project": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "cloudfoundry": { + "properties": { + "app": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "container": { + "properties": { + "cpu": { + "properties": { + "pct": { + "type": "float" + } + } + }, + "disk": { + "properties": { + "bytes": { + "type": "long" + }, + "quota": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "instance_index": { + "type": "long" + }, + "memory": { + "properties": { + "bytes": { + "type": "long" + }, + "quota": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "counter": { + "properties": { + "delta": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "total": { + "type": "long" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "value": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "value": { + "type": "float" + } + } + } + } + }, + "cockroachdb": { + "type": "object" + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "consul": { + "properties": { + "agent": { + "properties": { + "autopilot": { + "properties": { + "healthy": { + "type": "boolean" + } + } + }, + "runtime": { + "properties": { + "alloc": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "garbage_collector": { + "properties": { + "pause": { + "properties": { + "current": { + "properties": { + "ns": { + "type": "long" + } + } + }, + "total": { + "properties": { + "ns": { + "type": "long" + } + } + } + } + }, + "runs": { + "type": "long" + } + } + }, + "goroutines": { + "type": "long" + }, + "heap_objects": { + "type": "long" + }, + "malloc_count": { + "type": "long" + }, + "sys": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "container": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "image": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "tag": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "labels": { + "properties": { + "GCB_BUILD_ID": { + "type": "keyword" + }, + "GCB_PROJECT_ID": { + "type": "keyword" + }, + "INCLUDES_SOURCE": { + "type": "keyword" + }, + "LICENSE": { + "type": "keyword" + }, + "LICENSE_CATEGORY": { + "type": "keyword" + }, + "NOTICES_PATH": { + "type": "keyword" + }, + "SOURCES_INCLUDED": { + "type": "keyword" + }, + "annotation_checksum/config": { + "type": "keyword" + }, + "annotation_checksum/configmap": { + "type": "keyword" + }, + "annotation_checksum/dashboards-json-config": { + "type": "keyword" + }, + "annotation_checksum/health": { + "type": "keyword" + }, + "annotation_checksum/sc-dashboard-provider-config": { + "type": "keyword" + }, + "annotation_checksum/secret": { + "type": "keyword" + }, + "annotation_configChecksum": { + "type": "keyword" + }, + "annotation_configchecksum": { + "type": "keyword" + }, + "annotation_io_kubernetes_container_hash": { + "type": "keyword" + }, + "annotation_io_kubernetes_container_ports": { + "type": "keyword" + }, + "annotation_io_kubernetes_container_restartCount": { + "type": "keyword" + }, + "annotation_io_kubernetes_container_terminationMessagePath": { + "type": "keyword" + }, + "annotation_io_kubernetes_container_terminationMessagePolicy": { + "type": "keyword" + }, + "annotation_io_kubernetes_pod_terminationGracePeriod": { + "type": "keyword" + }, + "annotation_kubernetes_io/config_hash": { + "type": "keyword" + }, + "annotation_kubernetes_io/config_seen": { + "type": "keyword" + }, + "annotation_kubernetes_io/config_source": { + "type": "keyword" + }, + "annotation_kubernetes_io/limit-ranger": { + "type": "keyword" + }, + "annotation_prometheus_io/path": { + "type": "keyword" + }, + "annotation_prometheus_io/port": { + "type": "keyword" + }, + "annotation_prometheus_io/scrape": { + "type": "keyword" + }, + "annotation_scheduler_alpha_kubernetes_io/critical-pod": { + "type": "keyword" + }, + "annotation_seccomp_security_alpha_kubernetes_io/pod": { + "type": "keyword" + }, + "app": { + "type": "keyword" + }, + "app_kubernetes_io/instance": { + "type": "keyword" + }, + "app_kubernetes_io/managed-by": { + "type": "keyword" + }, + "app_kubernetes_io/name": { + "type": "keyword" + }, + "chart": { + "type": "keyword" + }, + "component": { + "type": "keyword" + }, + "controller-revision-hash": { + "type": "keyword" + }, + "controller-uid": { + "type": "keyword" + }, + "description": { + "type": "keyword" + }, + "helm_sh/chart": { + "type": "keyword" + }, + "heritage": { + "type": "keyword" + }, + "io_kubernetes_container_logpath": { + "type": "keyword" + }, + "io_kubernetes_container_name": { + "type": "keyword" + }, + "io_kubernetes_docker_type": { + "type": "keyword" + }, + "io_kubernetes_pod_name": { + "type": "keyword" + }, + "io_kubernetes_pod_namespace": { + "type": "keyword" + }, + "io_kubernetes_pod_uid": { + "type": "keyword" + }, + "io_kubernetes_sandbox_id": { + "type": "keyword" + }, + "job-name": { + "type": "keyword" + }, + "k8s-app": { + "type": "keyword" + }, + "license": { + "type": "keyword" + }, + "maintainer": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "org_label-schema_build-date": { + "type": "keyword" + }, + "org_label-schema_license": { + "type": "keyword" + }, + "org_label-schema_name": { + "type": "keyword" + }, + "org_label-schema_schema-version": { + "type": "keyword" + }, + "org_label-schema_url": { + "type": "keyword" + }, + "org_label-schema_usage": { + "type": "keyword" + }, + "org_label-schema_vcs-ref": { + "type": "keyword" + }, + "org_label-schema_vcs-url": { + "type": "keyword" + }, + "org_label-schema_vendor": { + "type": "keyword" + }, + "org_label-schema_version": { + "type": "keyword" + }, + "org_opencontainers_image_documentation": { + "type": "keyword" + }, + "org_opencontainers_image_licenses": { + "type": "keyword" + }, + "org_opencontainers_image_source": { + "type": "keyword" + }, + "org_opencontainers_image_title": { + "type": "keyword" + }, + "org_opencontainers_image_vendor": { + "type": "keyword" + }, + "org_opencontainers_image_version": { + "type": "keyword" + }, + "pod-template-generation": { + "type": "keyword" + }, + "pod-template-hash": { + "type": "keyword" + }, + "release": { + "type": "keyword" + }, + "role": { + "type": "keyword" + }, + "service": { + "type": "keyword" + }, + "statefulset_kubernetes_io/pod-name": { + "type": "keyword" + }, + "tier": { + "type": "keyword" + }, + "version": { + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "runtime": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "coredns": { + "properties": { + "stats": { + "properties": { + "dns": { + "properties": { + "cache": { + "properties": { + "hits": { + "properties": { + "count": { + "type": "long" + } + } + }, + "misses": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "request": { + "properties": { + "count": { + "type": "long" + }, + "do": { + "properties": { + "count": { + "type": "long" + } + } + }, + "duration": { + "properties": { + "ns": { + "properties": { + "bucket": { + "properties": { + "*": { + "type": "object" + } + } + }, + "count": { + "type": "long" + }, + "sum": { + "type": "long" + } + } + } + } + }, + "size": { + "properties": { + "bytes": { + "properties": { + "bucket": { + "properties": { + "*": { + "type": "object" + } + } + }, + "count": { + "type": "long" + }, + "sum": { + "type": "long" + } + } + } + } + }, + "type": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "response": { + "properties": { + "rcode": { + "properties": { + "count": { + "type": "long" + } + } + }, + "size": { + "properties": { + "bytes": { + "properties": { + "bucket": { + "properties": { + "*": { + "type": "object" + } + } + }, + "count": { + "type": "long" + }, + "sum": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "panic": { + "properties": { + "count": { + "type": "long" + } + } + }, + "proto": { + "ignore_above": 1024, + "type": "keyword" + }, + "rcode": { + "ignore_above": 1024, + "type": "keyword" + }, + "server": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "zone": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "couchbase": { + "properties": { + "bucket": { + "properties": { + "data": { + "properties": { + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "disk": { + "properties": { + "fetches": { + "type": "long" + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "item_count": { + "type": "long" + }, + "memory": { + "properties": { + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "ops_per_sec": { + "type": "long" + }, + "quota": { + "properties": { + "ram": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "use": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "cluster": { + "properties": { + "hdd": { + "properties": { + "free": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "quota": { + "properties": { + "total": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "total": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "used": { + "properties": { + "by_data": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "value": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "max_bucket_count": { + "type": "long" + }, + "quota": { + "properties": { + "index_memory": { + "properties": { + "mb": { + "type": "long" + } + } + }, + "memory": { + "properties": { + "mb": { + "type": "long" + } + } + } + } + }, + "ram": { + "properties": { + "quota": { + "properties": { + "total": { + "properties": { + "per_node": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "value": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "used": { + "properties": { + "per_node": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "value": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "total": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "used": { + "properties": { + "by_data": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "value": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "node": { + "properties": { + "cmd_get": { + "type": "long" + }, + "couch": { + "properties": { + "docs": { + "properties": { + "data_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "disk_size": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "spatial": { + "properties": { + "data_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "disk_size": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "views": { + "properties": { + "data_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "disk_size": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "cpu_utilization_rate": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "current_items": { + "properties": { + "total": { + "type": "long" + }, + "value": { + "type": "long" + } + } + }, + "ep_bg_fetched": { + "type": "long" + }, + "get_hits": { + "type": "long" + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "mcd_memory": { + "properties": { + "allocated": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "reserved": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "memory": { + "properties": { + "free": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "total": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "ops": { + "type": "long" + }, + "swap": { + "properties": { + "total": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "uptime": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "vb_replica_curr_items": { + "type": "long" + } + } + } + } + }, + "couchdb": { + "properties": { + "server": { + "properties": { + "couchdb": { + "properties": { + "auth_cache_hits": { + "type": "long" + }, + "auth_cache_misses": { + "type": "long" + }, + "database_reads": { + "type": "long" + }, + "database_writes": { + "type": "long" + }, + "open_databases": { + "type": "long" + }, + "open_os_files": { + "type": "long" + }, + "request_time": { + "type": "long" + } + } + }, + "httpd": { + "properties": { + "bulk_requests": { + "type": "long" + }, + "clients_requesting_changes": { + "type": "long" + }, + "requests": { + "type": "long" + }, + "temporary_view_reads": { + "type": "long" + }, + "view_reads": { + "type": "long" + } + } + }, + "httpd_request_methods": { + "properties": { + "COPY": { + "type": "long" + }, + "DELETE": { + "type": "long" + }, + "GET": { + "type": "long" + }, + "HEAD": { + "type": "long" + }, + "POST": { + "type": "long" + }, + "PUT": { + "type": "long" + } + } + }, + "httpd_status_codes": { + "properties": { + "200": { + "type": "long" + }, + "201": { + "type": "long" + }, + "202": { + "type": "long" + }, + "301": { + "type": "long" + }, + "304": { + "type": "long" + }, + "400": { + "type": "long" + }, + "401": { + "type": "long" + }, + "403": { + "type": "long" + }, + "404": { + "type": "long" + }, + "405": { + "type": "long" + }, + "409": { + "type": "long" + }, + "412": { + "type": "long" + }, + "500": { + "type": "long" + } + } + } + } + } + } + }, + "destination": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "dll": { + "properties": { + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "pe": { + "properties": { + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "dns": { + "properties": { + "answers": { + "properties": { + "class": { + "ignore_above": 1024, + "type": "keyword" + }, + "data": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "ttl": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "header_flags": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "op_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "question": { + "properties": { + "class": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "resolved_ip": { + "type": "ip" + }, + "response_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "docker": { + "properties": { + "container": { + "properties": { + "command": { + "ignore_above": 1024, + "type": "keyword" + }, + "created": { + "type": "date" + }, + "ip_addresses": { + "type": "ip" + }, + "labels": { + "properties": { + "GCB_BUILD_ID": { + "type": "keyword" + }, + "GCB_PROJECT_ID": { + "type": "keyword" + }, + "INCLUDES_SOURCE": { + "type": "keyword" + }, + "LICENSE": { + "type": "keyword" + }, + "LICENSE_CATEGORY": { + "type": "keyword" + }, + "NOTICES_PATH": { + "type": "keyword" + }, + "SOURCES_INCLUDED": { + "type": "keyword" + }, + "annotation_checksum/config": { + "type": "keyword" + }, + "annotation_checksum/configmap": { + "type": "keyword" + }, + "annotation_checksum/dashboards-json-config": { + "type": "keyword" + }, + "annotation_checksum/health": { + "type": "keyword" + }, + "annotation_checksum/sc-dashboard-provider-config": { + "type": "keyword" + }, + "annotation_checksum/secret": { + "type": "keyword" + }, + "annotation_configChecksum": { + "type": "keyword" + }, + "annotation_configchecksum": { + "type": "keyword" + }, + "annotation_io_kubernetes_container_hash": { + "type": "keyword" + }, + "annotation_io_kubernetes_container_ports": { + "type": "keyword" + }, + "annotation_io_kubernetes_container_restartCount": { + "type": "keyword" + }, + "annotation_io_kubernetes_container_terminationMessagePath": { + "type": "keyword" + }, + "annotation_io_kubernetes_container_terminationMessagePolicy": { + "type": "keyword" + }, + "annotation_io_kubernetes_pod_terminationGracePeriod": { + "type": "keyword" + }, + "annotation_kubernetes_io/config_hash": { + "type": "keyword" + }, + "annotation_kubernetes_io/config_seen": { + "type": "keyword" + }, + "annotation_kubernetes_io/config_source": { + "type": "keyword" + }, + "annotation_kubernetes_io/limit-ranger": { + "type": "keyword" + }, + "annotation_prometheus_io/path": { + "type": "keyword" + }, + "annotation_prometheus_io/port": { + "type": "keyword" + }, + "annotation_prometheus_io/scrape": { + "type": "keyword" + }, + "annotation_scheduler_alpha_kubernetes_io/critical-pod": { + "type": "keyword" + }, + "annotation_seccomp_security_alpha_kubernetes_io/pod": { + "type": "keyword" + }, + "app": { + "type": "keyword" + }, + "app_kubernetes_io/instance": { + "type": "keyword" + }, + "app_kubernetes_io/managed-by": { + "type": "keyword" + }, + "app_kubernetes_io/name": { + "type": "keyword" + }, + "chart": { + "type": "keyword" + }, + "component": { + "type": "keyword" + }, + "controller-revision-hash": { + "type": "keyword" + }, + "controller-uid": { + "type": "keyword" + }, + "description": { + "type": "keyword" + }, + "helm_sh/chart": { + "type": "keyword" + }, + "heritage": { + "type": "keyword" + }, + "io_k8s_description": { + "type": "keyword" + }, + "io_k8s_display-name": { + "type": "keyword" + }, + "io_kubernetes_container_logpath": { + "type": "keyword" + }, + "io_kubernetes_container_name": { + "type": "keyword" + }, + "io_kubernetes_docker_type": { + "type": "keyword" + }, + "io_kubernetes_pod_name": { + "type": "keyword" + }, + "io_kubernetes_pod_namespace": { + "type": "keyword" + }, + "io_kubernetes_pod_uid": { + "type": "keyword" + }, + "io_kubernetes_sandbox_id": { + "type": "keyword" + }, + "job-name": { + "type": "keyword" + }, + "k8s-app": { + "type": "keyword" + }, + "license": { + "type": "keyword" + }, + "maintainer": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "org_label-schema_build-date": { + "type": "keyword" + }, + "org_label-schema_license": { + "type": "keyword" + }, + "org_label-schema_name": { + "type": "keyword" + }, + "org_label-schema_schema-version": { + "type": "keyword" + }, + "org_label-schema_url": { + "type": "keyword" + }, + "org_label-schema_usage": { + "type": "keyword" + }, + "org_label-schema_vcs-ref": { + "type": "keyword" + }, + "org_label-schema_vcs-url": { + "type": "keyword" + }, + "org_label-schema_vendor": { + "type": "keyword" + }, + "org_label-schema_version": { + "type": "keyword" + }, + "org_opencontainers_image_created": { + "type": "keyword" + }, + "org_opencontainers_image_documentation": { + "type": "keyword" + }, + "org_opencontainers_image_licenses": { + "type": "keyword" + }, + "org_opencontainers_image_revision": { + "type": "keyword" + }, + "org_opencontainers_image_source": { + "type": "keyword" + }, + "org_opencontainers_image_title": { + "type": "keyword" + }, + "org_opencontainers_image_url": { + "type": "keyword" + }, + "org_opencontainers_image_vendor": { + "type": "keyword" + }, + "org_opencontainers_image_version": { + "type": "keyword" + }, + "pod-template-generation": { + "type": "keyword" + }, + "pod-template-hash": { + "type": "keyword" + }, + "release": { + "type": "keyword" + }, + "role": { + "type": "keyword" + }, + "service": { + "type": "keyword" + }, + "statefulset_kubernetes_io/pod-name": { + "type": "keyword" + }, + "summary": { + "type": "keyword" + }, + "tier": { + "type": "keyword" + }, + "url": { + "type": "keyword" + }, + "vendor": { + "type": "keyword" + }, + "version": { + "type": "keyword" + } + } + }, + "size": { + "properties": { + "root_fs": { + "type": "long" + }, + "rw": { + "type": "long" + } + } + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "cpu": { + "properties": { + "core": { + "properties": { + "*": { + "properties": { + "norm": { + "properties": { + "pct": { + "type": "object" + } + } + }, + "pct": { + "type": "object" + }, + "ticks": { + "type": "object" + } + } + }, + "0": { + "properties": { + "norm": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "1": { + "properties": { + "norm": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "2": { + "properties": { + "norm": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "3": { + "properties": { + "norm": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + } + } + }, + "kernel": { + "properties": { + "norm": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "system": { + "properties": { + "norm": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "total": { + "properties": { + "norm": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "user": { + "properties": { + "norm": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + } + } + }, + "diskio": { + "properties": { + "read": { + "properties": { + "bytes": { + "type": "long" + }, + "ops": { + "type": "long" + }, + "queued": { + "type": "long" + }, + "rate": { + "type": "long" + }, + "service_time": { + "type": "long" + }, + "wait_time": { + "type": "long" + } + } + }, + "reads": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "summary": { + "properties": { + "bytes": { + "type": "long" + }, + "ops": { + "type": "long" + }, + "queued": { + "type": "long" + }, + "rate": { + "type": "long" + }, + "service_time": { + "type": "long" + }, + "wait_time": { + "type": "long" + } + } + }, + "total": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "write": { + "properties": { + "bytes": { + "type": "long" + }, + "ops": { + "type": "long" + }, + "queued": { + "type": "long" + }, + "rate": { + "type": "long" + }, + "service_time": { + "type": "long" + }, + "wait_time": { + "type": "long" + } + } + }, + "writes": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "event": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "actor": { + "properties": { + "attributes": { + "type": "object" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "from": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "healthcheck": { + "properties": { + "event": { + "properties": { + "end_date": { + "type": "date" + }, + "exit_code": { + "type": "long" + }, + "output": { + "ignore_above": 1024, + "type": "keyword" + }, + "start_date": { + "type": "date" + } + } + }, + "failingstreak": { + "type": "long" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "image": { + "properties": { + "created": { + "type": "date" + }, + "id": { + "properties": { + "current": { + "ignore_above": 1024, + "type": "keyword" + }, + "parent": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "labels": { + "type": "object" + }, + "size": { + "properties": { + "regular": { + "type": "long" + }, + "virtual": { + "type": "long" + } + } + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "info": { + "properties": { + "containers": { + "properties": { + "paused": { + "type": "long" + }, + "running": { + "type": "long" + }, + "stopped": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "images": { + "type": "long" + } + } + }, + "memory": { + "properties": { + "commit": { + "properties": { + "peak": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "fail": { + "properties": { + "count": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "limit": { + "type": "long" + }, + "private_working_set": { + "properties": { + "total": { + "type": "long" + } + } + }, + "rss": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "total": { + "type": "long" + } + } + }, + "stats": { + "properties": { + "*": { + "type": "object" + }, + "active_anon": { + "type": "long" + }, + "active_file": { + "type": "long" + }, + "cache": { + "type": "long" + }, + "dirty": { + "type": "long" + }, + "hierarchical_memory_limit": { + "type": "long" + }, + "hierarchical_memsw_limit": { + "type": "long" + }, + "inactive_anon": { + "type": "long" + }, + "inactive_file": { + "type": "long" + }, + "mapped_file": { + "type": "long" + }, + "pgfault": { + "type": "long" + }, + "pgmajfault": { + "type": "long" + }, + "pgpgin": { + "type": "long" + }, + "pgpgout": { + "type": "long" + }, + "rss": { + "type": "long" + }, + "rss_huge": { + "type": "long" + }, + "total_active_anon": { + "type": "long" + }, + "total_active_file": { + "type": "long" + }, + "total_cache": { + "type": "long" + }, + "total_dirty": { + "type": "long" + }, + "total_inactive_anon": { + "type": "long" + }, + "total_inactive_file": { + "type": "long" + }, + "total_mapped_file": { + "type": "long" + }, + "total_pgfault": { + "type": "long" + }, + "total_pgmajfault": { + "type": "long" + }, + "total_pgpgin": { + "type": "long" + }, + "total_pgpgout": { + "type": "long" + }, + "total_rss": { + "type": "long" + }, + "total_rss_huge": { + "type": "long" + }, + "total_unevictable": { + "type": "long" + }, + "total_writeback": { + "type": "long" + }, + "unevictable": { + "type": "long" + }, + "writeback": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "max": { + "type": "long" + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "total": { + "type": "long" + } + } + } + } + }, + "network": { + "properties": { + "in": { + "properties": { + "bytes": { + "type": "long" + }, + "dropped": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "errors": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + }, + "inbound": { + "properties": { + "bytes": { + "type": "long" + }, + "dropped": { + "type": "long" + }, + "errors": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + }, + "interface": { + "ignore_above": 1024, + "type": "keyword" + }, + "out": { + "properties": { + "bytes": { + "type": "long" + }, + "dropped": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "errors": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + }, + "outbound": { + "properties": { + "bytes": { + "type": "long" + }, + "dropped": { + "type": "long" + }, + "errors": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + } + } + } + } + }, + "dropwizard": { + "type": "object" + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "elasticsearch": { + "properties": { + "ccr": { + "properties": { + "follower": { + "properties": { + "global_checkpoint": { + "type": "long" + }, + "index": { + "ignore_above": 1024, + "type": "keyword" + }, + "operations_written": { + "type": "long" + }, + "shard": { + "properties": { + "number": { + "type": "long" + } + } + }, + "time_since_last_read": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "leader": { + "properties": { + "index": { + "ignore_above": 1024, + "type": "keyword" + }, + "max_seq_no": { + "type": "long" + } + } + } + } + }, + "cluster": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "pending_task": { + "properties": { + "insert_order": { + "type": "long" + }, + "priority": { + "type": "long" + }, + "source": { + "ignore_above": 1024, + "type": "keyword" + }, + "time_in_queue": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "state": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "stats": { + "properties": { + "indices": { + "properties": { + "count": { + "type": "long" + }, + "fielddata": { + "properties": { + "memory": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "shards": { + "properties": { + "count": { + "type": "long" + }, + "primaries": { + "type": "long" + } + } + } + } + }, + "nodes": { + "properties": { + "count": { + "type": "long" + }, + "data": { + "type": "long" + }, + "master": { + "type": "long" + } + } + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "enrich": { + "properties": { + "executed_searches": { + "properties": { + "total": { + "type": "long" + } + } + }, + "queue": { + "properties": { + "size": { + "type": "long" + } + } + }, + "remote_requests": { + "properties": { + "current": { + "type": "long" + }, + "total": { + "type": "long" + } + } + } + } + }, + "index": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "recovery": { + "properties": { + "id": { + "type": "long" + }, + "primary": { + "type": "boolean" + }, + "source": { + "properties": { + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "stage": { + "ignore_above": 1024, + "type": "keyword" + }, + "target": { + "properties": { + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "summary": { + "properties": { + "primaries": { + "properties": { + "docs": { + "properties": { + "count": { + "type": "long" + }, + "deleted": { + "type": "long" + } + } + }, + "segments": { + "properties": { + "count": { + "type": "long" + }, + "memory": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "store": { + "properties": { + "size": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "total": { + "properties": { + "docs": { + "properties": { + "count": { + "type": "long" + }, + "deleted": { + "type": "long" + } + } + }, + "segments": { + "properties": { + "count": { + "type": "long" + }, + "memory": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "store": { + "properties": { + "size": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "total": { + "properties": { + "docs": { + "properties": { + "count": { + "type": "long" + }, + "deleted": { + "type": "long" + } + } + }, + "segments": { + "properties": { + "count": { + "type": "long" + }, + "memory": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "store": { + "properties": { + "size": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "ml": { + "properties": { + "job": { + "properties": { + "data_counts": { + "properties": { + "invalid_date_count": { + "type": "long" + }, + "processed_record_count": { + "type": "long" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "node": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "jvm": { + "properties": { + "memory": { + "properties": { + "heap": { + "properties": { + "init": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "nonheap": { + "properties": { + "init": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "process": { + "properties": { + "mlockall": { + "type": "boolean" + } + } + }, + "stats": { + "properties": { + "fs": { + "properties": { + "summary": { + "properties": { + "available": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "free": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "total": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "indices": { + "properties": { + "docs": { + "properties": { + "count": { + "type": "long" + }, + "deleted": { + "type": "long" + } + } + }, + "segments": { + "properties": { + "count": { + "type": "long" + }, + "memory": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "store": { + "properties": { + "size": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "jvm": { + "properties": { + "gc": { + "properties": { + "collectors": { + "properties": { + "old": { + "properties": { + "collection": { + "properties": { + "count": { + "type": "long" + }, + "ms": { + "type": "long" + } + } + } + } + }, + "young": { + "properties": { + "collection": { + "properties": { + "count": { + "type": "long" + }, + "ms": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "mem": { + "properties": { + "pools": { + "properties": { + "old": { + "properties": { + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "peak": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "peak_max": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "survivor": { + "properties": { + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "peak": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "peak_max": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "young": { + "properties": { + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "peak": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "peak_max": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + } + } + } + } + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "shard": { + "properties": { + "number": { + "type": "long" + }, + "primary": { + "type": "boolean" + }, + "relocating_node": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "envoyproxy": { + "properties": { + "server": { + "properties": { + "cluster_manager": { + "properties": { + "active_clusters": { + "type": "long" + }, + "cluster_added": { + "type": "long" + }, + "cluster_modified": { + "type": "long" + }, + "cluster_removed": { + "type": "long" + }, + "cluster_updated": { + "type": "long" + }, + "cluster_updated_via_merge": { + "type": "long" + }, + "update_merge_cancelled": { + "type": "long" + }, + "update_out_of_merge_window": { + "type": "long" + }, + "warming_clusters": { + "type": "long" + } + } + }, + "filesystem": { + "properties": { + "flushed_by_timer": { + "type": "long" + }, + "reopen_failed": { + "type": "long" + }, + "write_buffered": { + "type": "long" + }, + "write_completed": { + "type": "long" + }, + "write_failed": { + "type": "long" + }, + "write_total_buffered": { + "type": "long" + } + } + }, + "http2": { + "properties": { + "header_overflow": { + "type": "long" + }, + "headers_cb_no_stream": { + "type": "long" + }, + "rx_messaging_error": { + "type": "long" + }, + "rx_reset": { + "type": "long" + }, + "too_many_header_frames": { + "type": "long" + }, + "trailers": { + "type": "long" + }, + "tx_reset": { + "type": "long" + } + } + }, + "listener_manager": { + "properties": { + "listener_added": { + "type": "long" + }, + "listener_create_failure": { + "type": "long" + }, + "listener_create_success": { + "type": "long" + }, + "listener_modified": { + "type": "long" + }, + "listener_removed": { + "type": "long" + }, + "listener_stopped": { + "type": "long" + }, + "total_listeners_active": { + "type": "long" + }, + "total_listeners_draining": { + "type": "long" + }, + "total_listeners_warming": { + "type": "long" + } + } + }, + "runtime": { + "properties": { + "admin_overrides_active": { + "type": "long" + }, + "deprecated_feature_use": { + "type": "long" + }, + "load_error": { + "type": "long" + }, + "load_success": { + "type": "long" + }, + "num_keys": { + "type": "long" + }, + "num_layers": { + "type": "long" + }, + "override_dir_exists": { + "type": "long" + }, + "override_dir_not_exists": { + "type": "long" + } + } + }, + "server": { + "properties": { + "concurrency": { + "type": "long" + }, + "days_until_first_cert_expiring": { + "type": "long" + }, + "debug_assertion_failures": { + "type": "long" + }, + "dynamic_unknown_fields": { + "type": "long" + }, + "hot_restart_epoch": { + "type": "long" + }, + "live": { + "type": "long" + }, + "memory_allocated": { + "type": "long" + }, + "memory_heap_size": { + "type": "long" + }, + "parent_connections": { + "type": "long" + }, + "state": { + "type": "long" + }, + "static_unknown_fields": { + "type": "long" + }, + "stats_recent_lookups": { + "type": "long" + }, + "total_connections": { + "type": "long" + }, + "uptime": { + "type": "long" + }, + "version": { + "type": "long" + }, + "watchdog_mega_miss": { + "type": "long" + }, + "watchdog_miss": { + "type": "long" + } + } + }, + "stats": { + "properties": { + "overflow": { + "type": "long" + } + } + } + } + } + } + }, + "error": { + "properties": { + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "stack_trace": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "etcd": { + "properties": { + "api_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "disk": { + "properties": { + "backend_commit_duration": { + "properties": { + "ns": { + "properties": { + "bucket": { + "properties": { + "*": { + "type": "object" + } + } + }, + "count": { + "type": "long" + }, + "sum": { + "type": "long" + } + } + } + } + }, + "mvcc_db_total_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "wal_fsync_duration": { + "properties": { + "ns": { + "properties": { + "bucket": { + "properties": { + "*": { + "type": "object" + } + } + }, + "count": { + "type": "long" + }, + "sum": { + "type": "long" + } + } + } + } + } + } + }, + "leader": { + "properties": { + "followers": { + "properties": { + "counts": { + "properties": { + "followers": { + "properties": { + "counts": { + "properties": { + "fail": { + "type": "long" + }, + "success": { + "type": "long" + } + } + } + } + } + } + }, + "latency": { + "properties": { + "follower": { + "properties": { + "latency": { + "properties": { + "standardDeviation": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + }, + "followers": { + "properties": { + "latency": { + "properties": { + "average": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "current": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "maximum": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "minimum": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "leader": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "memory": { + "properties": { + "go_memstats_alloc": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "network": { + "properties": { + "client_grpc_received": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "client_grpc_sent": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "self": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "leaderinfo": { + "properties": { + "leader": { + "ignore_above": 1024, + "type": "keyword" + }, + "starttime": { + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "recv": { + "properties": { + "appendrequest": { + "properties": { + "count": { + "type": "long" + } + } + }, + "bandwidthrate": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "pkgrate": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "send": { + "properties": { + "appendrequest": { + "properties": { + "count": { + "type": "long" + } + } + }, + "bandwidthrate": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "pkgrate": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "starttime": { + "ignore_above": 1024, + "type": "keyword" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "server": { + "properties": { + "grpc_handled": { + "properties": { + "count": { + "type": "long" + } + } + }, + "grpc_started": { + "properties": { + "count": { + "type": "long" + } + } + }, + "has_leader": { + "type": "byte" + }, + "leader_changes": { + "properties": { + "count": { + "type": "long" + } + } + }, + "proposals_committed": { + "properties": { + "count": { + "type": "long" + } + } + }, + "proposals_failed": { + "properties": { + "count": { + "type": "long" + } + } + }, + "proposals_pending": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "store": { + "properties": { + "compareanddelete": { + "properties": { + "fail": { + "type": "long" + }, + "success": { + "type": "long" + } + } + }, + "compareandswap": { + "properties": { + "fail": { + "type": "long" + }, + "success": { + "type": "long" + } + } + }, + "create": { + "properties": { + "fail": { + "type": "long" + }, + "success": { + "type": "long" + } + } + }, + "delete": { + "properties": { + "fail": { + "type": "long" + }, + "success": { + "type": "long" + } + } + }, + "expire": { + "properties": { + "count": { + "type": "long" + } + } + }, + "gets": { + "properties": { + "fail": { + "type": "long" + }, + "success": { + "type": "long" + } + } + }, + "sets": { + "properties": { + "fail": { + "type": "long" + }, + "success": { + "type": "long" + } + } + }, + "update": { + "properties": { + "fail": { + "type": "long" + }, + "success": { + "type": "long" + } + } + }, + "watchers": { + "type": "long" + } + } + } + } + }, + "event": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "created": { + "type": "date" + }, + "dataset": { + "ignore_above": 1024, + "type": "keyword" + }, + "duration": { + "type": "long" + }, + "end": { + "type": "date" + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ingested": { + "type": "date" + }, + "kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "ignore_above": 1024, + "type": "keyword" + }, + "outcome": { + "ignore_above": 1024, + "type": "keyword" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "risk_score": { + "type": "float" + }, + "risk_score_norm": { + "type": "float" + }, + "sequence": { + "type": "long" + }, + "severity": { + "type": "long" + }, + "start": { + "type": "date" + }, + "timezone": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "fields": { + "type": "object" + }, + "file": { + "properties": { + "accessed": { + "type": "date" + }, + "attributes": { + "ignore_above": 1024, + "type": "keyword" + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "created": { + "type": "date" + }, + "ctime": { + "type": "date" + }, + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "directory": { + "ignore_above": 1024, + "type": "keyword" + }, + "drive_letter": { + "ignore_above": 1, + "type": "keyword" + }, + "extension": { + "ignore_above": 1024, + "type": "keyword" + }, + "gid": { + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "inode": { + "ignore_above": 1024, + "type": "keyword" + }, + "mime_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "mode": { + "ignore_above": 1024, + "type": "keyword" + }, + "mtime": { + "type": "date" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "owner": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "pe": { + "properties": { + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "size": { + "type": "long" + }, + "target_path": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "golang": { + "properties": { + "expvar": { + "properties": { + "cmdline": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "heap": { + "properties": { + "allocations": { + "properties": { + "active": { + "type": "long" + }, + "allocated": { + "type": "long" + }, + "frees": { + "type": "long" + }, + "idle": { + "type": "long" + }, + "mallocs": { + "type": "long" + }, + "objects": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "cmdline": { + "ignore_above": 1024, + "type": "keyword" + }, + "gc": { + "properties": { + "cpu_fraction": { + "type": "float" + }, + "next_gc_limit": { + "type": "long" + }, + "pause": { + "properties": { + "avg": { + "properties": { + "ns": { + "type": "long" + } + } + }, + "count": { + "type": "long" + }, + "max": { + "properties": { + "ns": { + "type": "long" + } + } + }, + "sum": { + "properties": { + "ns": { + "type": "long" + } + } + } + } + }, + "total_count": { + "type": "long" + }, + "total_pause": { + "properties": { + "ns": { + "type": "long" + } + } + } + } + }, + "system": { + "properties": { + "obtained": { + "type": "long" + }, + "released": { + "type": "long" + }, + "stack": { + "type": "long" + }, + "total": { + "type": "long" + } + } + } + } + } + } + }, + "googlecloud": { + "properties": { + "labels": { + "type": "object" + } + } + }, + "graphite": { + "properties": { + "server": { + "properties": { + "example": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "haproxy": { + "properties": { + "info": { + "properties": { + "compress": { + "properties": { + "bps": { + "properties": { + "in": { + "type": "long" + }, + "out": { + "type": "long" + }, + "rate_limit": { + "type": "long" + } + } + } + } + }, + "connection": { + "properties": { + "current": { + "type": "long" + }, + "hard_max": { + "type": "long" + }, + "max": { + "type": "long" + }, + "rate": { + "properties": { + "limit": { + "type": "long" + }, + "max": { + "type": "long" + }, + "value": { + "type": "long" + } + } + }, + "ssl": { + "properties": { + "current": { + "type": "long" + }, + "max": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "total": { + "type": "long" + } + } + }, + "idle": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "memory": { + "properties": { + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "pipes": { + "properties": { + "free": { + "type": "long" + }, + "max": { + "type": "long" + }, + "used": { + "type": "long" + } + } + }, + "process_num": { + "type": "long" + }, + "processes": { + "type": "long" + }, + "requests": { + "properties": { + "max": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "run_queue": { + "type": "long" + }, + "session": { + "properties": { + "rate": { + "properties": { + "limit": { + "type": "long" + }, + "max": { + "type": "long" + }, + "value": { + "type": "long" + } + } + } + } + }, + "sockets": { + "properties": { + "max": { + "type": "long" + } + } + }, + "ssl": { + "properties": { + "backend": { + "properties": { + "key_rate": { + "properties": { + "max": { + "type": "long" + }, + "value": { + "type": "long" + } + } + } + } + }, + "cache_misses": { + "type": "long" + }, + "cached_lookups": { + "type": "long" + }, + "frontend": { + "properties": { + "key_rate": { + "properties": { + "max": { + "type": "long" + }, + "value": { + "type": "long" + } + } + }, + "session_reuse": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + }, + "rate": { + "properties": { + "limit": { + "type": "long" + }, + "max": { + "type": "long" + }, + "value": { + "type": "long" + } + } + } + } + }, + "tasks": { + "type": "long" + }, + "ulimit_n": { + "type": "long" + }, + "uptime": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "zlib_mem_usage": { + "properties": { + "max": { + "type": "long" + }, + "value": { + "type": "long" + } + } + } + } + }, + "stat": { + "properties": { + "agent": { + "properties": { + "check": { + "properties": { + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "health": { + "ignore_above": 1024, + "type": "keyword" + }, + "rise": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "fall": { + "ignore_above": 1024, + "type": "keyword" + }, + "health": { + "ignore_above": 1024, + "type": "keyword" + }, + "rise": { + "ignore_above": 1024, + "type": "keyword" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "check": { + "properties": { + "agent": { + "properties": { + "last": { + "type": "long" + } + } + }, + "code": { + "type": "long" + }, + "down": { + "type": "long" + }, + "duration": { + "type": "long" + }, + "failed": { + "type": "long" + }, + "health": { + "properties": { + "fail": { + "type": "long" + }, + "last": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "client": { + "properties": { + "aborted": { + "type": "long" + } + } + }, + "component_type": { + "type": "long" + }, + "compressor": { + "properties": { + "bypassed": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "in": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "out": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "response": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "connection": { + "properties": { + "cache": { + "type": "object" + }, + "idle": { + "type": "object" + }, + "retried": { + "type": "long" + }, + "time": { + "properties": { + "avg": { + "type": "long" + } + } + }, + "total": { + "type": "long" + } + } + }, + "cookie": { + "ignore_above": 1024, + "type": "keyword" + }, + "downtime": { + "type": "long" + }, + "header": { + "properties": { + "rewrite": { + "properties": { + "failed": { + "type": "object" + } + } + } + } + }, + "in": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "last_change": { + "type": "long" + }, + "load_balancing_algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "out": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "proxy": { + "properties": { + "id": { + "type": "long" + }, + "mode": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "queue": { + "properties": { + "limit": { + "type": "long" + }, + "time": { + "properties": { + "avg": { + "type": "long" + } + } + } + } + }, + "request": { + "properties": { + "connection": { + "properties": { + "errors": { + "type": "long" + } + } + }, + "denied": { + "type": "long" + }, + "errors": { + "type": "long" + }, + "queued": { + "properties": { + "current": { + "type": "long" + }, + "max": { + "type": "long" + } + } + }, + "rate": { + "properties": { + "max": { + "type": "long" + }, + "value": { + "type": "long" + } + } + }, + "redispatched": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "response": { + "properties": { + "denied": { + "type": "long" + }, + "errors": { + "type": "long" + }, + "http": { + "properties": { + "1xx": { + "type": "long" + }, + "2xx": { + "type": "long" + }, + "3xx": { + "type": "long" + }, + "4xx": { + "type": "long" + }, + "5xx": { + "type": "long" + }, + "other": { + "type": "long" + } + } + }, + "time": { + "properties": { + "avg": { + "type": "long" + } + } + } + } + }, + "selected": { + "properties": { + "total": { + "type": "long" + } + } + }, + "server": { + "properties": { + "aborted": { + "type": "long" + }, + "active": { + "type": "long" + }, + "backup": { + "type": "long" + }, + "id": { + "type": "long" + } + } + }, + "service_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "session": { + "properties": { + "current": { + "type": "long" + }, + "limit": { + "type": "long" + }, + "max": { + "type": "long" + }, + "rate": { + "properties": { + "limit": { + "type": "long" + }, + "max": { + "type": "long" + }, + "value": { + "type": "long" + } + } + } + } + }, + "source": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "throttle": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "tracked": { + "properties": { + "id": { + "type": "long" + } + } + }, + "weight": { + "type": "long" + } + } + } + } + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "host": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "containerized": { + "type": "boolean" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "build": { + "ignore_above": 1024, + "type": "keyword" + }, + "codename": { + "ignore_above": 1024, + "type": "keyword" + }, + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "http": { + "properties": { + "json": { + "type": "object" + }, + "request": { + "properties": { + "body": { + "properties": { + "bytes": { + "type": "long" + }, + "content": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bytes": { + "type": "long" + }, + "headers": { + "type": "object" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "referrer": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "response": { + "properties": { + "body": { + "properties": { + "bytes": { + "type": "long" + }, + "content": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bytes": { + "type": "long" + }, + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "headers": { + "type": "object" + }, + "phrase": { + "ignore_above": 1024, + "type": "keyword" + }, + "status_code": { + "type": "long" + } + } + }, + "server": { + "type": "object" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ibmmq": { + "type": "object" + }, + "iis": { + "properties": { + "application_pool": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "webserver": { + "properties": { + "*": { + "properties": { + "*": { + "type": "object" + } + } + } + } + }, + "website": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "interface": { + "properties": { + "alias": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "istio": { + "properties": { + "citadel": { + "properties": { + "grpc": { + "properties": { + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "server": { + "properties": { + "handled": { + "type": "long" + }, + "handling": { + "properties": { + "latency": { + "properties": { + "ms": { + "properties": { + "bucket": { + "properties": { + "*": { + "type": "object" + } + } + }, + "count": { + "type": "long" + }, + "sum": { + "type": "long" + } + } + } + } + } + } + }, + "msg": { + "properties": { + "received": { + "type": "long" + }, + "sent": { + "type": "long" + } + } + }, + "started": { + "type": "long" + } + } + }, + "service": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "secret_controller_svc_acc_created_cert": { + "properties": { + "count": { + "type": "long" + } + } + }, + "server_root_cert_expiry_seconds": { + "type": "float" + } + } + }, + "galley": { + "properties": { + "collection": { + "ignore_above": 1024, + "type": "keyword" + }, + "istio": { + "properties": { + "authentication": { + "properties": { + "meshpolicies": { + "type": "long" + }, + "policies": { + "type": "long" + } + } + }, + "mesh": { + "properties": { + "MeshConfig": { + "type": "long" + } + } + }, + "networking": { + "properties": { + "destinationrules": { + "type": "long" + }, + "envoyfilters": { + "type": "long" + }, + "gateways": { + "type": "long" + }, + "sidecars": { + "type": "long" + }, + "virtualservices": { + "type": "long" + } + } + }, + "policy": { + "properties": { + "attributemanifests": { + "type": "long" + }, + "handlers": { + "type": "long" + }, + "instances": { + "type": "long" + }, + "rules": { + "type": "long" + } + } + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "namespace": { + "ignore_above": 1024, + "type": "keyword" + }, + "runtime": { + "properties": { + "processor": { + "properties": { + "event_span": { + "properties": { + "duration": { + "properties": { + "ms": { + "properties": { + "bucket": { + "properties": { + "*": { + "type": "object" + } + } + }, + "count": { + "type": "long" + }, + "sum": { + "type": "long" + } + } + } + } + } + } + }, + "snapshot_events": { + "properties": { + "bucket": { + "properties": { + "*": { + "type": "object" + } + } + }, + "count": { + "type": "long" + }, + "sum": { + "type": "long" + } + } + }, + "snapshot_lifetime": { + "properties": { + "duration": { + "properties": { + "ms": { + "properties": { + "bucket": { + "properties": { + "*": { + "type": "object" + } + } + }, + "count": { + "type": "long" + }, + "sum": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "state_type_instances": { + "type": "long" + }, + "strategy": { + "properties": { + "on_change": { + "type": "long" + }, + "timer_quiesce_reached": { + "type": "long" + } + } + } + } + }, + "source_kube_event_success_total": { + "type": "long" + }, + "validation": { + "properties": { + "cert_key": { + "properties": { + "updates": { + "type": "long" + } + } + }, + "config": { + "properties": { + "load": { + "type": "long" + }, + "updates": { + "type": "long" + } + } + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "mesh": { + "properties": { + "connection": { + "properties": { + "security": { + "properties": { + "policy": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "destination": { + "properties": { + "app": { + "ignore_above": 1024, + "type": "keyword" + }, + "principal": { + "ignore_above": 1024, + "type": "keyword" + }, + "service": { + "properties": { + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "namespace": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "workload": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "namespace": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "instance": { + "norms": false, + "type": "text" + }, + "job": { + "ignore_above": 1024, + "type": "keyword" + }, + "reporter": { + "ignore_above": 1024, + "type": "keyword" + }, + "request": { + "properties": { + "duration": { + "properties": { + "ms": { + "properties": { + "bucket": { + "properties": { + "*": { + "type": "object" + } + } + }, + "count": { + "type": "long" + }, + "sum": { + "type": "long" + } + } + } + } + }, + "protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "size": { + "properties": { + "bytes": { + "properties": { + "bucket": { + "properties": { + "*": { + "type": "object" + } + } + }, + "count": { + "type": "long" + }, + "sum": { + "type": "long" + } + } + } + } + } + } + }, + "requests": { + "type": "long" + }, + "response": { + "properties": { + "code": { + "type": "long" + }, + "size": { + "properties": { + "bytes": { + "properties": { + "bucket": { + "properties": { + "*": { + "type": "object" + } + } + }, + "count": { + "type": "long" + }, + "sum": { + "type": "long" + } + } + } + } + } + } + }, + "source": { + "properties": { + "app": { + "ignore_above": 1024, + "type": "keyword" + }, + "principal": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "workload": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "namespace": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "mixer": { + "properties": { + "config": { + "properties": { + "adapter": { + "properties": { + "info": { + "properties": { + "configs": { + "type": "long" + }, + "errors": { + "properties": { + "config": { + "type": "long" + } + } + } + } + } + } + }, + "attributes": { + "type": "long" + }, + "handler": { + "properties": { + "configs": { + "type": "long" + }, + "errors": { + "properties": { + "validation": { + "type": "long" + } + } + } + } + }, + "instance": { + "properties": { + "configs": { + "type": "long" + }, + "errors": { + "properties": { + "config": { + "type": "long" + } + } + } + } + }, + "rule": { + "properties": { + "configs": { + "type": "long" + }, + "errors": { + "properties": { + "config": { + "type": "long" + }, + "match": { + "type": "long" + } + } + } + } + }, + "template": { + "properties": { + "configs": { + "type": "long" + }, + "errors": { + "properties": { + "config": { + "type": "long" + } + } + } + } + }, + "unsatisfied": { + "properties": { + "action_handler": { + "type": "long" + } + } + } + } + }, + "dispatcher_destinations_per_variety_total": { + "type": "long" + }, + "handler": { + "type": "long" + }, + "istio": { + "properties": { + "mcp": { + "properties": { + "request": { + "properties": { + "acks": { + "type": "long" + } + } + } + } + } + } + }, + "variety": { + "type": "long" + } + } + }, + "pilot": { + "properties": { + "cluster": { + "norms": false, + "type": "text" + }, + "conflict": { + "properties": { + "listener": { + "properties": { + "inbound": { + "type": "long" + }, + "outbound": { + "properties": { + "http": { + "properties": { + "over": { + "properties": { + "current": { + "properties": { + "tcp": { + "type": "long" + } + } + }, + "https": { + "type": "long" + } + } + } + } + }, + "tcp": { + "properties": { + "over": { + "properties": { + "current": { + "properties": { + "http": { + "type": "long" + }, + "tcp": { + "type": "long" + } + } + } + } + } + } + } + } + } + } + } + } + }, + "no": { + "properties": { + "ip": { + "type": "object" + } + } + }, + "proxy": { + "properties": { + "conv": { + "properties": { + "ms": { + "properties": { + "bucket": { + "properties": { + "*": { + "type": "object" + } + } + }, + "count": { + "type": "long" + }, + "sum": { + "type": "long" + } + } + } + } + } + } + }, + "services": { + "type": "long" + }, + "type": { + "norms": false, + "type": "text" + }, + "virt": { + "properties": { + "services": { + "type": "long" + } + } + }, + "xds": { + "properties": { + "count": { + "type": "long" + }, + "eds": { + "properties": { + "instances": { + "type": "object" + } + } + }, + "internal": { + "properties": { + "errors": { + "type": "long" + } + } + }, + "push": { + "properties": { + "context": { + "properties": { + "errors": { + "type": "long" + } + } + }, + "time": { + "properties": { + "sec": { + "type": "long" + } + } + } + } + }, + "pushes": { + "type": "long" + } + } + } + } + } + } + }, + "jolokia": { + "properties": { + "agent": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "secured": { + "type": "boolean" + }, + "server": { + "properties": { + "product": { + "ignore_above": 1024, + "type": "keyword" + }, + "vendor": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "kafka": { + "properties": { + "broker": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "type": "long" + }, + "log": { + "properties": { + "flush_rate": { + "type": "float" + } + } + }, + "mbean": { + "ignore_above": 1024, + "type": "keyword" + }, + "messages_in": { + "type": "float" + }, + "net": { + "properties": { + "in": { + "properties": { + "bytes_per_sec": { + "type": "float" + } + } + }, + "out": { + "properties": { + "bytes_per_sec": { + "type": "float" + } + } + }, + "rejected": { + "properties": { + "bytes_per_sec": { + "type": "float" + } + } + } + } + }, + "replication": { + "properties": { + "leader_elections": { + "type": "float" + }, + "unclean_leader_elections": { + "type": "float" + } + } + }, + "request": { + "properties": { + "channel": { + "properties": { + "queue": { + "properties": { + "size": { + "type": "long" + } + } + } + } + }, + "fetch": { + "properties": { + "failed": { + "type": "float" + }, + "failed_per_second": { + "type": "float" + } + } + }, + "produce": { + "properties": { + "failed": { + "type": "float" + }, + "failed_per_second": { + "type": "float" + } + } + } + } + }, + "session": { + "properties": { + "zookeeper": { + "properties": { + "disconnect": { + "type": "float" + }, + "expire": { + "type": "float" + }, + "readonly": { + "type": "float" + }, + "sync": { + "type": "float" + } + } + } + } + }, + "topic": { + "properties": { + "messages_in": { + "type": "float" + }, + "net": { + "properties": { + "in": { + "properties": { + "bytes_per_sec": { + "type": "float" + } + } + }, + "out": { + "properties": { + "bytes_per_sec": { + "type": "float" + } + } + }, + "rejected": { + "properties": { + "bytes_per_sec": { + "type": "float" + } + } + } + } + } + } + } + } + }, + "consumer": { + "properties": { + "bytes_consumed": { + "type": "float" + }, + "fetch_rate": { + "type": "float" + }, + "in": { + "properties": { + "bytes_per_sec": { + "type": "float" + } + } + }, + "kafka_commits": { + "type": "float" + }, + "max_lag": { + "type": "float" + }, + "mbean": { + "ignore_above": 1024, + "type": "keyword" + }, + "messages_in": { + "type": "float" + }, + "records_consumed": { + "type": "float" + }, + "zookeeper_commits": { + "type": "float" + } + } + }, + "consumergroup": { + "properties": { + "broker": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "type": "long" + } + } + }, + "client": { + "properties": { + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "member_id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "consumer_lag": { + "type": "long" + }, + "error": { + "properties": { + "code": { + "type": "long" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "meta": { + "ignore_above": 1024, + "type": "keyword" + }, + "offset": { + "type": "long" + }, + "partition": { + "type": "long" + }, + "topic": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "partition": { + "properties": { + "broker": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "type": "long" + } + } + }, + "id": { + "type": "long" + }, + "offset": { + "properties": { + "newest": { + "type": "long" + }, + "oldest": { + "type": "long" + } + } + }, + "partition": { + "properties": { + "error": { + "properties": { + "code": { + "type": "long" + } + } + }, + "id": { + "type": "long" + }, + "insync_replica": { + "type": "boolean" + }, + "is_leader": { + "type": "boolean" + }, + "isr": { + "ignore_above": 1024, + "type": "keyword" + }, + "leader": { + "type": "long" + }, + "replica": { + "type": "long" + } + } + }, + "topic": { + "properties": { + "error": { + "properties": { + "code": { + "type": "long" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "topic_broker_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "topic_id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "producer": { + "properties": { + "available_buffer_bytes": { + "type": "float" + }, + "batch_size_avg": { + "type": "float" + }, + "batch_size_max": { + "type": "long" + }, + "io_wait": { + "type": "float" + }, + "mbean": { + "ignore_above": 1024, + "type": "keyword" + }, + "message_rate": { + "type": "float" + }, + "out": { + "properties": { + "bytes_per_sec": { + "type": "float" + } + } + }, + "record_error_rate": { + "type": "float" + }, + "record_retry_rate": { + "type": "float" + }, + "record_send_rate": { + "type": "float" + }, + "record_size_avg": { + "type": "float" + }, + "record_size_max": { + "type": "long" + }, + "records_per_request": { + "type": "float" + }, + "request_rate": { + "type": "float" + }, + "response_rate": { + "type": "float" + } + } + }, + "topic": { + "properties": { + "error": { + "properties": { + "code": { + "type": "long" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "kibana": { + "properties": { + "stats": { + "properties": { + "concurrent_connections": { + "type": "long" + }, + "host": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "index": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "process": { + "properties": { + "event_loop_delay": { + "properties": { + "ms": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "memory": { + "properties": { + "heap": { + "properties": { + "size_limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "total": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "uptime": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "request": { + "properties": { + "disconnects": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "response_time": { + "properties": { + "avg": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "max": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "snapshot": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "status": { + "properties": { + "metrics": { + "properties": { + "concurrent_connections": { + "type": "long" + }, + "requests": { + "properties": { + "disconnects": { + "type": "long" + }, + "total": { + "type": "long" + } + } + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "status": { + "properties": { + "overall": { + "properties": { + "state": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + } + } + }, + "kubernetes": { + "properties": { + "annotations": { + "properties": { + "*": { + "type": "object" + } + } + }, + "apiserver": { + "properties": { + "audit": { + "properties": { + "event": { + "properties": { + "count": { + "type": "long" + } + } + }, + "rejected": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "client": { + "properties": { + "request": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "etcd": { + "properties": { + "object": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "http": { + "properties": { + "request": { + "properties": { + "count": { + "type": "long" + }, + "duration": { + "properties": { + "us": { + "properties": { + "count": { + "type": "long" + }, + "percentile": { + "properties": { + "*": { + "type": "object" + } + } + }, + "sum": { + "type": "double" + } + } + } + } + }, + "size": { + "properties": { + "bytes": { + "properties": { + "count": { + "type": "long" + }, + "percentile": { + "properties": { + "*": { + "type": "object" + } + } + }, + "sum": { + "type": "long" + } + } + } + } + } + } + }, + "response": { + "properties": { + "size": { + "properties": { + "bytes": { + "properties": { + "count": { + "type": "long" + }, + "percentile": { + "properties": { + "*": { + "type": "object" + } + } + }, + "sum": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "process": { + "properties": { + "cpu": { + "properties": { + "sec": { + "type": "double" + } + } + }, + "fds": { + "properties": { + "open": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "memory": { + "properties": { + "resident": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "virtual": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "started": { + "properties": { + "sec": { + "type": "double" + } + } + } + } + }, + "request": { + "properties": { + "client": { + "ignore_above": 1024, + "type": "keyword" + }, + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "component": { + "ignore_above": 1024, + "type": "keyword" + }, + "content_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "count": { + "type": "long" + }, + "current": { + "properties": { + "count": { + "type": "long" + } + } + }, + "dry_run": { + "ignore_above": 1024, + "type": "keyword" + }, + "duration": { + "properties": { + "us": { + "properties": { + "bucket": { + "properties": { + "*": { + "type": "object" + } + } + }, + "count": { + "type": "long" + }, + "sum": { + "type": "long" + } + } + } + } + }, + "group": { + "ignore_above": 1024, + "type": "keyword" + }, + "handler": { + "ignore_above": 1024, + "type": "keyword" + }, + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "latency": { + "properties": { + "bucket": { + "properties": { + "*": { + "type": "object" + } + } + }, + "count": { + "type": "long" + }, + "sum": { + "type": "long" + } + } + }, + "longrunning": { + "properties": { + "count": { + "type": "long" + } + } + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "resource": { + "ignore_above": 1024, + "type": "keyword" + }, + "scope": { + "ignore_above": 1024, + "type": "keyword" + }, + "subresource": { + "ignore_above": 1024, + "type": "keyword" + }, + "verb": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "container": { + "properties": { + "cpu": { + "properties": { + "limit": { + "properties": { + "cores": { + "type": "float" + }, + "nanocores": { + "type": "long" + } + } + }, + "request": { + "properties": { + "cores": { + "type": "float" + }, + "nanocores": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "core": { + "properties": { + "ns": { + "type": "long" + } + } + }, + "limit": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "nanocores": { + "type": "long" + }, + "node": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "image": { + "ignore_above": 1024, + "type": "keyword" + }, + "logs": { + "properties": { + "available": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "capacity": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "inodes": { + "properties": { + "count": { + "type": "long" + }, + "free": { + "type": "long" + }, + "used": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "memory": { + "properties": { + "available": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "majorpagefaults": { + "type": "long" + }, + "pagefaults": { + "type": "long" + }, + "request": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "rss": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "bytes": { + "type": "long" + }, + "limit": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "node": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + }, + "workingset": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "rootfs": { + "properties": { + "available": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "capacity": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "inodes": { + "properties": { + "used": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "start_time": { + "type": "date" + }, + "status": { + "properties": { + "phase": { + "ignore_above": 1024, + "type": "keyword" + }, + "ready": { + "type": "boolean" + }, + "reason": { + "ignore_above": 1024, + "type": "keyword" + }, + "restarts": { + "type": "long" + } + } + } + } + }, + "controllermanager": { + "properties": { + "client": { + "properties": { + "request": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "handler": { + "ignore_above": 1024, + "type": "keyword" + }, + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "http": { + "properties": { + "request": { + "properties": { + "count": { + "type": "long" + }, + "duration": { + "properties": { + "us": { + "properties": { + "count": { + "type": "long" + }, + "percentile": { + "properties": { + "*": { + "type": "object" + } + } + }, + "sum": { + "type": "double" + } + } + } + } + }, + "size": { + "properties": { + "bytes": { + "properties": { + "count": { + "type": "long" + }, + "percentile": { + "properties": { + "*": { + "type": "object" + } + } + }, + "sum": { + "type": "long" + } + } + } + } + } + } + }, + "response": { + "properties": { + "size": { + "properties": { + "bytes": { + "properties": { + "count": { + "type": "long" + }, + "percentile": { + "properties": { + "*": { + "type": "object" + } + } + }, + "sum": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "leader": { + "properties": { + "is_master": { + "type": "boolean" + } + } + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "node": { + "properties": { + "collector": { + "properties": { + "count": { + "type": "long" + }, + "eviction": { + "properties": { + "count": { + "type": "long" + } + } + }, + "health": { + "properties": { + "pct": { + "type": "long" + } + } + }, + "unhealthy": { + "properties": { + "count": { + "type": "long" + } + } + } + } + } + } + }, + "process": { + "properties": { + "cpu": { + "properties": { + "sec": { + "type": "double" + } + } + }, + "fds": { + "properties": { + "open": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "memory": { + "properties": { + "resident": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "virtual": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "started": { + "properties": { + "sec": { + "type": "double" + } + } + } + } + }, + "workqueue": { + "properties": { + "adds": { + "properties": { + "count": { + "type": "long" + } + } + }, + "depth": { + "properties": { + "count": { + "type": "long" + } + } + }, + "longestrunning": { + "properties": { + "sec": { + "type": "double" + } + } + }, + "retries": { + "properties": { + "count": { + "type": "long" + } + } + }, + "unfinished": { + "properties": { + "sec": { + "type": "double" + } + } + } + } + }, + "zone": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "cronjob": { + "properties": { + "active": { + "properties": { + "count": { + "type": "long" + } + } + }, + "concurrency": { + "ignore_above": 1024, + "type": "keyword" + }, + "created": { + "properties": { + "sec": { + "type": "double" + } + } + }, + "deadline": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "is_suspended": { + "type": "boolean" + }, + "last_schedule": { + "properties": { + "sec": { + "type": "double" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "next_schedule": { + "properties": { + "sec": { + "type": "double" + } + } + }, + "schedule": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "deployment": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "paused": { + "type": "boolean" + }, + "replicas": { + "properties": { + "available": { + "type": "long" + }, + "desired": { + "type": "long" + }, + "unavailable": { + "type": "long" + }, + "updated": { + "type": "long" + } + } + } + } + }, + "event": { + "properties": { + "count": { + "type": "long" + }, + "involved_object": { + "properties": { + "api_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "resource_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "message": { + "copy_to": [ + "message" + ], + "norms": false, + "type": "text" + }, + "metadata": { + "properties": { + "generate_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "namespace": { + "ignore_above": 1024, + "type": "keyword" + }, + "resource_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "self_link": { + "ignore_above": 1024, + "type": "keyword" + }, + "timestamp": { + "properties": { + "created": { + "type": "date" + } + } + }, + "uid": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "reason": { + "ignore_above": 1024, + "type": "keyword" + }, + "source": { + "properties": { + "component": { + "ignore_above": 1024, + "type": "keyword" + }, + "host": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "timestamp": { + "properties": { + "first_occurrence": { + "type": "date" + }, + "last_occurrence": { + "type": "date" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "labels": { + "properties": { + "*": { + "type": "object" + }, + "addonmanager_kubernetes_io/mode": { + "type": "keyword" + }, + "app": { + "type": "keyword" + }, + "app_kubernetes_io/instance": { + "type": "keyword" + }, + "app_kubernetes_io/managed-by": { + "type": "keyword" + }, + "app_kubernetes_io/name": { + "type": "keyword" + }, + "app_kubernetes_io/version": { + "type": "keyword" + }, + "beta_kubernetes_io/arch": { + "type": "keyword" + }, + "beta_kubernetes_io/fluentd-ds-ready": { + "type": "keyword" + }, + "beta_kubernetes_io/instance-type": { + "type": "keyword" + }, + "beta_kubernetes_io/os": { + "type": "keyword" + }, + "chart": { + "type": "keyword" + }, + "cloud_google_com/gke-nodepool": { + "type": "keyword" + }, + "cloud_google_com/gke-os-distribution": { + "type": "keyword" + }, + "component": { + "type": "keyword" + }, + "controller-revision-hash": { + "type": "keyword" + }, + "controller-uid": { + "type": "keyword" + }, + "failure-domain_beta_kubernetes_io/region": { + "type": "keyword" + }, + "failure-domain_beta_kubernetes_io/zone": { + "type": "keyword" + }, + "helm_sh/chart": { + "type": "keyword" + }, + "heritage": { + "type": "keyword" + }, + "job-name": { + "type": "keyword" + }, + "k8s-app": { + "type": "keyword" + }, + "kubernetes_io/arch": { + "type": "keyword" + }, + "kubernetes_io/cluster-service": { + "type": "keyword" + }, + "kubernetes_io/hostname": { + "type": "keyword" + }, + "kubernetes_io/name": { + "type": "keyword" + }, + "kubernetes_io/os": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "pod-template-generation": { + "type": "keyword" + }, + "pod-template-hash": { + "type": "keyword" + }, + "release": { + "type": "keyword" + }, + "role": { + "type": "keyword" + }, + "service": { + "type": "keyword" + }, + "statefulset_kubernetes_io/pod-name": { + "type": "keyword" + }, + "tier": { + "type": "keyword" + }, + "version": { + "type": "keyword" + } + } + }, + "namespace": { + "ignore_above": 1024, + "type": "keyword" + }, + "node": { + "properties": { + "cpu": { + "properties": { + "allocatable": { + "properties": { + "cores": { + "type": "float" + } + } + }, + "capacity": { + "properties": { + "cores": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "core": { + "properties": { + "ns": { + "type": "long" + } + } + }, + "nanocores": { + "type": "long" + } + } + } + } + }, + "fs": { + "properties": { + "available": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "capacity": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "inodes": { + "properties": { + "count": { + "type": "long" + }, + "free": { + "type": "long" + }, + "used": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "memory": { + "properties": { + "allocatable": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "available": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "capacity": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "majorpagefaults": { + "type": "long" + }, + "pagefaults": { + "type": "long" + }, + "rss": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "workingset": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "network": { + "properties": { + "rx": { + "properties": { + "bytes": { + "type": "long" + }, + "errors": { + "type": "long" + } + } + }, + "tx": { + "properties": { + "bytes": { + "type": "long" + }, + "errors": { + "type": "long" + } + } + } + } + }, + "pod": { + "properties": { + "allocatable": { + "properties": { + "total": { + "type": "long" + } + } + }, + "capacity": { + "properties": { + "total": { + "type": "long" + } + } + } + } + }, + "runtime": { + "properties": { + "imagefs": { + "properties": { + "available": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "capacity": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "start_time": { + "type": "date" + }, + "status": { + "properties": { + "ready": { + "ignore_above": 1024, + "type": "keyword" + }, + "unschedulable": { + "type": "boolean" + } + } + } + } + }, + "persistentvolume": { + "properties": { + "capacity": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "phase": { + "ignore_above": 1024, + "type": "keyword" + }, + "storage_class": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "persistentvolumeclaim": { + "properties": { + "access_mode": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "phase": { + "ignore_above": 1024, + "type": "keyword" + }, + "request_storage": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "storage_class": { + "ignore_above": 1024, + "type": "keyword" + }, + "volume_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pod": { + "properties": { + "cpu": { + "properties": { + "usage": { + "properties": { + "limit": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "nanocores": { + "type": "long" + }, + "node": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + } + } + }, + "host_ip": { + "type": "ip" + }, + "ip": { + "type": "ip" + }, + "memory": { + "properties": { + "available": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "major_page_faults": { + "type": "long" + }, + "page_faults": { + "type": "long" + }, + "rss": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "bytes": { + "type": "long" + }, + "limit": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "node": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + }, + "working_set": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "network": { + "properties": { + "rx": { + "properties": { + "bytes": { + "type": "long" + }, + "errors": { + "type": "long" + } + } + }, + "tx": { + "properties": { + "bytes": { + "type": "long" + }, + "errors": { + "type": "long" + } + } + } + } + }, + "start_time": { + "type": "date" + }, + "status": { + "properties": { + "phase": { + "ignore_above": 1024, + "type": "keyword" + }, + "ready": { + "ignore_above": 1024, + "type": "keyword" + }, + "scheduled": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "uid": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "proxy": { + "properties": { + "client": { + "properties": { + "request": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "handler": { + "ignore_above": 1024, + "type": "keyword" + }, + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "http": { + "properties": { + "request": { + "properties": { + "count": { + "type": "long" + }, + "duration": { + "properties": { + "us": { + "properties": { + "count": { + "type": "long" + }, + "percentile": { + "properties": { + "*": { + "type": "object" + } + } + }, + "sum": { + "type": "double" + } + } + } + } + }, + "size": { + "properties": { + "bytes": { + "properties": { + "count": { + "type": "long" + }, + "percentile": { + "properties": { + "*": { + "type": "object" + } + } + }, + "sum": { + "type": "long" + } + } + } + } + } + } + }, + "response": { + "properties": { + "size": { + "properties": { + "bytes": { + "properties": { + "count": { + "type": "long" + }, + "percentile": { + "properties": { + "*": { + "type": "object" + } + } + }, + "sum": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "process": { + "properties": { + "cpu": { + "properties": { + "sec": { + "type": "double" + } + } + }, + "fds": { + "properties": { + "open": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "memory": { + "properties": { + "resident": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "virtual": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "started": { + "properties": { + "sec": { + "type": "double" + } + } + } + } + }, + "sync": { + "properties": { + "networkprogramming": { + "properties": { + "duration": { + "properties": { + "us": { + "properties": { + "bucket": { + "properties": { + "*": { + "type": "object" + } + } + }, + "count": { + "type": "long" + }, + "sum": { + "type": "long" + } + } + } + } + } + } + }, + "rules": { + "properties": { + "duration": { + "properties": { + "us": { + "properties": { + "bucket": { + "properties": { + "*": { + "type": "object" + } + } + }, + "count": { + "type": "long" + }, + "sum": { + "type": "long" + } + } + } + } + } + } + } + } + } + } + }, + "replicaset": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "replicas": { + "properties": { + "available": { + "type": "long" + }, + "desired": { + "type": "long" + }, + "labeled": { + "type": "long" + }, + "observed": { + "type": "long" + }, + "ready": { + "type": "long" + } + } + } + } + }, + "resourcequota": { + "properties": { + "created": { + "properties": { + "sec": { + "type": "double" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "quota": { + "type": "double" + }, + "resource": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "scheduler": { + "properties": { + "client": { + "properties": { + "request": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "handler": { + "ignore_above": 1024, + "type": "keyword" + }, + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "http": { + "properties": { + "request": { + "properties": { + "count": { + "type": "long" + }, + "duration": { + "properties": { + "us": { + "properties": { + "count": { + "type": "long" + }, + "percentile": { + "properties": { + "*": { + "type": "object" + } + } + }, + "sum": { + "type": "double" + } + } + } + } + }, + "size": { + "properties": { + "bytes": { + "properties": { + "count": { + "type": "long" + }, + "percentile": { + "properties": { + "*": { + "type": "object" + } + } + }, + "sum": { + "type": "long" + } + } + } + } + } + } + }, + "response": { + "properties": { + "size": { + "properties": { + "bytes": { + "properties": { + "count": { + "type": "long" + }, + "percentile": { + "properties": { + "*": { + "type": "object" + } + } + }, + "sum": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "leader": { + "properties": { + "is_master": { + "type": "boolean" + } + } + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "operation": { + "ignore_above": 1024, + "type": "keyword" + }, + "process": { + "properties": { + "cpu": { + "properties": { + "sec": { + "type": "double" + } + } + }, + "fds": { + "properties": { + "open": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "memory": { + "properties": { + "resident": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "virtual": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "started": { + "properties": { + "sec": { + "type": "double" + } + } + } + } + }, + "result": { + "ignore_above": 1024, + "type": "keyword" + }, + "scheduling": { + "properties": { + "duration": { + "properties": { + "seconds": { + "properties": { + "count": { + "type": "long" + }, + "percentile": { + "properties": { + "*": { + "type": "object" + } + } + }, + "sum": { + "type": "double" + } + } + } + } + }, + "e2e": { + "properties": { + "duration": { + "properties": { + "us": { + "properties": { + "bucket": { + "properties": { + "*": { + "type": "object" + } + } + }, + "count": { + "type": "long" + }, + "sum": { + "type": "long" + } + } + } + } + } + } + }, + "pod": { + "properties": { + "attempts": { + "properties": { + "count": { + "type": "long" + } + } + }, + "preemption": { + "properties": { + "victims": { + "properties": { + "count": { + "type": "long" + } + } + } + } + } + } + } + } + } + } + }, + "service": { + "properties": { + "cluster_ip": { + "type": "ip" + }, + "created": { + "type": "date" + }, + "external_ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "external_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "ingress_hostname": { + "type": "ip" + }, + "ingress_ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "load_balancer_ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "statefulset": { + "properties": { + "created": { + "type": "long" + }, + "generation": { + "properties": { + "desired": { + "type": "long" + }, + "observed": { + "type": "long" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "replicas": { + "properties": { + "desired": { + "type": "long" + }, + "observed": { + "type": "long" + } + } + } + } + }, + "storageclass": { + "properties": { + "created": { + "type": "date" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "provisioner": { + "ignore_above": 1024, + "type": "keyword" + }, + "reclaim_policy": { + "ignore_above": 1024, + "type": "keyword" + }, + "volume_binding_mode": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "system": { + "properties": { + "container": { + "ignore_above": 1024, + "type": "keyword" + }, + "cpu": { + "properties": { + "usage": { + "properties": { + "core": { + "properties": { + "ns": { + "type": "long" + } + } + }, + "nanocores": { + "type": "long" + } + } + } + } + }, + "memory": { + "properties": { + "majorpagefaults": { + "type": "long" + }, + "pagefaults": { + "type": "long" + }, + "rss": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "workingset": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "start_time": { + "type": "date" + } + } + }, + "volume": { + "properties": { + "fs": { + "properties": { + "available": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "capacity": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "inodes": { + "properties": { + "count": { + "type": "long" + }, + "free": { + "type": "long" + }, + "used": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "kvm": { + "properties": { + "dommemstat": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "stat": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "value": { + "type": "long" + } + } + } + } + } + } + }, + "labels": { + "type": "object" + }, + "log": { + "properties": { + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "logger": { + "ignore_above": 1024, + "type": "keyword" + }, + "origin": { + "properties": { + "file": { + "properties": { + "line": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "function": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "original": { + "ignore_above": 1024, + "type": "keyword" + }, + "syslog": { + "properties": { + "facility": { + "properties": { + "code": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "priority": { + "type": "long" + }, + "severity": { + "properties": { + "code": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "logstash": { + "properties": { + "node": { + "properties": { + "jvm": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "stats": { + "properties": { + "events": { + "properties": { + "filtered": { + "type": "long" + }, + "in": { + "type": "long" + }, + "out": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "memcached": { + "properties": { + "stats": { + "properties": { + "bytes": { + "properties": { + "current": { + "type": "long" + }, + "limit": { + "type": "long" + } + } + }, + "cmd": { + "properties": { + "get": { + "type": "long" + }, + "set": { + "type": "long" + } + } + }, + "connections": { + "properties": { + "current": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "evictions": { + "type": "long" + }, + "get": { + "properties": { + "hits": { + "type": "long" + }, + "misses": { + "type": "long" + } + } + }, + "items": { + "properties": { + "current": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "pid": { + "type": "long" + }, + "read": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "threads": { + "type": "long" + }, + "uptime": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "written": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "message": { + "norms": false, + "type": "text" + }, + "metricset": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "period": { + "type": "long" + } + } + }, + "mongodb": { + "properties": { + "collstats": { + "properties": { + "collection": { + "ignore_above": 1024, + "type": "keyword" + }, + "commands": { + "properties": { + "count": { + "type": "long" + }, + "time": { + "properties": { + "us": { + "type": "long" + } + } + } + } + }, + "db": { + "ignore_above": 1024, + "type": "keyword" + }, + "getmore": { + "properties": { + "count": { + "type": "long" + }, + "time": { + "properties": { + "us": { + "type": "long" + } + } + } + } + }, + "insert": { + "properties": { + "count": { + "type": "long" + }, + "time": { + "properties": { + "us": { + "type": "long" + } + } + } + } + }, + "lock": { + "properties": { + "read": { + "properties": { + "count": { + "type": "long" + }, + "time": { + "properties": { + "us": { + "type": "long" + } + } + } + } + }, + "write": { + "properties": { + "count": { + "type": "long" + }, + "time": { + "properties": { + "us": { + "type": "long" + } + } + } + } + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "queries": { + "properties": { + "count": { + "type": "long" + }, + "time": { + "properties": { + "us": { + "type": "long" + } + } + } + } + }, + "remove": { + "properties": { + "count": { + "type": "long" + }, + "time": { + "properties": { + "us": { + "type": "long" + } + } + } + } + }, + "total": { + "properties": { + "count": { + "type": "long" + }, + "time": { + "properties": { + "us": { + "type": "long" + } + } + } + } + }, + "update": { + "properties": { + "count": { + "type": "long" + }, + "time": { + "properties": { + "us": { + "type": "long" + } + } + } + } + } + } + }, + "dbstats": { + "properties": { + "avg_obj_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "collections": { + "type": "long" + }, + "data_file_version": { + "properties": { + "major": { + "type": "long" + }, + "minor": { + "type": "long" + } + } + }, + "data_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "db": { + "ignore_above": 1024, + "type": "keyword" + }, + "extent_free_list": { + "properties": { + "num": { + "type": "long" + }, + "size": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "file_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "index_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "indexes": { + "type": "long" + }, + "ns_size_mb": { + "properties": { + "mb": { + "type": "long" + } + } + }, + "num_extents": { + "type": "long" + }, + "objects": { + "type": "long" + }, + "storage_size": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "metrics": { + "properties": { + "commands": { + "properties": { + "aggregate": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "build_info": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "coll_stats": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "connection_pool_stats": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "count": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "db_stats": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "distinct": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "find": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "get_cmd_line_opts": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "get_last_error": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "get_log": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "get_more": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "get_parameter": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "host_info": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "insert": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "is_master": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "is_self": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "last_collections": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "last_commands": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "list_databased": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "list_indexes": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "ping": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "profile": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "replset_get_rbid": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "replset_get_status": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "replset_heartbeat": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "replset_update_position": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "server_status": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "update": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "whatsmyuri": { + "properties": { + "failed": { + "type": "long" + }, + "total": { + "type": "long" + } + } + } + } + }, + "cursor": { + "properties": { + "open": { + "properties": { + "no_timeout": { + "type": "long" + }, + "pinned": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "timed_out": { + "type": "long" + } + } + }, + "document": { + "properties": { + "deleted": { + "type": "long" + }, + "inserted": { + "type": "long" + }, + "returned": { + "type": "long" + }, + "updated": { + "type": "long" + } + } + }, + "get_last_error": { + "properties": { + "write_timeouts": { + "type": "long" + }, + "write_wait": { + "properties": { + "count": { + "type": "long" + }, + "ms": { + "type": "long" + } + } + } + } + }, + "operation": { + "properties": { + "scan_and_order": { + "type": "long" + }, + "write_conflicts": { + "type": "long" + } + } + }, + "query_executor": { + "properties": { + "scanned_documents": { + "properties": { + "count": { + "type": "long" + } + } + }, + "scanned_indexes": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "replication": { + "properties": { + "apply": { + "properties": { + "attempts_to_become_secondary": { + "type": "long" + }, + "batches": { + "properties": { + "count": { + "type": "long" + }, + "time": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "ops": { + "type": "long" + } + } + }, + "buffer": { + "properties": { + "count": { + "type": "long" + }, + "max_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "size": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "executor": { + "properties": { + "counters": { + "properties": { + "cancels": { + "type": "long" + }, + "event_created": { + "type": "long" + }, + "event_wait": { + "type": "long" + }, + "scheduled": { + "properties": { + "dbwork": { + "type": "long" + }, + "exclusive": { + "type": "long" + }, + "failures": { + "type": "long" + }, + "netcmd": { + "type": "long" + }, + "work": { + "type": "long" + }, + "work_at": { + "type": "long" + } + } + }, + "waits": { + "type": "long" + } + } + }, + "event_waiters": { + "type": "long" + }, + "network_interface": { + "ignore_above": 1024, + "type": "keyword" + }, + "queues": { + "properties": { + "free": { + "type": "long" + }, + "in_progress": { + "properties": { + "dbwork": { + "type": "long" + }, + "exclusive": { + "type": "long" + }, + "network": { + "type": "long" + } + } + }, + "ready": { + "type": "long" + }, + "sleepers": { + "type": "long" + } + } + }, + "shutting_down": { + "type": "boolean" + }, + "unsignaled_events": { + "type": "long" + } + } + }, + "initial_sync": { + "properties": { + "completed": { + "type": "long" + }, + "failed_attempts": { + "type": "long" + }, + "failures": { + "type": "long" + } + } + }, + "network": { + "properties": { + "bytes": { + "type": "long" + }, + "getmores": { + "properties": { + "count": { + "type": "long" + }, + "time": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "ops": { + "type": "long" + }, + "reders_created": { + "type": "long" + } + } + }, + "preload": { + "properties": { + "docs": { + "properties": { + "count": { + "type": "long" + }, + "time": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "indexes": { + "properties": { + "count": { + "type": "long" + }, + "time": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "storage": { + "properties": { + "free_list": { + "properties": { + "search": { + "properties": { + "bucket_exhausted": { + "type": "long" + }, + "requests": { + "type": "long" + }, + "scanned": { + "type": "long" + } + } + } + } + } + } + }, + "ttl": { + "properties": { + "deleted_documents": { + "properties": { + "count": { + "type": "long" + } + } + }, + "passes": { + "properties": { + "count": { + "type": "long" + } + } + } + } + } + } + }, + "replstatus": { + "properties": { + "headroom": { + "properties": { + "max": { + "type": "long" + }, + "min": { + "type": "long" + } + } + }, + "lag": { + "properties": { + "max": { + "type": "long" + }, + "min": { + "type": "long" + } + } + }, + "members": { + "properties": { + "arbiter": { + "properties": { + "count": { + "type": "long" + }, + "hosts": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "down": { + "properties": { + "count": { + "type": "long" + }, + "hosts": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "primary": { + "properties": { + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "optime": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "recovering": { + "properties": { + "count": { + "type": "long" + }, + "hosts": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "rollback": { + "properties": { + "count": { + "type": "long" + }, + "hosts": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "secondary": { + "properties": { + "count": { + "type": "long" + }, + "hosts": { + "ignore_above": 1024, + "type": "keyword" + }, + "optimes": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "startup2": { + "properties": { + "count": { + "type": "long" + }, + "hosts": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "unhealthy": { + "properties": { + "count": { + "type": "long" + }, + "hosts": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "unknown": { + "properties": { + "count": { + "type": "long" + }, + "hosts": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "oplog": { + "properties": { + "first": { + "properties": { + "timestamp": { + "type": "long" + } + } + }, + "last": { + "properties": { + "timestamp": { + "type": "long" + } + } + }, + "size": { + "properties": { + "allocated": { + "type": "long" + }, + "used": { + "type": "long" + } + } + }, + "window": { + "type": "long" + } + } + }, + "optimes": { + "properties": { + "applied": { + "type": "long" + }, + "durable": { + "type": "long" + }, + "last_committed": { + "type": "long" + } + } + }, + "server_date": { + "type": "date" + }, + "set_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "status": { + "properties": { + "asserts": { + "properties": { + "msg": { + "type": "long" + }, + "regular": { + "type": "long" + }, + "rollovers": { + "type": "long" + }, + "user": { + "type": "long" + }, + "warning": { + "type": "long" + } + } + }, + "background_flushing": { + "properties": { + "average": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "flushes": { + "type": "long" + }, + "last": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "last_finished": { + "type": "date" + }, + "total": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "connections": { + "properties": { + "available": { + "type": "long" + }, + "current": { + "type": "long" + }, + "total_created": { + "type": "long" + } + } + }, + "extra_info": { + "properties": { + "heap_usage": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "page_faults": { + "type": "long" + } + } + }, + "global_lock": { + "properties": { + "active_clients": { + "properties": { + "readers": { + "type": "long" + }, + "total": { + "type": "long" + }, + "writers": { + "type": "long" + } + } + }, + "current_queue": { + "properties": { + "readers": { + "type": "long" + }, + "total": { + "type": "long" + }, + "writers": { + "type": "long" + } + } + }, + "total_time": { + "properties": { + "us": { + "type": "long" + } + } + } + } + }, + "journaling": { + "properties": { + "commits": { + "type": "long" + }, + "commits_in_write_lock": { + "type": "long" + }, + "compression": { + "type": "long" + }, + "early_commits": { + "type": "long" + }, + "journaled": { + "properties": { + "mb": { + "type": "long" + } + } + }, + "times": { + "properties": { + "commits": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "commits_in_write_lock": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "dt": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "prep_log_buffer": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "remap_private_view": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "write_to_data_files": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "write_to_journal": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "write_to_data_files": { + "properties": { + "mb": { + "type": "long" + } + } + } + } + }, + "local_time": { + "type": "date" + }, + "locks": { + "properties": { + "collection": { + "properties": { + "acquire": { + "properties": { + "count": { + "properties": { + "R": { + "type": "long" + }, + "W": { + "type": "long" + }, + "r": { + "type": "long" + }, + "w": { + "type": "long" + } + } + } + } + }, + "deadlock": { + "properties": { + "count": { + "properties": { + "R": { + "type": "long" + }, + "W": { + "type": "long" + }, + "r": { + "type": "long" + }, + "w": { + "type": "long" + } + } + } + } + }, + "wait": { + "properties": { + "count": { + "properties": { + "R": { + "type": "long" + }, + "W": { + "type": "long" + }, + "r": { + "type": "long" + }, + "w": { + "type": "long" + } + } + }, + "us": { + "properties": { + "R": { + "type": "long" + }, + "W": { + "type": "long" + }, + "r": { + "type": "long" + }, + "w": { + "type": "long" + } + } + } + } + } + } + }, + "database": { + "properties": { + "acquire": { + "properties": { + "count": { + "properties": { + "R": { + "type": "long" + }, + "W": { + "type": "long" + }, + "r": { + "type": "long" + }, + "w": { + "type": "long" + } + } + } + } + }, + "deadlock": { + "properties": { + "count": { + "properties": { + "R": { + "type": "long" + }, + "W": { + "type": "long" + }, + "r": { + "type": "long" + }, + "w": { + "type": "long" + } + } + } + } + }, + "wait": { + "properties": { + "count": { + "properties": { + "R": { + "type": "long" + }, + "W": { + "type": "long" + }, + "r": { + "type": "long" + }, + "w": { + "type": "long" + } + } + }, + "us": { + "properties": { + "R": { + "type": "long" + }, + "W": { + "type": "long" + }, + "r": { + "type": "long" + }, + "w": { + "type": "long" + } + } + } + } + } + } + }, + "global": { + "properties": { + "acquire": { + "properties": { + "count": { + "properties": { + "R": { + "type": "long" + }, + "W": { + "type": "long" + }, + "r": { + "type": "long" + }, + "w": { + "type": "long" + } + } + } + } + }, + "deadlock": { + "properties": { + "count": { + "properties": { + "R": { + "type": "long" + }, + "W": { + "type": "long" + }, + "r": { + "type": "long" + }, + "w": { + "type": "long" + } + } + } + } + }, + "wait": { + "properties": { + "count": { + "properties": { + "R": { + "type": "long" + }, + "W": { + "type": "long" + }, + "r": { + "type": "long" + }, + "w": { + "type": "long" + } + } + }, + "us": { + "properties": { + "R": { + "type": "long" + }, + "W": { + "type": "long" + }, + "r": { + "type": "long" + }, + "w": { + "type": "long" + } + } + } + } + } + } + }, + "meta_data": { + "properties": { + "acquire": { + "properties": { + "count": { + "properties": { + "R": { + "type": "long" + }, + "W": { + "type": "long" + }, + "r": { + "type": "long" + }, + "w": { + "type": "long" + } + } + } + } + }, + "deadlock": { + "properties": { + "count": { + "properties": { + "R": { + "type": "long" + }, + "W": { + "type": "long" + }, + "r": { + "type": "long" + }, + "w": { + "type": "long" + } + } + } + } + }, + "wait": { + "properties": { + "count": { + "properties": { + "R": { + "type": "long" + }, + "W": { + "type": "long" + }, + "r": { + "type": "long" + }, + "w": { + "type": "long" + } + } + }, + "us": { + "properties": { + "R": { + "type": "long" + }, + "W": { + "type": "long" + }, + "r": { + "type": "long" + }, + "w": { + "type": "long" + } + } + } + } + } + } + }, + "oplog": { + "properties": { + "acquire": { + "properties": { + "count": { + "properties": { + "R": { + "type": "long" + }, + "W": { + "type": "long" + }, + "r": { + "type": "long" + }, + "w": { + "type": "long" + } + } + } + } + }, + "deadlock": { + "properties": { + "count": { + "properties": { + "R": { + "type": "long" + }, + "W": { + "type": "long" + }, + "r": { + "type": "long" + }, + "w": { + "type": "long" + } + } + } + } + }, + "wait": { + "properties": { + "count": { + "properties": { + "R": { + "type": "long" + }, + "W": { + "type": "long" + }, + "r": { + "type": "long" + }, + "w": { + "type": "long" + } + } + }, + "us": { + "properties": { + "R": { + "type": "long" + }, + "W": { + "type": "long" + }, + "r": { + "type": "long" + }, + "w": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "memory": { + "properties": { + "bits": { + "type": "long" + }, + "mapped": { + "properties": { + "mb": { + "type": "long" + } + } + }, + "mapped_with_journal": { + "properties": { + "mb": { + "type": "long" + } + } + }, + "resident": { + "properties": { + "mb": { + "type": "long" + } + } + }, + "virtual": { + "properties": { + "mb": { + "type": "long" + } + } + } + } + }, + "network": { + "properties": { + "in": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "out": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "requests": { + "type": "long" + } + } + }, + "ops": { + "properties": { + "counters": { + "properties": { + "command": { + "type": "long" + }, + "delete": { + "type": "long" + }, + "getmore": { + "type": "long" + }, + "insert": { + "type": "long" + }, + "query": { + "type": "long" + }, + "update": { + "type": "long" + } + } + }, + "latencies": { + "properties": { + "commands": { + "properties": { + "count": { + "type": "long" + }, + "latency": { + "type": "long" + } + } + }, + "reads": { + "properties": { + "count": { + "type": "long" + }, + "latency": { + "type": "long" + } + } + }, + "writes": { + "properties": { + "count": { + "type": "long" + }, + "latency": { + "type": "long" + } + } + } + } + }, + "replicated": { + "properties": { + "command": { + "type": "long" + }, + "delete": { + "type": "long" + }, + "getmore": { + "type": "long" + }, + "insert": { + "type": "long" + }, + "query": { + "type": "long" + }, + "update": { + "type": "long" + } + } + } + } + }, + "process": { + "path": "process.name", + "type": "alias" + }, + "storage_engine": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "uptime": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "version": { + "path": "service.version", + "type": "alias" + }, + "wired_tiger": { + "properties": { + "cache": { + "properties": { + "dirty": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "maximum": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "pages": { + "properties": { + "evicted": { + "type": "long" + }, + "read": { + "type": "long" + }, + "write": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "concurrent_transactions": { + "properties": { + "read": { + "properties": { + "available": { + "type": "long" + }, + "out": { + "type": "long" + }, + "total_tickets": { + "type": "long" + } + } + }, + "write": { + "properties": { + "available": { + "type": "long" + }, + "out": { + "type": "long" + }, + "total_tickets": { + "type": "long" + } + } + } + } + }, + "log": { + "properties": { + "flushes": { + "type": "long" + }, + "max_file_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "scans": { + "type": "long" + }, + "size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "syncs": { + "type": "long" + }, + "write": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "writes": { + "type": "long" + } + } + } + } + }, + "write_backs_queued": { + "type": "boolean" + } + } + } + } + }, + "mssql": { + "properties": { + "database": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "performance": { + "properties": { + "active_temp_tables": { + "type": "long" + }, + "batch_requests_per_sec": { + "type": "long" + }, + "buffer": { + "properties": { + "cache_hit": { + "properties": { + "pct": { + "type": "double" + } + } + }, + "checkpoint_pages_per_sec": { + "type": "long" + }, + "database_pages": { + "type": "long" + }, + "page_life_expectancy": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "target_pages": { + "type": "long" + } + } + }, + "compilations_per_sec": { + "type": "long" + }, + "connections_reset_per_sec": { + "type": "long" + }, + "lock_waits_per_sec": { + "type": "long" + }, + "logins_per_sec": { + "type": "long" + }, + "logouts_per_sec": { + "type": "long" + }, + "page_splits_per_sec": { + "type": "long" + }, + "recompilations_per_sec": { + "type": "long" + }, + "transactions": { + "type": "long" + }, + "user_connections": { + "type": "long" + } + } + }, + "transaction_log": { + "properties": { + "space_usage": { + "properties": { + "since_last_backup": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "total": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "type": "float" + } + } + } + } + }, + "stats": { + "properties": { + "active_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "backup_time": { + "type": "date" + }, + "recovery_size": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "since_last_checkpoint": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "total_size": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "munin": { + "properties": { + "metrics": { + "properties": { + "*": { + "type": "object" + } + } + }, + "plugin": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "mysql": { + "properties": { + "galera_status": { + "properties": { + "apply": { + "properties": { + "oooe": { + "type": "double" + }, + "oool": { + "type": "double" + }, + "window": { + "type": "double" + } + } + }, + "cert": { + "properties": { + "deps_distance": { + "type": "double" + }, + "index_size": { + "type": "long" + }, + "interval": { + "type": "double" + } + } + }, + "cluster": { + "properties": { + "conf_id": { + "type": "long" + }, + "size": { + "type": "long" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "commit": { + "properties": { + "oooe": { + "type": "double" + }, + "window": { + "type": "long" + } + } + }, + "connected": { + "ignore_above": 1024, + "type": "keyword" + }, + "evs": { + "properties": { + "evict": { + "ignore_above": 1024, + "type": "keyword" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "flow_ctl": { + "properties": { + "paused": { + "type": "double" + }, + "paused_ns": { + "type": "long" + }, + "recv": { + "type": "long" + }, + "sent": { + "type": "long" + } + } + }, + "last_committed": { + "type": "long" + }, + "local": { + "properties": { + "bf_aborts": { + "type": "long" + }, + "cert_failures": { + "type": "long" + }, + "commits": { + "type": "long" + }, + "recv": { + "properties": { + "queue": { + "type": "long" + }, + "queue_avg": { + "type": "double" + }, + "queue_max": { + "type": "long" + }, + "queue_min": { + "type": "long" + } + } + }, + "replays": { + "type": "long" + }, + "send": { + "properties": { + "queue": { + "type": "long" + }, + "queue_avg": { + "type": "double" + }, + "queue_max": { + "type": "long" + }, + "queue_min": { + "type": "long" + } + } + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ready": { + "ignore_above": 1024, + "type": "keyword" + }, + "received": { + "properties": { + "bytes": { + "type": "long" + }, + "count": { + "type": "long" + } + } + }, + "repl": { + "properties": { + "bytes": { + "type": "long" + }, + "count": { + "type": "long" + }, + "data_bytes": { + "type": "long" + }, + "keys": { + "type": "long" + }, + "keys_bytes": { + "type": "long" + }, + "other_bytes": { + "type": "long" + } + } + } + } + }, + "status": { + "properties": { + "aborted": { + "properties": { + "clients": { + "type": "long" + }, + "connects": { + "type": "long" + } + } + }, + "binlog": { + "properties": { + "cache": { + "properties": { + "disk_use": { + "type": "long" + }, + "use": { + "type": "long" + } + } + } + } + }, + "bytes": { + "properties": { + "received": { + "type": "long" + }, + "sent": { + "type": "long" + } + } + }, + "cache": { + "properties": { + "ssl": { + "properties": { + "hits": { + "type": "long" + }, + "misses": { + "type": "long" + }, + "size": { + "type": "long" + } + } + }, + "table": { + "properties": { + "open_cache": { + "properties": { + "hits": { + "type": "long" + }, + "misses": { + "type": "long" + }, + "overflows": { + "type": "long" + } + } + } + } + } + } + }, + "command": { + "properties": { + "delete": { + "type": "long" + }, + "insert": { + "type": "long" + }, + "select": { + "type": "long" + }, + "update": { + "type": "long" + } + } + }, + "connection": { + "properties": { + "errors": { + "properties": { + "accept": { + "type": "long" + }, + "internal": { + "type": "long" + }, + "max": { + "type": "long" + }, + "peer_address": { + "type": "long" + }, + "select": { + "type": "long" + }, + "tcpwrap": { + "type": "long" + } + } + } + } + }, + "connections": { + "type": "long" + }, + "created": { + "properties": { + "tmp": { + "properties": { + "disk_tables": { + "type": "long" + }, + "files": { + "type": "long" + }, + "tables": { + "type": "long" + } + } + } + } + }, + "delayed": { + "properties": { + "errors": { + "type": "long" + }, + "insert_threads": { + "type": "long" + }, + "writes": { + "type": "long" + } + } + }, + "flush_commands": { + "type": "long" + }, + "handler": { + "properties": { + "commit": { + "type": "long" + }, + "delete": { + "type": "long" + }, + "external_lock": { + "type": "long" + }, + "mrr_init": { + "type": "long" + }, + "prepare": { + "type": "long" + }, + "read": { + "properties": { + "first": { + "type": "long" + }, + "key": { + "type": "long" + }, + "last": { + "type": "long" + }, + "next": { + "type": "long" + }, + "prev": { + "type": "long" + }, + "rnd": { + "type": "long" + }, + "rnd_next": { + "type": "long" + } + } + }, + "rollback": { + "type": "long" + }, + "savepoint": { + "type": "long" + }, + "savepoint_rollback": { + "type": "long" + }, + "update": { + "type": "long" + }, + "write": { + "type": "long" + } + } + }, + "innodb": { + "properties": { + "buffer_pool": { + "properties": { + "bytes": { + "properties": { + "data": { + "type": "long" + }, + "dirty": { + "type": "long" + } + } + }, + "dump_status": { + "type": "long" + }, + "load_status": { + "type": "long" + }, + "pages": { + "properties": { + "data": { + "type": "long" + }, + "dirty": { + "type": "long" + }, + "flushed": { + "type": "long" + }, + "free": { + "type": "long" + }, + "latched": { + "type": "long" + }, + "misc": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "pool": { + "properties": { + "reads": { + "type": "long" + }, + "resize_status": { + "type": "long" + }, + "wait_free": { + "type": "long" + } + } + }, + "read": { + "properties": { + "ahead": { + "type": "long" + }, + "ahead_evicted": { + "type": "long" + }, + "ahead_rnd": { + "type": "long" + }, + "requests": { + "type": "long" + } + } + }, + "write_requests": { + "type": "long" + } + } + }, + "rows": { + "properties": { + "deleted": { + "type": "long" + }, + "inserted": { + "type": "long" + }, + "reads": { + "type": "long" + }, + "updated": { + "type": "long" + } + } + } + } + }, + "max_used_connections": { + "type": "long" + }, + "open": { + "properties": { + "files": { + "type": "long" + }, + "streams": { + "type": "long" + }, + "tables": { + "type": "long" + } + } + }, + "opened_tables": { + "type": "long" + }, + "queries": { + "type": "long" + }, + "questions": { + "type": "long" + }, + "threads": { + "properties": { + "cached": { + "type": "long" + }, + "connected": { + "type": "long" + }, + "created": { + "type": "long" + }, + "running": { + "type": "long" + } + } + } + } + } + } + }, + "nats": { + "properties": { + "connections": { + "properties": { + "total": { + "type": "long" + } + } + }, + "routes": { + "properties": { + "total": { + "type": "long" + } + } + }, + "server": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "time": { + "type": "date" + } + } + }, + "stats": { + "properties": { + "cores": { + "type": "long" + }, + "cpu": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "http": { + "properties": { + "req_stats": { + "properties": { + "uri": { + "properties": { + "connz": { + "type": "long" + }, + "root": { + "type": "long" + }, + "routez": { + "type": "long" + }, + "subsz": { + "type": "long" + }, + "varz": { + "type": "long" + } + } + } + } + } + } + }, + "in": { + "properties": { + "bytes": { + "type": "long" + }, + "messages": { + "type": "long" + } + } + }, + "mem": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "out": { + "properties": { + "bytes": { + "type": "long" + }, + "messages": { + "type": "long" + } + } + }, + "remotes": { + "type": "long" + }, + "slow_consumers": { + "type": "long" + }, + "total_connections": { + "type": "long" + }, + "uptime": { + "type": "long" + } + } + }, + "subscriptions": { + "properties": { + "cache": { + "properties": { + "fanout": { + "properties": { + "avg": { + "type": "double" + }, + "max": { + "type": "long" + } + } + }, + "hit_rate": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "size": { + "type": "long" + } + } + }, + "inserts": { + "type": "long" + }, + "matches": { + "type": "long" + }, + "removes": { + "type": "long" + }, + "total": { + "type": "long" + } + } + } + } + }, + "network": { + "properties": { + "application": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "community_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "direction": { + "ignore_above": 1024, + "type": "keyword" + }, + "forwarded_ip": { + "type": "ip" + }, + "iana_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "inner": { + "properties": { + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "transport": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "nginx": { + "properties": { + "stubstatus": { + "properties": { + "accepts": { + "type": "long" + }, + "active": { + "type": "long" + }, + "current": { + "type": "long" + }, + "dropped": { + "type": "long" + }, + "handled": { + "type": "long" + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "reading": { + "type": "long" + }, + "requests": { + "type": "long" + }, + "waiting": { + "type": "long" + }, + "writing": { + "type": "long" + } + } + } + } + }, + "observer": { + "properties": { + "egress": { + "properties": { + "interface": { + "properties": { + "alias": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "zone": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "ingress": { + "properties": { + "interface": { + "properties": { + "alias": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "zone": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "vendor": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "openmetrics": { + "properties": { + "labels": { + "properties": { + "*": { + "type": "object" + } + } + }, + "metrics": { + "properties": { + "*": { + "type": "object" + } + } + } + } + }, + "oracle": { + "properties": { + "performance": { + "properties": { + "buffer_pool": { + "ignore_above": 1024, + "type": "keyword" + }, + "cache": { + "properties": { + "buffer": { + "properties": { + "hit": { + "properties": { + "pct": { + "type": "double" + } + } + } + } + }, + "get": { + "properties": { + "consistent": { + "type": "long" + }, + "db_blocks": { + "type": "long" + } + } + }, + "physical_reads": { + "type": "long" + } + } + }, + "cursors": { + "properties": { + "avg": { + "type": "double" + }, + "cache_hit": { + "properties": { + "pct": { + "type": "double" + } + } + }, + "max": { + "type": "double" + }, + "opened": { + "properties": { + "current": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "parse": { + "properties": { + "real": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "session": { + "properties": { + "cache_hits": { + "type": "long" + } + } + }, + "total": { + "type": "double" + } + } + }, + "io_reloads": { + "type": "double" + }, + "lock_requests": { + "type": "long" + }, + "machine": { + "ignore_above": 1024, + "type": "keyword" + }, + "pin_requests": { + "type": "double" + }, + "username": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "tablespace": { + "properties": { + "data_file": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "online_status": { + "ignore_above": 1024, + "type": "keyword" + }, + "size": { + "properties": { + "bytes": { + "type": "long" + }, + "free": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "space": { + "properties": { + "free": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "total": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "organization": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "package": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "build_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "checksum": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "install_scope": { + "ignore_above": 1024, + "type": "keyword" + }, + "installed": { + "type": "date" + }, + "license": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "size": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pe": { + "properties": { + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "php_fpm": { + "properties": { + "pool": { + "properties": { + "connections": { + "properties": { + "accepted": { + "type": "long" + }, + "listen_queue_len": { + "type": "long" + }, + "max_listen_queue": { + "type": "long" + }, + "queued": { + "type": "long" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "process_manager": { + "ignore_above": 1024, + "type": "keyword" + }, + "processes": { + "properties": { + "active": { + "type": "long" + }, + "idle": { + "type": "long" + }, + "max_active": { + "type": "long" + }, + "max_children_reached": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "slow_requests": { + "type": "long" + }, + "start_since": { + "type": "long" + }, + "start_time": { + "type": "date" + } + } + }, + "process": { + "properties": { + "last_request_cpu": { + "type": "long" + }, + "last_request_memory": { + "type": "long" + }, + "request_duration": { + "type": "long" + }, + "requests": { + "type": "long" + }, + "script": { + "ignore_above": 1024, + "type": "keyword" + }, + "start_since": { + "type": "long" + }, + "start_time": { + "type": "date" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "postgresql": { + "properties": { + "activity": { + "properties": { + "application_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "backend_start": { + "type": "date" + }, + "client": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + } + } + }, + "database": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "oid": { + "type": "long" + } + } + }, + "pid": { + "type": "long" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "query_start": { + "type": "date" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_change": { + "type": "date" + }, + "transaction_start": { + "type": "date" + }, + "user": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "waiting": { + "type": "boolean" + } + } + }, + "bgwriter": { + "properties": { + "buffers": { + "properties": { + "allocated": { + "type": "long" + }, + "backend": { + "type": "long" + }, + "backend_fsync": { + "type": "long" + }, + "checkpoints": { + "type": "long" + }, + "clean": { + "type": "long" + }, + "clean_full": { + "type": "long" + } + } + }, + "checkpoints": { + "properties": { + "requested": { + "type": "long" + }, + "scheduled": { + "type": "long" + }, + "times": { + "properties": { + "sync": { + "properties": { + "ms": { + "type": "float" + } + } + }, + "write": { + "properties": { + "ms": { + "type": "float" + } + } + } + } + } + } + }, + "stats_reset": { + "type": "date" + } + } + }, + "database": { + "properties": { + "blocks": { + "properties": { + "hit": { + "type": "long" + }, + "read": { + "type": "long" + }, + "time": { + "properties": { + "read": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "write": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + } + } + }, + "conflicts": { + "type": "long" + }, + "deadlocks": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "number_of_backends": { + "type": "long" + }, + "oid": { + "type": "long" + }, + "rows": { + "properties": { + "deleted": { + "type": "long" + }, + "fetched": { + "type": "long" + }, + "inserted": { + "type": "long" + }, + "returned": { + "type": "long" + }, + "updated": { + "type": "long" + } + } + }, + "stats_reset": { + "type": "date" + }, + "temporary": { + "properties": { + "bytes": { + "type": "long" + }, + "files": { + "type": "long" + } + } + }, + "transactions": { + "properties": { + "commit": { + "type": "long" + }, + "rollback": { + "type": "long" + } + } + } + } + }, + "statement": { + "properties": { + "database": { + "properties": { + "oid": { + "type": "long" + } + } + }, + "query": { + "properties": { + "calls": { + "type": "long" + }, + "id": { + "type": "long" + }, + "memory": { + "properties": { + "local": { + "properties": { + "dirtied": { + "type": "long" + }, + "hit": { + "type": "long" + }, + "read": { + "type": "long" + }, + "written": { + "type": "long" + } + } + }, + "shared": { + "properties": { + "dirtied": { + "type": "long" + }, + "hit": { + "type": "long" + }, + "read": { + "type": "long" + }, + "written": { + "type": "long" + } + } + }, + "temp": { + "properties": { + "read": { + "type": "long" + }, + "written": { + "type": "long" + } + } + } + } + }, + "rows": { + "type": "long" + }, + "text": { + "ignore_above": 1024, + "type": "keyword" + }, + "time": { + "properties": { + "max": { + "properties": { + "ms": { + "type": "float" + } + } + }, + "mean": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "min": { + "properties": { + "ms": { + "type": "float" + } + } + }, + "stddev": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "total": { + "properties": { + "ms": { + "type": "float" + } + } + } + } + } + } + }, + "user": { + "properties": { + "id": { + "type": "long" + } + } + } + } + } + } + }, + "process": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "args_count": { + "type": "long" + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "command_line": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "entity_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "executable": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "exit_code": { + "type": "long" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "parent": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "args_count": { + "type": "long" + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "command_line": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "entity_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "executable": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "exit_code": { + "type": "long" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "pgid": { + "type": "long" + }, + "pid": { + "type": "long" + }, + "ppid": { + "type": "long" + }, + "start": { + "type": "date" + }, + "thread": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "title": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + }, + "working_directory": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pe": { + "properties": { + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pgid": { + "type": "long" + }, + "pid": { + "type": "long" + }, + "ppid": { + "type": "long" + }, + "start": { + "type": "date" + }, + "thread": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "title": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + }, + "working_directory": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "prometheus": { + "properties": { + "*": { + "properties": { + "counter": { + "type": "object" + }, + "histogram": { + "type": "object" + }, + "rate": { + "type": "object" + }, + "value": { + "type": "object" + } + } + }, + "labels": { + "properties": { + "*": { + "type": "object" + } + } + }, + "metrics": { + "properties": { + "*": { + "type": "object" + } + } + }, + "query": { + "properties": { + "*": { + "type": "object" + } + } + }, + "remote_write": { + "type": "object" + } + } + }, + "rabbitmq": { + "properties": { + "connection": { + "properties": { + "channel_max": { + "type": "long" + }, + "channels": { + "type": "long" + }, + "client_provided": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "frame_max": { + "type": "long" + }, + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "octet_count": { + "properties": { + "received": { + "type": "long" + }, + "sent": { + "type": "long" + } + } + }, + "packet_count": { + "properties": { + "pending": { + "type": "long" + }, + "received": { + "type": "long" + }, + "sent": { + "type": "long" + } + } + }, + "peer": { + "properties": { + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + } + } + }, + "port": { + "type": "long" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "exchange": { + "properties": { + "auto_delete": { + "type": "boolean" + }, + "durable": { + "type": "boolean" + }, + "internal": { + "type": "boolean" + }, + "messages": { + "properties": { + "publish_in": { + "properties": { + "count": { + "type": "long" + }, + "details": { + "properties": { + "rate": { + "type": "float" + } + } + } + } + }, + "publish_out": { + "properties": { + "count": { + "type": "long" + }, + "details": { + "properties": { + "rate": { + "type": "float" + } + } + } + } + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "node": { + "properties": { + "disk": { + "properties": { + "free": { + "properties": { + "bytes": { + "type": "long" + }, + "limit": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "fd": { + "properties": { + "total": { + "type": "long" + }, + "used": { + "type": "long" + } + } + }, + "gc": { + "properties": { + "num": { + "properties": { + "count": { + "type": "long" + } + } + }, + "reclaimed": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "io": { + "properties": { + "file_handle": { + "properties": { + "open_attempt": { + "properties": { + "avg": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "count": { + "type": "long" + } + } + } + } + }, + "read": { + "properties": { + "avg": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "bytes": { + "type": "long" + }, + "count": { + "type": "long" + } + } + }, + "reopen": { + "properties": { + "count": { + "type": "long" + } + } + }, + "seek": { + "properties": { + "avg": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "count": { + "type": "long" + } + } + }, + "sync": { + "properties": { + "avg": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "count": { + "type": "long" + } + } + }, + "write": { + "properties": { + "avg": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "bytes": { + "type": "long" + }, + "count": { + "type": "long" + } + } + } + } + }, + "mem": { + "properties": { + "limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "mnesia": { + "properties": { + "disk": { + "properties": { + "tx": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "ram": { + "properties": { + "tx": { + "properties": { + "count": { + "type": "long" + } + } + } + } + } + } + }, + "msg": { + "properties": { + "store_read": { + "properties": { + "count": { + "type": "long" + } + } + }, + "store_write": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "proc": { + "properties": { + "total": { + "type": "long" + }, + "used": { + "type": "long" + } + } + }, + "processors": { + "type": "long" + }, + "queue": { + "properties": { + "index": { + "properties": { + "journal_write": { + "properties": { + "count": { + "type": "long" + } + } + }, + "read": { + "properties": { + "count": { + "type": "long" + } + } + }, + "write": { + "properties": { + "count": { + "type": "long" + } + } + } + } + } + } + }, + "run": { + "properties": { + "queue": { + "type": "long" + } + } + }, + "socket": { + "properties": { + "total": { + "type": "long" + }, + "used": { + "type": "long" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + } + } + }, + "queue": { + "properties": { + "arguments": { + "properties": { + "max_priority": { + "type": "long" + } + } + }, + "auto_delete": { + "type": "boolean" + }, + "consumers": { + "properties": { + "count": { + "type": "long" + }, + "utilisation": { + "properties": { + "pct": { + "type": "long" + } + } + } + } + }, + "disk": { + "properties": { + "reads": { + "properties": { + "count": { + "type": "long" + } + } + }, + "writes": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "durable": { + "type": "boolean" + }, + "exclusive": { + "type": "boolean" + }, + "memory": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "messages": { + "properties": { + "persistent": { + "properties": { + "count": { + "type": "long" + } + } + }, + "ready": { + "properties": { + "count": { + "type": "long" + }, + "details": { + "properties": { + "rate": { + "type": "float" + } + } + } + } + }, + "total": { + "properties": { + "count": { + "type": "long" + }, + "details": { + "properties": { + "rate": { + "type": "float" + } + } + } + } + }, + "unacknowledged": { + "properties": { + "count": { + "type": "long" + }, + "details": { + "properties": { + "rate": { + "type": "float" + } + } + } + } + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vhost": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "redis": { + "properties": { + "info": { + "properties": { + "clients": { + "properties": { + "biggest_input_buf": { + "type": "long" + }, + "blocked": { + "type": "long" + }, + "connected": { + "type": "long" + }, + "longest_output_list": { + "type": "long" + }, + "max_input_buffer": { + "type": "long" + }, + "max_output_buffer": { + "type": "long" + } + } + }, + "cluster": { + "properties": { + "enabled": { + "type": "boolean" + } + } + }, + "cpu": { + "properties": { + "used": { + "properties": { + "sys": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "sys_children": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "user": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "user_children": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + }, + "memory": { + "properties": { + "active_defrag": { + "properties": { + "is_running": { + "type": "boolean" + } + } + }, + "allocator": { + "ignore_above": 1024, + "type": "keyword" + }, + "allocator_stats": { + "properties": { + "active": { + "type": "long" + }, + "allocated": { + "type": "long" + }, + "fragmentation": { + "properties": { + "bytes": { + "type": "long" + }, + "ratio": { + "type": "float" + } + } + }, + "resident": { + "type": "long" + }, + "rss": { + "properties": { + "bytes": { + "type": "long" + }, + "ratio": { + "type": "float" + } + } + } + } + }, + "fragmentation": { + "properties": { + "bytes": { + "type": "long" + }, + "ratio": { + "type": "float" + } + } + }, + "max": { + "properties": { + "policy": { + "ignore_above": 1024, + "type": "keyword" + }, + "value": { + "type": "long" + } + } + }, + "used": { + "properties": { + "dataset": { + "type": "long" + }, + "lua": { + "type": "long" + }, + "peak": { + "type": "long" + }, + "rss": { + "type": "long" + }, + "value": { + "type": "long" + } + } + } + } + }, + "persistence": { + "properties": { + "aof": { + "properties": { + "bgrewrite": { + "properties": { + "last_status": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "buffer": { + "properties": { + "size": { + "type": "long" + } + } + }, + "copy_on_write": { + "properties": { + "last_size": { + "type": "long" + } + } + }, + "enabled": { + "type": "boolean" + }, + "fsync": { + "properties": { + "delayed": { + "type": "long" + }, + "pending": { + "type": "long" + } + } + }, + "rewrite": { + "properties": { + "buffer": { + "properties": { + "size": { + "type": "long" + } + } + }, + "current_time": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "in_progress": { + "type": "boolean" + }, + "last_time": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "scheduled": { + "type": "boolean" + } + } + }, + "size": { + "properties": { + "base": { + "type": "long" + }, + "current": { + "type": "long" + } + } + }, + "write": { + "properties": { + "last_status": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "loading": { + "type": "boolean" + }, + "rdb": { + "properties": { + "bgsave": { + "properties": { + "current_time": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "in_progress": { + "type": "boolean" + }, + "last_status": { + "ignore_above": 1024, + "type": "keyword" + }, + "last_time": { + "properties": { + "sec": { + "type": "long" + } + } + } + } + }, + "copy_on_write": { + "properties": { + "last_size": { + "type": "long" + } + } + }, + "last_save": { + "properties": { + "changes_since": { + "type": "long" + }, + "time": { + "type": "long" + } + } + } + } + } + } + }, + "replication": { + "properties": { + "backlog": { + "properties": { + "active": { + "type": "long" + }, + "first_byte_offset": { + "type": "long" + }, + "histlen": { + "type": "long" + }, + "size": { + "type": "long" + } + } + }, + "connected_slaves": { + "type": "long" + }, + "master": { + "properties": { + "last_io_seconds_ago": { + "type": "long" + }, + "link_status": { + "ignore_above": 1024, + "type": "keyword" + }, + "offset": { + "type": "long" + }, + "second_offset": { + "type": "long" + }, + "sync": { + "properties": { + "in_progress": { + "type": "boolean" + }, + "last_io_seconds_ago": { + "type": "long" + }, + "left_bytes": { + "type": "long" + } + } + } + } + }, + "master_offset": { + "type": "long" + }, + "role": { + "ignore_above": 1024, + "type": "keyword" + }, + "slave": { + "properties": { + "is_readonly": { + "type": "boolean" + }, + "offset": { + "type": "long" + }, + "priority": { + "type": "long" + } + } + } + } + }, + "server": { + "properties": { + "arch_bits": { + "ignore_above": 1024, + "type": "keyword" + }, + "build_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "config_file": { + "ignore_above": 1024, + "type": "keyword" + }, + "gcc_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "git_dirty": { + "ignore_above": 1024, + "type": "keyword" + }, + "git_sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "hz": { + "type": "long" + }, + "lru_clock": { + "type": "long" + }, + "mode": { + "ignore_above": 1024, + "type": "keyword" + }, + "multiplexing_api": { + "ignore_above": 1024, + "type": "keyword" + }, + "run_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "tcp_port": { + "type": "long" + }, + "uptime": { + "type": "long" + } + } + }, + "slowlog": { + "properties": { + "count": { + "type": "long" + } + } + }, + "stats": { + "properties": { + "active_defrag": { + "properties": { + "hits": { + "type": "long" + }, + "key_hits": { + "type": "long" + }, + "key_misses": { + "type": "long" + }, + "misses": { + "type": "long" + } + } + }, + "commands_processed": { + "type": "long" + }, + "connections": { + "properties": { + "received": { + "type": "long" + }, + "rejected": { + "type": "long" + } + } + }, + "instantaneous": { + "properties": { + "input_kbps": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ops_per_sec": { + "type": "long" + }, + "output_kbps": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "keys": { + "properties": { + "evicted": { + "type": "long" + }, + "expired": { + "type": "long" + } + } + }, + "keyspace": { + "properties": { + "hits": { + "type": "long" + }, + "misses": { + "type": "long" + } + } + }, + "latest_fork_usec": { + "type": "long" + }, + "migrate_cached_sockets": { + "type": "long" + }, + "net": { + "properties": { + "input": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "output": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "pubsub": { + "properties": { + "channels": { + "type": "long" + }, + "patterns": { + "type": "long" + } + } + }, + "slave_expires_tracked_keys": { + "type": "long" + }, + "sync": { + "properties": { + "full": { + "type": "long" + }, + "partial": { + "properties": { + "err": { + "type": "long" + }, + "ok": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "key": { + "properties": { + "expire": { + "properties": { + "ttl": { + "type": "long" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "length": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "keyspace": { + "properties": { + "avg_ttl": { + "type": "long" + }, + "expires": { + "type": "long" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "keys": { + "type": "long" + } + } + } + } + }, + "redisenterprise": { + "properties": { + "node": { + "type": "object" + }, + "proxy": { + "type": "object" + } + } + }, + "registry": { + "properties": { + "data": { + "properties": { + "bytes": { + "ignore_above": 1024, + "type": "keyword" + }, + "strings": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hive": { + "ignore_above": 1024, + "type": "keyword" + }, + "key": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "value": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "related": { + "properties": { + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "rule": { + "properties": { + "author": { + "ignore_above": 1024, + "type": "keyword" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "license": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "ruleset": { + "ignore_above": 1024, + "type": "keyword" + }, + "uuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "server": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "service": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "ephemeral_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "node": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "source": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "sql": { + "properties": { + "driver": { + "ignore_above": 1024, + "type": "keyword" + }, + "metrics": { + "properties": { + "numeric": { + "properties": { + "*": { + "type": "object" + } + } + }, + "string": { + "properties": { + "*": { + "type": "object" + } + } + } + } + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "stan": { + "properties": { + "channels": { + "properties": { + "bytes": { + "type": "long" + }, + "depth": { + "type": "long" + }, + "first_seq": { + "type": "long" + }, + "last_seq": { + "type": "long" + }, + "messages": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "cluster": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "server": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "stats": { + "properties": { + "bytes": { + "type": "long" + }, + "channels": { + "type": "long" + }, + "clients": { + "type": "long" + }, + "messages": { + "type": "long" + }, + "role": { + "ignore_above": 1024, + "type": "keyword" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "subscriptions": { + "type": "long" + } + } + }, + "subscriptions": { + "properties": { + "channel": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "last_sent": { + "type": "long" + }, + "offline": { + "type": "boolean" + }, + "pending": { + "type": "long" + }, + "queue": { + "ignore_above": 1024, + "type": "keyword" + }, + "stalled": { + "type": "boolean" + } + } + } + } + }, + "statsd": { + "properties": { + "*": { + "properties": { + "*": { + "type": "object" + }, + "count": { + "type": "object" + } + } + } + } + }, + "system": { + "properties": { + "core": { + "properties": { + "id": { + "type": "long" + }, + "idle": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "iowait": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "irq": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "nice": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "softirq": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "steal": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "system": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "user": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + } + } + }, + "cpu": { + "properties": { + "cores": { + "type": "long" + }, + "idle": { + "properties": { + "norm": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "iowait": { + "properties": { + "norm": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "irq": { + "properties": { + "norm": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "nice": { + "properties": { + "norm": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "softirq": { + "properties": { + "norm": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "steal": { + "properties": { + "norm": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "system": { + "properties": { + "norm": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + }, + "total": { + "properties": { + "norm": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "user": { + "properties": { + "norm": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + } + } + } + } + }, + "diskio": { + "properties": { + "io": { + "properties": { + "time": { + "type": "long" + } + } + }, + "iostat": { + "properties": { + "await": { + "type": "float" + }, + "busy": { + "type": "float" + }, + "queue": { + "properties": { + "avg_size": { + "type": "float" + } + } + }, + "read": { + "properties": { + "await": { + "type": "float" + }, + "per_sec": { + "properties": { + "bytes": { + "type": "float" + } + } + }, + "request": { + "properties": { + "merges_per_sec": { + "type": "float" + }, + "per_sec": { + "type": "float" + } + } + } + } + }, + "request": { + "properties": { + "avg_size": { + "type": "float" + } + } + }, + "service_time": { + "type": "float" + }, + "write": { + "properties": { + "await": { + "type": "float" + }, + "per_sec": { + "properties": { + "bytes": { + "type": "float" + } + } + }, + "request": { + "properties": { + "merges_per_sec": { + "type": "float" + }, + "per_sec": { + "type": "float" + } + } + } + } + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "read": { + "properties": { + "bytes": { + "type": "long" + }, + "count": { + "type": "long" + }, + "time": { + "type": "long" + } + } + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "write": { + "properties": { + "bytes": { + "type": "long" + }, + "count": { + "type": "long" + }, + "time": { + "type": "long" + } + } + } + } + }, + "entropy": { + "properties": { + "available_bits": { + "type": "long" + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "filesystem": { + "properties": { + "available": { + "type": "long" + }, + "device_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "files": { + "type": "long" + }, + "free": { + "type": "long" + }, + "free_files": { + "type": "long" + }, + "mount_point": { + "ignore_above": 1024, + "type": "keyword" + }, + "total": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "used": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + }, + "fsstat": { + "properties": { + "count": { + "type": "long" + }, + "total_files": { + "type": "long" + }, + "total_size": { + "properties": { + "free": { + "type": "long" + }, + "total": { + "type": "long" + }, + "used": { + "type": "long" + } + } + } + } + }, + "load": { + "properties": { + "1": { + "scaling_factor": 100, + "type": "scaled_float" + }, + "15": { + "scaling_factor": 100, + "type": "scaled_float" + }, + "5": { + "scaling_factor": 100, + "type": "scaled_float" + }, + "cores": { + "type": "long" + }, + "norm": { + "properties": { + "1": { + "scaling_factor": 100, + "type": "scaled_float" + }, + "15": { + "scaling_factor": 100, + "type": "scaled_float" + }, + "5": { + "scaling_factor": 100, + "type": "scaled_float" + } + } + } + } + }, + "memory": { + "properties": { + "actual": { + "properties": { + "free": { + "type": "long" + }, + "used": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + }, + "free": { + "type": "long" + }, + "hugepages": { + "properties": { + "default_size": { + "type": "long" + }, + "free": { + "type": "long" + }, + "reserved": { + "type": "long" + }, + "surplus": { + "type": "long" + }, + "swap": { + "properties": { + "out": { + "properties": { + "fallback": { + "type": "long" + }, + "pages": { + "type": "long" + } + } + } + } + }, + "total": { + "type": "long" + }, + "used": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "type": "long" + } + } + } + } + }, + "page_stats": { + "properties": { + "direct_efficiency": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "kswapd_efficiency": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "pgfree": { + "properties": { + "pages": { + "type": "long" + } + } + }, + "pgscan_direct": { + "properties": { + "pages": { + "type": "long" + } + } + }, + "pgscan_kswapd": { + "properties": { + "pages": { + "type": "long" + } + } + }, + "pgsteal_direct": { + "properties": { + "pages": { + "type": "long" + } + } + }, + "pgsteal_kswapd": { + "properties": { + "pages": { + "type": "long" + } + } + } + } + }, + "swap": { + "properties": { + "free": { + "type": "long" + }, + "in": { + "properties": { + "pages": { + "type": "long" + } + } + }, + "out": { + "properties": { + "pages": { + "type": "long" + } + } + }, + "readahead": { + "properties": { + "cached": { + "type": "long" + }, + "pages": { + "type": "long" + } + } + }, + "total": { + "type": "long" + }, + "used": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + }, + "total": { + "type": "long" + }, + "used": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + } + } + }, + "network": { + "properties": { + "in": { + "properties": { + "bytes": { + "type": "long" + }, + "dropped": { + "type": "long" + }, + "errors": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "out": { + "properties": { + "bytes": { + "type": "long" + }, + "dropped": { + "type": "long" + }, + "errors": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + } + } + }, + "network_summary": { + "properties": { + "icmp": { + "properties": { + "*": { + "type": "object" + } + } + }, + "ip": { + "properties": { + "*": { + "type": "object" + } + } + }, + "tcp": { + "properties": { + "*": { + "type": "object" + } + } + }, + "udp": { + "properties": { + "*": { + "type": "object" + } + } + }, + "udp_lite": { + "properties": { + "*": { + "type": "object" + } + } + } + } + }, + "process": { + "properties": { + "cgroup": { + "properties": { + "blkio": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "total": { + "properties": { + "bytes": { + "type": "long" + }, + "ios": { + "type": "long" + } + } + } + } + }, + "cpu": { + "properties": { + "cfs": { + "properties": { + "period": { + "properties": { + "us": { + "type": "long" + } + } + }, + "quota": { + "properties": { + "us": { + "type": "long" + } + } + }, + "shares": { + "type": "long" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "rt": { + "properties": { + "period": { + "properties": { + "us": { + "type": "long" + } + } + }, + "runtime": { + "properties": { + "us": { + "type": "long" + } + } + } + } + }, + "stats": { + "properties": { + "periods": { + "type": "long" + }, + "throttled": { + "properties": { + "ns": { + "type": "long" + }, + "periods": { + "type": "long" + } + } + } + } + } + } + }, + "cpuacct": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "percpu": { + "properties": { + "1": { + "type": "long" + }, + "2": { + "type": "long" + }, + "3": { + "type": "long" + }, + "4": { + "type": "long" + } + } + }, + "stats": { + "properties": { + "system": { + "properties": { + "ns": { + "type": "long" + } + } + }, + "user": { + "properties": { + "ns": { + "type": "long" + } + } + } + } + }, + "total": { + "properties": { + "ns": { + "type": "long" + } + } + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "memory": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "kmem": { + "properties": { + "failures": { + "type": "long" + }, + "limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "bytes": { + "type": "long" + }, + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "kmem_tcp": { + "properties": { + "failures": { + "type": "long" + }, + "limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "bytes": { + "type": "long" + }, + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "mem": { + "properties": { + "failures": { + "type": "long" + }, + "limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "bytes": { + "type": "long" + }, + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "memsw": { + "properties": { + "failures": { + "type": "long" + }, + "limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "usage": { + "properties": { + "bytes": { + "type": "long" + }, + "max": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "stats": { + "properties": { + "active_anon": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "active_file": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "cache": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "hierarchical_memory_limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "hierarchical_memsw_limit": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "inactive_anon": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "inactive_file": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "major_page_faults": { + "type": "long" + }, + "mapped_file": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "page_faults": { + "type": "long" + }, + "pages_in": { + "type": "long" + }, + "pages_out": { + "type": "long" + }, + "rss": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "rss_huge": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "swap": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "unevictable": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "cmdline": { + "ignore_above": 2048, + "type": "keyword" + }, + "cpu": { + "properties": { + "start_time": { + "type": "date" + }, + "system": { + "properties": { + "ticks": { + "type": "long" + } + } + }, + "total": { + "properties": { + "norm": { + "properties": { + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + }, + "ticks": { + "type": "long" + }, + "value": { + "type": "long" + } + } + }, + "user": { + "properties": { + "ticks": { + "type": "long" + } + } + } + } + }, + "env": { + "type": "object" + }, + "fd": { + "properties": { + "limit": { + "properties": { + "hard": { + "type": "long" + }, + "soft": { + "type": "long" + } + } + }, + "open": { + "type": "long" + } + } + }, + "memory": { + "properties": { + "rss": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "scaling_factor": 1000, + "type": "scaled_float" + } + } + }, + "share": { + "type": "long" + }, + "size": { + "type": "long" + } + } + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "summary": { + "properties": { + "dead": { + "type": "long" + }, + "idle": { + "type": "long" + }, + "running": { + "type": "long" + }, + "sleeping": { + "type": "long" + }, + "stopped": { + "type": "long" + }, + "total": { + "type": "long" + }, + "unknown": { + "type": "long" + }, + "zombie": { + "type": "long" + } + } + } + } + }, + "raid": { + "properties": { + "blocks": { + "properties": { + "synced": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "disks": { + "properties": { + "active": { + "type": "long" + }, + "failed": { + "type": "long" + }, + "spare": { + "type": "long" + }, + "states": { + "properties": { + "*": { + "type": "object" + } + } + }, + "total": { + "type": "long" + } + } + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "sync_action": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "service": { + "properties": { + "exec_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "load_state": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "resources": { + "properties": { + "cpu": { + "properties": { + "usage": { + "properties": { + "ns": { + "type": "long" + } + } + } + } + }, + "memory": { + "properties": { + "usage": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "network": { + "properties": { + "in": { + "properties": { + "bytes": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + }, + "out": { + "properties": { + "bytes": { + "type": "long" + }, + "packets": { + "type": "long" + } + } + } + } + }, + "tasks": { + "properties": { + "count": { + "type": "long" + } + } + } + } + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_since": { + "type": "date" + }, + "sub_state": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "socket": { + "properties": { + "local": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "process": { + "properties": { + "cmdline": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "remote": { + "properties": { + "etld_plus_one": { + "ignore_above": 1024, + "type": "keyword" + }, + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "host_error": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "summary": { + "properties": { + "all": { + "properties": { + "count": { + "type": "long" + }, + "listening": { + "type": "long" + } + } + }, + "tcp": { + "properties": { + "all": { + "properties": { + "close_wait": { + "type": "long" + }, + "closing": { + "type": "long" + }, + "count": { + "type": "long" + }, + "established": { + "type": "long" + }, + "fin_wait1": { + "type": "long" + }, + "fin_wait2": { + "type": "long" + }, + "last_ack": { + "type": "long" + }, + "listening": { + "type": "long" + }, + "orphan": { + "type": "long" + }, + "syn_recv": { + "type": "long" + }, + "syn_sent": { + "type": "long" + }, + "time_wait": { + "type": "long" + } + } + }, + "memory": { + "type": "long" + } + } + }, + "udp": { + "properties": { + "all": { + "properties": { + "count": { + "type": "long" + } + } + }, + "memory": { + "type": "long" + } + } + } + } + } + } + }, + "uptime": { + "properties": { + "duration": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "users": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "leader": { + "type": "long" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "remote": { + "type": "boolean" + }, + "remote_host": { + "ignore_above": 1024, + "type": "keyword" + }, + "scope": { + "ignore_above": 1024, + "type": "keyword" + }, + "seat": { + "ignore_above": 1024, + "type": "keyword" + }, + "service": { + "ignore_above": 1024, + "type": "keyword" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "systemd": { + "properties": { + "fragment_path": { + "ignore_above": 1024, + "type": "keyword" + }, + "unit": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "threat": { + "properties": { + "framework": { + "ignore_above": 1024, + "type": "keyword" + }, + "tactic": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "technique": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "timeseries": { + "properties": { + "instance": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "tls": { + "properties": { + "cipher": { + "ignore_above": 1024, + "type": "keyword" + }, + "client": { + "properties": { + "certificate": { + "ignore_above": 1024, + "type": "keyword" + }, + "certificate_chain": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "issuer": { + "ignore_above": 1024, + "type": "keyword" + }, + "ja3": { + "ignore_above": 1024, + "type": "keyword" + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "server_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject": { + "ignore_above": 1024, + "type": "keyword" + }, + "supported_ciphers": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "curve": { + "ignore_above": 1024, + "type": "keyword" + }, + "established": { + "type": "boolean" + }, + "next_protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "resumed": { + "type": "boolean" + }, + "server": { + "properties": { + "certificate": { + "ignore_above": 1024, + "type": "keyword" + }, + "certificate_chain": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "issuer": { + "ignore_above": 1024, + "type": "keyword" + }, + "ja3s": { + "ignore_above": 1024, + "type": "keyword" + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "subject": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "version_protocol": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "tomcat": { + "properties": { + "cache": { + "properties": { + "hit": { + "properties": { + "total": { + "type": "long" + } + } + }, + "lookup": { + "properties": { + "total": { + "type": "long" + } + } + }, + "mbean": { + "ignore_above": 1024, + "type": "keyword" + }, + "size": { + "properties": { + "max": { + "properties": { + "kb": { + "type": "long" + } + } + }, + "total": { + "properties": { + "kb": { + "type": "long" + } + } + } + } + }, + "ttl": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "memory": { + "properties": { + "heap": { + "properties": { + "usage": { + "properties": { + "committed": { + "type": "long" + }, + "init": { + "type": "long" + }, + "max": { + "type": "long" + }, + "used": { + "type": "long" + } + } + } + } + }, + "mbean": { + "ignore_above": 1024, + "type": "keyword" + }, + "other": { + "properties": { + "usage": { + "properties": { + "committed": { + "type": "long" + }, + "init": { + "type": "long" + }, + "max": { + "type": "long" + }, + "used": { + "type": "long" + } + } + } + } + } + } + }, + "requests": { + "properties": { + "bytes": { + "properties": { + "received": { + "type": "long" + }, + "sent": { + "type": "long" + } + } + }, + "errors": { + "properties": { + "total": { + "type": "long" + } + } + }, + "mbean": { + "ignore_above": 1024, + "type": "keyword" + }, + "processing": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "total": { + "type": "long" + } + } + }, + "threading": { + "properties": { + "busy": { + "type": "long" + }, + "cpu": { + "properties": { + "time": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, + "current": { + "type": "long" + }, + "keep_alive": { + "properties": { + "timeout": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "total": { + "type": "long" + } + } + }, + "max": { + "type": "long" + }, + "peak": { + "type": "long" + }, + "started": { + "properties": { + "total": { + "type": "long" + } + } + }, + "total": { + "type": "long" + }, + "user": { + "properties": { + "time": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + } + } + } + } + }, + "tracing": { + "properties": { + "trace": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "transaction": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "traefik": { + "properties": { + "health": { + "properties": { + "response": { + "properties": { + "avg_time": { + "properties": { + "us": { + "type": "long" + } + } + }, + "count": { + "type": "long" + }, + "status_codes": { + "properties": { + "*": { + "type": "object" + } + } + } + } + }, + "uptime": { + "properties": { + "sec": { + "type": "long" + } + } + } + } + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "url": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "extension": { + "ignore_above": 1024, + "type": "keyword" + }, + "fragment": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "password": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "scheme": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "username": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user_agent": { + "properties": { + "device": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "uwsgi": { + "properties": { + "status": { + "properties": { + "core": { + "properties": { + "id": { + "type": "long" + }, + "read_errors": { + "type": "long" + }, + "requests": { + "properties": { + "offloaded": { + "type": "long" + }, + "routed": { + "type": "long" + }, + "static": { + "type": "long" + }, + "total": { + "type": "long" + } + } + }, + "worker_pid": { + "type": "long" + }, + "write_errors": { + "type": "long" + } + } + }, + "total": { + "properties": { + "exceptions": { + "type": "long" + }, + "pid": { + "type": "long" + }, + "read_errors": { + "type": "long" + }, + "requests": { + "type": "long" + }, + "write_errors": { + "type": "long" + } + } + }, + "worker": { + "properties": { + "accepting": { + "type": "long" + }, + "avg_rt": { + "type": "long" + }, + "delta_requests": { + "type": "long" + }, + "exceptions": { + "type": "long" + }, + "harakiri_count": { + "type": "long" + }, + "id": { + "type": "long" + }, + "pid": { + "type": "long" + }, + "requests": { + "type": "long" + }, + "respawn_count": { + "type": "long" + }, + "rss": { + "ignore_above": 1024, + "type": "keyword" + }, + "running_time": { + "type": "long" + }, + "signal_queue": { + "type": "long" + }, + "signals": { + "type": "long" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "tx": { + "type": "long" + }, + "vsz": { + "type": "long" + } + } + } + } + } + } + }, + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vsphere": { + "properties": { + "datastore": { + "properties": { + "capacity": { + "properties": { + "free": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "total": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + }, + "pct": { + "type": "long" + } + } + } + } + }, + "fstype": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "host": { + "properties": { + "cpu": { + "properties": { + "free": { + "properties": { + "mhz": { + "type": "long" + } + } + }, + "total": { + "properties": { + "mhz": { + "type": "long" + } + } + }, + "used": { + "properties": { + "mhz": { + "type": "long" + } + } + } + } + }, + "memory": { + "properties": { + "free": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "total": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "used": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "network_names": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "virtualmachine": { + "properties": { + "cpu": { + "properties": { + "used": { + "properties": { + "mhz": { + "type": "long" + } + } + } + } + }, + "custom_fields": { + "type": "object" + }, + "host": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "memory": { + "properties": { + "free": { + "properties": { + "guest": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "total": { + "properties": { + "guest": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + }, + "used": { + "properties": { + "guest": { + "properties": { + "bytes": { + "type": "long" + } + } + }, + "host": { + "properties": { + "bytes": { + "type": "long" + } + } + } + } + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "network_names": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "vulnerability": { + "properties": { + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "classification": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "enumeration": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "report_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "scanner": { + "properties": { + "vendor": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "score": { + "properties": { + "base": { + "type": "float" + }, + "environmental": { + "type": "float" + }, + "temporal": { + "type": "float" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "severity": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "windows": { + "properties": { + "service": { + "properties": { + "display_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "exit_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "path_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "pid": { + "type": "long" + }, + "start_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "start_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + } + } + }, + "zookeeper": { + "properties": { + "connection": { + "properties": { + "interest_ops": { + "type": "long" + }, + "queued": { + "type": "long" + }, + "received": { + "type": "long" + }, + "sent": { + "type": "long" + } + } + }, + "mntr": { + "properties": { + "approximate_data_size": { + "type": "long" + }, + "ephemerals_count": { + "type": "long" + }, + "followers": { + "type": "long" + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "latency": { + "properties": { + "avg": { + "type": "long" + }, + "max": { + "type": "long" + }, + "min": { + "type": "long" + } + } + }, + "max_file_descriptor_count": { + "type": "long" + }, + "num_alive_connections": { + "type": "long" + }, + "open_file_descriptor_count": { + "type": "long" + }, + "outstanding_requests": { + "type": "long" + }, + "packets": { + "properties": { + "received": { + "type": "long" + }, + "sent": { + "type": "long" + } + } + }, + "pending_syncs": { + "type": "long" + }, + "server_state": { + "ignore_above": 1024, + "type": "keyword" + }, + "synced_followers": { + "type": "long" + }, + "version": { + "path": "service.version", + "type": "alias" + }, + "watch_count": { + "type": "long" + }, + "znode_count": { + "type": "long" + } + } + }, + "server": { + "properties": { + "connections": { + "type": "long" + }, + "count": { + "type": "long" + }, + "epoch": { + "type": "long" + }, + "latency": { + "properties": { + "avg": { + "type": "long" + }, + "max": { + "type": "long" + }, + "min": { + "type": "long" + } + } + }, + "mode": { + "ignore_above": 1024, + "type": "keyword" + }, + "node_count": { + "type": "long" + }, + "outstanding": { + "type": "long" + }, + "received": { + "type": "long" + }, + "sent": { + "type": "long" + }, + "version_date": { + "type": "date" + }, + "zxid": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "settings": { + "index": { + "mapping": { + "total_fields": { + "limit": "10000" + } + }, + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} \ No newline at end of file diff --git a/x-pack/test/functional/es_archives/ml/module_nginx_data_stream/data.json.gz b/x-pack/test/functional/es_archives/ml/module_nginx_data_stream/data.json.gz new file mode 100644 index 0000000000000..010e00dfd1d89 Binary files /dev/null and b/x-pack/test/functional/es_archives/ml/module_nginx_data_stream/data.json.gz differ diff --git a/x-pack/test/functional/es_archives/ml/module_nginx_data_stream/mappings.json b/x-pack/test/functional/es_archives/ml/module_nginx_data_stream/mappings.json new file mode 100644 index 0000000000000..6553c6b3ac73d --- /dev/null +++ b/x-pack/test/functional/es_archives/ml/module_nginx_data_stream/mappings.json @@ -0,0 +1,2675 @@ +{ + "type": "index", + "value": { + "aliases": { + }, + "index": "ft_module_nginx_data_stream", + "mappings": { + "_meta": { + "beat": "filebeat", + "version": "7.0.0" + }, + "date_detection": false, + "dynamic_templates": [ + { + "container.labels": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "container.labels.*" + } + }, + { + "fields": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "fields.*" + } + }, + { + "docker.container.labels": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "docker.container.labels.*" + } + }, + { + "kibana.log.meta": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "kibana.log.meta.*" + } + }, + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "agent": { + "properties": { + "ephemeral_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "apache": { + "properties": { + "access": { + "properties": { + "ssl": { + "properties": { + "cipher": { + "ignore_above": 1024, + "type": "keyword" + }, + "protocol": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "error": { + "properties": { + "module": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "apache2": { + "properties": { + "access": { + "properties": { + "geoip": { + "type": "object" + }, + "user_agent": { + "type": "object" + } + } + }, + "error": { + "type": "object" + } + } + }, + "auditd": { + "properties": { + "log": { + "properties": { + "a0": { + "ignore_above": 1024, + "type": "keyword" + }, + "addr": { + "type": "ip" + }, + "geoip": { + "type": "object" + }, + "item": { + "ignore_above": 1024, + "type": "keyword" + }, + "items": { + "ignore_above": 1024, + "type": "keyword" + }, + "laddr": { + "type": "ip" + }, + "lport": { + "type": "long" + }, + "new_auid": { + "ignore_above": 1024, + "type": "keyword" + }, + "new_ses": { + "ignore_above": 1024, + "type": "keyword" + }, + "old_auid": { + "ignore_above": 1024, + "type": "keyword" + }, + "old_ses": { + "ignore_above": 1024, + "type": "keyword" + }, + "rport": { + "type": "long" + }, + "sequence": { + "type": "long" + }, + "tty": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "certificate": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "client": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + } + } + }, + "cloud": { + "properties": { + "account": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "instance": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "machine": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "project": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "container": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "image": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "tag": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "labels": { + "type": "object" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "runtime": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "destination": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + } + } + }, + "docker": { + "properties": { + "container": { + "properties": { + "labels": { + "type": "object" + } + } + } + } + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "elasticsearch": { + "properties": { + "audit": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "indices": { + "ignore_above": 1024, + "type": "keyword" + }, + "layer": { + "ignore_above": 1024, + "type": "keyword" + }, + "origin": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "realm": { + "ignore_above": 1024, + "type": "keyword" + }, + "request": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "url": { + "properties": { + "params": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user": { + "properties": { + "realm": { + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "cluster": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "uuid": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "component": { + "ignore_above": 1024, + "type": "keyword" + }, + "deprecation": { + "type": "object" + }, + "gc": { + "properties": { + "heap": { + "properties": { + "size_kb": { + "type": "long" + }, + "used_kb": { + "type": "long" + } + } + }, + "jvm_runtime_sec": { + "type": "float" + }, + "old_gen": { + "properties": { + "size_kb": { + "type": "long" + }, + "used_kb": { + "type": "long" + } + } + }, + "phase": { + "properties": { + "class_unload_time_sec": { + "type": "float" + }, + "cpu_time": { + "properties": { + "real_sec": { + "type": "float" + }, + "sys_sec": { + "type": "float" + }, + "user_sec": { + "type": "float" + } + } + }, + "duration_sec": { + "type": "float" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "parallel_rescan_time_sec": { + "type": "float" + }, + "scrub_string_table_time_sec": { + "type": "float" + }, + "scrub_symbol_table_time_sec": { + "type": "float" + }, + "weak_refs_processing_time_sec": { + "type": "float" + } + } + }, + "stopping_threads_time_sec": { + "type": "float" + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "threads_total_stop_time_sec": { + "type": "float" + }, + "young_gen": { + "properties": { + "size_kb": { + "type": "long" + }, + "used_kb": { + "type": "long" + } + } + } + } + }, + "index": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "node": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "server": { + "properties": { + "component": { + "ignore_above": 1024, + "type": "keyword" + }, + "gc": { + "properties": { + "collection_duration": { + "properties": { + "ms": { + "type": "float" + } + } + }, + "observation_duration": { + "properties": { + "ms": { + "type": "float" + } + } + }, + "overhead_seq": { + "type": "long" + }, + "young": { + "properties": { + "one": { + "type": "long" + }, + "two": { + "type": "long" + } + } + } + } + } + } + }, + "shard": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "slowlog": { + "properties": { + "extra_source": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "logger": { + "ignore_above": 1024, + "type": "keyword" + }, + "routing": { + "ignore_above": 1024, + "type": "keyword" + }, + "search_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "source_query": { + "ignore_above": 1024, + "type": "keyword" + }, + "stats": { + "ignore_above": 1024, + "type": "keyword" + }, + "took": { + "ignore_above": 1024, + "type": "keyword" + }, + "total_hits": { + "ignore_above": 1024, + "type": "keyword" + }, + "total_shards": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "types": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "error": { + "properties": { + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "event": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "created": { + "type": "date" + }, + "dataset": { + "ignore_above": 1024, + "type": "keyword" + }, + "duration": { + "type": "long" + }, + "end": { + "type": "date" + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "doc_values": false, + "ignore_above": 1024, + "index": false, + "type": "keyword" + }, + "outcome": { + "ignore_above": 1024, + "type": "keyword" + }, + "risk_score": { + "type": "float" + }, + "risk_score_norm": { + "type": "float" + }, + "severity": { + "type": "long" + }, + "start": { + "type": "date" + }, + "timezone": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "fields": { + "type": "object" + }, + "file": { + "properties": { + "ctime": { + "type": "date" + }, + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "extension": { + "ignore_above": 1024, + "type": "keyword" + }, + "gid": { + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "ignore_above": 1024, + "type": "keyword" + }, + "inode": { + "ignore_above": 1024, + "type": "keyword" + }, + "mode": { + "ignore_above": 1024, + "type": "keyword" + }, + "mtime": { + "type": "date" + }, + "owner": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "size": { + "type": "long" + }, + "target_path": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "fileset": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "haproxy": { + "properties": { + "backend_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "backend_queue": { + "type": "long" + }, + "bind_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes_read": { + "type": "long" + }, + "client": { + "type": "object" + }, + "connection_wait_time_ms": { + "type": "long" + }, + "connections": { + "properties": { + "active": { + "type": "long" + }, + "backend": { + "type": "long" + }, + "frontend": { + "type": "long" + }, + "retries": { + "type": "long" + }, + "server": { + "type": "long" + } + } + }, + "destination": { + "type": "object" + }, + "error_message": { + "norms": false, + "type": "text" + }, + "frontend_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "geoip": { + "type": "object" + }, + "http": { + "properties": { + "request": { + "properties": { + "captured_cookie": { + "ignore_above": 1024, + "type": "keyword" + }, + "captured_headers": { + "ignore_above": 1024, + "type": "keyword" + }, + "raw_request_line": { + "ignore_above": 1024, + "type": "keyword" + }, + "time_wait_ms": { + "type": "long" + }, + "time_wait_without_data_ms": { + "type": "long" + } + } + }, + "response": { + "properties": { + "captured_cookie": { + "ignore_above": 1024, + "type": "keyword" + }, + "captured_headers": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "mode": { + "ignore_above": 1024, + "type": "keyword" + }, + "server_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "server_queue": { + "type": "long" + }, + "source": { + "norms": false, + "type": "text" + }, + "tcp": { + "properties": { + "connection_waiting_time_ms": { + "type": "long" + } + } + }, + "termination_state": { + "ignore_above": 1024, + "type": "keyword" + }, + "time_backend_connect": { + "type": "long" + }, + "time_queue": { + "type": "long" + }, + "total_waiting_time_ms": { + "type": "long" + } + } + }, + "hash": { + "properties": { + "sha256": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "host": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "build": { + "ignore_above": 1024, + "type": "keyword" + }, + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "http": { + "properties": { + "request": { + "properties": { + "body": { + "properties": { + "bytes": { + "type": "long" + }, + "content": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bytes": { + "type": "long" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "referrer": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "response": { + "properties": { + "body": { + "properties": { + "bytes": { + "type": "long" + }, + "content": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bytes": { + "type": "long" + }, + "status_code": { + "type": "long" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "icinga": { + "properties": { + "debug": { + "properties": { + "facility": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "main": { + "properties": { + "facility": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "startup": { + "properties": { + "facility": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "iis": { + "properties": { + "access": { + "properties": { + "cookie": { + "ignore_above": 1024, + "type": "keyword" + }, + "geoip": { + "type": "object" + }, + "server_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "site_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "sub_status": { + "type": "long" + }, + "user_agent": { + "type": "object" + }, + "win32_status": { + "type": "long" + } + } + }, + "error": { + "properties": { + "geoip": { + "type": "object" + }, + "queue_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "reason_phrase": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "input": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "kafka": { + "properties": { + "log": { + "properties": { + "class": { + "norms": false, + "type": "text" + }, + "component": { + "ignore_above": 1024, + "type": "keyword" + }, + "trace": { + "properties": { + "class": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "norms": false, + "type": "text" + }, + "message": { + "norms": false, + "type": "text" + } + } + } + } + } + } + }, + "kibana": { + "properties": { + "log": { + "properties": { + "meta": { + "type": "object" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "kubernetes": { + "properties": { + "annotations": { + "type": "object" + }, + "container": { + "properties": { + "image": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "labels": { + "type": "object" + }, + "namespace": { + "ignore_above": 1024, + "type": "keyword" + }, + "node": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pod": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "labels": { + "type": "object" + }, + "log": { + "properties": { + "file": { + "properties": { + "path": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "flags": { + "ignore_above": 1024, + "type": "keyword" + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "offset": { + "type": "long" + }, + "original": { + "doc_values": false, + "ignore_above": 1024, + "index": false, + "type": "keyword" + }, + "source": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "logstash": { + "properties": { + "log": { + "properties": { + "log_event": { + "type": "object" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "thread": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "slowlog": { + "properties": { + "event": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "plugin_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "plugin_params": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "plugin_params_object": { + "type": "object" + }, + "plugin_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "thread": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "took_in_millis": { + "type": "long" + } + } + } + } + }, + "message": { + "norms": false, + "type": "text" + }, + "mongodb": { + "properties": { + "log": { + "properties": { + "component": { + "ignore_above": 1024, + "type": "keyword" + }, + "context": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "mysql": { + "properties": { + "error": { + "type": "object" + }, + "slowlog": { + "properties": { + "bytes_sent": { + "type": "long" + }, + "current_user": { + "ignore_above": 1024, + "type": "keyword" + }, + "filesort": { + "type": "boolean" + }, + "filesort_on_disk": { + "type": "boolean" + }, + "full_join": { + "type": "boolean" + }, + "full_scan": { + "type": "boolean" + }, + "innodb": { + "properties": { + "io_r_bytes": { + "type": "long" + }, + "io_r_ops": { + "type": "long" + }, + "io_r_wait": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "pages_distinct": { + "type": "long" + }, + "queue_wait": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "rec_lock_wait": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "trx_id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "killed": { + "ignore_above": 1024, + "type": "keyword" + }, + "last_errno": { + "ignore_above": 1024, + "type": "keyword" + }, + "lock_time": { + "properties": { + "sec": { + "type": "float" + } + } + }, + "log_slow_rate_limit": { + "ignore_above": 1024, + "type": "keyword" + }, + "log_slow_rate_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "merge_passes": { + "type": "long" + }, + "priority_queue": { + "type": "boolean" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "query_cache_hit": { + "type": "boolean" + }, + "rows_affected": { + "type": "long" + }, + "rows_examined": { + "type": "long" + }, + "rows_sent": { + "type": "long" + }, + "schema": { + "ignore_above": 1024, + "type": "keyword" + }, + "tmp_disk_tables": { + "type": "long" + }, + "tmp_table": { + "type": "boolean" + }, + "tmp_table_on_disk": { + "type": "boolean" + }, + "tmp_table_sizes": { + "type": "long" + }, + "tmp_tables": { + "type": "long" + } + } + }, + "thread_id": { + "type": "long" + } + } + }, + "network": { + "properties": { + "application": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "community_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "direction": { + "ignore_above": 1024, + "type": "keyword" + }, + "forwarded_ip": { + "type": "ip" + }, + "iana_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "transport": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "nginx": { + "properties": { + "access": { + "properties": { + "agent": { + "ignore_above": 1024, + "type": "keyword" + }, + "geoip": { + "type": "object" + }, + "remote_ip_list": { + "ignore_above": 1024, + "type": "keyword" + }, + "user_agent": { + "type": "object" + } + } + }, + "error": { + "properties": { + "connection_id": { + "type": "long" + } + } + } + } + }, + "observer": { + "properties": { + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "vendor": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "organization": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "osquery": { + "properties": { + "result": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "calendar_time": { + "ignore_above": 1024, + "type": "keyword" + }, + "host_identifier": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "unix_time": { + "type": "long" + } + } + } + } + }, + "postgresql": { + "properties": { + "log": { + "properties": { + "core_id": { + "type": "long" + }, + "database": { + "ignore_above": 1024, + "type": "keyword" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "timestamp": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "process": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "executable": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "pid": { + "type": "long" + }, + "ppid": { + "type": "long" + }, + "program": { + "ignore_above": 1024, + "type": "keyword" + }, + "start": { + "type": "date" + }, + "thread": { + "properties": { + "id": { + "type": "long" + } + } + }, + "title": { + "ignore_above": 1024, + "type": "keyword" + }, + "working_directory": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "redis": { + "properties": { + "log": { + "properties": { + "role": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "slowlog": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "cmd": { + "ignore_above": 1024, + "type": "keyword" + }, + "duration": { + "properties": { + "us": { + "type": "long" + } + } + }, + "id": { + "type": "long" + }, + "key": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "related": { + "properties": { + "ip": { + "type": "ip" + } + } + }, + "santa": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "decision": { + "ignore_above": 1024, + "type": "keyword" + }, + "disk": { + "properties": { + "bsdname": { + "ignore_above": 1024, + "type": "keyword" + }, + "bus": { + "ignore_above": 1024, + "type": "keyword" + }, + "fs": { + "ignore_above": 1024, + "type": "keyword" + }, + "model": { + "ignore_above": 1024, + "type": "keyword" + }, + "mount": { + "ignore_above": 1024, + "type": "keyword" + }, + "serial": { + "ignore_above": 1024, + "type": "keyword" + }, + "volume": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "mode": { + "ignore_above": 1024, + "type": "keyword" + }, + "reason": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "server": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + } + } + }, + "service": { + "properties": { + "ephemeral_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "source": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + } + } + }, + "stream": { + "ignore_above": 1024, + "type": "keyword" + }, + "syslog": { + "properties": { + "facility": { + "type": "long" + }, + "facility_label": { + "ignore_above": 1024, + "type": "keyword" + }, + "priority": { + "type": "long" + }, + "severity_label": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "system": { + "properties": { + "auth": { + "properties": { + "groupadd": { + "type": "object" + }, + "ssh": { + "properties": { + "dropped_ip": { + "type": "ip" + }, + "geoip": { + "type": "object" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "signature": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "sudo": { + "properties": { + "command": { + "ignore_above": 1024, + "type": "keyword" + }, + "error": { + "ignore_above": 1024, + "type": "keyword" + }, + "pwd": { + "ignore_above": 1024, + "type": "keyword" + }, + "tty": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "useradd": { + "properties": { + "home": { + "ignore_above": 1024, + "type": "keyword" + }, + "shell": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "syslog": { + "type": "object" + } + } + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "traefik": { + "properties": { + "access": { + "properties": { + "backend_url": { + "ignore_above": 1024, + "type": "keyword" + }, + "frontend_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "geoip": { + "properties": { + "city_name": { + "path": "source.geo.city_name", + "type": "alias" + }, + "continent_name": { + "path": "source.geo.continent_name", + "type": "alias" + }, + "country_iso_code": { + "path": "source.geo.country_iso_code", + "type": "alias" + }, + "location": { + "path": "source.geo.location", + "type": "alias" + }, + "region_iso_code": { + "path": "source.geo.region_iso_code", + "type": "alias" + }, + "region_name": { + "path": "source.geo.region_name", + "type": "alias" + } + } + }, + "request_count": { + "type": "long" + }, + "user_agent": { + "properties": { + "device": { + "path": "user_agent.device.name", + "type": "alias" + }, + "major": { + "path": "user_agent.major", + "type": "alias" + }, + "minor": { + "path": "user_agent.minor", + "type": "alias" + }, + "name": { + "path": "user_agent.name", + "type": "alias" + }, + "original": { + "path": "user_agent.original", + "type": "alias" + }, + "os": { + "path": "user_agent.os.full_name", + "type": "alias" + }, + "os_major": { + "path": "user_agent.os.major", + "type": "alias" + }, + "os_minor": { + "path": "user_agent.os.minor", + "type": "alias" + }, + "os_name": { + "path": "user_agent.os.name", + "type": "alias" + }, + "patch": { + "path": "user_agent.patch", + "type": "alias" + } + } + }, + "user_identifier": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "url": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "fragment": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "ignore_above": 1024, + "type": "keyword" + }, + "password": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "scheme": { + "ignore_above": 1024, + "type": "keyword" + }, + "username": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user": { + "properties": { + "audit": { + "properties": { + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "effective": { + "properties": { + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "filesystem": { + "properties": { + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "full_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "owner": { + "properties": { + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "saved": { + "properties": { + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "terminal": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user_agent": { + "properties": { + "device": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "major": { + "ignore_above": 1024, + "type": "keyword" + }, + "minor": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "major": { + "type": "long" + }, + "minor": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "patch": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} diff --git a/x-pack/test/functional/es_archives/ml/module_siem_cloudtrail/data.json.gz b/x-pack/test/functional/es_archives/ml/module_siem_cloudtrail/data.json.gz new file mode 100644 index 0000000000000..9ec312602a5dd Binary files /dev/null and b/x-pack/test/functional/es_archives/ml/module_siem_cloudtrail/data.json.gz differ diff --git a/x-pack/test/functional/es_archives/ml/module_siem_cloudtrail/mappings.json b/x-pack/test/functional/es_archives/ml/module_siem_cloudtrail/mappings.json new file mode 100644 index 0000000000000..1df167c60dd48 --- /dev/null +++ b/x-pack/test/functional/es_archives/ml/module_siem_cloudtrail/mappings.json @@ -0,0 +1,12950 @@ +{ + "type": "index", + "value": { + "aliases": { + }, + "index": "ft_module_siem_cloudtrail", + "mappings": { + "_meta": { + "beat": "filebeat", + "version": "7.7.0" + }, + "date_detection": false, + "dynamic_templates": [ + { + "labels": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "labels.*" + } + }, + { + "container.labels": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "container.labels.*" + } + }, + { + "dns.answers": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "dns.answers.*" + } + }, + { + "log.syslog": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "log.syslog.*" + } + }, + { + "network.inner": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "network.inner.*" + } + }, + { + "observer.egress": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "observer.egress.*" + } + }, + { + "observer.ingress": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "observer.ingress.*" + } + }, + { + "fields": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "fields.*" + } + }, + { + "docker.container.labels": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "docker.container.labels.*" + } + }, + { + "kubernetes.labels.*": { + "mapping": { + "type": "keyword" + }, + "path_match": "kubernetes.labels.*" + } + }, + { + "kubernetes.annotations.*": { + "mapping": { + "type": "keyword" + }, + "path_match": "kubernetes.annotations.*" + } + }, + { + "docker.attrs": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "docker.attrs.*" + } + }, + { + "azure.activitylogs.identity.claims.*": { + "mapping": { + "type": "keyword" + }, + "path_match": "azure.activitylogs.identity.claims.*" + } + }, + { + "kibana.log.meta": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "kibana.log.meta.*" + } + }, + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "activemq": { + "properties": { + "audit": { + "type": "object" + }, + "caller": { + "ignore_above": 1024, + "type": "keyword" + }, + "log": { + "properties": { + "stack_trace": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "thread": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "agent": { + "properties": { + "ephemeral_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "apache": { + "properties": { + "access": { + "properties": { + "ssl": { + "properties": { + "cipher": { + "ignore_above": 1024, + "type": "keyword" + }, + "protocol": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "error": { + "properties": { + "module": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "apache2": { + "properties": { + "access": { + "properties": { + "geoip": { + "type": "object" + }, + "user_agent": { + "type": "object" + } + } + }, + "error": { + "type": "object" + } + } + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "auditd": { + "properties": { + "log": { + "properties": { + "a0": { + "ignore_above": 1024, + "type": "keyword" + }, + "addr": { + "type": "ip" + }, + "geoip": { + "type": "object" + }, + "item": { + "ignore_above": 1024, + "type": "keyword" + }, + "items": { + "ignore_above": 1024, + "type": "keyword" + }, + "laddr": { + "type": "ip" + }, + "lport": { + "type": "long" + }, + "new_auid": { + "ignore_above": 1024, + "type": "keyword" + }, + "new_ses": { + "ignore_above": 1024, + "type": "keyword" + }, + "old_auid": { + "ignore_above": 1024, + "type": "keyword" + }, + "old_ses": { + "ignore_above": 1024, + "type": "keyword" + }, + "rport": { + "type": "long" + }, + "sequence": { + "type": "long" + }, + "tty": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "aws": { + "properties": { + "cloudtrail": { + "properties": { + "additional_eventdata": { + "ignore_above": 1024, + "type": "keyword" + }, + "api_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "console_login": { + "properties": { + "additional_eventdata": { + "properties": { + "login_to": { + "ignore_above": 1024, + "type": "keyword" + }, + "mfa_used": { + "type": "boolean" + }, + "mobile_version": { + "type": "boolean" + } + } + } + } + }, + "error_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "error_message": { + "ignore_above": 1024, + "type": "keyword" + }, + "event_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "event_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "management_event": { + "ignore_above": 1024, + "type": "keyword" + }, + "read_only": { + "ignore_above": 1024, + "type": "keyword" + }, + "recipient_account_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "request_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "request_parameters": { + "ignore_above": 1024, + "type": "keyword" + }, + "resources": { + "properties": { + "account_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "arn": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "response_elements": { + "ignore_above": 1024, + "type": "keyword" + }, + "service_event_details": { + "ignore_above": 1024, + "type": "keyword" + }, + "shared_event_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "user_identity": { + "properties": { + "access_key_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "arn": { + "ignore_above": 1024, + "type": "keyword" + }, + "invoked_by": { + "ignore_above": 1024, + "type": "keyword" + }, + "session_context": { + "properties": { + "creation_date": { + "type": "date" + }, + "mfa_authenticated": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "session_issuer": { + "properties": { + "account_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "arn": { + "ignore_above": 1024, + "type": "keyword" + }, + "principal_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vpc_endpoint_id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "cloudwatch": { + "type": "object" + }, + "ec2": { + "properties": { + "ip_address": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "elb": { + "properties": { + "action_executed": { + "ignore_above": 1024, + "type": "keyword" + }, + "backend": { + "properties": { + "http": { + "properties": { + "response": { + "properties": { + "status_code": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "ip": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "backend_processing_time": { + "properties": { + "sec": { + "type": "float" + } + } + }, + "chosen_cert": { + "properties": { + "arn": { + "ignore_above": 1024, + "type": "keyword" + }, + "serial": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "connection_time": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "error": { + "properties": { + "reason": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "incoming_tls_alert": { + "ignore_above": 1024, + "type": "keyword" + }, + "listener": { + "ignore_above": 1024, + "type": "keyword" + }, + "matched_rule_priority": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "redirect_url": { + "ignore_above": 1024, + "type": "keyword" + }, + "request_processing_time": { + "properties": { + "sec": { + "type": "float" + } + } + }, + "response_processing_time": { + "properties": { + "sec": { + "type": "float" + } + } + }, + "ssl_cipher": { + "ignore_above": 1024, + "type": "keyword" + }, + "ssl_protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "target_group": { + "properties": { + "arn": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "tls_handshake_time": { + "properties": { + "ms": { + "type": "long" + } + } + }, + "tls_named_group": { + "ignore_above": 1024, + "type": "keyword" + }, + "trace_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "s3": { + "properties": { + "bucket": { + "properties": { + "arn": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "object": { + "properties": { + "key": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "s3access": { + "properties": { + "authentication_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "bucket": { + "ignore_above": 1024, + "type": "keyword" + }, + "bucket_owner": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes_sent": { + "type": "long" + }, + "cipher_suite": { + "ignore_above": 1024, + "type": "keyword" + }, + "error_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "host_header": { + "ignore_above": 1024, + "type": "keyword" + }, + "host_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "http_status": { + "type": "long" + }, + "key": { + "ignore_above": 1024, + "type": "keyword" + }, + "object_size": { + "type": "long" + }, + "operation": { + "ignore_above": 1024, + "type": "keyword" + }, + "referrer": { + "ignore_above": 1024, + "type": "keyword" + }, + "remote_ip": { + "type": "ip" + }, + "request_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "request_uri": { + "ignore_above": 1024, + "type": "keyword" + }, + "requester": { + "ignore_above": 1024, + "type": "keyword" + }, + "signature_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "tls_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "total_time": { + "type": "long" + }, + "turn_around_time": { + "type": "long" + }, + "user_agent": { + "ignore_above": 1024, + "type": "keyword" + }, + "version_id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vpcflow": { + "properties": { + "account_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "instance_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "interface_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "log_status": { + "ignore_above": 1024, + "type": "keyword" + }, + "pkt_dstaddr": { + "type": "ip" + }, + "pkt_srcaddr": { + "type": "ip" + }, + "subnet_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "tcp_flags": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "vpc_id": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "azure": { + "properties": { + "activitylogs": { + "properties": { + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "identity": { + "properties": { + "authorization": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "evidence": { + "properties": { + "principal_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "principal_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "role": { + "ignore_above": 1024, + "type": "keyword" + }, + "role_assignment_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "role_assignment_scope": { + "ignore_above": 1024, + "type": "keyword" + }, + "role_definition_id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "scope": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "claims": { + "properties": { + "*": { + "type": "object" + } + } + }, + "claims_initiated_by_user": { + "properties": { + "fullname": { + "ignore_above": 1024, + "type": "keyword" + }, + "givenname": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "schema": { + "ignore_above": 1024, + "type": "keyword" + }, + "surname": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "operation_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "properties": { + "properties": { + "service_request_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "status_code": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "result_signature": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "auditlogs": { + "properties": { + "identity": { + "ignore_above": 1024, + "type": "keyword" + }, + "operation_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "operation_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "properties": { + "properties": { + "activity_datetime": { + "type": "date" + }, + "activity_display_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "correlation_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "initiated_by": { + "properties": { + "app": { + "properties": { + "appId": { + "ignore_above": 1024, + "type": "keyword" + }, + "displayName": { + "ignore_above": 1024, + "type": "keyword" + }, + "servicePrincipalId": { + "ignore_above": 1024, + "type": "keyword" + }, + "servicePrincipalName": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user": { + "properties": { + "displayName": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ipAddress": { + "ignore_above": 1024, + "type": "keyword" + }, + "userPrincipalName": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "logged_by_service": { + "ignore_above": 1024, + "type": "keyword" + }, + "operation_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "result": { + "ignore_above": 1024, + "type": "keyword" + }, + "result_reason": { + "ignore_above": 1024, + "type": "keyword" + }, + "target_resources": { + "properties": { + "*": { + "properties": { + "display_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "modified_properties": { + "properties": { + "*": { + "properties": { + "display_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "new_value": { + "ignore_above": 1024, + "type": "keyword" + }, + "old_value": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "user_principal_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "result_signature": { + "ignore_above": 1024, + "type": "keyword" + }, + "tenant_id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "consumer_group": { + "ignore_above": 1024, + "type": "keyword" + }, + "correlation_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "enqueued_time": { + "type": "date" + }, + "eventhub": { + "ignore_above": 1024, + "type": "keyword" + }, + "offset": { + "type": "long" + }, + "partition_id": { + "type": "long" + }, + "resource": { + "properties": { + "authorization_rule": { + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "namespace": { + "ignore_above": 1024, + "type": "keyword" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "sequence_number": { + "type": "long" + }, + "signinlogs": { + "properties": { + "identity": { + "ignore_above": 1024, + "type": "keyword" + }, + "operation_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "operation_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "properties": { + "properties": { + "app_display_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "app_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "client_app_used": { + "ignore_above": 1024, + "type": "keyword" + }, + "conditional_access_status": { + "ignore_above": 1024, + "type": "keyword" + }, + "correlation_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "created_at": { + "type": "date" + }, + "device_detail": { + "properties": { + "browser": { + "ignore_above": 1024, + "type": "keyword" + }, + "device_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "display_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "operating_system": { + "ignore_above": 1024, + "type": "keyword" + }, + "trust_type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "is_interactive": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_request_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "processing_time_ms": { + "type": "float" + }, + "resource_display_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "risk_detail": { + "ignore_above": 1024, + "type": "keyword" + }, + "risk_level_aggregated": { + "ignore_above": 1024, + "type": "keyword" + }, + "risk_level_during_signin": { + "ignore_above": 1024, + "type": "keyword" + }, + "risk_state": { + "ignore_above": 1024, + "type": "keyword" + }, + "service_principal_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "status": { + "properties": { + "error_code": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "token_issuer_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "token_issuer_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "user_display_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "user_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "user_principal_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "result_description": { + "ignore_above": 1024, + "type": "keyword" + }, + "result_signature": { + "ignore_above": 1024, + "type": "keyword" + }, + "tenant_id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "subscription_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "tenant_id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bucket_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "cef": { + "properties": { + "device": { + "properties": { + "event_class_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + }, + "vendor": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "extensions": { + "properties": { + "Reason": { + "ignore_above": 1024, + "type": "keyword" + }, + "agentAddress": { + "type": "ip" + }, + "agentDnsDomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "agentHostName": { + "ignore_above": 1024, + "type": "keyword" + }, + "agentId": { + "ignore_above": 1024, + "type": "keyword" + }, + "agentMacAddress": { + "ignore_above": 1024, + "type": "keyword" + }, + "agentNtDomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "agentReceiptTime": { + "type": "date" + }, + "agentTimeZone": { + "ignore_above": 1024, + "type": "keyword" + }, + "agentTranslatedAddress": { + "type": "ip" + }, + "agentTranslatedZoneExternalID": { + "ignore_above": 1024, + "type": "keyword" + }, + "agentTranslatedZoneURI": { + "ignore_above": 1024, + "type": "keyword" + }, + "agentType": { + "ignore_above": 1024, + "type": "keyword" + }, + "agentVersion": { + "ignore_above": 1024, + "type": "keyword" + }, + "agentZoneExternalID": { + "ignore_above": 1024, + "type": "keyword" + }, + "agentZoneURI": { + "ignore_above": 1024, + "type": "keyword" + }, + "applicationProtocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "baseEventCount": { + "type": "long" + }, + "bytesIn": { + "type": "long" + }, + "bytesOut": { + "type": "long" + }, + "categoryBehavior": { + "ignore_above": 1024, + "type": "keyword" + }, + "categoryDeviceGroup": { + "ignore_above": 1024, + "type": "keyword" + }, + "categoryDeviceType": { + "ignore_above": 1024, + "type": "keyword" + }, + "categoryObject": { + "ignore_above": 1024, + "type": "keyword" + }, + "categoryOutcome": { + "ignore_above": 1024, + "type": "keyword" + }, + "categorySignificance": { + "ignore_above": 1024, + "type": "keyword" + }, + "categoryTechnique": { + "ignore_above": 1024, + "type": "keyword" + }, + "cp_app_risk": { + "ignore_above": 1024, + "type": "keyword" + }, + "cp_severity": { + "ignore_above": 1024, + "type": "keyword" + }, + "customerExternalID": { + "ignore_above": 1024, + "type": "keyword" + }, + "customerURI": { + "ignore_above": 1024, + "type": "keyword" + }, + "destinationAddress": { + "type": "ip" + }, + "destinationDnsDomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "destinationGeoLatitude": { + "type": "double" + }, + "destinationGeoLongitude": { + "type": "double" + }, + "destinationHostName": { + "ignore_above": 1024, + "type": "keyword" + }, + "destinationMacAddress": { + "ignore_above": 1024, + "type": "keyword" + }, + "destinationNtDomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "destinationPort": { + "type": "long" + }, + "destinationProcessId": { + "type": "long" + }, + "destinationProcessName": { + "ignore_above": 1024, + "type": "keyword" + }, + "destinationServiceName": { + "ignore_above": 1024, + "type": "keyword" + }, + "destinationTranslatedAddress": { + "type": "ip" + }, + "destinationTranslatedPort": { + "type": "long" + }, + "destinationTranslatedZoneExternalID": { + "ignore_above": 1024, + "type": "keyword" + }, + "destinationTranslatedZoneURI": { + "ignore_above": 1024, + "type": "keyword" + }, + "destinationUserId": { + "ignore_above": 1024, + "type": "keyword" + }, + "destinationUserName": { + "ignore_above": 1024, + "type": "keyword" + }, + "destinationUserPrivileges": { + "ignore_above": 1024, + "type": "keyword" + }, + "destinationZoneExternalID": { + "ignore_above": 1024, + "type": "keyword" + }, + "destinationZoneURI": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceAction": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceAddress": { + "type": "ip" + }, + "deviceCustomDate1": { + "type": "date" + }, + "deviceCustomDate1Label": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceCustomDate2": { + "type": "date" + }, + "deviceCustomDate2Label": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceCustomFloatingPoint1": { + "type": "double" + }, + "deviceCustomFloatingPoint1Label": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceCustomFloatingPoint2": { + "type": "double" + }, + "deviceCustomFloatingPoint2Label": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceCustomFloatingPoint3": { + "type": "double" + }, + "deviceCustomFloatingPoint3Label": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceCustomFloatingPoint4": { + "type": "double" + }, + "deviceCustomFloatingPoint4Label": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceCustomIPv6Address1": { + "type": "ip" + }, + "deviceCustomIPv6Address1Label": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceCustomIPv6Address2": { + "type": "ip" + }, + "deviceCustomIPv6Address2Label": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceCustomIPv6Address3": { + "type": "ip" + }, + "deviceCustomIPv6Address3Label": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceCustomIPv6Address4": { + "type": "ip" + }, + "deviceCustomIPv6Address4Label": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceCustomNumber1": { + "type": "long" + }, + "deviceCustomNumber1Label": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceCustomNumber2": { + "type": "long" + }, + "deviceCustomNumber2Label": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceCustomNumber3": { + "type": "long" + }, + "deviceCustomNumber3Label": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceCustomString1": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceCustomString1Label": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceCustomString2": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceCustomString2Label": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceCustomString3": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceCustomString3Label": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceCustomString4": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceCustomString4Label": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceCustomString5": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceCustomString5Label": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceCustomString6": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceCustomString6Label": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceDirection": { + "type": "long" + }, + "deviceDnsDomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceEventCategory": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceExternalId": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceFacility": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceFlexNumber1": { + "type": "long" + }, + "deviceFlexNumber1Label": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceFlexNumber2": { + "type": "long" + }, + "deviceFlexNumber2Label": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceHostName": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceInboundInterface": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceMacAddress": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceNtDomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceOutboundInterface": { + "ignore_above": 1024, + "type": "keyword" + }, + "devicePayloadId": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceProcessId": { + "type": "long" + }, + "deviceProcessName": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceReceiptTime": { + "type": "date" + }, + "deviceTimeZone": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceTranslatedAddress": { + "type": "ip" + }, + "deviceTranslatedZoneExternalID": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceTranslatedZoneURI": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceZoneExternalID": { + "ignore_above": 1024, + "type": "keyword" + }, + "deviceZoneURI": { + "ignore_above": 1024, + "type": "keyword" + }, + "endTime": { + "type": "date" + }, + "eventId": { + "type": "long" + }, + "eventOutcome": { + "ignore_above": 1024, + "type": "keyword" + }, + "externalId": { + "ignore_above": 1024, + "type": "keyword" + }, + "fileCreateTime": { + "type": "date" + }, + "fileHash": { + "ignore_above": 1024, + "type": "keyword" + }, + "fileId": { + "ignore_above": 1024, + "type": "keyword" + }, + "fileModificationTime": { + "type": "date" + }, + "filePath": { + "ignore_above": 1024, + "type": "keyword" + }, + "filePermission": { + "ignore_above": 1024, + "type": "keyword" + }, + "fileSize": { + "type": "long" + }, + "fileType": { + "ignore_above": 1024, + "type": "keyword" + }, + "filename": { + "ignore_above": 1024, + "type": "keyword" + }, + "flexDate1": { + "type": "date" + }, + "flexDate1Label": { + "ignore_above": 1024, + "type": "keyword" + }, + "flexString1": { + "ignore_above": 1024, + "type": "keyword" + }, + "flexString1Label": { + "ignore_above": 1024, + "type": "keyword" + }, + "flexString2": { + "ignore_above": 1024, + "type": "keyword" + }, + "flexString2Label": { + "ignore_above": 1024, + "type": "keyword" + }, + "ifname": { + "ignore_above": 1024, + "type": "keyword" + }, + "inzone": { + "ignore_above": 1024, + "type": "keyword" + }, + "layer_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "layer_uuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "logid": { + "ignore_above": 1024, + "type": "keyword" + }, + "loguid": { + "ignore_above": 1024, + "type": "keyword" + }, + "managerReceiptTime": { + "type": "date" + }, + "match_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat_addtnl_rulenum": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat_rulenum": { + "ignore_above": 1024, + "type": "keyword" + }, + "oldFileCreateTime": { + "type": "date" + }, + "oldFileHash": { + "ignore_above": 1024, + "type": "keyword" + }, + "oldFileId": { + "ignore_above": 1024, + "type": "keyword" + }, + "oldFileModificationTime": { + "type": "date" + }, + "oldFileName": { + "ignore_above": 1024, + "type": "keyword" + }, + "oldFilePath": { + "ignore_above": 1024, + "type": "keyword" + }, + "oldFilePermission": { + "ignore_above": 1024, + "type": "keyword" + }, + "oldFileSize": { + "type": "long" + }, + "oldFileType": { + "ignore_above": 1024, + "type": "keyword" + }, + "origin": { + "ignore_above": 1024, + "type": "keyword" + }, + "originsicname": { + "ignore_above": 1024, + "type": "keyword" + }, + "outzone": { + "ignore_above": 1024, + "type": "keyword" + }, + "parent_rule": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + }, + "rawEvent": { + "ignore_above": 1024, + "type": "keyword" + }, + "requestClientApplication": { + "ignore_above": 1024, + "type": "keyword" + }, + "requestContext": { + "ignore_above": 1024, + "type": "keyword" + }, + "requestCookies": { + "ignore_above": 1024, + "type": "keyword" + }, + "requestMethod": { + "ignore_above": 1024, + "type": "keyword" + }, + "requestUrl": { + "ignore_above": 1024, + "type": "keyword" + }, + "rule_action": { + "ignore_above": 1024, + "type": "keyword" + }, + "rule_uid": { + "ignore_above": 1024, + "type": "keyword" + }, + "sequencenum": { + "ignore_above": 1024, + "type": "keyword" + }, + "service_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "sourceAddress": { + "type": "ip" + }, + "sourceDnsDomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "sourceGeoLatitude": { + "type": "double" + }, + "sourceGeoLongitude": { + "type": "double" + }, + "sourceHostName": { + "ignore_above": 1024, + "type": "keyword" + }, + "sourceMacAddress": { + "ignore_above": 1024, + "type": "keyword" + }, + "sourceNtDomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "sourcePort": { + "type": "long" + }, + "sourceProcessId": { + "type": "long" + }, + "sourceProcessName": { + "ignore_above": 1024, + "type": "keyword" + }, + "sourceServiceName": { + "ignore_above": 1024, + "type": "keyword" + }, + "sourceTranslatedAddress": { + "type": "ip" + }, + "sourceTranslatedPort": { + "type": "long" + }, + "sourceTranslatedZoneExternalID": { + "ignore_above": 1024, + "type": "keyword" + }, + "sourceTranslatedZoneURI": { + "ignore_above": 1024, + "type": "keyword" + }, + "sourceUserId": { + "ignore_above": 1024, + "type": "keyword" + }, + "sourceUserName": { + "ignore_above": 1024, + "type": "keyword" + }, + "sourceUserPrivileges": { + "ignore_above": 1024, + "type": "keyword" + }, + "sourceZoneExternalID": { + "ignore_above": 1024, + "type": "keyword" + }, + "sourceZoneURI": { + "ignore_above": 1024, + "type": "keyword" + }, + "startTime": { + "type": "date" + }, + "transportProtocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "type": "long" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "severity": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "certificate": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "checkpoint": { + "properties": { + "app_risk": { + "ignore_above": 1024, + "type": "keyword" + }, + "app_severity": { + "ignore_above": 1024, + "type": "keyword" + }, + "app_sig_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "auth_method": { + "ignore_above": 1024, + "type": "keyword" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "confidence_level": { + "type": "long" + }, + "connectivity_state": { + "ignore_above": 1024, + "type": "keyword" + }, + "cookie": { + "ignore_above": 1024, + "type": "keyword" + }, + "dst_phone_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "email_control": { + "ignore_above": 1024, + "type": "keyword" + }, + "email_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "email_recipients_num": { + "type": "long" + }, + "email_session_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "email_spool_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "email_subject": { + "ignore_above": 1024, + "type": "keyword" + }, + "event_count": { + "type": "long" + }, + "frequency": { + "ignore_above": 1024, + "type": "keyword" + }, + "icmp_code": { + "type": "long" + }, + "icmp_type": { + "type": "long" + }, + "identity_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "incident_extension": { + "ignore_above": 1024, + "type": "keyword" + }, + "integrity_av_invoke_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "malware_family": { + "ignore_above": 1024, + "type": "keyword" + }, + "peer_gateway": { + "type": "ip" + }, + "performance_impact": { + "type": "long" + }, + "protection_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "protection_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "protection_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "scan_result": { + "ignore_above": 1024, + "type": "keyword" + }, + "sensor_mode": { + "ignore_above": 1024, + "type": "keyword" + }, + "severity": { + "ignore_above": 1024, + "type": "keyword" + }, + "spyware_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "spyware_status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subs_exp": { + "type": "date" + }, + "tcp_flags": { + "ignore_above": 1024, + "type": "keyword" + }, + "termination_reason": { + "ignore_above": 1024, + "type": "keyword" + }, + "update_status": { + "ignore_above": 1024, + "type": "keyword" + }, + "user_status": { + "ignore_above": 1024, + "type": "keyword" + }, + "uuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "virus_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "voip_log_type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "cisco": { + "properties": { + "asa": { + "properties": { + "connection_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "connection_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "dap_records": { + "ignore_above": 1024, + "type": "keyword" + }, + "destination_interface": { + "ignore_above": 1024, + "type": "keyword" + }, + "destination_username": { + "ignore_above": 1024, + "type": "keyword" + }, + "icmp_code": { + "type": "short" + }, + "icmp_type": { + "type": "short" + }, + "mapped_destination_ip": { + "type": "ip" + }, + "mapped_destination_port": { + "type": "long" + }, + "mapped_source_ip": { + "type": "ip" + }, + "mapped_source_port": { + "type": "long" + }, + "message_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "rule_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "source_interface": { + "ignore_above": 1024, + "type": "keyword" + }, + "source_username": { + "ignore_above": 1024, + "type": "keyword" + }, + "suffix": { + "ignore_above": 1024, + "type": "keyword" + }, + "threat_category": { + "ignore_above": 1024, + "type": "keyword" + }, + "threat_level": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ftd": { + "properties": { + "connection_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "connection_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "dap_records": { + "ignore_above": 1024, + "type": "keyword" + }, + "destination_interface": { + "ignore_above": 1024, + "type": "keyword" + }, + "destination_username": { + "ignore_above": 1024, + "type": "keyword" + }, + "icmp_code": { + "type": "short" + }, + "icmp_type": { + "type": "short" + }, + "mapped_destination_ip": { + "type": "ip" + }, + "mapped_destination_port": { + "type": "long" + }, + "mapped_source_ip": { + "type": "ip" + }, + "mapped_source_port": { + "type": "long" + }, + "message_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "rule_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "security": { + "type": "object" + }, + "source_interface": { + "ignore_above": 1024, + "type": "keyword" + }, + "source_username": { + "ignore_above": 1024, + "type": "keyword" + }, + "suffix": { + "ignore_above": 1024, + "type": "keyword" + }, + "threat_category": { + "ignore_above": 1024, + "type": "keyword" + }, + "threat_level": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ios": { + "properties": { + "access_list": { + "ignore_above": 1024, + "type": "keyword" + }, + "facility": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "client": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "cloud": { + "properties": { + "account": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "availability_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "image": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "instance": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "machine": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "project": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "container": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "image": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "tag": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "labels": { + "type": "object" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "runtime": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "coredns": { + "properties": { + "dnssec_ok": { + "type": "boolean" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "query": { + "properties": { + "class": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "size": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "response": { + "properties": { + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "flags": { + "ignore_above": 1024, + "type": "keyword" + }, + "size": { + "type": "long" + } + } + } + } + }, + "destination": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "service": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "dll": { + "properties": { + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "pe": { + "properties": { + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "dns": { + "properties": { + "answers": { + "properties": { + "class": { + "ignore_above": 1024, + "type": "keyword" + }, + "data": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "ttl": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "header_flags": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "op_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "question": { + "properties": { + "class": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "subdomain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "resolved_ip": { + "type": "ip" + }, + "response_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "docker": { + "properties": { + "attrs": { + "type": "object" + }, + "container": { + "properties": { + "labels": { + "type": "object" + } + } + } + } + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "elasticsearch": { + "properties": { + "audit": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "event_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "indices": { + "ignore_above": 1024, + "type": "keyword" + }, + "layer": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "origin": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "realm": { + "ignore_above": 1024, + "type": "keyword" + }, + "request": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "url": { + "properties": { + "params": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user": { + "properties": { + "realm": { + "ignore_above": 1024, + "type": "keyword" + }, + "roles": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "cluster": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "uuid": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "component": { + "ignore_above": 1024, + "type": "keyword" + }, + "deprecation": { + "type": "object" + }, + "gc": { + "properties": { + "heap": { + "properties": { + "size_kb": { + "type": "long" + }, + "used_kb": { + "type": "long" + } + } + }, + "jvm_runtime_sec": { + "type": "float" + }, + "old_gen": { + "properties": { + "size_kb": { + "type": "long" + }, + "used_kb": { + "type": "long" + } + } + }, + "phase": { + "properties": { + "class_unload_time_sec": { + "type": "float" + }, + "cpu_time": { + "properties": { + "real_sec": { + "type": "float" + }, + "sys_sec": { + "type": "float" + }, + "user_sec": { + "type": "float" + } + } + }, + "duration_sec": { + "type": "float" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "parallel_rescan_time_sec": { + "type": "float" + }, + "scrub_string_table_time_sec": { + "type": "float" + }, + "scrub_symbol_table_time_sec": { + "type": "float" + }, + "weak_refs_processing_time_sec": { + "type": "float" + } + } + }, + "stopping_threads_time_sec": { + "type": "float" + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "threads_total_stop_time_sec": { + "type": "float" + }, + "young_gen": { + "properties": { + "size_kb": { + "type": "long" + }, + "used_kb": { + "type": "long" + } + } + } + } + }, + "index": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "node": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "server": { + "properties": { + "gc": { + "properties": { + "collection_duration": { + "properties": { + "ms": { + "type": "float" + } + } + }, + "observation_duration": { + "properties": { + "ms": { + "type": "float" + } + } + }, + "overhead_seq": { + "type": "long" + }, + "young": { + "properties": { + "one": { + "type": "long" + }, + "two": { + "type": "long" + } + } + } + } + }, + "stacktrace": { + "ignore_above": 1024, + "index": false, + "type": "keyword" + } + } + }, + "shard": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "slowlog": { + "properties": { + "extra_source": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "logger": { + "ignore_above": 1024, + "type": "keyword" + }, + "routing": { + "ignore_above": 1024, + "type": "keyword" + }, + "search_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "source": { + "ignore_above": 1024, + "type": "keyword" + }, + "source_query": { + "ignore_above": 1024, + "type": "keyword" + }, + "stats": { + "ignore_above": 1024, + "type": "keyword" + }, + "took": { + "ignore_above": 1024, + "type": "keyword" + }, + "total_hits": { + "ignore_above": 1024, + "type": "keyword" + }, + "total_shards": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "types": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "envoyproxy": { + "properties": { + "authority": { + "ignore_above": 1024, + "type": "keyword" + }, + "log_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "proxy_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "request_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "response_flags": { + "ignore_above": 1024, + "type": "keyword" + }, + "upstream_service_time": { + "type": "long" + } + } + }, + "error": { + "properties": { + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "stack_trace": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "event": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "created": { + "type": "date" + }, + "dataset": { + "ignore_above": 1024, + "type": "keyword" + }, + "duration": { + "type": "long" + }, + "end": { + "type": "date" + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ingested": { + "type": "date" + }, + "kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "ignore_above": 1024, + "type": "keyword" + }, + "outcome": { + "ignore_above": 1024, + "type": "keyword" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "risk_score": { + "type": "float" + }, + "risk_score_norm": { + "type": "float" + }, + "sequence": { + "type": "long" + }, + "severity": { + "type": "long" + }, + "start": { + "type": "date" + }, + "timezone": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "fields": { + "type": "object" + }, + "file": { + "properties": { + "accessed": { + "type": "date" + }, + "attributes": { + "ignore_above": 1024, + "type": "keyword" + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "created": { + "type": "date" + }, + "ctime": { + "type": "date" + }, + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "directory": { + "ignore_above": 1024, + "type": "keyword" + }, + "drive_letter": { + "ignore_above": 1, + "type": "keyword" + }, + "extension": { + "ignore_above": 1024, + "type": "keyword" + }, + "gid": { + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "inode": { + "ignore_above": 1024, + "type": "keyword" + }, + "mime_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "mode": { + "ignore_above": 1024, + "type": "keyword" + }, + "mtime": { + "type": "date" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "owner": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "pe": { + "properties": { + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "size": { + "type": "long" + }, + "target_path": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "fileset": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "forcepoint": { + "properties": { + "virus_id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "googlecloud": { + "properties": { + "audit": { + "properties": { + "authentication_info": { + "properties": { + "authority_selector": { + "ignore_above": 1024, + "type": "keyword" + }, + "principal_email": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "method_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "num_response_items": { + "type": "long" + }, + "request": { + "properties": { + "filter": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "proto_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "resource_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "request_metadata": { + "properties": { + "caller_ip": { + "type": "ip" + }, + "caller_supplied_user_agent": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "resource_location": { + "properties": { + "current_locations": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "resource_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "service_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "status": { + "properties": { + "code": { + "type": "long" + }, + "message": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "destination": { + "properties": { + "instance": { + "properties": { + "project_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + }, + "zone": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vpc": { + "properties": { + "project_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "subnetwork_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "vpc_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "firewall": { + "properties": { + "rule_details": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "destination_range": { + "ignore_above": 1024, + "type": "keyword" + }, + "direction": { + "ignore_above": 1024, + "type": "keyword" + }, + "priority": { + "type": "long" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "source_range": { + "ignore_above": 1024, + "type": "keyword" + }, + "source_service_account": { + "ignore_above": 1024, + "type": "keyword" + }, + "source_tag": { + "ignore_above": 1024, + "type": "keyword" + }, + "target_service_account": { + "ignore_above": 1024, + "type": "keyword" + }, + "target_tag": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "source": { + "properties": { + "instance": { + "properties": { + "project_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "region": { + "ignore_above": 1024, + "type": "keyword" + }, + "zone": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vpc": { + "properties": { + "project_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "subnetwork_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "vpc_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "vpcflow": { + "properties": { + "reporter": { + "ignore_above": 1024, + "type": "keyword" + }, + "rtt": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + } + } + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "haproxy": { + "properties": { + "backend_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "backend_queue": { + "type": "long" + }, + "bind_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes_read": { + "type": "long" + }, + "client": { + "type": "object" + }, + "connection_wait_time_ms": { + "type": "long" + }, + "connections": { + "properties": { + "active": { + "type": "long" + }, + "backend": { + "type": "long" + }, + "frontend": { + "type": "long" + }, + "retries": { + "type": "long" + }, + "server": { + "type": "long" + } + } + }, + "destination": { + "type": "object" + }, + "error_message": { + "norms": false, + "type": "text" + }, + "frontend_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "geoip": { + "type": "object" + }, + "http": { + "properties": { + "request": { + "properties": { + "captured_cookie": { + "ignore_above": 1024, + "type": "keyword" + }, + "captured_headers": { + "ignore_above": 1024, + "type": "keyword" + }, + "raw_request_line": { + "ignore_above": 1024, + "type": "keyword" + }, + "time_wait_ms": { + "type": "long" + }, + "time_wait_without_data_ms": { + "type": "long" + } + } + }, + "response": { + "properties": { + "captured_cookie": { + "ignore_above": 1024, + "type": "keyword" + }, + "captured_headers": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "mode": { + "ignore_above": 1024, + "type": "keyword" + }, + "server_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "server_queue": { + "type": "long" + }, + "source": { + "ignore_above": 1024, + "type": "keyword" + }, + "tcp": { + "properties": { + "connection_waiting_time_ms": { + "type": "long" + } + } + }, + "termination_state": { + "ignore_above": 1024, + "type": "keyword" + }, + "time_backend_connect": { + "type": "long" + }, + "time_queue": { + "type": "long" + }, + "total_waiting_time_ms": { + "type": "long" + } + } + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "host": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "containerized": { + "type": "boolean" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "build": { + "ignore_above": 1024, + "type": "keyword" + }, + "codename": { + "ignore_above": 1024, + "type": "keyword" + }, + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "http": { + "properties": { + "request": { + "properties": { + "body": { + "properties": { + "bytes": { + "type": "long" + }, + "content": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bytes": { + "type": "long" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "referrer": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "response": { + "properties": { + "body": { + "properties": { + "bytes": { + "type": "long" + }, + "content": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bytes": { + "type": "long" + }, + "status_code": { + "type": "long" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ibmmq": { + "properties": { + "errorlog": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "arithinsert": { + "ignore_above": 1024, + "type": "keyword" + }, + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "commentinsert": { + "ignore_above": 1024, + "type": "keyword" + }, + "errordescription": { + "norms": false, + "type": "text" + }, + "explanation": { + "ignore_above": 1024, + "type": "keyword" + }, + "installation": { + "ignore_above": 1024, + "type": "keyword" + }, + "qmgr": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "icinga": { + "properties": { + "debug": { + "properties": { + "facility": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "main": { + "properties": { + "facility": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "startup": { + "properties": { + "facility": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "icmp": { + "properties": { + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "igmp": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "iis": { + "properties": { + "access": { + "properties": { + "cookie": { + "ignore_above": 1024, + "type": "keyword" + }, + "geoip": { + "type": "object" + }, + "server_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "site_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "sub_status": { + "type": "long" + }, + "user_agent": { + "type": "object" + }, + "win32_status": { + "type": "long" + } + } + }, + "error": { + "properties": { + "geoip": { + "type": "object" + }, + "queue_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "reason_phrase": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "input": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "interface": { + "properties": { + "alias": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "iptables": { + "properties": { + "ether_type": { + "type": "long" + }, + "flow_label": { + "type": "long" + }, + "fragment_flags": { + "ignore_above": 1024, + "type": "keyword" + }, + "fragment_offset": { + "type": "long" + }, + "icmp": { + "properties": { + "code": { + "type": "long" + }, + "id": { + "type": "long" + }, + "parameter": { + "type": "long" + }, + "redirect": { + "type": "ip" + }, + "seq": { + "type": "long" + }, + "type": { + "type": "long" + } + } + }, + "id": { + "type": "long" + }, + "incomplete_bytes": { + "type": "long" + }, + "input_device": { + "ignore_above": 1024, + "type": "keyword" + }, + "length": { + "type": "long" + }, + "output_device": { + "ignore_above": 1024, + "type": "keyword" + }, + "precedence_bits": { + "type": "short" + }, + "tcp": { + "properties": { + "ack": { + "type": "long" + }, + "flags": { + "ignore_above": 1024, + "type": "keyword" + }, + "reserved_bits": { + "type": "short" + }, + "seq": { + "type": "long" + }, + "window": { + "type": "long" + } + } + }, + "tos": { + "type": "long" + }, + "ttl": { + "type": "long" + }, + "ubiquiti": { + "properties": { + "input_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "output_zone": { + "ignore_above": 1024, + "type": "keyword" + }, + "rule_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "rule_set": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "udp": { + "properties": { + "length": { + "type": "long" + } + } + } + } + }, + "jolokia": { + "properties": { + "agent": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "secured": { + "type": "boolean" + }, + "server": { + "properties": { + "product": { + "ignore_above": 1024, + "type": "keyword" + }, + "vendor": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "kafka": { + "properties": { + "block_timestamp": { + "type": "date" + }, + "key": { + "ignore_above": 1024, + "type": "keyword" + }, + "log": { + "properties": { + "class": { + "ignore_above": 1024, + "type": "keyword" + }, + "component": { + "ignore_above": 1024, + "type": "keyword" + }, + "trace": { + "properties": { + "class": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + } + } + } + } + }, + "offset": { + "type": "long" + }, + "partition": { + "type": "long" + }, + "topic": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "kibana": { + "properties": { + "log": { + "properties": { + "meta": { + "type": "object" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "kubernetes": { + "properties": { + "annotations": { + "properties": { + "*": { + "type": "object" + } + } + }, + "container": { + "properties": { + "image": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "deployment": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "labels": { + "properties": { + "*": { + "type": "object" + } + } + }, + "namespace": { + "ignore_above": 1024, + "type": "keyword" + }, + "node": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pod": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "replicaset": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "statefulset": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "labels": { + "type": "object" + }, + "log": { + "properties": { + "file": { + "properties": { + "path": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "flags": { + "ignore_above": 1024, + "type": "keyword" + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "logger": { + "ignore_above": 1024, + "type": "keyword" + }, + "offset": { + "type": "long" + }, + "origin": { + "properties": { + "file": { + "properties": { + "line": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "function": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "original": { + "ignore_above": 1024, + "type": "keyword" + }, + "source": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "syslog": { + "properties": { + "facility": { + "properties": { + "code": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "priority": { + "type": "long" + }, + "severity": { + "properties": { + "code": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "logstash": { + "properties": { + "log": { + "properties": { + "log_event": { + "type": "object" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "pipeline_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "thread": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "slowlog": { + "properties": { + "event": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "plugin_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "plugin_params": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "plugin_params_object": { + "type": "object" + }, + "plugin_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "thread": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "took_in_millis": { + "type": "long" + } + } + } + } + }, + "message": { + "norms": false, + "type": "text" + }, + "misp": { + "properties": { + "attack_pattern": { + "properties": { + "description": { + "norms": false, + "type": "text" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "kill_chain_phases": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "campaign": { + "properties": { + "aliases": { + "norms": false, + "type": "text" + }, + "description": { + "norms": false, + "type": "text" + }, + "first_seen": { + "type": "date" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "last_seen": { + "type": "date" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "objective": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "course_of_action": { + "properties": { + "description": { + "norms": false, + "type": "text" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "identity": { + "properties": { + "contact_information": { + "norms": false, + "type": "text" + }, + "description": { + "norms": false, + "type": "text" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "identity_class": { + "ignore_above": 1024, + "type": "keyword" + }, + "labels": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "sectors": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "intrusion_set": { + "properties": { + "aliases": { + "norms": false, + "type": "text" + }, + "description": { + "norms": false, + "type": "text" + }, + "first_seen": { + "type": "date" + }, + "goals": { + "norms": false, + "type": "text" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "last_seen": { + "type": "date" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "primary_motivation": { + "norms": false, + "type": "text" + }, + "resource_level": { + "norms": false, + "type": "text" + }, + "secondary_motivations": { + "norms": false, + "type": "text" + } + } + }, + "malware": { + "properties": { + "description": { + "norms": false, + "type": "text" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "kill_chain_phases": { + "ignore_above": 1024, + "type": "keyword" + }, + "labels": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "note": { + "properties": { + "authors": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "norms": false, + "type": "text" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "object_refs": { + "ignore_above": 1024, + "type": "keyword" + }, + "summary": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "observed_data": { + "properties": { + "first_observed": { + "type": "date" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "last_observed": { + "type": "date" + }, + "number_observed": { + "type": "long" + }, + "objects": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "report": { + "properties": { + "description": { + "norms": false, + "type": "text" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "labels": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "object_refs": { + "norms": false, + "type": "text" + }, + "published": { + "type": "date" + } + } + }, + "threat_actor": { + "properties": { + "aliases": { + "norms": false, + "type": "text" + }, + "description": { + "norms": false, + "type": "text" + }, + "goals": { + "norms": false, + "type": "text" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "labels": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "personal_motivations": { + "norms": false, + "type": "text" + }, + "primary_motivation": { + "norms": false, + "type": "text" + }, + "resource_level": { + "norms": false, + "type": "text" + }, + "roles": { + "norms": false, + "type": "text" + }, + "secondary_motivations": { + "norms": false, + "type": "text" + }, + "sophistication": { + "norms": false, + "type": "text" + } + } + }, + "threat_indicator": { + "properties": { + "attack_pattern": { + "ignore_above": 1024, + "type": "keyword" + }, + "attack_pattern_kql": { + "ignore_above": 1024, + "type": "keyword" + }, + "campaign": { + "ignore_above": 1024, + "type": "keyword" + }, + "confidence": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "norms": false, + "type": "text" + }, + "feed": { + "norms": false, + "type": "text" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "intrusion_set": { + "ignore_above": 1024, + "type": "keyword" + }, + "kill_chain_phases": { + "ignore_above": 1024, + "type": "keyword" + }, + "labels": { + "ignore_above": 1024, + "type": "keyword" + }, + "mitre_tactic": { + "ignore_above": 1024, + "type": "keyword" + }, + "mitre_technique": { + "ignore_above": 1024, + "type": "keyword" + }, + "negate": { + "type": "boolean" + }, + "severity": { + "ignore_above": 1024, + "type": "keyword" + }, + "threat_actor": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "valid_from": { + "type": "date" + }, + "valid_until": { + "type": "date" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "tool": { + "properties": { + "description": { + "norms": false, + "type": "text" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "kill_chain_phases": { + "norms": false, + "type": "text" + }, + "labels": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "tool_version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vulnerability": { + "properties": { + "description": { + "norms": false, + "type": "text" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "mongodb": { + "properties": { + "log": { + "properties": { + "component": { + "ignore_above": 1024, + "type": "keyword" + }, + "context": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "mssql": { + "properties": { + "log": { + "properties": { + "origin": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "mysql": { + "properties": { + "error": { + "type": "object" + }, + "slowlog": { + "properties": { + "bytes_received": { + "type": "long" + }, + "bytes_sent": { + "type": "long" + }, + "current_user": { + "ignore_above": 1024, + "type": "keyword" + }, + "filesort": { + "type": "boolean" + }, + "filesort_on_disk": { + "type": "boolean" + }, + "full_join": { + "type": "boolean" + }, + "full_scan": { + "type": "boolean" + }, + "innodb": { + "properties": { + "io_r_bytes": { + "type": "long" + }, + "io_r_ops": { + "type": "long" + }, + "io_r_wait": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "pages_distinct": { + "type": "long" + }, + "queue_wait": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "rec_lock_wait": { + "properties": { + "sec": { + "type": "long" + } + } + }, + "trx_id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "killed": { + "ignore_above": 1024, + "type": "keyword" + }, + "last_errno": { + "ignore_above": 1024, + "type": "keyword" + }, + "lock_time": { + "properties": { + "sec": { + "type": "float" + } + } + }, + "log_slow_rate_limit": { + "ignore_above": 1024, + "type": "keyword" + }, + "log_slow_rate_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "merge_passes": { + "type": "long" + }, + "priority_queue": { + "type": "boolean" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "query_cache_hit": { + "type": "boolean" + }, + "read_first": { + "type": "long" + }, + "read_key": { + "type": "long" + }, + "read_last": { + "type": "long" + }, + "read_next": { + "type": "long" + }, + "read_prev": { + "type": "long" + }, + "read_rnd": { + "type": "long" + }, + "read_rnd_next": { + "type": "long" + }, + "rows_affected": { + "type": "long" + }, + "rows_examined": { + "type": "long" + }, + "rows_sent": { + "type": "long" + }, + "schema": { + "ignore_above": 1024, + "type": "keyword" + }, + "sort_merge_passes": { + "type": "long" + }, + "sort_range_count": { + "type": "long" + }, + "sort_rows": { + "type": "long" + }, + "sort_scan_count": { + "type": "long" + }, + "tmp_disk_tables": { + "type": "long" + }, + "tmp_table": { + "type": "boolean" + }, + "tmp_table_on_disk": { + "type": "boolean" + }, + "tmp_table_sizes": { + "type": "long" + }, + "tmp_tables": { + "type": "long" + } + } + }, + "thread_id": { + "type": "long" + } + } + }, + "nats": { + "properties": { + "log": { + "properties": { + "client": { + "properties": { + "id": { + "type": "long" + } + } + }, + "msg": { + "properties": { + "bytes": { + "type": "long" + }, + "error": { + "properties": { + "message": { + "norms": false, + "type": "text" + } + } + }, + "max_messages": { + "type": "long" + }, + "queue_group": { + "norms": false, + "type": "text" + }, + "reply_to": { + "ignore_above": 1024, + "type": "keyword" + }, + "sid": { + "type": "long" + }, + "subject": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "netflow": { + "properties": { + "absolute_error": { + "type": "double" + }, + "address_pool_high_threshold": { + "type": "long" + }, + "address_pool_low_threshold": { + "type": "long" + }, + "address_port_mapping_high_threshold": { + "type": "long" + }, + "address_port_mapping_low_threshold": { + "type": "long" + }, + "address_port_mapping_per_user_high_threshold": { + "type": "long" + }, + "anonymization_flags": { + "type": "long" + }, + "anonymization_technique": { + "type": "long" + }, + "application_category_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "application_description": { + "ignore_above": 1024, + "type": "keyword" + }, + "application_group_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "application_id": { + "type": "short" + }, + "application_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "application_sub_category_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "bgp_destination_as_number": { + "type": "long" + }, + "bgp_next_adjacent_as_number": { + "type": "long" + }, + "bgp_next_hop_ipv4_address": { + "type": "ip" + }, + "bgp_next_hop_ipv6_address": { + "type": "ip" + }, + "bgp_prev_adjacent_as_number": { + "type": "long" + }, + "bgp_source_as_number": { + "type": "long" + }, + "bgp_validity_state": { + "type": "short" + }, + "biflow_direction": { + "type": "short" + }, + "class_id": { + "type": "long" + }, + "class_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "classification_engine_id": { + "type": "short" + }, + "collection_time_milliseconds": { + "type": "date" + }, + "collector_certificate": { + "type": "short" + }, + "collector_ipv4_address": { + "type": "ip" + }, + "collector_ipv6_address": { + "type": "ip" + }, + "collector_transport_port": { + "type": "long" + }, + "common_properties_id": { + "type": "long" + }, + "confidence_level": { + "type": "double" + }, + "connection_sum_duration_seconds": { + "type": "long" + }, + "connection_transaction_id": { + "type": "long" + }, + "data_link_frame_section": { + "type": "short" + }, + "data_link_frame_size": { + "type": "long" + }, + "data_link_frame_type": { + "type": "long" + }, + "data_records_reliability": { + "type": "boolean" + }, + "delta_flow_count": { + "type": "long" + }, + "destination_ipv4_address": { + "type": "ip" + }, + "destination_ipv4_prefix": { + "type": "ip" + }, + "destination_ipv4_prefix_length": { + "type": "short" + }, + "destination_ipv6_address": { + "type": "ip" + }, + "destination_ipv6_prefix": { + "type": "ip" + }, + "destination_ipv6_prefix_length": { + "type": "short" + }, + "destination_mac_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "destination_transport_port": { + "type": "long" + }, + "digest_hash_value": { + "type": "long" + }, + "distinct_count_of_destination_ip_address": { + "type": "long" + }, + "distinct_count_of_destination_ipv4_address": { + "type": "long" + }, + "distinct_count_of_destination_ipv6_address": { + "type": "long" + }, + "distinct_count_of_source_ip_address": { + "type": "long" + }, + "distinct_count_of_source_ipv4_address": { + "type": "long" + }, + "distinct_count_of_source_ipv6_address": { + "type": "long" + }, + "dot1q_customer_dei": { + "type": "boolean" + }, + "dot1q_customer_destination_mac_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "dot1q_customer_priority": { + "type": "short" + }, + "dot1q_customer_source_mac_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "dot1q_customer_vlan_id": { + "type": "long" + }, + "dot1q_dei": { + "type": "boolean" + }, + "dot1q_priority": { + "type": "short" + }, + "dot1q_service_instance_id": { + "type": "long" + }, + "dot1q_service_instance_priority": { + "type": "short" + }, + "dot1q_service_instance_tag": { + "type": "short" + }, + "dot1q_vlan_id": { + "type": "long" + }, + "dropped_layer2_octet_delta_count": { + "type": "long" + }, + "dropped_layer2_octet_total_count": { + "type": "long" + }, + "dropped_octet_delta_count": { + "type": "long" + }, + "dropped_octet_total_count": { + "type": "long" + }, + "dropped_packet_delta_count": { + "type": "long" + }, + "dropped_packet_total_count": { + "type": "long" + }, + "dst_traffic_index": { + "type": "long" + }, + "egress_broadcast_packet_total_count": { + "type": "long" + }, + "egress_interface": { + "type": "long" + }, + "egress_interface_type": { + "type": "long" + }, + "egress_physical_interface": { + "type": "long" + }, + "egress_unicast_packet_total_count": { + "type": "long" + }, + "egress_vrfid": { + "type": "long" + }, + "encrypted_technology": { + "ignore_above": 1024, + "type": "keyword" + }, + "engine_id": { + "type": "short" + }, + "engine_type": { + "type": "short" + }, + "ethernet_header_length": { + "type": "short" + }, + "ethernet_payload_length": { + "type": "long" + }, + "ethernet_total_length": { + "type": "long" + }, + "ethernet_type": { + "type": "long" + }, + "export_interface": { + "type": "long" + }, + "export_protocol_version": { + "type": "short" + }, + "export_sctp_stream_id": { + "type": "long" + }, + "export_transport_protocol": { + "type": "short" + }, + "exported_flow_record_total_count": { + "type": "long" + }, + "exported_message_total_count": { + "type": "long" + }, + "exported_octet_total_count": { + "type": "long" + }, + "exporter": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "source_id": { + "type": "long" + }, + "timestamp": { + "type": "date" + }, + "uptime_millis": { + "type": "long" + }, + "version": { + "type": "long" + } + } + }, + "exporter_certificate": { + "type": "short" + }, + "exporter_ipv4_address": { + "type": "ip" + }, + "exporter_ipv6_address": { + "type": "ip" + }, + "exporter_transport_port": { + "type": "long" + }, + "exporting_process_id": { + "type": "long" + }, + "external_address_realm": { + "type": "short" + }, + "firewall_event": { + "type": "short" + }, + "flags_and_sampler_id": { + "type": "long" + }, + "flow_active_timeout": { + "type": "long" + }, + "flow_direction": { + "type": "short" + }, + "flow_duration_microseconds": { + "type": "long" + }, + "flow_duration_milliseconds": { + "type": "long" + }, + "flow_end_delta_microseconds": { + "type": "long" + }, + "flow_end_microseconds": { + "type": "date" + }, + "flow_end_milliseconds": { + "type": "date" + }, + "flow_end_nanoseconds": { + "type": "date" + }, + "flow_end_reason": { + "type": "short" + }, + "flow_end_seconds": { + "type": "date" + }, + "flow_end_sys_up_time": { + "type": "long" + }, + "flow_id": { + "type": "long" + }, + "flow_idle_timeout": { + "type": "long" + }, + "flow_key_indicator": { + "type": "long" + }, + "flow_label_ipv6": { + "type": "long" + }, + "flow_sampling_time_interval": { + "type": "long" + }, + "flow_sampling_time_spacing": { + "type": "long" + }, + "flow_selected_flow_delta_count": { + "type": "long" + }, + "flow_selected_octet_delta_count": { + "type": "long" + }, + "flow_selected_packet_delta_count": { + "type": "long" + }, + "flow_selector_algorithm": { + "type": "long" + }, + "flow_start_delta_microseconds": { + "type": "long" + }, + "flow_start_microseconds": { + "type": "date" + }, + "flow_start_milliseconds": { + "type": "date" + }, + "flow_start_nanoseconds": { + "type": "date" + }, + "flow_start_seconds": { + "type": "date" + }, + "flow_start_sys_up_time": { + "type": "long" + }, + "forwarding_status": { + "type": "short" + }, + "fragment_flags": { + "type": "short" + }, + "fragment_identification": { + "type": "long" + }, + "fragment_offset": { + "type": "long" + }, + "global_address_mapping_high_threshold": { + "type": "long" + }, + "gre_key": { + "type": "long" + }, + "hash_digest_output": { + "type": "boolean" + }, + "hash_flow_domain": { + "type": "long" + }, + "hash_initialiser_value": { + "type": "long" + }, + "hash_ip_payload_offset": { + "type": "long" + }, + "hash_ip_payload_size": { + "type": "long" + }, + "hash_output_range_max": { + "type": "long" + }, + "hash_output_range_min": { + "type": "long" + }, + "hash_selected_range_max": { + "type": "long" + }, + "hash_selected_range_min": { + "type": "long" + }, + "http_content_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "http_message_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "http_reason_phrase": { + "ignore_above": 1024, + "type": "keyword" + }, + "http_request_host": { + "ignore_above": 1024, + "type": "keyword" + }, + "http_request_method": { + "ignore_above": 1024, + "type": "keyword" + }, + "http_request_target": { + "ignore_above": 1024, + "type": "keyword" + }, + "http_status_code": { + "type": "long" + }, + "http_user_agent": { + "ignore_above": 1024, + "type": "keyword" + }, + "icmp_code_ipv4": { + "type": "short" + }, + "icmp_code_ipv6": { + "type": "short" + }, + "icmp_type_code_ipv4": { + "type": "long" + }, + "icmp_type_code_ipv6": { + "type": "long" + }, + "icmp_type_ipv4": { + "type": "short" + }, + "icmp_type_ipv6": { + "type": "short" + }, + "igmp_type": { + "type": "short" + }, + "ignored_data_record_total_count": { + "type": "long" + }, + "ignored_layer2_frame_total_count": { + "type": "long" + }, + "ignored_layer2_octet_total_count": { + "type": "long" + }, + "ignored_octet_total_count": { + "type": "long" + }, + "ignored_packet_total_count": { + "type": "long" + }, + "information_element_data_type": { + "type": "short" + }, + "information_element_description": { + "ignore_above": 1024, + "type": "keyword" + }, + "information_element_id": { + "type": "long" + }, + "information_element_index": { + "type": "long" + }, + "information_element_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "information_element_range_begin": { + "type": "long" + }, + "information_element_range_end": { + "type": "long" + }, + "information_element_semantics": { + "type": "short" + }, + "information_element_units": { + "type": "long" + }, + "ingress_broadcast_packet_total_count": { + "type": "long" + }, + "ingress_interface": { + "type": "long" + }, + "ingress_interface_type": { + "type": "long" + }, + "ingress_multicast_packet_total_count": { + "type": "long" + }, + "ingress_physical_interface": { + "type": "long" + }, + "ingress_unicast_packet_total_count": { + "type": "long" + }, + "ingress_vrfid": { + "type": "long" + }, + "initiator_octets": { + "type": "long" + }, + "initiator_packets": { + "type": "long" + }, + "interface_description": { + "ignore_above": 1024, + "type": "keyword" + }, + "interface_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "intermediate_process_id": { + "type": "long" + }, + "internal_address_realm": { + "type": "short" + }, + "ip_class_of_service": { + "type": "short" + }, + "ip_diff_serv_code_point": { + "type": "short" + }, + "ip_header_length": { + "type": "short" + }, + "ip_header_packet_section": { + "type": "short" + }, + "ip_next_hop_ipv4_address": { + "type": "ip" + }, + "ip_next_hop_ipv6_address": { + "type": "ip" + }, + "ip_payload_length": { + "type": "long" + }, + "ip_payload_packet_section": { + "type": "short" + }, + "ip_precedence": { + "type": "short" + }, + "ip_sec_spi": { + "type": "long" + }, + "ip_total_length": { + "type": "long" + }, + "ip_ttl": { + "type": "short" + }, + "ip_version": { + "type": "short" + }, + "ipv4_ihl": { + "type": "short" + }, + "ipv4_options": { + "type": "long" + }, + "ipv4_router_sc": { + "type": "ip" + }, + "ipv6_extension_headers": { + "type": "long" + }, + "is_multicast": { + "type": "short" + }, + "layer2_frame_delta_count": { + "type": "long" + }, + "layer2_frame_total_count": { + "type": "long" + }, + "layer2_octet_delta_count": { + "type": "long" + }, + "layer2_octet_delta_sum_of_squares": { + "type": "long" + }, + "layer2_octet_total_count": { + "type": "long" + }, + "layer2_octet_total_sum_of_squares": { + "type": "long" + }, + "layer2_segment_id": { + "type": "long" + }, + "layer2packet_section_data": { + "type": "short" + }, + "layer2packet_section_offset": { + "type": "long" + }, + "layer2packet_section_size": { + "type": "long" + }, + "line_card_id": { + "type": "long" + }, + "lower_ci_limit": { + "type": "double" + }, + "max_bib_entries": { + "type": "long" + }, + "max_entries_per_user": { + "type": "long" + }, + "max_export_seconds": { + "type": "date" + }, + "max_flow_end_microseconds": { + "type": "date" + }, + "max_flow_end_milliseconds": { + "type": "date" + }, + "max_flow_end_nanoseconds": { + "type": "date" + }, + "max_flow_end_seconds": { + "type": "date" + }, + "max_fragments_pending_reassembly": { + "type": "long" + }, + "max_session_entries": { + "type": "long" + }, + "max_subscribers": { + "type": "long" + }, + "maximum_ip_total_length": { + "type": "long" + }, + "maximum_layer2_total_length": { + "type": "long" + }, + "maximum_ttl": { + "type": "short" + }, + "message_md5_checksum": { + "type": "short" + }, + "message_scope": { + "type": "short" + }, + "metering_process_id": { + "type": "long" + }, + "metro_evc_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "metro_evc_type": { + "type": "short" + }, + "mib_capture_time_semantics": { + "type": "short" + }, + "mib_context_engine_id": { + "type": "short" + }, + "mib_context_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "mib_index_indicator": { + "type": "long" + }, + "mib_module_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "mib_object_description": { + "ignore_above": 1024, + "type": "keyword" + }, + "mib_object_identifier": { + "type": "short" + }, + "mib_object_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "mib_object_syntax": { + "ignore_above": 1024, + "type": "keyword" + }, + "mib_object_value_bits": { + "type": "short" + }, + "mib_object_value_counter": { + "type": "long" + }, + "mib_object_value_gauge": { + "type": "long" + }, + "mib_object_value_integer": { + "type": "long" + }, + "mib_object_value_ip_address": { + "type": "ip" + }, + "mib_object_value_octet_string": { + "type": "short" + }, + "mib_object_value_oid": { + "type": "short" + }, + "mib_object_value_time_ticks": { + "type": "long" + }, + "mib_object_value_unsigned": { + "type": "long" + }, + "mib_sub_identifier": { + "type": "long" + }, + "min_export_seconds": { + "type": "date" + }, + "min_flow_start_microseconds": { + "type": "date" + }, + "min_flow_start_milliseconds": { + "type": "date" + }, + "min_flow_start_nanoseconds": { + "type": "date" + }, + "min_flow_start_seconds": { + "type": "date" + }, + "minimum_ip_total_length": { + "type": "long" + }, + "minimum_layer2_total_length": { + "type": "long" + }, + "minimum_ttl": { + "type": "short" + }, + "mobile_imsi": { + "ignore_above": 1024, + "type": "keyword" + }, + "mobile_msisdn": { + "ignore_above": 1024, + "type": "keyword" + }, + "monitoring_interval_end_milli_seconds": { + "type": "date" + }, + "monitoring_interval_start_milli_seconds": { + "type": "date" + }, + "mpls_label_stack_depth": { + "type": "long" + }, + "mpls_label_stack_length": { + "type": "long" + }, + "mpls_label_stack_section": { + "type": "short" + }, + "mpls_label_stack_section10": { + "type": "short" + }, + "mpls_label_stack_section2": { + "type": "short" + }, + "mpls_label_stack_section3": { + "type": "short" + }, + "mpls_label_stack_section4": { + "type": "short" + }, + "mpls_label_stack_section5": { + "type": "short" + }, + "mpls_label_stack_section6": { + "type": "short" + }, + "mpls_label_stack_section7": { + "type": "short" + }, + "mpls_label_stack_section8": { + "type": "short" + }, + "mpls_label_stack_section9": { + "type": "short" + }, + "mpls_payload_length": { + "type": "long" + }, + "mpls_payload_packet_section": { + "type": "short" + }, + "mpls_top_label_exp": { + "type": "short" + }, + "mpls_top_label_ipv4_address": { + "type": "ip" + }, + "mpls_top_label_ipv6_address": { + "type": "ip" + }, + "mpls_top_label_prefix_length": { + "type": "short" + }, + "mpls_top_label_stack_section": { + "type": "short" + }, + "mpls_top_label_ttl": { + "type": "short" + }, + "mpls_top_label_type": { + "type": "short" + }, + "mpls_vpn_route_distinguisher": { + "type": "short" + }, + "multicast_replication_factor": { + "type": "long" + }, + "nat_event": { + "type": "short" + }, + "nat_instance_id": { + "type": "long" + }, + "nat_originating_address_realm": { + "type": "short" + }, + "nat_pool_id": { + "type": "long" + }, + "nat_pool_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat_quota_exceeded_event": { + "type": "long" + }, + "nat_threshold_event": { + "type": "long" + }, + "nat_type": { + "type": "short" + }, + "new_connection_delta_count": { + "type": "long" + }, + "next_header_ipv6": { + "type": "short" + }, + "not_sent_flow_total_count": { + "type": "long" + }, + "not_sent_layer2_octet_total_count": { + "type": "long" + }, + "not_sent_octet_total_count": { + "type": "long" + }, + "not_sent_packet_total_count": { + "type": "long" + }, + "observation_domain_id": { + "type": "long" + }, + "observation_domain_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "observation_point_id": { + "type": "long" + }, + "observation_point_type": { + "type": "short" + }, + "observation_time_microseconds": { + "type": "date" + }, + "observation_time_milliseconds": { + "type": "date" + }, + "observation_time_nanoseconds": { + "type": "date" + }, + "observation_time_seconds": { + "type": "date" + }, + "observed_flow_total_count": { + "type": "long" + }, + "octet_delta_count": { + "type": "long" + }, + "octet_delta_sum_of_squares": { + "type": "long" + }, + "octet_total_count": { + "type": "long" + }, + "octet_total_sum_of_squares": { + "type": "long" + }, + "opaque_octets": { + "type": "short" + }, + "original_exporter_ipv4_address": { + "type": "ip" + }, + "original_exporter_ipv6_address": { + "type": "ip" + }, + "original_flows_completed": { + "type": "long" + }, + "original_flows_initiated": { + "type": "long" + }, + "original_flows_present": { + "type": "long" + }, + "original_observation_domain_id": { + "type": "long" + }, + "p2p_technology": { + "ignore_above": 1024, + "type": "keyword" + }, + "packet_delta_count": { + "type": "long" + }, + "packet_total_count": { + "type": "long" + }, + "padding_octets": { + "type": "short" + }, + "payload_length_ipv6": { + "type": "long" + }, + "port_id": { + "type": "long" + }, + "port_range_end": { + "type": "long" + }, + "port_range_num_ports": { + "type": "long" + }, + "port_range_start": { + "type": "long" + }, + "port_range_step_size": { + "type": "long" + }, + "post_destination_mac_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "post_dot1q_customer_vlan_id": { + "type": "long" + }, + "post_dot1q_vlan_id": { + "type": "long" + }, + "post_ip_class_of_service": { + "type": "short" + }, + "post_ip_diff_serv_code_point": { + "type": "short" + }, + "post_ip_precedence": { + "type": "short" + }, + "post_layer2_octet_delta_count": { + "type": "long" + }, + "post_layer2_octet_total_count": { + "type": "long" + }, + "post_mcast_layer2_octet_delta_count": { + "type": "long" + }, + "post_mcast_layer2_octet_total_count": { + "type": "long" + }, + "post_mcast_octet_delta_count": { + "type": "long" + }, + "post_mcast_octet_total_count": { + "type": "long" + }, + "post_mcast_packet_delta_count": { + "type": "long" + }, + "post_mcast_packet_total_count": { + "type": "long" + }, + "post_mpls_top_label_exp": { + "type": "short" + }, + "post_napt_destination_transport_port": { + "type": "long" + }, + "post_napt_source_transport_port": { + "type": "long" + }, + "post_nat_destination_ipv4_address": { + "type": "ip" + }, + "post_nat_destination_ipv6_address": { + "type": "ip" + }, + "post_nat_source_ipv4_address": { + "type": "ip" + }, + "post_nat_source_ipv6_address": { + "type": "ip" + }, + "post_octet_delta_count": { + "type": "long" + }, + "post_octet_total_count": { + "type": "long" + }, + "post_packet_delta_count": { + "type": "long" + }, + "post_packet_total_count": { + "type": "long" + }, + "post_source_mac_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "post_vlan_id": { + "type": "long" + }, + "private_enterprise_number": { + "type": "long" + }, + "protocol_identifier": { + "type": "short" + }, + "pseudo_wire_control_word": { + "type": "long" + }, + "pseudo_wire_destination_ipv4_address": { + "type": "ip" + }, + "pseudo_wire_id": { + "type": "long" + }, + "pseudo_wire_type": { + "type": "long" + }, + "relative_error": { + "type": "double" + }, + "responder_octets": { + "type": "long" + }, + "responder_packets": { + "type": "long" + }, + "rfc3550_jitter_microseconds": { + "type": "long" + }, + "rfc3550_jitter_milliseconds": { + "type": "long" + }, + "rfc3550_jitter_nanoseconds": { + "type": "long" + }, + "rtp_sequence_number": { + "type": "long" + }, + "sampler_id": { + "type": "short" + }, + "sampler_mode": { + "type": "short" + }, + "sampler_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "sampler_random_interval": { + "type": "long" + }, + "sampling_algorithm": { + "type": "short" + }, + "sampling_flow_interval": { + "type": "long" + }, + "sampling_flow_spacing": { + "type": "long" + }, + "sampling_interval": { + "type": "long" + }, + "sampling_packet_interval": { + "type": "long" + }, + "sampling_packet_space": { + "type": "long" + }, + "sampling_population": { + "type": "long" + }, + "sampling_probability": { + "type": "double" + }, + "sampling_size": { + "type": "long" + }, + "sampling_time_interval": { + "type": "long" + }, + "sampling_time_space": { + "type": "long" + }, + "section_exported_octets": { + "type": "long" + }, + "section_offset": { + "type": "long" + }, + "selection_sequence_id": { + "type": "long" + }, + "selector_algorithm": { + "type": "long" + }, + "selector_id": { + "type": "long" + }, + "selector_id_total_flows_observed": { + "type": "long" + }, + "selector_id_total_flows_selected": { + "type": "long" + }, + "selector_id_total_pkts_observed": { + "type": "long" + }, + "selector_id_total_pkts_selected": { + "type": "long" + }, + "selector_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "session_scope": { + "type": "short" + }, + "source_ipv4_address": { + "type": "ip" + }, + "source_ipv4_prefix": { + "type": "ip" + }, + "source_ipv4_prefix_length": { + "type": "short" + }, + "source_ipv6_address": { + "type": "ip" + }, + "source_ipv6_prefix": { + "type": "ip" + }, + "source_ipv6_prefix_length": { + "type": "short" + }, + "source_mac_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "source_transport_port": { + "type": "long" + }, + "source_transport_ports_limit": { + "type": "long" + }, + "src_traffic_index": { + "type": "long" + }, + "sta_ipv4_address": { + "type": "ip" + }, + "sta_mac_address": { + "ignore_above": 1024, + "type": "keyword" + }, + "system_init_time_milliseconds": { + "type": "date" + }, + "tcp_ack_total_count": { + "type": "long" + }, + "tcp_acknowledgement_number": { + "type": "long" + }, + "tcp_control_bits": { + "type": "long" + }, + "tcp_destination_port": { + "type": "long" + }, + "tcp_fin_total_count": { + "type": "long" + }, + "tcp_header_length": { + "type": "short" + }, + "tcp_options": { + "type": "long" + }, + "tcp_psh_total_count": { + "type": "long" + }, + "tcp_rst_total_count": { + "type": "long" + }, + "tcp_sequence_number": { + "type": "long" + }, + "tcp_source_port": { + "type": "long" + }, + "tcp_syn_total_count": { + "type": "long" + }, + "tcp_urg_total_count": { + "type": "long" + }, + "tcp_urgent_pointer": { + "type": "long" + }, + "tcp_window_scale": { + "type": "long" + }, + "tcp_window_size": { + "type": "long" + }, + "template_id": { + "type": "long" + }, + "total_length_ipv4": { + "type": "long" + }, + "transport_octet_delta_count": { + "type": "long" + }, + "transport_packet_delta_count": { + "type": "long" + }, + "tunnel_technology": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "udp_destination_port": { + "type": "long" + }, + "udp_message_length": { + "type": "long" + }, + "udp_source_port": { + "type": "long" + }, + "upper_ci_limit": { + "type": "double" + }, + "user_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "value_distribution_method": { + "type": "short" + }, + "virtual_station_interface_id": { + "type": "short" + }, + "virtual_station_interface_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "virtual_station_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "virtual_station_uuid": { + "type": "short" + }, + "vlan_id": { + "type": "long" + }, + "vpn_identifier": { + "type": "short" + }, + "vr_fname": { + "ignore_above": 1024, + "type": "keyword" + }, + "wlan_channel_id": { + "type": "short" + }, + "wlan_ssid": { + "ignore_above": 1024, + "type": "keyword" + }, + "wtp_mac_address": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "network": { + "properties": { + "application": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "community_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "direction": { + "ignore_above": 1024, + "type": "keyword" + }, + "forwarded_ip": { + "type": "ip" + }, + "iana_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "inner": { + "properties": { + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "transport": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "nginx": { + "properties": { + "access": { + "properties": { + "geoip": { + "type": "object" + }, + "user_agent": { + "type": "object" + } + } + }, + "error": { + "properties": { + "connection_id": { + "type": "long" + } + } + }, + "ingress_controller": { + "properties": { + "geoip": { + "type": "object" + }, + "http": { + "properties": { + "request": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "length": { + "type": "long" + }, + "time": { + "type": "double" + } + } + } + } + }, + "upstream": { + "properties": { + "alternative_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "response": { + "properties": { + "length": { + "type": "long" + }, + "status_code": { + "type": "long" + }, + "time": { + "type": "double" + } + } + } + } + }, + "user_agent": { + "type": "object" + } + } + } + } + }, + "o365": { + "properties": { + "audit": { + "properties": { + "ActorContextId": { + "ignore_above": 1024, + "type": "keyword" + }, + "ActorIpAddress": { + "ignore_above": 1024, + "type": "keyword" + }, + "ActorUserId": { + "ignore_above": 1024, + "type": "keyword" + }, + "ActorYammerUserId": { + "ignore_above": 1024, + "type": "keyword" + }, + "AlertEntityId": { + "ignore_above": 1024, + "type": "keyword" + }, + "AlertId": { + "ignore_above": 1024, + "type": "keyword" + }, + "AlertType": { + "ignore_above": 1024, + "type": "keyword" + }, + "AppId": { + "ignore_above": 1024, + "type": "keyword" + }, + "ApplicationDisplayName": { + "ignore_above": 1024, + "type": "keyword" + }, + "ApplicationId": { + "ignore_above": 1024, + "type": "keyword" + }, + "AzureActiveDirectoryEventType": { + "ignore_above": 1024, + "type": "keyword" + }, + "Category": { + "ignore_above": 1024, + "type": "keyword" + }, + "ClientAppId": { + "ignore_above": 1024, + "type": "keyword" + }, + "ClientIP": { + "ignore_above": 1024, + "type": "keyword" + }, + "ClientIPAddress": { + "ignore_above": 1024, + "type": "keyword" + }, + "ClientInfoString": { + "ignore_above": 1024, + "type": "keyword" + }, + "Comments": { + "norms": false, + "type": "text" + }, + "CorrelationId": { + "ignore_above": 1024, + "type": "keyword" + }, + "CreationTime": { + "ignore_above": 1024, + "type": "keyword" + }, + "CustomUniqueId": { + "ignore_above": 1024, + "type": "keyword" + }, + "Data": { + "ignore_above": 1024, + "type": "keyword" + }, + "DataType": { + "ignore_above": 1024, + "type": "keyword" + }, + "EntityType": { + "ignore_above": 1024, + "type": "keyword" + }, + "EventData": { + "ignore_above": 1024, + "type": "keyword" + }, + "EventSource": { + "ignore_above": 1024, + "type": "keyword" + }, + "ExceptionInfo": { + "properties": { + "*": { + "type": "object" + } + } + }, + "ExchangeMetaData": { + "properties": { + "*": { + "type": "object" + } + } + }, + "ExtendedProperties": { + "properties": { + "*": { + "type": "object" + } + } + }, + "ExternalAccess": { + "ignore_above": 1024, + "type": "keyword" + }, + "GroupName": { + "ignore_above": 1024, + "type": "keyword" + }, + "Id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ImplicitShare": { + "ignore_above": 1024, + "type": "keyword" + }, + "IncidentId": { + "ignore_above": 1024, + "type": "keyword" + }, + "InterSystemsId": { + "ignore_above": 1024, + "type": "keyword" + }, + "InternalLogonType": { + "ignore_above": 1024, + "type": "keyword" + }, + "IntraSystemId": { + "ignore_above": 1024, + "type": "keyword" + }, + "Item": { + "properties": { + "*": { + "properties": { + "*": { + "type": "object" + } + } + } + } + }, + "ItemName": { + "ignore_above": 1024, + "type": "keyword" + }, + "ItemType": { + "ignore_above": 1024, + "type": "keyword" + }, + "ListId": { + "ignore_above": 1024, + "type": "keyword" + }, + "ListItemUniqueId": { + "ignore_above": 1024, + "type": "keyword" + }, + "LogonError": { + "ignore_above": 1024, + "type": "keyword" + }, + "LogonType": { + "ignore_above": 1024, + "type": "keyword" + }, + "LogonUserSid": { + "ignore_above": 1024, + "type": "keyword" + }, + "MailboxGuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "MailboxOwnerMasterAccountSid": { + "ignore_above": 1024, + "type": "keyword" + }, + "MailboxOwnerSid": { + "ignore_above": 1024, + "type": "keyword" + }, + "MailboxOwnerUPN": { + "ignore_above": 1024, + "type": "keyword" + }, + "Members": { + "properties": { + "*": { + "type": "object" + } + } + }, + "ModifiedProperties": { + "properties": { + "*": { + "properties": { + "*": { + "type": "object" + } + } + } + } + }, + "Name": { + "ignore_above": 1024, + "type": "keyword" + }, + "ObjectId": { + "ignore_above": 1024, + "type": "keyword" + }, + "Operation": { + "ignore_above": 1024, + "type": "keyword" + }, + "OrganizationId": { + "ignore_above": 1024, + "type": "keyword" + }, + "OrganizationName": { + "ignore_above": 1024, + "type": "keyword" + }, + "OriginatingServer": { + "ignore_above": 1024, + "type": "keyword" + }, + "Parameters": { + "properties": { + "*": { + "type": "object" + } + } + }, + "PolicyId": { + "ignore_above": 1024, + "type": "keyword" + }, + "RecordType": { + "ignore_above": 1024, + "type": "keyword" + }, + "ResultStatus": { + "ignore_above": 1024, + "type": "keyword" + }, + "SensitiveInfoDetectionIsIncluded": { + "ignore_above": 1024, + "type": "keyword" + }, + "SessionId": { + "ignore_above": 1024, + "type": "keyword" + }, + "Severity": { + "ignore_above": 1024, + "type": "keyword" + }, + "SharePointMetaData": { + "properties": { + "*": { + "type": "object" + } + } + }, + "Site": { + "ignore_above": 1024, + "type": "keyword" + }, + "SiteUrl": { + "ignore_above": 1024, + "type": "keyword" + }, + "Source": { + "ignore_above": 1024, + "type": "keyword" + }, + "SourceFileExtension": { + "ignore_above": 1024, + "type": "keyword" + }, + "SourceFileName": { + "ignore_above": 1024, + "type": "keyword" + }, + "SourceRelativeUrl": { + "ignore_above": 1024, + "type": "keyword" + }, + "Status": { + "ignore_above": 1024, + "type": "keyword" + }, + "SupportTicketId": { + "ignore_above": 1024, + "type": "keyword" + }, + "TargetContextId": { + "ignore_above": 1024, + "type": "keyword" + }, + "TargetUserOrGroupName": { + "ignore_above": 1024, + "type": "keyword" + }, + "TargetUserOrGroupType": { + "ignore_above": 1024, + "type": "keyword" + }, + "TeamGuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "TeamName": { + "ignore_above": 1024, + "type": "keyword" + }, + "UniqueSharingId": { + "ignore_above": 1024, + "type": "keyword" + }, + "UserAgent": { + "ignore_above": 1024, + "type": "keyword" + }, + "UserId": { + "ignore_above": 1024, + "type": "keyword" + }, + "UserKey": { + "ignore_above": 1024, + "type": "keyword" + }, + "UserType": { + "ignore_above": 1024, + "type": "keyword" + }, + "Version": { + "ignore_above": 1024, + "type": "keyword" + }, + "WebId": { + "ignore_above": 1024, + "type": "keyword" + }, + "Workload": { + "ignore_above": 1024, + "type": "keyword" + }, + "YammerNetworkId": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "object_key": { + "ignore_above": 1024, + "type": "keyword" + }, + "observer": { + "properties": { + "egress": { + "properties": { + "interface": { + "properties": { + "alias": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "zone": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "ingress": { + "properties": { + "interface": { + "properties": { + "alias": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "zone": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "vendor": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "okta": { + "properties": { + "actor": { + "properties": { + "alternate_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "display_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "authentication_context": { + "properties": { + "authentication_provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "authentication_step": { + "type": "long" + }, + "credential_provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "credential_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "external_session_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "interface": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "client": { + "properties": { + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "user_agent": { + "properties": { + "browser": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "ignore_above": 1024, + "type": "keyword" + }, + "raw_user_agent": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "zone": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "debug_context": { + "properties": { + "debug_data": { + "properties": { + "device_fingerprint": { + "ignore_above": 1024, + "type": "keyword" + }, + "request_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "request_uri": { + "ignore_above": 1024, + "type": "keyword" + }, + "threat_suspected": { + "ignore_above": 1024, + "type": "keyword" + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "display_message": { + "ignore_above": 1024, + "type": "keyword" + }, + "event_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "outcome": { + "properties": { + "reason": { + "ignore_above": 1024, + "type": "keyword" + }, + "result": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "request": { + "properties": { + "ip_chain": { + "properties": { + "geographical_context": { + "properties": { + "city": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "geolocation": { + "type": "geo_point" + }, + "postal_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "source": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "security_context": { + "properties": { + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "is_proxy": { + "type": "boolean" + }, + "isp": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "severity": { + "ignore_above": 1024, + "type": "keyword" + }, + "transaction": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "uuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "organization": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "osquery": { + "properties": { + "result": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "calendar_time": { + "ignore_above": 1024, + "type": "keyword" + }, + "host_identifier": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "unix_time": { + "type": "long" + } + } + } + } + }, + "package": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "build_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "checksum": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "install_scope": { + "ignore_above": 1024, + "type": "keyword" + }, + "installed": { + "type": "date" + }, + "license": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "size": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "panw": { + "properties": { + "panos": { + "properties": { + "destination": { + "properties": { + "interface": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "zone": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "file": { + "properties": { + "hash": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "flow_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "network": { + "properties": { + "nat": { + "properties": { + "community_id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pcap_id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ruleset": { + "ignore_above": 1024, + "type": "keyword" + }, + "sequence_number": { + "type": "long" + }, + "source": { + "properties": { + "interface": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "zone": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "threat": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "resource": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "url": { + "properties": { + "category": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "pe": { + "properties": { + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "postgresql": { + "properties": { + "log": { + "properties": { + "core_id": { + "type": "long" + }, + "database": { + "ignore_above": 1024, + "type": "keyword" + }, + "error": { + "properties": { + "code": { + "type": "long" + } + } + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "query_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "query_step": { + "ignore_above": 1024, + "type": "keyword" + }, + "timestamp": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "process": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "args_count": { + "type": "long" + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "command_line": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "entity_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "executable": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "exit_code": { + "type": "long" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "parent": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "args_count": { + "type": "long" + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "command_line": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "entity_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "executable": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "exit_code": { + "type": "long" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "pgid": { + "type": "long" + }, + "pid": { + "type": "long" + }, + "ppid": { + "type": "long" + }, + "start": { + "type": "date" + }, + "thread": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "title": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + }, + "working_directory": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pe": { + "properties": { + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pgid": { + "type": "long" + }, + "pid": { + "type": "long" + }, + "ppid": { + "type": "long" + }, + "program": { + "ignore_above": 1024, + "type": "keyword" + }, + "start": { + "type": "date" + }, + "thread": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "title": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + }, + "working_directory": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "rabbitmq": { + "properties": { + "log": { + "properties": { + "pid": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "redis": { + "properties": { + "log": { + "properties": { + "role": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "slowlog": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "cmd": { + "ignore_above": 1024, + "type": "keyword" + }, + "duration": { + "properties": { + "us": { + "type": "long" + } + } + }, + "id": { + "type": "long" + }, + "key": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "registry": { + "properties": { + "data": { + "properties": { + "bytes": { + "ignore_above": 1024, + "type": "keyword" + }, + "strings": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hive": { + "ignore_above": 1024, + "type": "keyword" + }, + "key": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "value": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "related": { + "properties": { + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "rule": { + "properties": { + "author": { + "ignore_above": 1024, + "type": "keyword" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "license": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "ruleset": { + "ignore_above": 1024, + "type": "keyword" + }, + "uuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "santa": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "decision": { + "ignore_above": 1024, + "type": "keyword" + }, + "disk": { + "properties": { + "bsdname": { + "ignore_above": 1024, + "type": "keyword" + }, + "bus": { + "ignore_above": 1024, + "type": "keyword" + }, + "fs": { + "ignore_above": 1024, + "type": "keyword" + }, + "model": { + "ignore_above": 1024, + "type": "keyword" + }, + "mount": { + "ignore_above": 1024, + "type": "keyword" + }, + "serial": { + "ignore_above": 1024, + "type": "keyword" + }, + "volume": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "mode": { + "ignore_above": 1024, + "type": "keyword" + }, + "reason": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "server": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "service": { + "properties": { + "ephemeral_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "node": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "source": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "service": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "stream": { + "ignore_above": 1024, + "type": "keyword" + }, + "suricata": { + "properties": { + "eve": { + "properties": { + "alert": { + "properties": { + "action": { + "path": "event.outcome", + "type": "alias" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "gid": { + "type": "long" + }, + "rev": { + "type": "long" + }, + "severity": { + "path": "event.severity", + "type": "alias" + }, + "signature": { + "ignore_above": 1024, + "type": "keyword" + }, + "signature_id": { + "type": "long" + } + } + }, + "app_proto": { + "path": "network.protocol", + "type": "alias" + }, + "app_proto_expected": { + "ignore_above": 1024, + "type": "keyword" + }, + "app_proto_orig": { + "ignore_above": 1024, + "type": "keyword" + }, + "app_proto_tc": { + "ignore_above": 1024, + "type": "keyword" + }, + "app_proto_ts": { + "ignore_above": 1024, + "type": "keyword" + }, + "dest_ip": { + "path": "destination.ip", + "type": "alias" + }, + "dest_port": { + "path": "destination.port", + "type": "alias" + }, + "dns": { + "properties": { + "id": { + "type": "long" + }, + "rcode": { + "ignore_above": 1024, + "type": "keyword" + }, + "rdata": { + "ignore_above": 1024, + "type": "keyword" + }, + "rrname": { + "ignore_above": 1024, + "type": "keyword" + }, + "rrtype": { + "ignore_above": 1024, + "type": "keyword" + }, + "ttl": { + "type": "long" + }, + "tx_id": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "email": { + "properties": { + "status": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "event_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "fileinfo": { + "properties": { + "filename": { + "path": "file.path", + "type": "alias" + }, + "gaps": { + "type": "boolean" + }, + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "size": { + "path": "file.size", + "type": "alias" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "stored": { + "type": "boolean" + }, + "tx_id": { + "type": "long" + } + } + }, + "flags": { + "type": "object" + }, + "flow": { + "properties": { + "age": { + "type": "long" + }, + "alerted": { + "type": "boolean" + }, + "bytes_toclient": { + "path": "destination.bytes", + "type": "alias" + }, + "bytes_toserver": { + "path": "source.bytes", + "type": "alias" + }, + "end": { + "type": "date" + }, + "pkts_toclient": { + "path": "destination.packets", + "type": "alias" + }, + "pkts_toserver": { + "path": "source.packets", + "type": "alias" + }, + "reason": { + "ignore_above": 1024, + "type": "keyword" + }, + "start": { + "path": "event.start", + "type": "alias" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "flow_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "http": { + "properties": { + "hostname": { + "path": "url.domain", + "type": "alias" + }, + "http_content_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "http_method": { + "path": "http.request.method", + "type": "alias" + }, + "http_refer": { + "path": "http.request.referrer", + "type": "alias" + }, + "http_user_agent": { + "path": "user_agent.original", + "type": "alias" + }, + "length": { + "path": "http.response.body.bytes", + "type": "alias" + }, + "protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "redirect": { + "ignore_above": 1024, + "type": "keyword" + }, + "status": { + "path": "http.response.status_code", + "type": "alias" + }, + "url": { + "path": "url.original", + "type": "alias" + } + } + }, + "icmp_code": { + "type": "long" + }, + "icmp_type": { + "type": "long" + }, + "in_iface": { + "ignore_above": 1024, + "type": "keyword" + }, + "pcap_cnt": { + "type": "long" + }, + "proto": { + "path": "network.transport", + "type": "alias" + }, + "smtp": { + "properties": { + "helo": { + "ignore_above": 1024, + "type": "keyword" + }, + "mail_from": { + "ignore_above": 1024, + "type": "keyword" + }, + "rcpt_to": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "src_ip": { + "path": "source.ip", + "type": "alias" + }, + "src_port": { + "path": "source.port", + "type": "alias" + }, + "ssh": { + "properties": { + "client": { + "properties": { + "proto_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "software_version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "server": { + "properties": { + "proto_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "software_version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "stats": { + "properties": { + "app_layer": { + "properties": { + "flow": { + "properties": { + "dcerpc_tcp": { + "type": "long" + }, + "dcerpc_udp": { + "type": "long" + }, + "dns_tcp": { + "type": "long" + }, + "dns_udp": { + "type": "long" + }, + "failed_tcp": { + "type": "long" + }, + "failed_udp": { + "type": "long" + }, + "ftp": { + "type": "long" + }, + "http": { + "type": "long" + }, + "imap": { + "type": "long" + }, + "msn": { + "type": "long" + }, + "smb": { + "type": "long" + }, + "smtp": { + "type": "long" + }, + "ssh": { + "type": "long" + }, + "tls": { + "type": "long" + } + } + }, + "tx": { + "properties": { + "dcerpc_tcp": { + "type": "long" + }, + "dcerpc_udp": { + "type": "long" + }, + "dns_tcp": { + "type": "long" + }, + "dns_udp": { + "type": "long" + }, + "ftp": { + "type": "long" + }, + "http": { + "type": "long" + }, + "smb": { + "type": "long" + }, + "smtp": { + "type": "long" + }, + "ssh": { + "type": "long" + }, + "tls": { + "type": "long" + } + } + } + } + }, + "capture": { + "properties": { + "kernel_drops": { + "type": "long" + }, + "kernel_ifdrops": { + "type": "long" + }, + "kernel_packets": { + "type": "long" + } + } + }, + "decoder": { + "properties": { + "avg_pkt_size": { + "type": "long" + }, + "bytes": { + "type": "long" + }, + "dce": { + "properties": { + "pkt_too_small": { + "type": "long" + } + } + }, + "erspan": { + "type": "long" + }, + "ethernet": { + "type": "long" + }, + "gre": { + "type": "long" + }, + "icmpv4": { + "type": "long" + }, + "icmpv6": { + "type": "long" + }, + "ieee8021ah": { + "type": "long" + }, + "invalid": { + "type": "long" + }, + "ipraw": { + "properties": { + "invalid_ip_version": { + "type": "long" + } + } + }, + "ipv4": { + "type": "long" + }, + "ipv4_in_ipv6": { + "type": "long" + }, + "ipv6": { + "type": "long" + }, + "ipv6_in_ipv6": { + "type": "long" + }, + "ltnull": { + "properties": { + "pkt_too_small": { + "type": "long" + }, + "unsupported_type": { + "type": "long" + } + } + }, + "max_pkt_size": { + "type": "long" + }, + "mpls": { + "type": "long" + }, + "null": { + "type": "long" + }, + "pkts": { + "type": "long" + }, + "ppp": { + "type": "long" + }, + "pppoe": { + "type": "long" + }, + "raw": { + "type": "long" + }, + "sctp": { + "type": "long" + }, + "sll": { + "type": "long" + }, + "tcp": { + "type": "long" + }, + "teredo": { + "type": "long" + }, + "udp": { + "type": "long" + }, + "vlan": { + "type": "long" + }, + "vlan_qinq": { + "type": "long" + } + } + }, + "defrag": { + "properties": { + "ipv4": { + "properties": { + "fragments": { + "type": "long" + }, + "reassembled": { + "type": "long" + }, + "timeouts": { + "type": "long" + } + } + }, + "ipv6": { + "properties": { + "fragments": { + "type": "long" + }, + "reassembled": { + "type": "long" + }, + "timeouts": { + "type": "long" + } + } + }, + "max_frag_hits": { + "type": "long" + } + } + }, + "detect": { + "properties": { + "alert": { + "type": "long" + } + } + }, + "dns": { + "properties": { + "memcap_global": { + "type": "long" + }, + "memcap_state": { + "type": "long" + }, + "memuse": { + "type": "long" + } + } + }, + "file_store": { + "properties": { + "open_files": { + "type": "long" + } + } + }, + "flow": { + "properties": { + "emerg_mode_entered": { + "type": "long" + }, + "emerg_mode_over": { + "type": "long" + }, + "icmpv4": { + "type": "long" + }, + "icmpv6": { + "type": "long" + }, + "memcap": { + "type": "long" + }, + "memuse": { + "type": "long" + }, + "spare": { + "type": "long" + }, + "tcp": { + "type": "long" + }, + "tcp_reuse": { + "type": "long" + }, + "udp": { + "type": "long" + } + } + }, + "flow_mgr": { + "properties": { + "bypassed_pruned": { + "type": "long" + }, + "closed_pruned": { + "type": "long" + }, + "est_pruned": { + "type": "long" + }, + "flows_checked": { + "type": "long" + }, + "flows_notimeout": { + "type": "long" + }, + "flows_removed": { + "type": "long" + }, + "flows_timeout": { + "type": "long" + }, + "flows_timeout_inuse": { + "type": "long" + }, + "new_pruned": { + "type": "long" + }, + "rows_busy": { + "type": "long" + }, + "rows_checked": { + "type": "long" + }, + "rows_empty": { + "type": "long" + }, + "rows_maxlen": { + "type": "long" + }, + "rows_skipped": { + "type": "long" + } + } + }, + "http": { + "properties": { + "memcap": { + "type": "long" + }, + "memuse": { + "type": "long" + } + } + }, + "tcp": { + "properties": { + "insert_data_normal_fail": { + "type": "long" + }, + "insert_data_overlap_fail": { + "type": "long" + }, + "insert_list_fail": { + "type": "long" + }, + "invalid_checksum": { + "type": "long" + }, + "memuse": { + "type": "long" + }, + "no_flow": { + "type": "long" + }, + "overlap": { + "type": "long" + }, + "overlap_diff_data": { + "type": "long" + }, + "pseudo": { + "type": "long" + }, + "pseudo_failed": { + "type": "long" + }, + "reassembly_gap": { + "type": "long" + }, + "reassembly_memuse": { + "type": "long" + }, + "rst": { + "type": "long" + }, + "segment_memcap_drop": { + "type": "long" + }, + "sessions": { + "type": "long" + }, + "ssn_memcap_drop": { + "type": "long" + }, + "stream_depth_reached": { + "type": "long" + }, + "syn": { + "type": "long" + }, + "synack": { + "type": "long" + } + } + }, + "uptime": { + "type": "long" + } + } + }, + "tcp": { + "properties": { + "ack": { + "type": "boolean" + }, + "fin": { + "type": "boolean" + }, + "psh": { + "type": "boolean" + }, + "rst": { + "type": "boolean" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "syn": { + "type": "boolean" + }, + "tcp_flags": { + "ignore_above": 1024, + "type": "keyword" + }, + "tcp_flags_tc": { + "ignore_above": 1024, + "type": "keyword" + }, + "tcp_flags_ts": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "timestamp": { + "path": "@timestamp", + "type": "alias" + }, + "tls": { + "properties": { + "fingerprint": { + "ignore_above": 1024, + "type": "keyword" + }, + "issuerdn": { + "ignore_above": 1024, + "type": "keyword" + }, + "notafter": { + "type": "date" + }, + "notbefore": { + "type": "date" + }, + "serial": { + "ignore_above": 1024, + "type": "keyword" + }, + "session_resumed": { + "type": "boolean" + }, + "sni": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "tx_id": { + "type": "long" + } + } + } + } + }, + "syslog": { + "properties": { + "facility": { + "type": "long" + }, + "facility_label": { + "ignore_above": 1024, + "type": "keyword" + }, + "priority": { + "type": "long" + }, + "severity_label": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "system": { + "properties": { + "auth": { + "properties": { + "groupadd": { + "type": "object" + }, + "ssh": { + "properties": { + "dropped_ip": { + "type": "ip" + }, + "event": { + "ignore_above": 1024, + "type": "keyword" + }, + "geoip": { + "type": "object" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "signature": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "sudo": { + "properties": { + "command": { + "ignore_above": 1024, + "type": "keyword" + }, + "error": { + "ignore_above": 1024, + "type": "keyword" + }, + "pwd": { + "ignore_above": 1024, + "type": "keyword" + }, + "tty": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "useradd": { + "properties": { + "home": { + "ignore_above": 1024, + "type": "keyword" + }, + "shell": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "syslog": { + "type": "object" + } + } + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "threat": { + "properties": { + "framework": { + "ignore_above": 1024, + "type": "keyword" + }, + "tactic": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "technique": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "timeseries": { + "properties": { + "instance": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "tls": { + "properties": { + "cipher": { + "ignore_above": 1024, + "type": "keyword" + }, + "client": { + "properties": { + "certificate": { + "ignore_above": 1024, + "type": "keyword" + }, + "certificate_chain": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "issuer": { + "ignore_above": 1024, + "type": "keyword" + }, + "ja3": { + "ignore_above": 1024, + "type": "keyword" + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "server_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject": { + "ignore_above": 1024, + "type": "keyword" + }, + "supported_ciphers": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "curve": { + "ignore_above": 1024, + "type": "keyword" + }, + "established": { + "type": "boolean" + }, + "next_protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "resumed": { + "type": "boolean" + }, + "server": { + "properties": { + "certificate": { + "ignore_above": 1024, + "type": "keyword" + }, + "certificate_chain": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "issuer": { + "ignore_above": 1024, + "type": "keyword" + }, + "ja3s": { + "ignore_above": 1024, + "type": "keyword" + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "subject": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "version_protocol": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "tracing": { + "properties": { + "trace": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "transaction": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "traefik": { + "properties": { + "access": { + "properties": { + "backend_url": { + "ignore_above": 1024, + "type": "keyword" + }, + "frontend_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "geoip": { + "properties": { + "city_name": { + "path": "source.geo.city_name", + "type": "alias" + }, + "continent_name": { + "path": "source.geo.continent_name", + "type": "alias" + }, + "country_iso_code": { + "path": "source.geo.country_iso_code", + "type": "alias" + }, + "location": { + "path": "source.geo.location", + "type": "alias" + }, + "region_iso_code": { + "path": "source.geo.region_iso_code", + "type": "alias" + }, + "region_name": { + "path": "source.geo.region_name", + "type": "alias" + } + } + }, + "request_count": { + "type": "long" + }, + "user_agent": { + "properties": { + "device": { + "path": "user_agent.device.name", + "type": "alias" + }, + "name": { + "path": "user_agent.name", + "type": "alias" + }, + "original": { + "path": "user_agent.original", + "type": "alias" + }, + "os": { + "path": "user_agent.os.full_name", + "type": "alias" + }, + "os_name": { + "path": "user_agent.os.name", + "type": "alias" + } + } + }, + "user_identifier": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "url": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "extension": { + "ignore_above": 1024, + "type": "keyword" + }, + "fragment": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "password": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "scheme": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "username": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user": { + "properties": { + "audit": { + "properties": { + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "effective": { + "properties": { + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "filesystem": { + "properties": { + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "owner": { + "properties": { + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "saved": { + "properties": { + "group": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "terminal": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user_agent": { + "properties": { + "device": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vulnerability": { + "properties": { + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "classification": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "enumeration": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "report_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "scanner": { + "properties": { + "vendor": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "score": { + "properties": { + "base": { + "type": "float" + }, + "environmental": { + "type": "float" + }, + "temporal": { + "type": "float" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "severity": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "zeek": { + "properties": { + "capture_loss": { + "properties": { + "acks": { + "type": "long" + }, + "gaps": { + "type": "long" + }, + "peer": { + "ignore_above": 1024, + "type": "keyword" + }, + "percent_lost": { + "type": "double" + }, + "ts_delta": { + "type": "long" + } + } + }, + "connection": { + "properties": { + "history": { + "ignore_above": 1024, + "type": "keyword" + }, + "icmp": { + "properties": { + "code": { + "type": "long" + }, + "type": { + "type": "long" + } + } + }, + "inner_vlan": { + "type": "long" + }, + "local_orig": { + "type": "boolean" + }, + "local_resp": { + "type": "boolean" + }, + "missed_bytes": { + "type": "long" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "state_message": { + "ignore_above": 1024, + "type": "keyword" + }, + "vlan": { + "type": "long" + } + } + }, + "dce_rpc": { + "properties": { + "endpoint": { + "ignore_above": 1024, + "type": "keyword" + }, + "named_pipe": { + "ignore_above": 1024, + "type": "keyword" + }, + "operation": { + "ignore_above": 1024, + "type": "keyword" + }, + "rtt": { + "type": "long" + } + } + }, + "dhcp": { + "properties": { + "address": { + "properties": { + "assigned": { + "type": "ip" + }, + "client": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "requested": { + "type": "ip" + }, + "server": { + "type": "ip" + } + } + }, + "client_fqdn": { + "ignore_above": 1024, + "type": "keyword" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "duration": { + "type": "double" + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "properties": { + "circuit": { + "ignore_above": 1024, + "type": "keyword" + }, + "remote_agent": { + "ignore_above": 1024, + "type": "keyword" + }, + "subscriber": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "lease_time": { + "type": "long" + }, + "msg": { + "properties": { + "client": { + "ignore_above": 1024, + "type": "keyword" + }, + "origin": { + "type": "ip" + }, + "server": { + "ignore_above": 1024, + "type": "keyword" + }, + "types": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "software": { + "properties": { + "client": { + "ignore_above": 1024, + "type": "keyword" + }, + "server": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "dnp3": { + "properties": { + "function": { + "properties": { + "reply": { + "ignore_above": 1024, + "type": "keyword" + }, + "request": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "id": { + "type": "long" + } + } + }, + "dns": { + "properties": { + "AA": { + "type": "boolean" + }, + "RA": { + "type": "boolean" + }, + "RD": { + "type": "boolean" + }, + "TC": { + "type": "boolean" + }, + "TTLs": { + "type": "double" + }, + "answers": { + "ignore_above": 1024, + "type": "keyword" + }, + "qclass": { + "type": "long" + }, + "qclass_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "qtype": { + "type": "long" + }, + "qtype_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "rcode": { + "type": "long" + }, + "rcode_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "rejected": { + "type": "boolean" + }, + "rtt": { + "type": "double" + }, + "saw_query": { + "type": "boolean" + }, + "saw_reply": { + "type": "boolean" + }, + "total_answers": { + "type": "long" + }, + "total_replies": { + "type": "long" + }, + "trans_id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "dpd": { + "properties": { + "analyzer": { + "ignore_above": 1024, + "type": "keyword" + }, + "failure_reason": { + "ignore_above": 1024, + "type": "keyword" + }, + "packet_segment": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "files": { + "properties": { + "analyzers": { + "ignore_above": 1024, + "type": "keyword" + }, + "depth": { + "type": "long" + }, + "duration": { + "type": "double" + }, + "entropy": { + "type": "double" + }, + "extracted": { + "ignore_above": 1024, + "type": "keyword" + }, + "extracted_cutoff": { + "type": "boolean" + }, + "extracted_size": { + "type": "long" + }, + "filename": { + "ignore_above": 1024, + "type": "keyword" + }, + "fuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "is_orig": { + "type": "boolean" + }, + "local_orig": { + "type": "boolean" + }, + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "mime_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "missing_bytes": { + "type": "long" + }, + "overflow_bytes": { + "type": "long" + }, + "parent_fuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "rx_host": { + "type": "ip" + }, + "seen_bytes": { + "type": "long" + }, + "session_ids": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "source": { + "ignore_above": 1024, + "type": "keyword" + }, + "timedout": { + "type": "boolean" + }, + "total_bytes": { + "type": "long" + }, + "tx_host": { + "type": "ip" + } + } + }, + "ftp": { + "properties": { + "arg": { + "ignore_above": 1024, + "type": "keyword" + }, + "capture_password": { + "type": "boolean" + }, + "cmdarg": { + "properties": { + "arg": { + "ignore_above": 1024, + "type": "keyword" + }, + "cmd": { + "ignore_above": 1024, + "type": "keyword" + }, + "seq": { + "type": "long" + } + } + }, + "command": { + "ignore_above": 1024, + "type": "keyword" + }, + "cwd": { + "ignore_above": 1024, + "type": "keyword" + }, + "data_channel": { + "properties": { + "originating_host": { + "type": "ip" + }, + "passive": { + "type": "boolean" + }, + "response_host": { + "type": "ip" + }, + "response_port": { + "type": "long" + } + } + }, + "file": { + "properties": { + "fuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "mime_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "size": { + "type": "long" + } + } + }, + "last_auth_requested": { + "ignore_above": 1024, + "type": "keyword" + }, + "passive": { + "type": "boolean" + }, + "password": { + "ignore_above": 1024, + "type": "keyword" + }, + "pending_commands": { + "type": "long" + }, + "reply": { + "properties": { + "code": { + "type": "long" + }, + "msg": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "http": { + "properties": { + "captured_password": { + "type": "boolean" + }, + "client_header_names": { + "ignore_above": 1024, + "type": "keyword" + }, + "info_code": { + "type": "long" + }, + "info_msg": { + "ignore_above": 1024, + "type": "keyword" + }, + "orig_filenames": { + "ignore_above": 1024, + "type": "keyword" + }, + "orig_fuids": { + "ignore_above": 1024, + "type": "keyword" + }, + "orig_mime_depth": { + "type": "long" + }, + "orig_mime_types": { + "ignore_above": 1024, + "type": "keyword" + }, + "password": { + "ignore_above": 1024, + "type": "keyword" + }, + "proxied": { + "ignore_above": 1024, + "type": "keyword" + }, + "range_request": { + "type": "boolean" + }, + "resp_filenames": { + "ignore_above": 1024, + "type": "keyword" + }, + "resp_fuids": { + "ignore_above": 1024, + "type": "keyword" + }, + "resp_mime_depth": { + "type": "long" + }, + "resp_mime_types": { + "ignore_above": 1024, + "type": "keyword" + }, + "server_header_names": { + "ignore_above": 1024, + "type": "keyword" + }, + "status_msg": { + "ignore_above": 1024, + "type": "keyword" + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "trans_depth": { + "type": "long" + } + } + }, + "intel": { + "properties": { + "file_desc": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_mime_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "fuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "matched": { + "ignore_above": 1024, + "type": "keyword" + }, + "seen": { + "properties": { + "conn": { + "ignore_above": 1024, + "type": "keyword" + }, + "f": { + "type": "object" + }, + "fuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "indicator": { + "ignore_above": 1024, + "type": "keyword" + }, + "indicator_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "node": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "ignore_above": 1024, + "type": "keyword" + }, + "where": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "sources": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "irc": { + "properties": { + "addl": { + "ignore_above": 1024, + "type": "keyword" + }, + "command": { + "ignore_above": 1024, + "type": "keyword" + }, + "dcc": { + "properties": { + "file": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "size": { + "type": "long" + } + } + }, + "mime_type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "fuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "nick": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + }, + "value": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "kerberos": { + "properties": { + "cert": { + "properties": { + "client": { + "properties": { + "fuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject": { + "ignore_above": 1024, + "type": "keyword" + }, + "value": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "server": { + "properties": { + "fuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject": { + "ignore_above": 1024, + "type": "keyword" + }, + "value": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "cipher": { + "ignore_above": 1024, + "type": "keyword" + }, + "client": { + "ignore_above": 1024, + "type": "keyword" + }, + "error": { + "properties": { + "code": { + "type": "long" + }, + "msg": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "forwardable": { + "type": "boolean" + }, + "renewable": { + "type": "boolean" + }, + "request_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "service": { + "ignore_above": 1024, + "type": "keyword" + }, + "success": { + "type": "boolean" + }, + "ticket": { + "properties": { + "auth": { + "ignore_above": 1024, + "type": "keyword" + }, + "new": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "valid": { + "properties": { + "days": { + "type": "long" + }, + "from": { + "type": "date" + }, + "until": { + "type": "date" + } + } + } + } + }, + "modbus": { + "properties": { + "exception": { + "ignore_above": 1024, + "type": "keyword" + }, + "function": { + "ignore_above": 1024, + "type": "keyword" + }, + "track_address": { + "type": "long" + } + } + }, + "mysql": { + "properties": { + "arg": { + "ignore_above": 1024, + "type": "keyword" + }, + "cmd": { + "ignore_above": 1024, + "type": "keyword" + }, + "response": { + "ignore_above": 1024, + "type": "keyword" + }, + "rows": { + "type": "long" + }, + "success": { + "type": "boolean" + } + } + }, + "notice": { + "properties": { + "actions": { + "ignore_above": 1024, + "type": "keyword" + }, + "connection_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "dropped": { + "type": "boolean" + }, + "email_body_sections": { + "norms": false, + "type": "text" + }, + "email_delay_tokens": { + "ignore_above": 1024, + "type": "keyword" + }, + "false": { + "type": "long" + }, + "ffile": { + "properties": { + "total_bytes": { + "type": "long" + } + } + }, + "file": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "is_orig": { + "type": "boolean" + }, + "mime_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "missing_bytes": { + "type": "long" + }, + "overflow_bytes": { + "type": "long" + }, + "parent_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "seen_bytes": { + "type": "long" + }, + "source": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "fuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "icmp_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "identifier": { + "ignore_above": 1024, + "type": "keyword" + }, + "msg": { + "ignore_above": 1024, + "type": "keyword" + }, + "note": { + "ignore_above": 1024, + "type": "keyword" + }, + "peer_descr": { + "norms": false, + "type": "text" + }, + "peer_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "sub": { + "ignore_above": 1024, + "type": "keyword" + }, + "suppress_for": { + "type": "double" + } + } + }, + "ntlm": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "server": { + "properties": { + "name": { + "properties": { + "dns": { + "ignore_above": 1024, + "type": "keyword" + }, + "netbios": { + "ignore_above": 1024, + "type": "keyword" + }, + "tree": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "success": { + "type": "boolean" + }, + "username": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ocsp": { + "properties": { + "file_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "properties": { + "algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "issuer": { + "properties": { + "key": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "revoke": { + "properties": { + "reason": { + "ignore_above": 1024, + "type": "keyword" + }, + "time": { + "type": "date" + } + } + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "update": { + "properties": { + "next": { + "type": "date" + }, + "this": { + "type": "date" + } + } + } + } + }, + "pe": { + "properties": { + "client": { + "ignore_above": 1024, + "type": "keyword" + }, + "compile_time": { + "type": "date" + }, + "has_cert_table": { + "type": "boolean" + }, + "has_debug_data": { + "type": "boolean" + }, + "has_export_table": { + "type": "boolean" + }, + "has_import_table": { + "type": "boolean" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "is_64bit": { + "type": "boolean" + }, + "is_exe": { + "type": "boolean" + }, + "machine": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "ignore_above": 1024, + "type": "keyword" + }, + "section_names": { + "ignore_above": 1024, + "type": "keyword" + }, + "subsystem": { + "ignore_above": 1024, + "type": "keyword" + }, + "uses_aslr": { + "type": "boolean" + }, + "uses_code_integrity": { + "type": "boolean" + }, + "uses_dep": { + "type": "boolean" + }, + "uses_seh": { + "type": "boolean" + } + } + }, + "radius": { + "properties": { + "connect_info": { + "ignore_above": 1024, + "type": "keyword" + }, + "framed_addr": { + "type": "ip" + }, + "logged": { + "type": "boolean" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "remote_ip": { + "type": "ip" + }, + "reply_msg": { + "ignore_above": 1024, + "type": "keyword" + }, + "result": { + "ignore_above": 1024, + "type": "keyword" + }, + "ttl": { + "type": "long" + }, + "username": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "rdp": { + "properties": { + "cert": { + "properties": { + "count": { + "type": "long" + }, + "permanent": { + "type": "boolean" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "client": { + "properties": { + "build": { + "ignore_above": 1024, + "type": "keyword" + }, + "client_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product_id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "cookie": { + "ignore_above": 1024, + "type": "keyword" + }, + "desktop": { + "properties": { + "color_depth": { + "ignore_above": 1024, + "type": "keyword" + }, + "height": { + "type": "long" + }, + "width": { + "type": "long" + } + } + }, + "done": { + "type": "boolean" + }, + "encryption": { + "properties": { + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "keyboard_layout": { + "ignore_above": 1024, + "type": "keyword" + }, + "result": { + "ignore_above": 1024, + "type": "keyword" + }, + "security_protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "ssl": { + "type": "boolean" + } + } + }, + "rfb": { + "properties": { + "auth": { + "properties": { + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "success": { + "type": "boolean" + } + } + }, + "desktop_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "height": { + "type": "long" + }, + "share_flag": { + "type": "boolean" + }, + "version": { + "properties": { + "client": { + "properties": { + "major": { + "ignore_above": 1024, + "type": "keyword" + }, + "minor": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "server": { + "properties": { + "major": { + "ignore_above": 1024, + "type": "keyword" + }, + "minor": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "width": { + "type": "long" + } + } + }, + "session_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "sip": { + "properties": { + "call_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "content_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "date": { + "ignore_above": 1024, + "type": "keyword" + }, + "reply_to": { + "ignore_above": 1024, + "type": "keyword" + }, + "request": { + "properties": { + "body_length": { + "type": "long" + }, + "from": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "to": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "response": { + "properties": { + "body_length": { + "type": "long" + }, + "from": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "to": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "sequence": { + "properties": { + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "number": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "status": { + "properties": { + "code": { + "type": "long" + }, + "msg": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "subject": { + "ignore_above": 1024, + "type": "keyword" + }, + "transaction_depth": { + "type": "long" + }, + "uri": { + "ignore_above": 1024, + "type": "keyword" + }, + "user_agent": { + "ignore_above": 1024, + "type": "keyword" + }, + "warning": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "smb_cmd": { + "properties": { + "argument": { + "ignore_above": 1024, + "type": "keyword" + }, + "command": { + "ignore_above": 1024, + "type": "keyword" + }, + "file": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "host": { + "properties": { + "rx": { + "type": "ip" + }, + "tx": { + "type": "ip" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "rtt": { + "type": "double" + }, + "smb1_offered_dialects": { + "ignore_above": 1024, + "type": "keyword" + }, + "smb2_offered_dialects": { + "type": "long" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "sub_command": { + "ignore_above": 1024, + "type": "keyword" + }, + "tree": { + "ignore_above": 1024, + "type": "keyword" + }, + "tree_service": { + "ignore_above": 1024, + "type": "keyword" + }, + "username": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "smb_files": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "fid": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "previous_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "size": { + "type": "long" + }, + "times": { + "properties": { + "accessed": { + "type": "date" + }, + "changed": { + "type": "date" + }, + "created": { + "type": "date" + }, + "modified": { + "type": "date" + } + } + }, + "uuid": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "smb_mapping": { + "properties": { + "native_file_system": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "service": { + "ignore_above": 1024, + "type": "keyword" + }, + "share_type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "smtp": { + "properties": { + "cc": { + "ignore_above": 1024, + "type": "keyword" + }, + "date": { + "type": "date" + }, + "first_received": { + "ignore_above": 1024, + "type": "keyword" + }, + "from": { + "ignore_above": 1024, + "type": "keyword" + }, + "fuids": { + "ignore_above": 1024, + "type": "keyword" + }, + "has_client_activity": { + "type": "boolean" + }, + "helo": { + "ignore_above": 1024, + "type": "keyword" + }, + "in_reply_to": { + "ignore_above": 1024, + "type": "keyword" + }, + "is_webmail": { + "type": "boolean" + }, + "last_reply": { + "ignore_above": 1024, + "type": "keyword" + }, + "mail_from": { + "ignore_above": 1024, + "type": "keyword" + }, + "msg_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "type": "ip" + }, + "process_received_from": { + "type": "boolean" + }, + "rcpt_to": { + "ignore_above": 1024, + "type": "keyword" + }, + "reply_to": { + "ignore_above": 1024, + "type": "keyword" + }, + "second_received": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject": { + "ignore_above": 1024, + "type": "keyword" + }, + "tls": { + "type": "boolean" + }, + "to": { + "ignore_above": 1024, + "type": "keyword" + }, + "transaction_depth": { + "type": "long" + }, + "user_agent": { + "ignore_above": 1024, + "type": "keyword" + }, + "x_originating_ip": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "snmp": { + "properties": { + "community": { + "ignore_above": 1024, + "type": "keyword" + }, + "display_string": { + "ignore_above": 1024, + "type": "keyword" + }, + "duration": { + "type": "double" + }, + "get": { + "properties": { + "bulk_requests": { + "type": "long" + }, + "requests": { + "type": "long" + }, + "responses": { + "type": "long" + } + } + }, + "set": { + "properties": { + "requests": { + "type": "long" + } + } + }, + "up_since": { + "type": "date" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "socks": { + "properties": { + "bound": { + "properties": { + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + } + } + }, + "capture_password": { + "type": "boolean" + }, + "password": { + "ignore_above": 1024, + "type": "keyword" + }, + "request": { + "properties": { + "host": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + } + } + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "type": "long" + } + } + }, + "ssh": { + "properties": { + "algorithm": { + "properties": { + "cipher": { + "ignore_above": 1024, + "type": "keyword" + }, + "compression": { + "ignore_above": 1024, + "type": "keyword" + }, + "host_key": { + "ignore_above": 1024, + "type": "keyword" + }, + "key_exchange": { + "ignore_above": 1024, + "type": "keyword" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "auth": { + "properties": { + "attempts": { + "type": "long" + }, + "success": { + "type": "boolean" + } + } + }, + "client": { + "ignore_above": 1024, + "type": "keyword" + }, + "direction": { + "ignore_above": 1024, + "type": "keyword" + }, + "host_key": { + "ignore_above": 1024, + "type": "keyword" + }, + "server": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "type": "long" + } + } + }, + "ssl": { + "properties": { + "cipher": { + "ignore_above": 1024, + "type": "keyword" + }, + "client": { + "properties": { + "cert_chain": { + "ignore_above": 1024, + "type": "keyword" + }, + "cert_chain_fuids": { + "ignore_above": 1024, + "type": "keyword" + }, + "issuer": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "subject": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "curve": { + "ignore_above": 1024, + "type": "keyword" + }, + "established": { + "type": "boolean" + }, + "last_alert": { + "ignore_above": 1024, + "type": "keyword" + }, + "next_protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "resumed": { + "type": "boolean" + }, + "server": { + "properties": { + "cert_chain": { + "ignore_above": 1024, + "type": "keyword" + }, + "cert_chain_fuids": { + "ignore_above": 1024, + "type": "keyword" + }, + "issuer": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "validation": { + "properties": { + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "stats": { + "properties": { + "bytes": { + "properties": { + "received": { + "type": "long" + } + } + }, + "connections": { + "properties": { + "icmp": { + "properties": { + "active": { + "type": "long" + }, + "count": { + "type": "long" + } + } + }, + "tcp": { + "properties": { + "active": { + "type": "long" + }, + "count": { + "type": "long" + } + } + }, + "udp": { + "properties": { + "active": { + "type": "long" + }, + "count": { + "type": "long" + } + } + } + } + }, + "dns_requests": { + "properties": { + "active": { + "type": "long" + }, + "count": { + "type": "long" + } + } + }, + "events": { + "properties": { + "processed": { + "type": "long" + }, + "queued": { + "type": "long" + } + } + }, + "files": { + "properties": { + "active": { + "type": "long" + }, + "count": { + "type": "long" + } + } + }, + "memory": { + "type": "long" + }, + "packets": { + "properties": { + "dropped": { + "type": "long" + }, + "processed": { + "type": "long" + }, + "received": { + "type": "long" + } + } + }, + "peer": { + "ignore_above": 1024, + "type": "keyword" + }, + "reassembly_size": { + "properties": { + "file": { + "type": "long" + }, + "frag": { + "type": "long" + }, + "tcp": { + "type": "long" + }, + "unknown": { + "type": "long" + } + } + }, + "timers": { + "properties": { + "active": { + "type": "long" + }, + "count": { + "type": "long" + } + } + }, + "timestamp_lag": { + "type": "long" + } + } + }, + "syslog": { + "properties": { + "facility": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "ignore_above": 1024, + "type": "keyword" + }, + "severity": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "tunnel": { + "properties": { + "action": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "weird": { + "properties": { + "additional_info": { + "ignore_above": 1024, + "type": "keyword" + }, + "identifier": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "notice": { + "type": "boolean" + }, + "peer": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "x509": { + "properties": { + "basic_constraints": { + "properties": { + "certificate_authority": { + "type": "boolean" + }, + "path_length": { + "type": "long" + } + } + }, + "certificate": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "curve": { + "ignore_above": 1024, + "type": "keyword" + }, + "exponent": { + "ignore_above": 1024, + "type": "keyword" + }, + "issuer": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "key": { + "properties": { + "algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "length": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "serial": { + "ignore_above": 1024, + "type": "keyword" + }, + "signature_algorithm": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject": { + "properties": { + "common_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country": { + "ignore_above": 1024, + "type": "keyword" + }, + "locality": { + "ignore_above": 1024, + "type": "keyword" + }, + "organization": { + "ignore_above": 1024, + "type": "keyword" + }, + "organizational_unit": { + "ignore_above": 1024, + "type": "keyword" + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "valid": { + "properties": { + "from": { + "type": "date" + }, + "until": { + "type": "date" + } + } + }, + "version": { + "type": "long" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "log_cert": { + "type": "boolean" + }, + "san": { + "properties": { + "dns": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "other_fields": { + "type": "boolean" + }, + "uri": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + } + } + }, + "settings": { + "index": { + "mapping": { + "total_fields": { + "limit": "5000" + } + }, + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} \ No newline at end of file diff --git a/x-pack/test/functional/es_archives/ml/module_siem_winlogbeat/data.json.gz b/x-pack/test/functional/es_archives/ml/module_siem_winlogbeat/data.json.gz index 238fb3b6e79c0..a8ddcd19aad1d 100644 Binary files a/x-pack/test/functional/es_archives/ml/module_siem_winlogbeat/data.json.gz and b/x-pack/test/functional/es_archives/ml/module_siem_winlogbeat/data.json.gz differ diff --git a/x-pack/test/functional/es_archives/ml/module_siem_winlogbeat/mappings.json b/x-pack/test/functional/es_archives/ml/module_siem_winlogbeat/mappings.json index e931063040815..3eeda43fbebbe 100644 --- a/x-pack/test/functional/es_archives/ml/module_siem_winlogbeat/mappings.json +++ b/x-pack/test/functional/es_archives/ml/module_siem_winlogbeat/mappings.json @@ -5,6 +5,137 @@ }, "index": "ft_module_siem_winlogbeat", "mappings": { + "_meta": { + "beat": "winlogbeat", + "version": "7.7.0" + }, + "date_detection": false, + "dynamic_templates": [ + { + "labels": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "labels.*" + } + }, + { + "container.labels": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "container.labels.*" + } + }, + { + "dns.answers": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "dns.answers.*" + } + }, + { + "log.syslog": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "log.syslog.*" + } + }, + { + "network.inner": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "network.inner.*" + } + }, + { + "observer.egress": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "observer.egress.*" + } + }, + { + "observer.ingress": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "observer.ingress.*" + } + }, + { + "fields": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "fields.*" + } + }, + { + "docker.container.labels": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "docker.container.labels.*" + } + }, + { + "kubernetes.labels.*": { + "mapping": { + "type": "keyword" + }, + "path_match": "kubernetes.labels.*" + } + }, + { + "kubernetes.annotations.*": { + "mapping": { + "type": "keyword" + }, + "path_match": "kubernetes.annotations.*" + } + }, + { + "winlog.event_data": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "winlog.event_data.*" + } + }, + { + "winlog.user_data": { + "mapping": { + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "winlog.user_data.*" + } + }, + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], "properties": { "@timestamp": { "type": "date" @@ -37,12 +168,54 @@ } } }, - "destination": { + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "client": { "properties": { "address": { "ignore_above": 1024, "type": "keyword" }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, "bytes": { "type": "long" }, @@ -92,28 +265,56 @@ "ignore_above": 1024, "type": "keyword" }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, "packets": { "type": "long" }, - "path": { + "port": { + "type": "long" + }, + "registered_domain": { "ignore_above": 1024, "type": "keyword" }, - "port": { - "type": "long" + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" }, "user": { "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, "email": { "ignore_above": 1024, "type": "keyword" }, "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" }, "group": { "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, "id": { "ignore_above": 1024, "type": "keyword" @@ -133,6 +334,12 @@ "type": "keyword" }, "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" } @@ -140,90 +347,151 @@ } } }, - "event": { + "cloud": { "properties": { - "action": { - "ignore_above": 1024, - "type": "keyword" + "account": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } }, - "category": { + "availability_zone": { "ignore_above": 1024, "type": "keyword" }, - "code": { - "type": "long" - }, - "created": { - "type": "date" - }, - "dataset": { - "ignore_above": 1024, - "type": "keyword" + "image": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } }, - "duration": { - "type": "long" + "instance": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } }, - "end": { - "type": "date" + "machine": { + "properties": { + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } }, - "hash": { - "ignore_above": 1024, - "type": "keyword" + "project": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } }, - "id": { + "provider": { "ignore_above": 1024, "type": "keyword" }, - "kind": { + "region": { "ignore_above": 1024, "type": "keyword" + } + } + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" }, - "module": { + "status": { "ignore_above": 1024, "type": "keyword" }, - "origin": { + "subject_name": { "ignore_above": 1024, "type": "keyword" }, - "original": { - "ignore_above": 1024, - "type": "keyword" + "trusted": { + "type": "boolean" }, - "outcome": { + "valid": { + "type": "boolean" + } + } + }, + "container": { + "properties": { + "id": { "ignore_above": 1024, "type": "keyword" }, - "risk_score": { - "type": "float" - }, - "risk_score_norm": { - "type": "float" - }, - "severity": { - "type": "long" + "image": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "tag": { + "ignore_above": 1024, + "type": "keyword" + } + } }, - "start": { - "type": "date" + "labels": { + "type": "object" }, - "timezone": { + "name": { "ignore_above": 1024, "type": "keyword" }, - "type": { + "runtime": { "ignore_above": 1024, "type": "keyword" } } }, - "host": { + "destination": { "properties": { - "architecture": { + "address": { "ignore_above": 1024, "type": "keyword" }, - "containerized": { - "type": "boolean" + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" }, "geo": { "properties": { @@ -260,540 +528,3561 @@ } } }, - "hostname": { - "ignore_above": 1024, - "type": "keyword" + "ip": { + "type": "ip" }, - "id": { + "mac": { "ignore_above": 1024, "type": "keyword" }, - "ip": { - "type": "ip" + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } }, - "mac": { + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "registered_domain": { "ignore_above": 1024, "type": "keyword" }, - "name": { + "top_level_domain": { "ignore_above": 1024, "type": "keyword" }, - "os": { + "user": { "properties": { - "build": { + "domain": { "ignore_above": 1024, "type": "keyword" }, - "codename": { + "email": { "ignore_above": 1024, "type": "keyword" }, - "family": { + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" }, - "full": { + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { "ignore_above": 1024, "type": "keyword" }, - "kernel": { + "id": { "ignore_above": 1024, "type": "keyword" }, "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, "ignore_above": 1024, "type": "keyword" + } + } + } + } + }, + "dll": { + "properties": { + "code_signature": { + "properties": { + "exists": { + "type": "boolean" }, - "platform": { + "status": { "ignore_above": 1024, "type": "keyword" }, - "version": { + "subject_name": { "ignore_above": 1024, "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" } } }, - "type": { - "ignore_above": 1024, - "type": "keyword" - }, - "user": { + "hash": { "properties": { - "email": { - "ignore_above": 1024, - "type": "keyword" - }, - "full_name": { + "md5": { "ignore_above": 1024, "type": "keyword" }, - "group": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "hash": { + "sha1": { "ignore_above": 1024, "type": "keyword" }, - "id": { + "sha256": { "ignore_above": 1024, "type": "keyword" }, - "name": { + "sha512": { "ignore_above": 1024, "type": "keyword" } } - } - } - }, - "process": { - "properties": { - "args": { - "ignore_above": 1024, - "type": "keyword" }, - "entity_id": { + "name": { "ignore_above": 1024, "type": "keyword" }, - "executable": { + "path": { "ignore_above": 1024, "type": "keyword" }, - "hash": { + "pe": { "properties": { - "blake2b_256": { + "company": { "ignore_above": 1024, "type": "keyword" }, - "blake2b_384": { + "description": { "ignore_above": 1024, "type": "keyword" }, - "blake2b_512": { + "file_version": { "ignore_above": 1024, "type": "keyword" }, - "md5": { + "original_file_name": { "ignore_above": 1024, "type": "keyword" }, - "sha1": { + "product": { "ignore_above": 1024, "type": "keyword" - }, - "sha224": { + } + } + } + } + }, + "dns": { + "properties": { + "answers": { + "properties": { + "class": { "ignore_above": 1024, "type": "keyword" }, - "sha256": { + "data": { "ignore_above": 1024, "type": "keyword" }, - "sha384": { + "name": { "ignore_above": 1024, "type": "keyword" }, - "sha3_224": { - "ignore_above": 1024, - "type": "keyword" + "ttl": { + "type": "long" }, - "sha3_256": { + "type": { "ignore_above": 1024, "type": "keyword" - }, - "sha3_384": { + } + } + }, + "header_flags": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "op_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "question": { + "properties": { + "class": { "ignore_above": 1024, "type": "keyword" }, - "sha3_512": { + "name": { "ignore_above": 1024, "type": "keyword" }, - "sha512": { + "registered_domain": { "ignore_above": 1024, "type": "keyword" }, - "sha512_224": { + "subdomain": { "ignore_above": 1024, "type": "keyword" }, - "sha512_256": { + "top_level_domain": { "ignore_above": 1024, "type": "keyword" }, - "xxh64": { + "type": { "ignore_above": 1024, "type": "keyword" } } }, - "name": { + "resolved_ip": { + "type": "ip" + }, + "response_code": { "ignore_above": 1024, "type": "keyword" }, - "parent": { + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "docker": { + "properties": { + "container": { "properties": { - "name": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" + "labels": { + "type": "object" } } + } + } + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "error": { + "properties": { + "code": { + "ignore_above": 1024, + "type": "keyword" }, - "pid": { - "type": "long" - }, - "ppid": { - "type": "long" + "id": { + "ignore_above": 1024, + "type": "keyword" }, - "start": { - "type": "date" + "message": { + "norms": false, + "type": "text" }, - "thread": { - "properties": { - "id": { - "type": "long" + "stack_trace": { + "fields": { + "text": { + "norms": false, + "type": "text" } - } - }, - "title": { + }, "ignore_above": 1024, "type": "keyword" }, - "working_directory": { + "type": { "ignore_above": 1024, "type": "keyword" } } }, - "source": { + "event": { "properties": { - "address": { + "action": { "ignore_above": 1024, "type": "keyword" }, - "bytes": { + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "created": { + "type": "date" + }, + "dataset": { + "ignore_above": 1024, + "type": "keyword" + }, + "duration": { "type": "long" }, - "domain": { + "end": { + "type": "date" + }, + "hash": { "ignore_above": 1024, "type": "keyword" }, - "geo": { - "properties": { - "city_name": { - "ignore_above": 1024, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ingested": { + "type": "date" + }, + "kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "ignore_above": 1024, + "type": "keyword" + }, + "outcome": { + "ignore_above": 1024, + "type": "keyword" + }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "risk_score": { + "type": "float" + }, + "risk_score_norm": { + "type": "float" + }, + "sequence": { + "type": "long" + }, + "severity": { + "type": "long" + }, + "start": { + "type": "date" + }, + "timezone": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "fields": { + "type": "object" + }, + "file": { + "properties": { + "accessed": { + "type": "date" + }, + "attributes": { + "ignore_above": 1024, + "type": "keyword" + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "created": { + "type": "date" + }, + "ctime": { + "type": "date" + }, + "device": { + "ignore_above": 1024, + "type": "keyword" + }, + "directory": { + "ignore_above": 1024, + "type": "keyword" + }, + "drive_letter": { + "ignore_above": 1, + "type": "keyword" + }, + "extension": { + "ignore_above": 1024, + "type": "keyword" + }, + "gid": { + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "inode": { + "ignore_above": 1024, + "type": "keyword" + }, + "mime_type": { + "ignore_above": 1024, + "type": "keyword" + }, + "mode": { + "ignore_above": 1024, + "type": "keyword" + }, + "mtime": { + "type": "date" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "owner": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "pe": { + "properties": { + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "size": { + "type": "long" + }, + "target_path": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "properties": { + "imphash": { + "ignore_above": 1024, + "type": "keyword" + }, + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "host": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "containerized": { + "type": "boolean" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "build": { + "ignore_above": 1024, + "type": "keyword" + }, + "codename": { + "ignore_above": 1024, + "type": "keyword" + }, + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "http": { + "properties": { + "request": { + "properties": { + "body": { + "properties": { + "bytes": { + "type": "long" + }, + "content": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bytes": { + "type": "long" + }, + "method": { + "ignore_above": 1024, + "type": "keyword" + }, + "referrer": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "response": { + "properties": { + "body": { + "properties": { + "bytes": { + "type": "long" + }, + "content": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "bytes": { + "type": "long" + }, + "status_code": { + "type": "long" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "interface": { + "properties": { + "alias": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "jolokia": { + "properties": { + "agent": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "secured": { + "type": "boolean" + }, + "server": { + "properties": { + "product": { + "ignore_above": 1024, + "type": "keyword" + }, + "vendor": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "url": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "kubernetes": { + "properties": { + "annotations": { + "properties": { + "*": { + "type": "object" + } + } + }, + "container": { + "properties": { + "image": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "deployment": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "labels": { + "properties": { + "*": { + "type": "object" + } + } + }, + "namespace": { + "ignore_above": 1024, + "type": "keyword" + }, + "node": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pod": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "uid": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "replicaset": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "statefulset": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "labels": { + "type": "object" + }, + "log": { + "properties": { + "file": { + "properties": { + "path": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "logger": { + "ignore_above": 1024, + "type": "keyword" + }, + "origin": { + "properties": { + "file": { + "properties": { + "line": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "function": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "original": { + "ignore_above": 1024, + "type": "keyword" + }, + "syslog": { + "properties": { + "facility": { + "properties": { + "code": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "priority": { + "type": "long" + }, + "severity": { + "properties": { + "code": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + } + }, + "message": { + "norms": false, + "type": "text" + }, + "network": { + "properties": { + "application": { + "ignore_above": 1024, + "type": "keyword" + }, + "bytes": { + "type": "long" + }, + "community_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "direction": { + "ignore_above": 1024, + "type": "keyword" + }, + "forwarded_ip": { + "type": "ip" + }, + "iana_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "inner": { + "properties": { + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "packets": { + "type": "long" + }, + "protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "transport": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "observer": { + "properties": { + "egress": { + "properties": { + "interface": { + "properties": { + "alias": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "zone": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "ingress": { + "properties": { + "interface": { + "properties": { + "alias": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "zone": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + }, + "serial_number": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "vendor": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "organization": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "package": { + "properties": { + "architecture": { + "ignore_above": 1024, + "type": "keyword" + }, + "build_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "checksum": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "install_scope": { + "ignore_above": 1024, + "type": "keyword" + }, + "installed": { + "type": "date" + }, + "license": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "size": { + "type": "long" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pe": { + "properties": { + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "process": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "args_count": { + "type": "long" + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "command_line": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "entity_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "executable": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "exit_code": { + "type": "long" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "parent": { + "properties": { + "args": { + "ignore_above": 1024, + "type": "keyword" + }, + "args_count": { + "type": "long" + }, + "code_signature": { + "properties": { + "exists": { + "type": "boolean" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "trusted": { + "type": "boolean" + }, + "valid": { + "type": "boolean" + } + } + }, + "command_line": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "entity_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "executable": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "exit_code": { + "type": "long" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha512": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "pgid": { + "type": "long" + }, + "pid": { + "type": "long" + }, + "ppid": { + "type": "long" + }, + "start": { + "type": "date" + }, + "thread": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "title": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + }, + "working_directory": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pe": { + "properties": { + "company": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "file_version": { + "ignore_above": 1024, + "type": "keyword" + }, + "original_file_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "product": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "pgid": { + "type": "long" + }, + "pid": { + "type": "long" + }, + "ppid": { + "type": "long" + }, + "start": { + "type": "date" + }, + "thread": { + "properties": { + "id": { + "type": "long" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "title": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "uptime": { + "type": "long" + }, + "working_directory": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "registry": { + "properties": { + "data": { + "properties": { + "bytes": { + "ignore_above": 1024, + "type": "keyword" + }, + "strings": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hive": { + "ignore_above": 1024, + "type": "keyword" + }, + "key": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "value": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "related": { + "properties": { + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "ip": { + "type": "ip" + }, + "user": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "rule": { + "properties": { + "author": { + "ignore_above": 1024, + "type": "keyword" + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "license": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "ruleset": { + "ignore_above": 1024, + "type": "keyword" + }, + "uuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "server": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "service": { + "properties": { + "ephemeral_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "node": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "state": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "source": { + "properties": { + "address": { + "ignore_above": 1024, + "type": "keyword" + }, + "as": { + "properties": { + "number": { + "type": "long" + }, + "organization": { + "properties": { + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "bytes": { + "type": "long" + }, + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "geo": { + "properties": { + "city_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "continent_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "country_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "location": { + "type": "geo_point" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_iso_code": { + "ignore_above": 1024, + "type": "keyword" + }, + "region_name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "mac": { + "ignore_above": 1024, + "type": "keyword" + }, + "nat": { + "properties": { + "ip": { + "type": "ip" + }, + "port": { + "type": "long" + } + } + }, + "packets": { + "type": "long" + }, + "port": { + "type": "long" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "sysmon": { + "properties": { + "dns": { + "properties": { + "status": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + }, + "threat": { + "properties": { + "framework": { + "ignore_above": 1024, + "type": "keyword" + }, + "tactic": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "technique": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "timeseries": { + "properties": { + "instance": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "tls": { + "properties": { + "cipher": { + "ignore_above": 1024, + "type": "keyword" + }, + "client": { + "properties": { + "certificate": { + "ignore_above": 1024, + "type": "keyword" + }, + "certificate_chain": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "issuer": { + "ignore_above": 1024, + "type": "keyword" + }, + "ja3": { + "ignore_above": 1024, + "type": "keyword" + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "server_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "subject": { + "ignore_above": 1024, + "type": "keyword" + }, + "supported_ciphers": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "curve": { + "ignore_above": 1024, + "type": "keyword" + }, + "established": { + "type": "boolean" + }, + "next_protocol": { + "ignore_above": 1024, + "type": "keyword" + }, + "resumed": { + "type": "boolean" + }, + "server": { + "properties": { + "certificate": { + "ignore_above": 1024, + "type": "keyword" + }, + "certificate_chain": { + "ignore_above": 1024, + "type": "keyword" + }, + "hash": { + "properties": { + "md5": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha1": { + "ignore_above": 1024, + "type": "keyword" + }, + "sha256": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "issuer": { + "ignore_above": 1024, + "type": "keyword" + }, + "ja3s": { + "ignore_above": 1024, + "type": "keyword" + }, + "not_after": { + "type": "date" + }, + "not_before": { + "type": "date" + }, + "subject": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + }, + "version_protocol": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "tracing": { + "properties": { + "trace": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "transaction": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + }, + "url": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "extension": { + "ignore_above": 1024, + "type": "keyword" + }, + "fragment": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "password": { + "ignore_above": 1024, + "type": "keyword" + }, + "path": { + "ignore_above": 1024, + "type": "keyword" + }, + "port": { + "type": "long" + }, + "query": { + "ignore_above": 1024, + "type": "keyword" + }, + "registered_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "scheme": { + "ignore_above": 1024, + "type": "keyword" + }, + "top_level_domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "username": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "email": { + "ignore_above": 1024, + "type": "keyword" + }, + "full_name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "group": { + "properties": { + "domain": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "user_agent": { + "properties": { + "device": { + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "os": { + "properties": { + "family": { + "ignore_above": 1024, + "type": "keyword" + }, + "full": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "kernel": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "platform": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vlan": { + "properties": { + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "vulnerability": { + "properties": { + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "classification": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "type": "keyword" + }, + "enumeration": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "report_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "scanner": { + "properties": { + "vendor": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "score": { + "properties": { + "base": { + "type": "float" + }, + "environmental": { + "type": "float" + }, + "temporal": { + "type": "float" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "severity": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "winlog": { + "properties": { + "activity_id": { + "ignore_above": 1024, + "type": "keyword" + }, + "api": { + "ignore_above": 1024, + "type": "keyword" + }, + "channel": { + "ignore_above": 1024, + "type": "keyword" + }, + "computer_name": { + "ignore_above": 1024, + "type": "keyword" + }, + "event_data": { + "properties": { + "AccessList": { + "type": "keyword" + }, + "AccessMask": { + "type": "keyword" + }, + "AccountName": { + "type": "keyword" + }, + "AdvancedOptions": { + "type": "keyword" + }, + "AlgorithmName": { + "type": "keyword" + }, + "AppPoolID": { + "type": "keyword" + }, + "AuditPolicyChanges": { + "type": "keyword" + }, + "AuditPolicyChangesDescription": { + "type": "keyword" + }, + "AuthenticationPackageName": { + "ignore_above": 1024, + "type": "keyword" + }, + "Binary": { + "ignore_above": 1024, + "type": "keyword" + }, + "BitlockerUserInputTime": { + "ignore_above": 1024, + "type": "keyword" + }, + "BootMenuPolicy": { + "type": "keyword" + }, + "BootMode": { + "ignore_above": 1024, + "type": "keyword" + }, + "BootStatusPolicy": { + "type": "keyword" + }, + "BootType": { + "ignore_above": 1024, + "type": "keyword" + }, + "BuildVersion": { + "ignore_above": 1024, + "type": "keyword" + }, + "CallerProcessId": { + "type": "keyword" + }, + "CallerProcessName": { + "type": "keyword" + }, + "Category": { + "type": "keyword" + }, + "CategoryId": { + "type": "keyword" + }, + "ClientCreationTime": { + "type": "keyword" + }, + "ClientProcessId": { + "type": "keyword" + }, + "Company": { + "ignore_above": 1024, + "type": "keyword" + }, + "Config": { + "type": "keyword" + }, + "ConfigAccessPolicy": { + "type": "keyword" + }, + "ConfigurationReader": { + "type": "keyword" + }, + "CorruptionActionState": { + "ignore_above": 1024, + "type": "keyword" + }, + "CountNew": { + "type": "keyword" + }, + "CountOfCredentialsReturned": { + "type": "keyword" + }, + "CountOld": { + "type": "keyword" + }, + "CreationUtcTime": { + "ignore_above": 1024, + "type": "keyword" + }, + "CurrentStratumNumber": { + "type": "keyword" + }, + "Default SD String:": { + "type": "keyword" + }, + "Description": { + "ignore_above": 1024, + "type": "keyword" + }, + "DestinationIsIpv6": { + "type": "keyword" + }, + "Detail": { + "ignore_above": 1024, + "type": "keyword" + }, + "Details": { + "type": "keyword" + }, + "DeviceName": { + "ignore_above": 1024, + "type": "keyword" + }, + "DeviceNameLength": { + "ignore_above": 1024, + "type": "keyword" + }, + "DeviceTime": { + "ignore_above": 1024, + "type": "keyword" + }, + "DeviceVersionMajor": { + "ignore_above": 1024, + "type": "keyword" + }, + "DeviceVersionMinor": { + "ignore_above": 1024, + "type": "keyword" + }, + "DirtyPages": { + "type": "keyword" + }, + "DisableIntegrityChecks": { + "type": "keyword" + }, + "DriveName": { + "ignore_above": 1024, + "type": "keyword" + }, + "DriverName": { + "ignore_above": 1024, + "type": "keyword" + }, + "DriverNameLength": { + "ignore_above": 1024, + "type": "keyword" + }, + "DwordVal": { + "ignore_above": 1024, + "type": "keyword" + }, + "ElevatedToken": { + "type": "keyword" + }, + "EnableDisableReason": { + "type": "keyword" + }, + "EnabledNew": { + "type": "keyword" + }, + "EntryCount": { + "ignore_above": 1024, + "type": "keyword" + }, + "EventType": { + "type": "keyword" + }, + "ExtraInfo": { + "ignore_above": 1024, + "type": "keyword" + }, + "ExtraString": { + "type": "keyword" + }, + "ExtraStringLength": { + "type": "keyword" + }, + "FailureName": { + "ignore_above": 1024, + "type": "keyword" + }, + "FailureNameLength": { + "ignore_above": 1024, + "type": "keyword" + }, + "FailureReason": { + "type": "keyword" + }, + "FileName": { + "type": "keyword" + }, + "FileVersion": { + "ignore_above": 1024, + "type": "keyword" + }, + "FinalStatus": { + "ignore_above": 1024, + "type": "keyword" + }, + "FlightSigning": { + "type": "keyword" + }, + "Group": { + "ignore_above": 1024, + "type": "keyword" + }, + "HandleId": { + "type": "keyword" + }, + "HiveName": { + "type": "keyword" + }, + "HiveNameLength": { + "type": "keyword" + }, + "HypervisorDebug": { + "type": "keyword" + }, + "HypervisorLaunchType": { + "type": "keyword" + }, + "HypervisorLoadOptions": { + "type": "keyword" + }, + "IdleImplementation": { + "ignore_above": 1024, + "type": "keyword" + }, + "IdleStateCount": { + "ignore_above": 1024, + "type": "keyword" + }, + "ImagePath": { + "type": "keyword" + }, + "ImpersonationLevel": { + "ignore_above": 1024, + "type": "keyword" + }, + "Initiated": { + "type": "keyword" + }, + "IntegrityLevel": { + "ignore_above": 1024, + "type": "keyword" + }, + "InternalCode": { + "type": "keyword" + }, + "IpAddress": { + "ignore_above": 1024, + "type": "keyword" + }, + "IpPort": { + "ignore_above": 1024, + "type": "keyword" + }, + "IsTestConfig": { + "type": "keyword" + }, + "KernelDebug": { + "type": "keyword" + }, + "KeyFilePath": { + "type": "keyword" + }, + "KeyLength": { + "ignore_above": 1024, + "type": "keyword" + }, + "KeyName": { + "type": "keyword" + }, + "KeyType": { + "type": "keyword" + }, + "KeysUpdated": { + "type": "keyword" + }, + "LastBootGood": { + "ignore_above": 1024, + "type": "keyword" + }, + "LastBootId": { + "type": "keyword" + }, + "LastShutdownGood": { + "ignore_above": 1024, + "type": "keyword" + }, + "LinkName": { + "type": "keyword" + }, + "ListenerAdapterProtocol": { + "type": "keyword" + }, + "LmPackageName": { + "ignore_above": 1024, + "type": "keyword" + }, + "LoadOptions": { + "type": "keyword" + }, + "LogonGuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "LogonId": { + "ignore_above": 1024, + "type": "keyword" + }, + "LogonProcessName": { + "ignore_above": 1024, + "type": "keyword" + }, + "LogonType": { + "ignore_above": 1024, + "type": "keyword" + }, + "MajorVersion": { + "ignore_above": 1024, + "type": "keyword" + }, + "MandatoryLabel": { + "type": "keyword" + }, + "MaximumPerformancePercent": { + "ignore_above": 1024, + "type": "keyword" + }, + "MemberName": { + "ignore_above": 1024, + "type": "keyword" + }, + "MemberSid": { + "ignore_above": 1024, + "type": "keyword" + }, + "MinimumPerformancePercent": { + "ignore_above": 1024, + "type": "keyword" + }, + "MinimumThrottlePercent": { + "ignore_above": 1024, + "type": "keyword" + }, + "MinorVersion": { + "ignore_above": 1024, + "type": "keyword" + }, + "Minutes": { + "type": "keyword" + }, + "NewProcessId": { + "ignore_above": 1024, + "type": "keyword" + }, + "NewProcessName": { + "ignore_above": 1024, + "type": "keyword" + }, + "NewSchemeGuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "NewSd": { + "type": "keyword" + }, + "NewSize": { + "type": "keyword" + }, + "NewState": { + "type": "keyword" + }, + "NewThreadId": { + "type": "keyword" + }, + "NewTime": { + "ignore_above": 1024, + "type": "keyword" + }, + "NominalFrequency": { + "ignore_above": 1024, + "type": "keyword" + }, + "Number": { + "ignore_above": 1024, + "type": "keyword" + }, + "NumberOfGroupPolicyObjects": { + "type": "keyword" + }, + "ObjectName": { + "type": "keyword" + }, + "ObjectServer": { + "type": "keyword" + }, + "ObjectType": { + "type": "keyword" + }, + "OldSchemeGuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "OldSd": { + "type": "keyword" + }, + "OldTime": { + "ignore_above": 1024, + "type": "keyword" + }, + "Operation": { + "type": "keyword" + }, + "OriginalFileName": { + "ignore_above": 1024, + "type": "keyword" + }, + "OriginalSize": { + "type": "keyword" + }, + "PackageName": { + "type": "keyword" + }, + "Path": { + "ignore_above": 1024, + "type": "keyword" + }, + "PerformanceImplementation": { + "ignore_above": 1024, + "type": "keyword" + }, + "PreviousCreationUtcTime": { + "ignore_above": 1024, + "type": "keyword" + }, + "PreviousTime": { + "ignore_above": 1024, + "type": "keyword" + }, + "PrivilegeList": { + "ignore_above": 1024, + "type": "keyword" + }, + "ProcessCreationTime": { + "type": "keyword" + }, + "ProcessID": { + "type": "keyword" + }, + "ProcessId": { + "ignore_above": 1024, + "type": "keyword" + }, + "ProcessName": { + "ignore_above": 1024, + "type": "keyword" + }, + "ProcessPath": { + "ignore_above": 1024, + "type": "keyword" + }, + "ProcessPid": { + "ignore_above": 1024, + "type": "keyword" + }, + "ProcessingMode": { + "type": "keyword" + }, + "ProcessingTimeInMilliseconds": { + "type": "keyword" + }, + "Product": { + "ignore_above": 1024, + "type": "keyword" + }, + "Protocol": { + "type": "keyword" + }, + "ProviderName": { + "type": "keyword" + }, + "PuaCount": { + "ignore_above": 1024, + "type": "keyword" + }, + "PuaPolicyId": { + "ignore_above": 1024, + "type": "keyword" + }, + "QfeVersion": { + "ignore_above": 1024, + "type": "keyword" + }, + "ReadOperation": { + "type": "keyword" + }, + "Reason": { + "ignore_above": 1024, + "type": "keyword" + }, + "RemoteEventLogging": { + "type": "keyword" + }, + "ResourceAttributes": { + "type": "keyword" + }, + "ResourceManager": { + "type": "keyword" + }, + "RestrictedAdminMode": { + "type": "keyword" + }, + "ReturnCode": { + "type": "keyword" + }, + "RmId": { + "type": "keyword" + }, + "RuleName": { + "type": "keyword" + }, + "RunningMode": { + "type": "keyword" + }, + "SchemaVersion": { + "ignore_above": 1024, + "type": "keyword" + }, + "ScriptBlockText": { + "ignore_above": 1024, + "type": "keyword" + }, + "ServiceName": { + "ignore_above": 1024, + "type": "keyword" + }, + "ServiceType": { + "type": "keyword" + }, + "ServiceVersion": { + "ignore_above": 1024, + "type": "keyword" + }, + "ShutdownActionType": { + "ignore_above": 1024, + "type": "keyword" + }, + "ShutdownEventCode": { + "ignore_above": 1024, + "type": "keyword" + }, + "ShutdownReason": { + "ignore_above": 1024, + "type": "keyword" + }, + "Signature": { + "ignore_above": 1024, + "type": "keyword" + }, + "SignatureStatus": { + "ignore_above": 1024, + "type": "keyword" + }, + "Signed": { + "ignore_above": 1024, + "type": "keyword" + }, + "SourceIsIpv6": { + "type": "keyword" + }, + "SourcePortName": { + "type": "keyword" + }, + "StartAddress": { + "type": "keyword" + }, + "StartFunction": { + "type": "keyword" + }, + "StartModule": { + "type": "keyword" + }, + "StartTime": { + "ignore_above": 1024, + "type": "keyword" + }, + "StartType": { + "type": "keyword" + }, + "State": { + "ignore_above": 1024, + "type": "keyword" + }, + "Status": { + "ignore_above": 1024, + "type": "keyword" + }, + "StopTime": { + "ignore_above": 1024, + "type": "keyword" + }, + "SubCategory": { + "type": "keyword" + }, + "SubStatus": { + "type": "keyword" + }, + "SubcategoryGuid": { + "type": "keyword" + }, + "SubcategoryId": { + "type": "keyword" + }, + "SubjectDomainName": { + "ignore_above": 1024, + "type": "keyword" + }, + "SubjectLogonId": { + "ignore_above": 1024, + "type": "keyword" + }, + "SubjectUserName": { + "ignore_above": 1024, + "type": "keyword" + }, + "SubjectUserSid": { + "ignore_above": 1024, + "type": "keyword" + }, + "SupportInfo1": { + "type": "keyword" + }, + "SupportInfo2": { + "type": "keyword" + }, + "TSId": { + "ignore_above": 1024, + "type": "keyword" + }, + "TargetDomainName": { + "ignore_above": 1024, + "type": "keyword" + }, + "TargetImage": { + "type": "keyword" + }, + "TargetInfo": { + "ignore_above": 1024, + "type": "keyword" + }, + "TargetLinkedLogonId": { + "type": "keyword" + }, + "TargetLogonGuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "TargetLogonId": { + "ignore_above": 1024, + "type": "keyword" + }, + "TargetName": { + "type": "keyword" + }, + "TargetObject": { + "type": "keyword" + }, + "TargetOutboundDomainName": { + "type": "keyword" + }, + "TargetOutboundUserName": { + "type": "keyword" + }, + "TargetProcessGuid": { "type": "keyword" }, - "continent_name": { - "ignore_above": 1024, + "TargetProcessId": { "type": "keyword" }, - "country_iso_code": { - "ignore_above": 1024, + "TargetProcessName": { "type": "keyword" }, - "country_name": { + "TargetServerName": { "ignore_above": 1024, "type": "keyword" }, - "location": { - "type": "geo_point" + "TargetSid": { + "type": "keyword" }, - "name": { + "TargetUserName": { "ignore_above": 1024, "type": "keyword" }, - "region_iso_code": { + "TargetUserSid": { "ignore_above": 1024, "type": "keyword" }, - "region_name": { + "TerminalSessionId": { "ignore_above": 1024, "type": "keyword" - } - } - }, - "ip": { - "type": "ip" - }, - "mac": { - "ignore_above": 1024, - "type": "keyword" - }, - "packets": { - "type": "long" - }, - "path": { - "ignore_above": 1024, - "type": "keyword" - }, - "port": { - "type": "long" - }, - "user": { - "properties": { - "email": { - "ignore_above": 1024, + }, + "TestSigning": { "type": "keyword" }, - "full_name": { - "ignore_above": 1024, + "TimeSource": { "type": "keyword" }, - "group": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } + "TimeSourceRefId": { + "type": "keyword" }, - "hash": { - "ignore_above": 1024, + "TmId": { "type": "keyword" }, - "id": { + "TokenElevationType": { "ignore_above": 1024, "type": "keyword" }, - "name": { - "ignore_above": 1024, + "TransactionId": { "type": "keyword" - } - } - } - } - }, - "user": { - "properties": { - "audit": { - "properties": { - "id": { + }, + "TransmittedServices": { "ignore_above": 1024, "type": "keyword" }, - "name": { - "ignore_above": 1024, + "Type": { "type": "keyword" - } - } - }, - "effective": { - "properties": { - "group": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } }, - "id": { - "ignore_above": 1024, + "UpdateReason": { "type": "keyword" }, - "name": { - "ignore_above": 1024, + "Url": { "type": "keyword" - } - } - }, - "email": { - "ignore_above": 1024, - "type": "keyword" - }, - "entity_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "filesystem": { - "properties": { - "group": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } }, - "id": { - "ignore_above": 1024, + "User": { "type": "keyword" }, - "name": { + "UserSid": { "ignore_above": 1024, "type": "keyword" - } - } - }, - "full_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "group": { - "properties": { - "id": { + }, + "Version": { "ignore_above": 1024, "type": "keyword" }, - "name": { - "ignore_above": 1024, + "VirtualAccount": { "type": "keyword" - } - } - }, - "hash": { - "ignore_above": 1024, - "type": "keyword" - }, - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - }, - "name_map": { - "type": "object" - }, - "saved": { - "properties": { - "group": { - "properties": { - "id": { - "ignore_above": 1024, - "type": "keyword" - }, - "name": { - "ignore_above": 1024, - "type": "keyword" - } - } }, - "id": { - "ignore_above": 1024, + "VsmLaunchType": { "type": "keyword" }, - "name": { - "ignore_above": 1024, + "VsmPolicy": { "type": "keyword" - } - } - }, - "selinux": { - "properties": { - "category": { + }, + "Workstation": { "ignore_above": 1024, "type": "keyword" }, - "domain": { + "param1": { "ignore_above": 1024, "type": "keyword" }, - "level": { - "ignore_above": 1024, + "param10": { + "type": "keyword" + }, + "param11": { "type": "keyword" }, - "role": { + "param12": { + "type": "keyword" + }, + "param2": { "ignore_above": 1024, "type": "keyword" }, - "user": { + "param3": { "ignore_above": 1024, "type": "keyword" - } - } - }, - "terminal": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "winlog": { - "properties": { - "activity_id": { - "ignore_above": 1024, - "type": "keyword" - }, - "api": { - "ignore_above": 1024, - "type": "keyword" - }, - "channel": { - "ignore_above": 1024, - "type": "keyword" - }, - "computer_name": { - "ignore_above": 1024, - "type": "keyword" - }, - "event_data": { - "properties": { - "ContextInfo": { + }, + "param4": { + "ignore_above": 1024, "type": "keyword" }, - "LogonType": { - "fields": { - "keyword": { - "ignore_above": 256, - "type": "keyword" - } - }, - "type": "text" + "param5": { + "ignore_above": 1024, + "type": "keyword" }, - "MessageNumber": { + "param6": { + "ignore_above": 1024, "type": "keyword" }, - "MessageTotal": { + "param7": { + "ignore_above": 1024, "type": "keyword" }, - "Path": { + "param8": { + "ignore_above": 1024, "type": "keyword" }, - "Payload": { + "param9": { "type": "keyword" }, - "ScriptBlockId": { + "serviceGuid": { "type": "keyword" }, - "ScriptBlockText": { + "updateGuid": { "type": "keyword" }, - "param1": { + "updateRevisionNumber": { "type": "keyword" }, - "param2": { + "updateTitle": { "type": "keyword" } } @@ -806,6 +4095,34 @@ "ignore_above": 1024, "type": "keyword" }, + "logon": { + "properties": { + "failure": { + "properties": { + "reason": { + "ignore_above": 1024, + "type": "keyword" + }, + "status": { + "ignore_above": 1024, + "type": "keyword" + }, + "sub_status": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "opcode": { "ignore_above": 1024, "type": "keyword" @@ -865,7 +4182,23 @@ } }, "user_data": { - "type": "object" + "properties": { + "binaryData": { + "type": "keyword" + }, + "binaryDataSize": { + "type": "keyword" + }, + "param1": { + "type": "keyword" + }, + "param2": { + "type": "keyword" + }, + "xml_name": { + "type": "keyword" + } + } }, "version": { "type": "long" @@ -876,6 +4209,11 @@ }, "settings": { "index": { + "mapping": { + "total_fields": { + "limit": "5000" + } + }, "number_of_replicas": "1", "number_of_shards": "1" } diff --git a/x-pack/test/functional/page_objects/index_lifecycle_management_page.ts b/x-pack/test/functional/page_objects/index_lifecycle_management_page.ts index 525e0d91e2f4d..2dd70f8a95717 100644 --- a/x-pack/test/functional/page_objects/index_lifecycle_management_page.ts +++ b/x-pack/test/functional/page_objects/index_lifecycle_management_page.ts @@ -69,6 +69,11 @@ export function IndexLifecycleManagementPageProvider({ getService }: FtrProvider await this.saveNewPolicy(); }, + async increasePolicyListPageSize() { + await testSubjects.click('tablePaginationPopoverButton'); + await testSubjects.click(`tablePagination-100-rows`); + }, + async getPolicyList() { const policies = await testSubjects.findAll('policyTableRow'); return mapAsync(policies, async (policy) => { diff --git a/x-pack/test/functional/page_objects/lens_page.ts b/x-pack/test/functional/page_objects/lens_page.ts index 100ed8e079d37..f73440e331466 100644 --- a/x-pack/test/functional/page_objects/lens_page.ts +++ b/x-pack/test/functional/page_objects/lens_page.ts @@ -174,6 +174,19 @@ export function LensPageProvider({ getService, getPageObjects }: FtrProviderCont await PageObjects.header.waitUntilLoadingHasFinished(); }, + /** + * Drags field to geo field workspace + * + * @param field - the desired geo_point or geo_shape field + * */ + async dragFieldToGeoFieldWorkspace(field: string) { + await browser.html5DragAndDrop( + testSubjects.getCssSelector(`lnsFieldListPanelField-${field}`), + testSubjects.getCssSelector('lnsGeoFieldWorkspace') + ); + await PageObjects.header.waitUntilLoadingHasFinished(); + }, + /** * Drags field to workspace * diff --git a/x-pack/test/functional/services/ml/dashboard_embeddables.ts b/x-pack/test/functional/services/ml/dashboard_embeddables.ts index db5c3f35a2e14..03b69854744d4 100644 --- a/x-pack/test/functional/services/ml/dashboard_embeddables.ts +++ b/x-pack/test/functional/services/ml/dashboard_embeddables.ts @@ -35,46 +35,40 @@ export function MachineLearningDashboardEmbeddablesProvider( async assertSelectMaxSeriesToPlotValue(expectedValue: number) { const subj = 'mlAnomalyChartsInitializerMaxSeries'; - await testSubjects.existOrFail(subj); - const input = await testSubjects.find(subj); - const actualValue = await input.getAttribute('value'); - - expect(actualValue).to.eql( - expectedValue, - `Expected max series to plot value to be ${expectedValue}, got ${actualValue}` - ); - }, - - async assertPanelTitle(expectedValue: string) { - const actualValue = await testSubjects.getAttribute('panelTitleInput', 'value'); - expect(actualValue).to.eql( - expectedValue, - `Panel title should be '${expectedValue}' (got '${actualValue}')` - ); - }, - - async setPanelTitle(panelTitle: string) { - await mlCommonUI.setValueWithChecks('panelTitleInput', panelTitle, { - clearWithKeyboard: true, + await retry.tryForTime(60 * 1000, async () => { + await testSubjects.existOrFail(subj); + const input = await testSubjects.find(subj); + const actualValue = await input.getAttribute('value'); + + expect(actualValue).to.eql( + expectedValue, + `Expected max series to plot value to be ${expectedValue}, got ${actualValue}` + ); }); - await this.assertPanelTitle(panelTitle); }, async assertInitializerConfirmButtonEnabled() { const subj = 'mlAnomalyChartsInitializerConfirmButton'; - await testSubjects.existOrFail(subj); - await testSubjects.isEnabled(subj); + + await retry.tryForTime(60 * 1000, async () => { + await testSubjects.existOrFail(subj); + await testSubjects.isEnabled(subj); + }); }, async clickInitializerConfirmButtonEnabled() { const subj = 'mlAnomalyChartsInitializerConfirmButton'; - await this.assertInitializerConfirmButtonEnabled(); - await testSubjects.clickWhenNotDisabled(subj); - await this.assertAnomalyChartsEmbeddableInitializerNotExists(); + await retry.tryForTime(60 * 1000, async () => { + await this.assertInitializerConfirmButtonEnabled(); + await testSubjects.clickWhenNotDisabled(subj); + await this.assertAnomalyChartsEmbeddableInitializerNotExists(); + }); }, async assertDashboardIsEmpty() { - await testSubjects.existOrFail('emptyDashboardWidget'); + await retry.tryForTime(60 * 1000, async () => { + await testSubjects.existOrFail('emptyDashboardWidget'); + }); }, async assertDashboardPanelExists(title: string) { @@ -84,15 +78,21 @@ export function MachineLearningDashboardEmbeddablesProvider( }, async assertAnomalyChartsSeverityThresholdControlExists() { - await testSubjects.existOrFail(`mlAnomalySeverityThresholdControls`); + await retry.tryForTime(60 * 1000, async () => { + await testSubjects.existOrFail(`mlAnomalySeverityThresholdControls`); + }); }, async assertNoMatchingAnomaliesMessageExists() { - await testSubjects.existOrFail(`mlNoMatchingAnomaliesMessage`); + await retry.tryForTime(60 * 1000, async () => { + await testSubjects.existOrFail(`mlNoMatchingAnomaliesMessage`); + }); }, async assertAnomalyChartsExists() { - await testSubjects.existOrFail(`mlExplorerChartsContainer`); + await retry.tryForTime(60 * 1000, async () => { + await testSubjects.existOrFail(`mlExplorerChartsContainer`); + }); }, async openJobSelectionFlyout() { diff --git a/x-pack/test/functional/services/ml/test_resources.ts b/x-pack/test/functional/services/ml/test_resources.ts index c801c077c1319..547ff782bcbe5 100644 --- a/x-pack/test/functional/services/ml/test_resources.ts +++ b/x-pack/test/functional/services/ml/test_resources.ts @@ -528,5 +528,27 @@ export function MachineLearningTestResourcesProvider({ getService }: FtrProvider } log.debug('> ML saved objects deleted.'); }, + + async installFleetPackage(packageIdentifier: string) { + log.debug(`Installing Fleet package'${packageIdentifier}'`); + + await supertest + .post(`/api/fleet/epm/packages/${packageIdentifier}`) + .set(COMMON_REQUEST_HEADERS) + .expect(200); + + log.debug(` > Installed`); + }, + + async removeFleetPackage(packageIdentifier: string) { + log.debug(`Removing Fleet package'${packageIdentifier}'`); + + await supertest + .delete(`/api/fleet/epm/packages/${packageIdentifier}`) + .set(COMMON_REQUEST_HEADERS) + .expect(200); + + log.debug(` > Removed`); + }, }; } diff --git a/x-pack/test/security_api_integration/kerberos.config.ts b/x-pack/test/security_api_integration/kerberos.config.ts index bdd96bae0d9fb..c3f647aadc6d1 100644 --- a/x-pack/test/security_api_integration/kerberos.config.ts +++ b/x-pack/test/security_api_integration/kerberos.config.ts @@ -32,11 +32,10 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { 'xpack.security.authc.realms.kerberos.kerb1.order=0', `xpack.security.authc.realms.kerberos.kerb1.keytab.path=${kerberosKeytabPath}`, ], - serverEnvVars: { - // We're going to use the same TGT multiple times and during a short period of time, so we - // have to disable replay cache so that ES doesn't complain about that. - ES_JAVA_OPTS: `-Djava.security.krb5.conf=${kerberosConfigPath} -Dsun.security.krb5.rcache=none`, - }, + + // We're going to use the same TGT multiple times and during a short period of time, so we + // have to disable replay cache so that ES doesn't complain about that. + esJavaOpts: `-Djava.security.krb5.conf=${kerberosConfigPath} -Dsun.security.krb5.rcache=none`, }, kbnTestServer: { diff --git a/x-pack/test/security_api_integration/login_selector.config.ts b/x-pack/test/security_api_integration/login_selector.config.ts index 9603356568011..f9ef0903b39aa 100644 --- a/x-pack/test/security_api_integration/login_selector.config.ts +++ b/x-pack/test/security_api_integration/login_selector.config.ts @@ -96,11 +96,10 @@ export default async function ({ readConfigFile }: FtrConfigProviderContext) { `xpack.security.authc.realms.saml.saml2.sp.acs=http://localhost:${kibanaPort}/api/security/saml/callback`, 'xpack.security.authc.realms.saml.saml2.attributes.principal=urn:oid:0.0.7', ], - serverEnvVars: { - // We're going to use the same TGT multiple times and during a short period of time, so we - // have to disable replay cache so that ES doesn't complain about that. - ES_JAVA_OPTS: `-Djava.security.krb5.conf=${kerberosConfigPath} -Dsun.security.krb5.rcache=none`, - }, + + // We're going to use the same TGT multiple times and during a short period of time, so we + // have to disable replay cache so that ES doesn't complain about that. + esJavaOpts: `-Djava.security.krb5.conf=${kerberosConfigPath} -Dsun.security.krb5.rcache=none`, }, kbnTestServer: { diff --git a/x-pack/test/security_api_integration/tests/session_idle/extension.ts b/x-pack/test/security_api_integration/tests/session_idle/extension.ts index dfd72dae82346..71621f4e3db8a 100644 --- a/x-pack/test/security_api_integration/tests/session_idle/extension.ts +++ b/x-pack/test/security_api_integration/tests/session_idle/extension.ts @@ -59,17 +59,15 @@ export default function ({ getService }: FtrProviderContext) { describe('GET /internal/security/session', () => { it('should return current session information', async () => { const { body } = await getSessionInfo(); - expect(body.now).to.be.a('number'); - expect(body.idleTimeoutExpiration).to.be.a('number'); - expect(body.lifespanExpiration).to.be(null); + expect(body.expiresInMs).to.be.a('number'); + expect(body.canBeExtended).to.be(true); expect(body.provider).to.eql({ type: 'basic', name: 'basic1' }); }); it('should not extend the session', async () => { const { body } = await getSessionInfo(); const { body: body2 } = await getSessionInfo(); - expect(body2.now).to.be.greaterThan(body.now); - expect(body2.idleTimeoutExpiration).to.equal(body.idleTimeoutExpiration); + expect(body2.expiresInMs).to.be.lessThan(body.expiresInMs); }); }); @@ -85,8 +83,7 @@ export default function ({ getService }: FtrProviderContext) { const { body } = await getSessionInfo(); await extendSession(); const { body: body2 } = await getSessionInfo(); - expect(body2.now).to.be.greaterThan(body.now); - expect(body2.idleTimeoutExpiration).to.be.greaterThan(body.idleTimeoutExpiration); + expect(body2.expiresInMs).not.to.be.lessThan(body.expiresInMs); }); }); }); diff --git a/x-pack/test/stack_functional_integration/apps/ccs/ccs_discover.js b/x-pack/test/stack_functional_integration/apps/ccs/ccs_discover.js index 588ff9a6e9f92..7de23c2899b67 100644 --- a/x-pack/test/stack_functional_integration/apps/ccs/ccs_discover.js +++ b/x-pack/test/stack_functional_integration/apps/ccs/ccs_discover.js @@ -24,6 +24,7 @@ export default ({ getService, getPageObjects }) => { const kibanaServer = getService('kibanaServer'); const queryBar = getService('queryBar'); const filterBar = getService('filterBar'); + const supertest = getService('supertest'); before(async () => { await browser.setWindowSize(1200, 800); @@ -97,6 +98,8 @@ export default ({ getService, getPageObjects }) => { ); await PageObjects.security.logout(); } + // visit app/security so to create .siem-signals-* as side effect + await PageObjects.common.navigateToApp('security', { insertTimestamp: false }); const url = await browser.getCurrentUrl(); log.debug(url); if (!url.includes('kibana')) { @@ -135,6 +138,35 @@ export default ({ getService, getPageObjects }) => { expect(patternName).to.be('*:makelogs工程-*'); }); + it('create local siem signals index pattern', async () => { + log.debug('Add index pattern: .siem-signals-*'); + await supertest + .post('/api/index_patterns/index_pattern') + .set('kbn-xsrf', 'true') + .send({ + index_pattern: { + title: '.siem-signals-*', + }, + override: true, + }) + .expect(200); + }); + + it('create remote monitoring ES index pattern', async () => { + log.debug('Add index pattern: data:.monitoring-es-*'); + await supertest + .post('/api/index_patterns/index_pattern') + .set('kbn-xsrf', 'true') + .send({ + index_pattern: { + title: 'data:.monitoring-es-*', + timeFieldName: 'timestamp', + }, + override: true, + }) + .expect(200); + }); + it('local:makelogs(star) should discover data from the local cluster', async () => { await PageObjects.common.navigateToApp('discover', { insertTimestamp: false }); @@ -203,5 +235,36 @@ export default ({ getService, getPageObjects }) => { expect(hitCount).to.be.lessThan(originalHitCount); }); }); + + it('should generate alerts based on remote events', async () => { + log.debug('Add detection rule type:shards on data:.monitoring-es-*'); + await supertest + .post('/api/detection_engine/rules') + .set('kbn-xsrf', 'true') + .send({ + description: 'This is the description of the rule', + risk_score: 17, + severity: 'low', + interval: '10s', + name: 'CCS_Detection_test', + type: 'query', + from: 'now-1d', + index: ['data:.monitoring-es-*'], + timestamp_override: 'timestamp', + query: 'type:shards', + language: 'kuery', + enabled: true, + }) + .expect(200); + + log.debug('Check if any alert got to .siem-signals-*'); + await PageObjects.common.navigateToApp('discover', { insertTimestamp: false }); + await PageObjects.discover.selectIndexPattern('.siem-signals-*'); + await retry.tryForTime(40000, async () => { + const hitCount = await PageObjects.discover.getHitCount(); + log.debug('### hit count = ' + hitCount); + expect(hitCount).to.be.greaterThan('0'); + }); + }); }); }; diff --git a/yarn.lock b/yarn.lock index ccf722c7728f4..69d5c9553a3b6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1204,10 +1204,10 @@ resolved "https://registry.yarnpkg.com/@bazel/ibazel/-/ibazel-0.15.10.tgz#cf0cff1aec6d8e7bb23e1fc618d09fbd39b7a13f" integrity sha512-0v+OwCQ6fsGFa50r6MXWbUkSGuWOoZ22K4pMSdtWiL5LKFIE4kfmMmtQS+M7/ICNwk2EIYob+NRreyi/DGUz5A== -"@bazel/typescript@^3.4.2": - version "3.4.2" - resolved "https://registry.yarnpkg.com/@bazel/typescript/-/typescript-3.4.2.tgz#183cb14d1f4149cc67ed2723c4b8a7366da5ec36" - integrity sha512-JtLdPOC7rytALJBxawxTCnxVopGstk2eXFs56zHBy+JWSeqrnwujeWZyK5qZHzpag02/JtIQ/ZKkM/DQtrXC8Q== +"@bazel/typescript@^3.5.0": + version "3.5.0" + resolved "https://registry.yarnpkg.com/@bazel/typescript/-/typescript-3.5.0.tgz#605493f4f0a5297df8a7fcccb86a1a80ea2090bb" + integrity sha512-BtGFp4nYFkQTmnONCzomk7dkmOwaINBL3piq+lykBlcc6UxLe9iCAnZpOyPypB1ReN3k3SRNAa53x6oGScQxMg== dependencies: protobufjs "6.8.8" semver "5.6.0" @@ -2686,7 +2686,7 @@ version "0.0.0" uid "" -"@kbn/plugin-generator@link:packages/kbn-plugin-generator": +"@kbn/plugin-generator@link:bazel-bin/packages/kbn-plugin-generator/npm_module": version "0.0.0" uid "" @@ -2698,10 +2698,18 @@ version "0.0.0" uid "" +"@kbn/rule-data-utils@link:packages/kbn-rule-data-utils": + version "0.0.0" + uid "" + "@kbn/securitysolution-constants@link:bazel-bin/packages/kbn-securitysolution-constants/npm_module": version "0.0.0" uid "" +"@kbn/securitysolution-es-utils@link:bazel-bin/packages/kbn-securitysolution-es-utils/npm_module": + version "0.0.0" + uid "" + "@kbn/securitysolution-io-ts-utils@link:bazel-bin/packages/kbn-securitysolution-io-ts-utils/npm_module": version "0.0.0" uid "" @@ -2726,7 +2734,7 @@ version "0.0.0" uid "" -"@kbn/telemetry-tools@link:packages/kbn-telemetry-tools": +"@kbn/telemetry-tools@link:bazel-bin/packages/kbn-telemetry-tools/npm_module": version "0.0.0" uid ""