diff --git a/src/plugins/data/common/search/search_source/index.ts b/src/plugins/data/common/search/search_source/index.ts index 189538415ab53..0bc25a0fe85d0 100644 --- a/src/plugins/data/common/search/search_source/index.ts +++ b/src/plugins/data/common/search/search_source/index.ts @@ -15,3 +15,4 @@ export * from './fetch'; export * from './search_source'; export * from './search_source_service'; export * from './types'; +export * from './query_to_fields'; diff --git a/src/plugins/data/common/search/search_source/query_to_fields.ts b/src/plugins/data/common/search/search_source/query_to_fields.ts new file mode 100644 index 0000000000000..5c64a95a0a927 --- /dev/null +++ b/src/plugins/data/common/search/search_source/query_to_fields.ts @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { DataViewLazy } from '@kbn/data-views-plugin/common'; +import { fromKueryExpression, getKqlFieldNames } from '@kbn/es-query'; +import type { SearchRequest } from './fetch'; +import { EsQuerySortValue } from '../..'; + +export async function queryToFields({ + dataView, + sort, + request, +}: { + dataView: DataViewLazy; + sort?: EsQuerySortValue | EsQuerySortValue[]; + request: SearchRequest; +}) { + let fields = dataView.timeFieldName ? [dataView.timeFieldName] : []; + if (sort) { + const sortArr = Array.isArray(sort) ? sort : [sort]; + fields.push(...sortArr.flatMap((s) => Object.keys(s))); + } + for (const query of request.query) { + if (query.query) { + const nodes = fromKueryExpression(query.query); + const queryFields = getKqlFieldNames(nodes); + fields = fields.concat(queryFields); + } + } + const filters = request.filters; + if (filters) { + const filtersArr = Array.isArray(filters) ? filters : [filters]; + for (const f of filtersArr) { + // unified search bar filters have meta object and key (regular filters) + // unified search bar "custom" filters ("Edit as query DSL", where meta.key is not present but meta is) + // Any other Elasticsearch query DSL filter that gets passed in by consumers (not coming from unified search, and these probably won't have a meta key at all) + if (f?.meta?.key && f.meta.disabled !== true) { + fields.push(f.meta.key); + } + } + } + + // if source filtering is enabled, we need to fetch all the fields + const fieldName = + dataView.getSourceFiltering() && dataView.getSourceFiltering().excludes.length ? ['*'] : fields; + + if (fieldName.length) { + return (await dataView.getFields({ fieldName })).getFieldMapSorted(); + } + + // no fields needed to be loaded for query + return {}; +} diff --git a/src/plugins/data/common/search/search_source/search_source.ts b/src/plugins/data/common/search/search_source/search_source.ts index 6f0807cb42248..a5009f350bd3a 100644 --- a/src/plugins/data/common/search/search_source/search_source.ts +++ b/src/plugins/data/common/search/search_source/search_source.ts @@ -77,11 +77,9 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { buildEsQuery, Filter, - fromKueryExpression, isOfQueryType, isPhraseFilter, isPhrasesFilter, - getKqlFieldNames, } from '@kbn/es-query'; import { fieldWildcardFilter } from '@kbn/kibana-utils-plugin/common'; import { getHighlightRequest } from '@kbn/field-formats-plugin/common'; @@ -95,6 +93,7 @@ import type { ISearchGeneric, IKibanaSearchResponse, IEsSearchResponse } from '@ import { normalizeSortRequest } from './normalize_sort_request'; import { AggConfigSerialized, DataViewField, SerializedSearchSourceFields } from '../..'; +import { queryToFields } from './query_to_fields'; import { AggConfigs, EsQuerySortValue } from '../..'; import type { @@ -778,43 +777,7 @@ export class SearchSource { public async loadDataViewFields(dataView: DataViewLazy) { const request = this.mergeProps(this, { body: {} }); - let fields = dataView.timeFieldName ? [dataView.timeFieldName] : []; - const sort = this.getField('sort'); - if (sort) { - const sortArr = Array.isArray(sort) ? sort : [sort]; - for (const s of sortArr) { - const keys = Object.keys(s); - fields = fields.concat(keys); - } - } - for (const query of request.query) { - if (query.query) { - const nodes = fromKueryExpression(query.query); - const queryFields = getKqlFieldNames(nodes); - fields = fields.concat(queryFields); - } - } - const filters = request.filters; - if (filters) { - const filtersArr = Array.isArray(filters) ? filters : [filters]; - for (const f of filtersArr) { - fields = fields.concat(f.meta.key); - } - } - fields = fields.filter((f) => Boolean(f)); - - if (dataView.getSourceFiltering() && dataView.getSourceFiltering().excludes.length) { - // if source filtering is enabled, we need to fetch all the fields - return (await dataView.getFields({ fieldName: ['*'] })).getFieldMapSorted(); - } else if (fields.length) { - return ( - await dataView.getFields({ - fieldName: fields, - }) - ).getFieldMapSorted(); - } - // no fields needed to be loaded for query - return {}; + return await queryToFields({ dataView, request }); } private flatten() { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/rule_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/rule_type.ts index 6a863728cd746..6cf237ca583e2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/rule_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/rule_type.ts @@ -104,6 +104,14 @@ export const createRuleTypeMocks = ( alertWithPersistence: jest.fn(), logger: loggerMock, shouldWriteAlerts: () => true, + dataViews: { + createDataViewLazy: jest.fn().mockResolvedValue({ + getFields: jest.fn().mockResolvedValue({ + getFieldMapSorted: jest.fn().mockReturnValue({}), + }), + getSourceFiltering: jest.fn().mockReturnValue({ excludes: [] }), + }), + }, }; return { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts index 72caa1d61ff63..471087fa77a4a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/create_security_rule_type_wrapper.ts @@ -26,6 +26,8 @@ import { hasTimestampFields, isMachineLearningParams, isEsqlParams, + isQueryParams, + isEqlParams, getDisabledActionsWarningText, } from './utils/utils'; import { DEFAULT_MAX_SIGNALS, DEFAULT_SEARCH_AFTER_PAGE_SIZE } from '../../../../common/constants'; @@ -341,7 +343,12 @@ export const createSecurityRuleTypeWrapper: CreateSecurityRuleTypeWrapper = }); } - if (!isMachineLearningParams(params) && !isEsqlParams(params)) { + if ( + !isMachineLearningParams(params) && + !isEsqlParams(params) && + !isQueryParams(params) && + !isEqlParams(params) + ) { inputIndexFields = await getFieldsForWildcard({ index: inputIndex, dataViews: services.dataViews, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/query.ts index b3de5a39d829f..272184dbf1e58 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/query/query.ts @@ -59,6 +59,7 @@ export const queryExecutor = async ({ index: runOpts.inputIndex, exceptionFilter: runOpts.exceptionFilter, fields: runOpts.inputIndexFields, + loadFields: true, }); const license = await firstValueFrom(licensing.license$); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_filter.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_filter.ts index 7d7492bd17e2b..920e2c2df9c5e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_filter.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_filter.ts @@ -26,7 +26,8 @@ import type { SavedIdOrUndefined } from '../../../../../common/api/detection_eng import type { PartialFilter } from '../../types'; import { withSecuritySpan } from '../../../../utils/with_security_span'; import type { ESBoolQuery } from '../../../../../common/typed_json'; -import { getQueryFilter } from './get_query_filter'; +import { getQueryFilter as getQueryFilterNoLoadFields } from './get_query_filter'; +import { getQueryFilterLoadFields } from './get_query_filter_load_fields'; export interface GetFilterArgs { type: Type; @@ -38,6 +39,7 @@ export interface GetFilterArgs { index: IndexPatternArray | undefined; exceptionFilter: Filter | undefined; fields?: DataViewFieldBase[]; + loadFields?: boolean; } interface QueryAttributes { @@ -59,7 +61,11 @@ export const getFilter = async ({ query, exceptionFilter, fields = [], + loadFields = false, }: GetFilterArgs): Promise => { + const getQueryFilter = loadFields + ? getQueryFilterLoadFields(services.dataViews) + : getQueryFilterNoLoadFields; const queryFilter = () => { if (query != null && language != null && index != null) { return getQueryFilter({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_query_filter_load_fields.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_query_filter_load_fields.ts new file mode 100644 index 0000000000000..d1c8eab7ac2c2 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/utils/get_query_filter_load_fields.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Language } from '@kbn/securitysolution-io-ts-alerting-types'; +import type { Filter, EsQueryConfig, DataViewFieldBase } from '@kbn/es-query'; +import { DataView } from '@kbn/data-views-plugin/server'; +import { queryToFields } from '@kbn/data-plugin/common'; +import type { DataViewsContract } from '@kbn/data-views-plugin/common'; +import type { FieldFormatsStartCommon } from '@kbn/field-formats-plugin/common'; +import { buildEsQuery } from '@kbn/es-query'; +import type { ESBoolQuery } from '../../../../../common/typed_json'; +import { getAllFilters } from './get_query_filter'; +import type { + IndexPatternArray, + RuleQuery, +} from '../../../../../common/api/detection_engine/model/rule_schema'; + +export const getQueryFilterLoadFields = + (dataViewsService: DataViewsContract) => + async ({ + query, + language, + filters, + index, + exceptionFilter, + }: { + query: RuleQuery; + language: Language; + filters: unknown; + index: IndexPatternArray; + exceptionFilter: Filter | undefined; + fields?: DataViewFieldBase[]; + }): Promise => { + const config: EsQueryConfig = { + allowLeadingWildcards: true, + queryStringOptions: { analyze_wildcard: true }, + ignoreFilterIfFieldNotInIndex: false, + dateFormatTZ: 'Zulu', + }; + + const initialQuery = { query, language }; + const allFilters = getAllFilters(filters as Filter[], exceptionFilter); + + const title = (index ?? []).join(); + + const dataViewLazy = await dataViewsService.createDataViewLazy({ title }); + + const flds = await queryToFields({ + dataView: dataViewLazy, + request: { query: [initialQuery], filters: allFilters }, + }); + + const dataViewLimitedFields = new DataView({ + spec: { title }, + fieldFormats: {} as unknown as FieldFormatsStartCommon, + shortDotsEnable: false, + metaFields: [], + }); + + dataViewLimitedFields.fields.replaceAll(Object.values(flds).map((fld) => fld.toSpec())); + + return buildEsQuery(dataViewLimitedFields, initialQuery, allFilters, config); + };