From 2fd4b5ab328ec68067aef7e7f16badcf22337753 Mon Sep 17 00:00:00 2001 From: Tomasz Ciecierski Date: Wed, 12 Jul 2023 22:03:37 +0200 Subject: [PATCH 01/10] wip change filterQuery to kql in osquery --- .../common/search_strategy/osquery/index.ts | 4 ++- .../osquery/cypress/e2e/all/cases.cy.ts | 1 + .../osquery/cypress/tasks/api_fixtures.ts | 1 + .../action_results/use_action_results.ts | 13 +++----- .../osquery/public/actions/actions_table.tsx | 6 +--- .../public/actions/use_all_live_queries.ts | 8 ++--- .../public/actions/use_live_query_details.ts | 7 ++-- .../osquery/public/agents/use_agent_groups.ts | 2 ++ .../osquery/public/common/helpers.test.ts | 30 ----------------- .../osquery/public/results/use_all_results.ts | 13 +++----- .../live_query/find_live_query_route.ts | 6 ++-- .../get_live_query_details_route.ts | 9 ++++-- .../get_live_query_results_route.ts | 8 ++--- .../osquery/server/routes/live_query/utils.ts | 2 +- .../actions/all/query.all_actions.dsl.ts | 8 +++-- .../details/query.action_details.dsl.ts | 22 ++++++------- .../results/query.action_results.dsl.ts | 22 ++++++------- .../factory/agents/query.all_agents.dsl.ts | 7 ++++ .../factory/results/query.all_results.dsl.ts | 32 +++++++------------ .../osquery/server/utils/build_query.ts | 25 +++++++++++++++ .../components/event_details/osquery_tab.tsx | 2 +- 21 files changed, 109 insertions(+), 119 deletions(-) delete mode 100644 x-pack/plugins/osquery/public/common/helpers.test.ts create mode 100644 x-pack/plugins/osquery/server/utils/build_query.ts diff --git a/x-pack/plugins/osquery/common/search_strategy/osquery/index.ts b/x-pack/plugins/osquery/common/search_strategy/osquery/index.ts index b8985297b3062..16940f0e88153 100644 --- a/x-pack/plugins/osquery/common/search_strategy/osquery/index.ts +++ b/x-pack/plugins/osquery/common/search_strategy/osquery/index.ts @@ -36,7 +36,9 @@ export enum OsqueryQueries { export type FactoryQueryTypes = OsqueryQueries; export interface RequestBasicOptions extends IEsSearchRequest { - filterQuery: ESQuery | string | undefined; + // filterQuery to be removed, when I solve the aggregations problem with kql + filterQuery?: ESQuery | string | undefined; + kql?: string; aggregations?: Record; docValueFields?: DocValueFields[]; factoryQueryType?: FactoryQueryTypes; diff --git a/x-pack/plugins/osquery/cypress/e2e/all/cases.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/cases.cy.ts index 00d7e6738cfb5..8509cae295d54 100644 --- a/x-pack/plugins/osquery/cypress/e2e/all/cases.cy.ts +++ b/x-pack/plugins/osquery/cypress/e2e/all/cases.cy.ts @@ -22,6 +22,7 @@ describe('Add to Cases', () => { loadLiveQuery({ agent_all: true, query: "SELECT * FROM os_version where name='Ubuntu';", + kql: '', }).then((liveQuery) => { liveQueryId = liveQuery.action_id; liveQueryQuery = liveQuery.queries[0].query; diff --git a/x-pack/plugins/osquery/cypress/tasks/api_fixtures.ts b/x-pack/plugins/osquery/cypress/tasks/api_fixtures.ts index a87869d693348..13948ee77b1bf 100644 --- a/x-pack/plugins/osquery/cypress/tasks/api_fixtures.ts +++ b/x-pack/plugins/osquery/cypress/tasks/api_fixtures.ts @@ -119,6 +119,7 @@ export const loadLiveQuery = ( payload = { agent_all: true, query: 'select * from uptime;', + kql: '', } ) => request<{ diff --git a/x-pack/plugins/osquery/public/action_results/use_action_results.ts b/x-pack/plugins/osquery/public/action_results/use_action_results.ts index f950b4f1907c3..b5d3b216ce2b5 100644 --- a/x-pack/plugins/osquery/public/action_results/use_action_results.ts +++ b/x-pack/plugins/osquery/public/action_results/use_action_results.ts @@ -11,11 +11,7 @@ import { useQuery } from '@tanstack/react-query'; import { i18n } from '@kbn/i18n'; import { lastValueFrom } from 'rxjs'; import type { InspectResponse } from '../common/helpers'; -import { - createFilter, - getInspectResponse, - generateTablePaginationOptions, -} from '../common/helpers'; +import { getInspectResponse, generateTablePaginationOptions } from '../common/helpers'; import { useKibana } from '../common/lib/kibana'; import type { ResultEdges, @@ -24,7 +20,6 @@ import type { Direction, } from '../../common/search_strategy'; import { OsqueryQueries } from '../../common/search_strategy'; -import type { ESTermQuery } from '../../common/typed_json'; import { queryClient } from '../query_client'; import { useErrorToast } from '../common/hooks/use_error_toast'; @@ -43,7 +38,7 @@ export interface UseActionResults { direction: Direction; limit: number; sortField: string; - filterQuery?: ESTermQuery | string; + kql?: string; skip?: boolean; isLive?: boolean; } @@ -55,7 +50,7 @@ export const useActionResults = ({ direction, limit, sortField, - filterQuery, + kql, skip = false, isLive = false, }: UseActionResults) => { @@ -70,7 +65,7 @@ export const useActionResults = ({ { actionId, factoryQueryType: OsqueryQueries.actionResults, - filterQuery: createFilter(filterQuery), + kql, pagination: generateTablePaginationOptions(activePage, limit), sort: { direction, diff --git a/x-pack/plugins/osquery/public/actions/actions_table.tsx b/x-pack/plugins/osquery/public/actions/actions_table.tsx index ef256d73946e3..d2bb19d0556c9 100644 --- a/x-pack/plugins/osquery/public/actions/actions_table.tsx +++ b/x-pack/plugins/osquery/public/actions/actions_table.tsx @@ -62,11 +62,7 @@ const ActionsTableComponent = () => { const { data: actionsData } = useAllLiveQueries({ activePage: pageIndex, limit: pageSize, - filterQuery: { - exists: { - field: 'user_id', - }, - }, + kql: 'user_id: *', }); const onTableChange = useCallback(({ page = {} }) => { diff --git a/x-pack/plugins/osquery/public/actions/use_all_live_queries.ts b/x-pack/plugins/osquery/public/actions/use_all_live_queries.ts index ec124b552b2aa..a90da2e6326a2 100644 --- a/x-pack/plugins/osquery/public/actions/use_all_live_queries.ts +++ b/x-pack/plugins/osquery/public/actions/use_all_live_queries.ts @@ -9,10 +9,8 @@ import { useQuery } from '@tanstack/react-query'; import { i18n } from '@kbn/i18n'; import { API_VERSIONS } from '../../common/constants'; -import { createFilter } from '../common/helpers'; import { useKibana } from '../common/lib/kibana'; import type { ActionEdges, ActionsStrategyResponse } from '../../common/search_strategy'; -import type { ESTermQuery, ESExistsQuery } from '../../common/typed_json'; import { useErrorToast } from '../common/hooks/use_error_toast'; import { Direction } from '../../common/search_strategy'; @@ -22,7 +20,7 @@ export interface UseAllLiveQueriesConfig { direction?: Direction; limit?: number; sortField?: string; - filterQuery?: ESTermQuery | ESExistsQuery | string; + kql?: string; skip?: boolean; alertId?: string; } @@ -35,7 +33,7 @@ export const useAllLiveQueries = ({ direction = Direction.desc, limit = 100, sortField = '@timestamp', - filterQuery, + kql, skip = false, alertId, }: UseAllLiveQueriesConfig) => { @@ -53,7 +51,7 @@ export const useAllLiveQueries = ({ { version: API_VERSIONS.public.v1, query: { - filterQuery: createFilter(filterQuery), + kql, page: activePage, pageSize: limit, sort: sortField, diff --git a/x-pack/plugins/osquery/public/actions/use_live_query_details.ts b/x-pack/plugins/osquery/public/actions/use_live_query_details.ts index bea2a0bc6b6fb..7f6d4b9231f0a 100644 --- a/x-pack/plugins/osquery/public/actions/use_live_query_details.ts +++ b/x-pack/plugins/osquery/public/actions/use_live_query_details.ts @@ -12,13 +12,12 @@ import { filter } from 'lodash'; import type { ECSMapping } from '@kbn/osquery-io-ts-types'; import { API_VERSIONS } from '../../common/constants'; import { useKibana } from '../common/lib/kibana'; -import type { ESTermQuery } from '../../common/typed_json'; import { useErrorToast } from '../common/hooks/use_error_toast'; interface UseLiveQueryDetails { actionId?: string; isLive?: boolean; - filterQuery?: ESTermQuery | string; + kql?: string; skip?: boolean; queryIds?: string[]; } @@ -54,7 +53,7 @@ export interface LiveQueryDetailsItem { export const useLiveQueryDetails = ({ actionId, - filterQuery, + kql, isLive = false, skip = false, queryIds, // enable finding out specific queries only, eg. in cases @@ -63,7 +62,7 @@ export const useLiveQueryDetails = ({ const setErrorToast = useErrorToast(); return useQuery<{ data: LiveQueryDetailsItem }, Error, LiveQueryDetailsItem>( - ['liveQueries', { actionId, filterQuery, queryIds }], + ['liveQueries', { actionId, kql, queryIds }], () => http.get(`/api/osquery/live_queries/${actionId}`, { version: API_VERSIONS.public.v1 }), { enabled: !skip && !!actionId, diff --git a/x-pack/plugins/osquery/public/agents/use_agent_groups.ts b/x-pack/plugins/osquery/public/agents/use_agent_groups.ts index 156b439dcc18d..ed077becd32b9 100644 --- a/x-pack/plugins/osquery/public/agents/use_agent_groups.ts +++ b/x-pack/plugins/osquery/public/agents/use_agent_groups.ts @@ -39,6 +39,8 @@ export const useAgentGroups = () => { data.search.search( { filterQuery: { terms: { policy_id: osqueryPolicies } }, + // response fails when kql provided, not sure if it's aggregations or kql + // kql: `policy_id: ${osqueryPolicies}`, factoryQueryType: OsqueryQueries.agents, aggregations: { platforms: { diff --git a/x-pack/plugins/osquery/public/common/helpers.test.ts b/x-pack/plugins/osquery/public/common/helpers.test.ts deleted file mode 100644 index 968023d726b3b..0000000000000 --- a/x-pack/plugins/osquery/public/common/helpers.test.ts +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { ESQuery } from '../../common/typed_json'; - -import { createFilter } from './helpers'; - -describe('Helpers', () => { - describe('#createFilter', () => { - test('if it is a string it returns untouched', () => { - const filter = createFilter('even invalid strings return the same'); - expect(filter).toBe('even invalid strings return the same'); - }); - - test('if it is an ESQuery object it will be returned as a string', () => { - const query: ESQuery = { term: { 'host.id': 'host-value' } }; - const filter = createFilter(query); - expect(filter).toBe(JSON.stringify(query)); - }); - - test('if it is undefined, then undefined is returned', () => { - const filter = createFilter(undefined); - expect(filter).toBe(undefined); - }); - }); -}); diff --git a/x-pack/plugins/osquery/public/results/use_all_results.ts b/x-pack/plugins/osquery/public/results/use_all_results.ts index ee545a2eaf9e9..c92cfdd12ff81 100644 --- a/x-pack/plugins/osquery/public/results/use_all_results.ts +++ b/x-pack/plugins/osquery/public/results/use_all_results.ts @@ -10,11 +10,7 @@ import { useQuery } from '@tanstack/react-query'; import { i18n } from '@kbn/i18n'; import { lastValueFrom } from 'rxjs'; import type { InspectResponse } from '../common/helpers'; -import { - createFilter, - generateTablePaginationOptions, - getInspectResponse, -} from '../common/helpers'; +import { generateTablePaginationOptions, getInspectResponse } from '../common/helpers'; import { useKibana } from '../common/lib/kibana'; import type { ResultEdges, @@ -23,7 +19,6 @@ import type { Direction, } from '../../common/search_strategy'; import { OsqueryQueries } from '../../common/search_strategy'; -import type { ESTermQuery } from '../../common/typed_json'; import { useErrorToast } from '../common/hooks/use_error_toast'; @@ -40,7 +35,7 @@ interface UseAllResults { activePage: number; limit: number; sort: Array<{ field: string; direction: Direction }>; - filterQuery?: ESTermQuery | string; + kql?: string; skip?: boolean; isLive?: boolean; } @@ -50,7 +45,7 @@ export const useAllResults = ({ activePage, limit, sort, - filterQuery, + kql, skip = false, isLive = false, }: UseAllResults) => { @@ -65,7 +60,7 @@ export const useAllResults = ({ { actionId, factoryQueryType: OsqueryQueries.results, - filterQuery: createFilter(filterQuery), + kql, pagination: generateTablePaginationOptions(activePage, limit), sort, }, diff --git a/x-pack/plugins/osquery/server/routes/live_query/find_live_query_route.ts b/x-pack/plugins/osquery/server/routes/live_query/find_live_query_route.ts index 858b76315c8e0..6645728581fae 100644 --- a/x-pack/plugins/osquery/server/routes/live_query/find_live_query_route.ts +++ b/x-pack/plugins/osquery/server/routes/live_query/find_live_query_route.ts @@ -20,7 +20,7 @@ import type { Direction, } from '../../../common/search_strategy'; import { OsqueryQueries } from '../../../common/search_strategy'; -import { createFilter, generateTablePaginationOptions } from '../../../common/utils/build_query'; +import { generateTablePaginationOptions } from '../../../common/utils/build_query'; export const findLiveQueryRoute = (router: IRouter) => { router.versioned @@ -36,7 +36,7 @@ export const findLiveQueryRoute = (router: IRouter) = request: { query: schema.object( { - filterQuery: schema.maybe(schema.string()), + kql: schema.maybe(schema.string()), page: schema.maybe(schema.number()), pageSize: schema.maybe(schema.number()), sort: schema.maybe(schema.string()), @@ -58,7 +58,7 @@ export const findLiveQueryRoute = (router: IRouter) = search.search( { factoryQueryType: OsqueryQueries.actions, - filterQuery: createFilter(request.query.filterQuery), + kql: request.query.kql, pagination: generateTablePaginationOptions( request.query.page ?? 0, request.query.pageSize ?? 100 diff --git a/x-pack/plugins/osquery/server/routes/live_query/get_live_query_details_route.ts b/x-pack/plugins/osquery/server/routes/live_query/get_live_query_details_route.ts index 6bd2bf3a11963..8d8fa0b38a879 100644 --- a/x-pack/plugins/osquery/server/routes/live_query/get_live_query_details_route.ts +++ b/x-pack/plugins/osquery/server/routes/live_query/get_live_query_details_route.ts @@ -39,7 +39,12 @@ export const getLiveQueryDetailsRoute = (router: IRouter( { actionId: request.params.id, - filterQuery: request.query, + kql: request.query.kql, factoryQueryType: OsqueryQueries.actionDetails, }, { abortSignal, strategy: 'osquerySearchStrategy' } diff --git a/x-pack/plugins/osquery/server/routes/live_query/get_live_query_results_route.ts b/x-pack/plugins/osquery/server/routes/live_query/get_live_query_results_route.ts index c7a86d440e344..b6a6b38ab5ab3 100644 --- a/x-pack/plugins/osquery/server/routes/live_query/get_live_query_results_route.ts +++ b/x-pack/plugins/osquery/server/routes/live_query/get_live_query_results_route.ts @@ -18,7 +18,7 @@ import type { ActionDetailsStrategyResponse, } from '../../../common/search_strategy'; import { OsqueryQueries } from '../../../common/search_strategy'; -import { createFilter, generateTablePaginationOptions } from '../../../common/utils/build_query'; +import { generateTablePaginationOptions } from '../../../common/utils/build_query'; import { getActionResponses } from './utils'; export const getLiveQueryResultsRoute = (router: IRouter) => { @@ -35,7 +35,7 @@ export const getLiveQueryResultsRoute = (router: IRouter( { actionId: request.params.id, - filterQuery: createFilter(request.query.filterQuery), + kql: request.query.kql, factoryQueryType: OsqueryQueries.actionDetails, }, { abortSignal, strategy: 'osquerySearchStrategy' } @@ -86,7 +86,7 @@ export const getLiveQueryResultsRoute = (router: IRouter { - const filter = [...createQueryFilterClauses(filterQuery)]; + const { + bool: { filter }, + } = getQueryFilter({ filter: kql }); const dslQuery = { allow_no_indices: true, diff --git a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/details/query.action_details.dsl.ts b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/details/query.action_details.dsl.ts index da5f0d216c686..21c3424a28a90 100644 --- a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/details/query.action_details.dsl.ts +++ b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/details/query.action_details.dsl.ts @@ -7,30 +7,30 @@ import type { ISearchRequestParams } from '@kbn/data-plugin/common'; import { AGENT_ACTIONS_INDEX } from '@kbn/fleet-plugin/common'; +import { isEmpty } from 'lodash'; +import { getQueryFilter } from '../../../../../utils/build_query'; import { ACTIONS_INDEX } from '../../../../../../common/constants'; import type { ActionDetailsRequestOptions } from '../../../../../../common/search_strategy'; -import { createQueryFilterClauses } from '../../../../../../common/utils/build_query'; export const buildActionDetailsQuery = ({ actionId, - filterQuery, + kql, componentTemplateExists, }: ActionDetailsRequestOptions): ISearchRequestParams => { - const filter = [ - ...createQueryFilterClauses(filterQuery), - { - match_phrase: { - action_id: actionId, - }, - }, - ]; + const actionIdQuery = `action_id: ${actionId}`; + let filter = actionIdQuery; + if (!isEmpty(kql)) { + filter = actionIdQuery + ` and ${kql}`; + } + + const query = getQueryFilter({ filter }); const dslQuery = { allow_no_indices: true, index: componentTemplateExists ? `${ACTIONS_INDEX}*` : AGENT_ACTIONS_INDEX, ignore_unavailable: true, body: { - query: { bool: { filter } }, + query, size: 1, fields: ['*'], }, diff --git a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/results/query.action_results.dsl.ts b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/results/query.action_results.dsl.ts index 366342e954fad..2e3554726755b 100644 --- a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/results/query.action_results.dsl.ts +++ b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/results/query.action_results.dsl.ts @@ -7,25 +7,25 @@ import type { ISearchRequestParams } from '@kbn/data-plugin/common'; import { AGENT_ACTIONS_RESULTS_INDEX } from '@kbn/fleet-plugin/common'; +import { isEmpty } from 'lodash'; import { ACTION_RESPONSES_INDEX } from '../../../../../../common/constants'; import type { ActionResultsRequestOptions } from '../../../../../../common/search_strategy'; -import { createQueryFilterClauses } from '../../../../../../common/utils/build_query'; +import { getQueryFilter } from '../../../../../utils/build_query'; export const buildActionResultsQuery = ({ actionId, - filterQuery, + kql, // pagination: { activePage, querySize }, sort, componentTemplateExists, }: ActionResultsRequestOptions): ISearchRequestParams => { - const filter = [ - ...createQueryFilterClauses(filterQuery), - { - match_phrase: { - action_id: actionId, - }, - }, - ]; + const actionIdQuery = `action_id: ${actionId}`; + let filter = actionIdQuery; + if (!isEmpty(kql)) { + filter = actionIdQuery + ` AND ${kql}`; + } + + const query = getQueryFilter({ filter }); const dslQuery = { allow_no_indices: true, @@ -70,7 +70,7 @@ export const buildActionResultsQuery = ({ }, }, }, - query: { bool: { filter } }, + query, // from: activePage * querySize, size: 10000, // querySize, track_total_hits: true, diff --git a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/agents/query.all_agents.dsl.ts b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/agents/query.all_agents.dsl.ts index 908c0cbdd32bf..4782f30d6247d 100644 --- a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/agents/query.all_agents.dsl.ts +++ b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/agents/query.all_agents.dsl.ts @@ -20,6 +20,13 @@ export const buildAgentsQuery = ({ { term: { active: { value: 'true' } } }, ...createQueryFilterClauses(filterQuery), ]; + // const activeQuery = `active: true`; + // let filter = activeQuery; + // if (!isEmpty(kql)) { + // filter = activeQuery + ` AND ${kql}`; + // } + + // const query = getQueryFilter({ filter }); const dslQuery = { allow_no_indices: true, diff --git a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/results/query.all_results.dsl.ts b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/results/query.all_results.dsl.ts index 8a33b4b5a8371..125148dc02c88 100644 --- a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/results/query.all_results.dsl.ts +++ b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/results/query.all_results.dsl.ts @@ -6,34 +6,26 @@ */ import type { ISearchRequestParams } from '@kbn/data-plugin/common'; +import { isEmpty } from 'lodash'; +import { getQueryFilter } from '../../../../utils/build_query'; import { OSQUERY_INTEGRATION_NAME } from '../../../../../common'; import type { ResultsRequestOptions } from '../../../../../common/search_strategy'; -import { createQueryFilterClauses } from '../../../../../common/utils/build_query'; export const buildResultsQuery = ({ actionId, agentId, - filterQuery, + kql, sort, pagination: { activePage, querySize }, }: ResultsRequestOptions): ISearchRequestParams => { - const filter = [ - ...createQueryFilterClauses(filterQuery), - { - match_phrase: { - action_id: actionId, - }, - }, - ...(agentId - ? [ - { - match_phrase: { - 'agent.id': agentId, - }, - }, - ] - : []), - ]; + const actionIdQuery = `action_id: ${actionId}`; + const agentQuery = agentId ? ` and agent.id: ${agentId}` : ''; + let filter = actionIdQuery + agentQuery; + if (!isEmpty(kql)) { + filter = actionIdQuery + ` and ${kql}`; + } + + const query = getQueryFilter({ filter }); const dslQuery = { allow_no_indices: true, @@ -53,7 +45,7 @@ export const buildResultsQuery = ({ }, }, }, - query: { bool: { filter } }, + query, from: activePage * querySize, size: querySize, track_total_hits: true, diff --git a/x-pack/plugins/osquery/server/utils/build_query.ts b/x-pack/plugins/osquery/server/utils/build_query.ts new file mode 100644 index 0000000000000..888e155658367 --- /dev/null +++ b/x-pack/plugins/osquery/server/utils/build_query.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { EsQueryConfig, Query } from '@kbn/es-query'; +import { buildEsQuery } from '@kbn/es-query'; + +export const getQueryFilter = ({ filter }: { filter: string }) => { + // export const getQueryFilter = ({ filter }: GetQueryFilterOptions): GetQueryFilterReturn => { + const kqlQuery: Query = { + language: 'kuery', + query: filter, + }; + const config: EsQueryConfig = { + allowLeadingWildcards: true, + dateFormatTZ: 'Zulu', + ignoreFilterIfFieldNotInIndex: false, + queryStringOptions: { analyze_wildcard: true }, + }; + + return buildEsQuery(undefined, kqlQuery, [], config); +}; diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/osquery_tab.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/osquery_tab.tsx index 67957f2a38133..9eca47d7a4c86 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/osquery_tab.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/osquery_tab.tsx @@ -61,7 +61,7 @@ export const useOsqueryTab = ({ const { OsqueryResults, fetchAllLiveQueries } = osquery; const { data: actionsData } = fetchAllLiveQueries({ - filterQuery: { term: { alert_ids: alertId } }, + kql: `alert_ids: ( ${alertId} )`, alertId, skip: shouldEarlyReturn, }); From 1930fb15bfe873002d15135e5ebbf7ad3fb9e5c4 Mon Sep 17 00:00:00 2001 From: Tomasz Ciecierski Date: Thu, 13 Jul 2023 11:45:27 +0200 Subject: [PATCH 02/10] remove filter query and aggregations --- .../common/search_strategy/osquery/index.ts | 5 --- .../common/utils/build_query/filters.ts | 8 ---- .../osquery/public/agents/use_agent_groups.ts | 25 ++--------- .../plugins/osquery/public/common/helpers.ts | 7 --- x-pack/plugins/osquery/public/common/index.ts | 8 ---- .../details/query.action_details.dsl.ts | 4 +- .../results/query.action_results.dsl.ts | 4 +- .../factory/agents/query.all_agents.dsl.ts | 44 ++++++++++++------- .../factory/results/query.all_results.dsl.ts | 8 ++-- 9 files changed, 40 insertions(+), 73 deletions(-) delete mode 100644 x-pack/plugins/osquery/public/common/index.ts diff --git a/x-pack/plugins/osquery/common/search_strategy/osquery/index.ts b/x-pack/plugins/osquery/common/search_strategy/osquery/index.ts index 16940f0e88153..bdf5e062300e5 100644 --- a/x-pack/plugins/osquery/common/search_strategy/osquery/index.ts +++ b/x-pack/plugins/osquery/common/search_strategy/osquery/index.ts @@ -5,9 +5,7 @@ * 2.0. */ -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import type { IEsSearchRequest } from '@kbn/data-plugin/common'; -import type { ESQuery } from '../../typed_json'; import type { ActionsStrategyResponse, ActionsRequestOptions, @@ -36,10 +34,7 @@ export enum OsqueryQueries { export type FactoryQueryTypes = OsqueryQueries; export interface RequestBasicOptions extends IEsSearchRequest { - // filterQuery to be removed, when I solve the aggregations problem with kql - filterQuery?: ESQuery | string | undefined; kql?: string; - aggregations?: Record; docValueFields?: DocValueFields[]; factoryQueryType?: FactoryQueryTypes; componentTemplateExists?: boolean; diff --git a/x-pack/plugins/osquery/common/utils/build_query/filters.ts b/x-pack/plugins/osquery/common/utils/build_query/filters.ts index 0d82a581e27d9..3ae72c25edf82 100644 --- a/x-pack/plugins/osquery/common/utils/build_query/filters.ts +++ b/x-pack/plugins/osquery/common/utils/build_query/filters.ts @@ -5,15 +5,7 @@ * 2.0. */ -import { isEmpty, isString } from 'lodash/fp'; import type { PaginationInputPaginated, Inspect } from '../../search_strategy'; -import type { ESQuery } from '../../typed_json'; - -export const createQueryFilterClauses = (filterQuery: ESQuery | string | undefined) => - !isEmpty(filterQuery) ? [isString(filterQuery) ? JSON.parse(filterQuery) : filterQuery] : []; - -export const createFilter = (filterQuery: ESQuery | string | undefined) => - isString(filterQuery) ? filterQuery : JSON.stringify(filterQuery); export type InspectResponse = Inspect & { response: string[] }; diff --git a/x-pack/plugins/osquery/public/agents/use_agent_groups.ts b/x-pack/plugins/osquery/public/agents/use_agent_groups.ts index ed077becd32b9..ccacdca1734d8 100644 --- a/x-pack/plugins/osquery/public/agents/use_agent_groups.ts +++ b/x-pack/plugins/osquery/public/agents/use_agent_groups.ts @@ -35,32 +35,13 @@ export const useAgentGroups = () => { >( ['agentGroups'], async () => { + const policiesQuery = osqueryPolicies?.reduce((acc, policy) => `${acc} OR ${policy}`); + const responseData = await lastValueFrom( data.search.search( { - filterQuery: { terms: { policy_id: osqueryPolicies } }, - // response fails when kql provided, not sure if it's aggregations or kql - // kql: `policy_id: ${osqueryPolicies}`, + kql: `policy_id: ( ${policiesQuery} )`, factoryQueryType: OsqueryQueries.agents, - aggregations: { - platforms: { - terms: { - field: 'local_metadata.os.platform', - }, - aggs: { - policies: { - terms: { - field: 'policy_id', - }, - }, - }, - }, - policies: { - terms: { - field: 'policy_id', - }, - }, - }, pagination: generateTablePaginationOptions(0, 9000), sort: { direction: 'asc', diff --git a/x-pack/plugins/osquery/public/common/helpers.ts b/x-pack/plugins/osquery/public/common/helpers.ts index 7697e1d59d5ce..db882c867326f 100644 --- a/x-pack/plugins/osquery/public/common/helpers.ts +++ b/x-pack/plugins/osquery/public/common/helpers.ts @@ -5,8 +5,6 @@ * 2.0. */ -import { isString } from 'lodash/fp'; - import type { PaginationInputPaginated, FactoryQueryTypes, @@ -14,11 +12,6 @@ import type { Inspect, } from '../../common/search_strategy'; -import type { ESQuery } from '../../common/typed_json'; - -export const createFilter = (filterQuery: ESQuery | string | undefined) => - isString(filterQuery) ? filterQuery : JSON.stringify(filterQuery); - export type InspectResponse = Inspect & { response: string[] }; export const generateTablePaginationOptions = ( diff --git a/x-pack/plugins/osquery/public/common/index.ts b/x-pack/plugins/osquery/public/common/index.ts deleted file mode 100644 index 377d7af6d8164..0000000000000 --- a/x-pack/plugins/osquery/public/common/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export { createFilter } from './helpers'; diff --git a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/details/query.action_details.dsl.ts b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/details/query.action_details.dsl.ts index 21c3424a28a90..9b7af953dd2c1 100644 --- a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/details/query.action_details.dsl.ts +++ b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/details/query.action_details.dsl.ts @@ -23,14 +23,14 @@ export const buildActionDetailsQuery = ({ filter = actionIdQuery + ` and ${kql}`; } - const query = getQueryFilter({ filter }); + const filterQuery = getQueryFilter({ filter }); const dslQuery = { allow_no_indices: true, index: componentTemplateExists ? `${ACTIONS_INDEX}*` : AGENT_ACTIONS_INDEX, ignore_unavailable: true, body: { - query, + query: { bool: { filter: filterQuery } }, size: 1, fields: ['*'], }, diff --git a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/results/query.action_results.dsl.ts b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/results/query.action_results.dsl.ts index 2e3554726755b..bba4c0d8c7b12 100644 --- a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/results/query.action_results.dsl.ts +++ b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/results/query.action_results.dsl.ts @@ -25,7 +25,7 @@ export const buildActionResultsQuery = ({ filter = actionIdQuery + ` AND ${kql}`; } - const query = getQueryFilter({ filter }); + const filterQuery = getQueryFilter({ filter }); const dslQuery = { allow_no_indices: true, @@ -70,7 +70,7 @@ export const buildActionResultsQuery = ({ }, }, }, - query, + query: { bool: { filter: filterQuery } }, // from: activePage * querySize, size: 10000, // querySize, track_total_hits: true, diff --git a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/agents/query.all_agents.dsl.ts b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/agents/query.all_agents.dsl.ts index 4782f30d6247d..56aec2a8bc8a5 100644 --- a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/agents/query.all_agents.dsl.ts +++ b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/agents/query.all_agents.dsl.ts @@ -7,26 +7,22 @@ import type { ISearchRequestParams } from '@kbn/data-plugin/common'; import { AGENTS_INDEX } from '@kbn/fleet-plugin/common'; +import { isEmpty } from 'lodash'; +import { getQueryFilter } from '../../../../utils/build_query'; import type { AgentsRequestOptions } from '../../../../../common/search_strategy'; -import { createQueryFilterClauses } from '../../../../../common/utils/build_query'; export const buildAgentsQuery = ({ - filterQuery, + kql, pagination: { cursorStart, querySize }, sort, - aggregations, }: AgentsRequestOptions): ISearchRequestParams => { - const filter = [ - { term: { active: { value: 'true' } } }, - ...createQueryFilterClauses(filterQuery), - ]; - // const activeQuery = `active: true`; - // let filter = activeQuery; - // if (!isEmpty(kql)) { - // filter = activeQuery + ` AND ${kql}`; - // } + const activeQuery = `active: true`; + let filter = activeQuery; + if (!isEmpty(kql)) { + filter = activeQuery + ` AND ${kql}`; + } - // const query = getQueryFilter({ filter }); + const filterQuery = getQueryFilter({ filter }); const dslQuery = { allow_no_indices: true, @@ -35,10 +31,28 @@ export const buildAgentsQuery = ({ body: { query: { bool: { - filter, + filter: filterQuery, + }, + }, + aggs: { + platforms: { + terms: { + field: 'local_metadata.os.platform', + }, + aggs: { + policies: { + terms: { + field: 'policy_id', + }, + }, + }, + }, + policies: { + terms: { + field: 'policy_id', + }, }, }, - aggs: aggregations, track_total_hits: true, sort: [ { diff --git a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/results/query.all_results.dsl.ts b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/results/query.all_results.dsl.ts index 125148dc02c88..40d2a2a849fa6 100644 --- a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/results/query.all_results.dsl.ts +++ b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/results/query.all_results.dsl.ts @@ -19,13 +19,13 @@ export const buildResultsQuery = ({ pagination: { activePage, querySize }, }: ResultsRequestOptions): ISearchRequestParams => { const actionIdQuery = `action_id: ${actionId}`; - const agentQuery = agentId ? ` and agent.id: ${agentId}` : ''; + const agentQuery = agentId ? ` AND agent.id: ${agentId}` : ''; let filter = actionIdQuery + agentQuery; if (!isEmpty(kql)) { - filter = actionIdQuery + ` and ${kql}`; + filter = actionIdQuery + ` AND ${kql}`; } - const query = getQueryFilter({ filter }); + const filterQuery = getQueryFilter({ filter }); const dslQuery = { allow_no_indices: true, @@ -45,7 +45,7 @@ export const buildResultsQuery = ({ }, }, }, - query, + query: { bool: { filter: filterQuery } }, from: activePage * querySize, size: querySize, track_total_hits: true, From 89a32fc9945d83e1aac18dd5a1244c8277a13446 Mon Sep 17 00:00:00 2001 From: Tomasz Ciecierski Date: Thu, 13 Jul 2023 12:17:15 +0200 Subject: [PATCH 03/10] change request --- .../server/search_strategy/osquery/index.ts | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/osquery/server/search_strategy/osquery/index.ts b/x-pack/plugins/osquery/server/search_strategy/osquery/index.ts index 3083e8c191786..bd0e45c89985b 100644 --- a/x-pack/plugins/osquery/server/search_strategy/osquery/index.ts +++ b/x-pack/plugins/osquery/server/search_strategy/osquery/index.ts @@ -15,9 +15,11 @@ import type { FactoryQueryTypes, StrategyResponseType, StrategyRequestType, + ResultsRequestOptions, } from '../../../common/search_strategy/osquery'; import { osqueryFactory } from './factory'; import type { OsqueryFactory } from './factory/types'; +import type { RequestOptionsPaginated } from '../../../common/search_strategy/osquery'; export const osquerySearchStrategyProvider = ( data: PluginStart, @@ -39,7 +41,24 @@ export const osquerySearchStrategyProvider = ( }) ).pipe( mergeMap((exists) => { - const dsl = queryFactory.buildDsl({ ...request, componentTemplateExists: exists }); + const requestWithOptionalTypes = { + factoryQueryType: request.factoryQueryType, + kql: request.kql, + ...((request as RequestOptionsPaginated).pagination + ? { pagination: (request as RequestOptionsPaginated).pagination } + : {}), + ...((request as RequestOptionsPaginated).sort + ? { sort: (request as RequestOptionsPaginated).sort } + : {}), + ...((request as ResultsRequestOptions).actionId + ? { actionId: (request as ResultsRequestOptions).actionId } + : {}), + ...((request as ResultsRequestOptions).agentId + ? { agentId: (request as ResultsRequestOptions).agentId } + : {}), + componentTemplateExists: exists, + } as StrategyRequestType; + const dsl = queryFactory.buildDsl(requestWithOptionalTypes); // use internal user for searching .fleet* indices es = dsl.index?.includes('fleet') || dsl.index?.includes('logs-osquery_manager.action') @@ -48,7 +67,7 @@ export const osquerySearchStrategyProvider = ( return es.search( { - ...request, + ...requestWithOptionalTypes, params: dsl, }, options, From e7939fc2d5bf5428ce0005f854b61f98db90a11c Mon Sep 17 00:00:00 2001 From: Tomasz Ciecierski Date: Thu, 13 Jul 2023 12:39:45 +0200 Subject: [PATCH 04/10] remove redundant config --- x-pack/plugins/osquery/server/utils/build_query.ts | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/osquery/server/utils/build_query.ts b/x-pack/plugins/osquery/server/utils/build_query.ts index 888e155658367..c085790febdec 100644 --- a/x-pack/plugins/osquery/server/utils/build_query.ts +++ b/x-pack/plugins/osquery/server/utils/build_query.ts @@ -5,21 +5,14 @@ * 2.0. */ -import type { EsQueryConfig, Query } from '@kbn/es-query'; +import type { Query } from '@kbn/es-query'; import { buildEsQuery } from '@kbn/es-query'; export const getQueryFilter = ({ filter }: { filter: string }) => { - // export const getQueryFilter = ({ filter }: GetQueryFilterOptions): GetQueryFilterReturn => { const kqlQuery: Query = { language: 'kuery', query: filter, }; - const config: EsQueryConfig = { - allowLeadingWildcards: true, - dateFormatTZ: 'Zulu', - ignoreFilterIfFieldNotInIndex: false, - queryStringOptions: { analyze_wildcard: true }, - }; - return buildEsQuery(undefined, kqlQuery, [], config); + return buildEsQuery(undefined, kqlQuery, []); }; From 64296497de81478f38a01eaa123c252acf49a740 Mon Sep 17 00:00:00 2001 From: Tomasz Ciecierski Date: Thu, 13 Jul 2023 13:38:22 +0200 Subject: [PATCH 05/10] fix optional types --- .../server/search_strategy/osquery/index.ts | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/osquery/server/search_strategy/osquery/index.ts b/x-pack/plugins/osquery/server/search_strategy/osquery/index.ts index bd0e45c89985b..d56acddbe8b90 100644 --- a/x-pack/plugins/osquery/server/search_strategy/osquery/index.ts +++ b/x-pack/plugins/osquery/server/search_strategy/osquery/index.ts @@ -15,11 +15,9 @@ import type { FactoryQueryTypes, StrategyResponseType, StrategyRequestType, - ResultsRequestOptions, } from '../../../common/search_strategy/osquery'; import { osqueryFactory } from './factory'; import type { OsqueryFactory } from './factory/types'; -import type { RequestOptionsPaginated } from '../../../common/search_strategy/osquery'; export const osquerySearchStrategyProvider = ( data: PluginStart, @@ -44,18 +42,10 @@ export const osquerySearchStrategyProvider = ( const requestWithOptionalTypes = { factoryQueryType: request.factoryQueryType, kql: request.kql, - ...((request as RequestOptionsPaginated).pagination - ? { pagination: (request as RequestOptionsPaginated).pagination } - : {}), - ...((request as RequestOptionsPaginated).sort - ? { sort: (request as RequestOptionsPaginated).sort } - : {}), - ...((request as ResultsRequestOptions).actionId - ? { actionId: (request as ResultsRequestOptions).actionId } - : {}), - ...((request as ResultsRequestOptions).agentId - ? { agentId: (request as ResultsRequestOptions).agentId } - : {}), + ...('pagination' in request ? { pagination: request.pagination } : {}), + ...('sort' in request ? { sort: request.sort } : {}), + ...('actionId' in request ? { actionId: request.actionId } : {}), + ...('agentId' in request ? { agentId: request.agentId } : {}), componentTemplateExists: exists, } as StrategyRequestType; const dsl = queryFactory.buildDsl(requestWithOptionalTypes); From 6d6f0092e41d8016a582b1fab771ca4d3181cf23 Mon Sep 17 00:00:00 2001 From: Tomasz Ciecierski Date: Thu, 13 Jul 2023 13:51:33 +0200 Subject: [PATCH 06/10] remove docValueFields --- x-pack/plugins/osquery/common/search_strategy/osquery/index.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugins/osquery/common/search_strategy/osquery/index.ts b/x-pack/plugins/osquery/common/search_strategy/osquery/index.ts index bdf5e062300e5..e400180455135 100644 --- a/x-pack/plugins/osquery/common/search_strategy/osquery/index.ts +++ b/x-pack/plugins/osquery/common/search_strategy/osquery/index.ts @@ -17,7 +17,7 @@ import type { import type { AgentsStrategyResponse, AgentsRequestOptions } from './agents'; import type { ResultsStrategyResponse, ResultsRequestOptions } from './results'; -import type { DocValueFields, SortField, PaginationInputPaginated } from '../common'; +import type { SortField, PaginationInputPaginated } from '../common'; export * from './actions'; export * from './agents'; @@ -35,7 +35,6 @@ export type FactoryQueryTypes = OsqueryQueries; export interface RequestBasicOptions extends IEsSearchRequest { kql?: string; - docValueFields?: DocValueFields[]; factoryQueryType?: FactoryQueryTypes; componentTemplateExists?: boolean; } From aa4c7e381af04b459a20f316ac661479fec1c884 Mon Sep 17 00:00:00 2001 From: Tomasz Ciecierski Date: Thu, 13 Jul 2023 13:55:09 +0200 Subject: [PATCH 07/10] rename --- .../plugins/osquery/server/search_strategy/osquery/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/osquery/server/search_strategy/osquery/index.ts b/x-pack/plugins/osquery/server/search_strategy/osquery/index.ts index d56acddbe8b90..628f3cf0dcf17 100644 --- a/x-pack/plugins/osquery/server/search_strategy/osquery/index.ts +++ b/x-pack/plugins/osquery/server/search_strategy/osquery/index.ts @@ -39,7 +39,7 @@ export const osquerySearchStrategyProvider = ( }) ).pipe( mergeMap((exists) => { - const requestWithOptionalTypes = { + const strictRequest = { factoryQueryType: request.factoryQueryType, kql: request.kql, ...('pagination' in request ? { pagination: request.pagination } : {}), @@ -48,7 +48,7 @@ export const osquerySearchStrategyProvider = ( ...('agentId' in request ? { agentId: request.agentId } : {}), componentTemplateExists: exists, } as StrategyRequestType; - const dsl = queryFactory.buildDsl(requestWithOptionalTypes); + const dsl = queryFactory.buildDsl(strictRequest); // use internal user for searching .fleet* indices es = dsl.index?.includes('fleet') || dsl.index?.includes('logs-osquery_manager.action') @@ -57,7 +57,7 @@ export const osquerySearchStrategyProvider = ( return es.search( { - ...requestWithOptionalTypes, + ...strictRequest, params: dsl, }, options, From 957b537d1fd1cde0d4ce15aa09b46c2edfef88da Mon Sep 17 00:00:00 2001 From: Tomasz Ciecierski Date: Fri, 14 Jul 2023 22:30:21 +0200 Subject: [PATCH 08/10] change filter string --- .../osquery/factory/actions/details/query.action_details.dsl.ts | 2 +- .../osquery/factory/actions/results/query.action_results.dsl.ts | 2 +- .../osquery/factory/results/query.all_results.dsl.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/details/query.action_details.dsl.ts b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/details/query.action_details.dsl.ts index 9b7af953dd2c1..63c26596c0213 100644 --- a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/details/query.action_details.dsl.ts +++ b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/details/query.action_details.dsl.ts @@ -20,7 +20,7 @@ export const buildActionDetailsQuery = ({ const actionIdQuery = `action_id: ${actionId}`; let filter = actionIdQuery; if (!isEmpty(kql)) { - filter = actionIdQuery + ` and ${kql}`; + filter = filter + ` AND ${kql}`; } const filterQuery = getQueryFilter({ filter }); diff --git a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/results/query.action_results.dsl.ts b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/results/query.action_results.dsl.ts index bba4c0d8c7b12..f0bc2c7ee6b2f 100644 --- a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/results/query.action_results.dsl.ts +++ b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/results/query.action_results.dsl.ts @@ -22,7 +22,7 @@ export const buildActionResultsQuery = ({ const actionIdQuery = `action_id: ${actionId}`; let filter = actionIdQuery; if (!isEmpty(kql)) { - filter = actionIdQuery + ` AND ${kql}`; + filter = filter + ` AND ${kql}`; } const filterQuery = getQueryFilter({ filter }); diff --git a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/results/query.all_results.dsl.ts b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/results/query.all_results.dsl.ts index 40d2a2a849fa6..b6990ff4a6204 100644 --- a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/results/query.all_results.dsl.ts +++ b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/results/query.all_results.dsl.ts @@ -22,7 +22,7 @@ export const buildResultsQuery = ({ const agentQuery = agentId ? ` AND agent.id: ${agentId}` : ''; let filter = actionIdQuery + agentQuery; if (!isEmpty(kql)) { - filter = actionIdQuery + ` AND ${kql}`; + filter = filter + ` AND ${kql}`; } const filterQuery = getQueryFilter({ filter }); From 6a127a9edc8dfcf5dcd11c02079f45b0059e43d5 Mon Sep 17 00:00:00 2001 From: Tomasz Ciecierski Date: Tue, 22 Aug 2023 12:46:56 +0200 Subject: [PATCH 09/10] adjust kql and rename to kuery --- .../osquery/common/api/live_query/find_live_query_route.ts | 2 +- .../common/api/live_query/get_live_query_results_route.ts | 2 +- .../plugins/osquery/common/search_strategy/osquery/index.ts | 2 +- x-pack/plugins/osquery/cypress/e2e/all/cases.cy.ts | 2 +- x-pack/plugins/osquery/cypress/tasks/api_fixtures.ts | 2 +- .../osquery/public/action_results/use_action_results.ts | 6 +++--- x-pack/plugins/osquery/public/actions/actions_table.tsx | 2 +- .../plugins/osquery/public/actions/use_all_live_queries.ts | 6 +++--- .../osquery/public/actions/use_live_query_details.ts | 6 +++--- x-pack/plugins/osquery/public/agents/use_agent_groups.ts | 2 +- x-pack/plugins/osquery/public/results/use_all_results.ts | 6 +++--- .../server/routes/live_query/find_live_query_route.ts | 2 +- .../routes/live_query/get_live_query_details_route.ts | 1 - .../routes/live_query/get_live_query_results_route.ts | 4 ++-- x-pack/plugins/osquery/server/routes/live_query/utils.ts | 2 +- .../osquery/factory/actions/all/query.all_actions.dsl.ts | 4 ++-- .../factory/actions/details/query.action_details.dsl.ts | 6 +++--- .../factory/actions/results/query.action_results.dsl.ts | 6 +++--- .../osquery/factory/agents/query.all_agents.dsl.ts | 6 +++--- .../osquery/factory/results/query.all_results.dsl.ts | 6 +++--- .../plugins/osquery/server/search_strategy/osquery/index.ts | 2 +- 21 files changed, 38 insertions(+), 39 deletions(-) diff --git a/x-pack/plugins/osquery/common/api/live_query/find_live_query_route.ts b/x-pack/plugins/osquery/common/api/live_query/find_live_query_route.ts index 110466a706457..be0787b85fd87 100644 --- a/x-pack/plugins/osquery/common/api/live_query/find_live_query_route.ts +++ b/x-pack/plugins/osquery/common/api/live_query/find_live_query_route.ts @@ -9,7 +9,7 @@ import * as t from 'io-ts'; import { toNumberRt } from '@kbn/io-ts-utils'; export const findLiveQueryRequestQuerySchema = t.type({ - filterQuery: t.union([t.string, t.undefined]), + kuery: t.union([t.string, t.undefined]), page: t.union([toNumberRt, t.undefined]), pageSize: t.union([toNumberRt, t.undefined]), sort: t.union([t.string, t.undefined]), diff --git a/x-pack/plugins/osquery/common/api/live_query/get_live_query_results_route.ts b/x-pack/plugins/osquery/common/api/live_query/get_live_query_results_route.ts index 553e3eae10cd8..ce09004ddb58f 100644 --- a/x-pack/plugins/osquery/common/api/live_query/get_live_query_results_route.ts +++ b/x-pack/plugins/osquery/common/api/live_query/get_live_query_results_route.ts @@ -9,7 +9,7 @@ import * as t from 'io-ts'; import { toNumberRt } from '@kbn/io-ts-utils'; export const getLiveQueryResultsRequestQuerySchema = t.type({ - filterQuery: t.union([t.string, t.undefined]), + kuery: t.union([t.string, t.undefined]), page: t.union([toNumberRt, t.undefined]), pageSize: t.union([toNumberRt, t.undefined]), sort: t.union([t.string, t.undefined]), diff --git a/x-pack/plugins/osquery/common/search_strategy/osquery/index.ts b/x-pack/plugins/osquery/common/search_strategy/osquery/index.ts index e400180455135..53508c17208b2 100644 --- a/x-pack/plugins/osquery/common/search_strategy/osquery/index.ts +++ b/x-pack/plugins/osquery/common/search_strategy/osquery/index.ts @@ -34,7 +34,7 @@ export enum OsqueryQueries { export type FactoryQueryTypes = OsqueryQueries; export interface RequestBasicOptions extends IEsSearchRequest { - kql?: string; + kuery?: string; factoryQueryType?: FactoryQueryTypes; componentTemplateExists?: boolean; } diff --git a/x-pack/plugins/osquery/cypress/e2e/all/cases.cy.ts b/x-pack/plugins/osquery/cypress/e2e/all/cases.cy.ts index 8509cae295d54..3c9a3133b238d 100644 --- a/x-pack/plugins/osquery/cypress/e2e/all/cases.cy.ts +++ b/x-pack/plugins/osquery/cypress/e2e/all/cases.cy.ts @@ -22,7 +22,7 @@ describe('Add to Cases', () => { loadLiveQuery({ agent_all: true, query: "SELECT * FROM os_version where name='Ubuntu';", - kql: '', + kuery: '', }).then((liveQuery) => { liveQueryId = liveQuery.action_id; liveQueryQuery = liveQuery.queries[0].query; diff --git a/x-pack/plugins/osquery/cypress/tasks/api_fixtures.ts b/x-pack/plugins/osquery/cypress/tasks/api_fixtures.ts index 499cfd94e0995..1012fdc9ff2d7 100644 --- a/x-pack/plugins/osquery/cypress/tasks/api_fixtures.ts +++ b/x-pack/plugins/osquery/cypress/tasks/api_fixtures.ts @@ -121,7 +121,7 @@ export const loadLiveQuery = ( payload = { agent_all: true, query: 'select * from uptime;', - kql: '', + kuery: '', } ) => request<{ diff --git a/x-pack/plugins/osquery/public/action_results/use_action_results.ts b/x-pack/plugins/osquery/public/action_results/use_action_results.ts index b5d3b216ce2b5..ef7dbad151d20 100644 --- a/x-pack/plugins/osquery/public/action_results/use_action_results.ts +++ b/x-pack/plugins/osquery/public/action_results/use_action_results.ts @@ -38,7 +38,7 @@ export interface UseActionResults { direction: Direction; limit: number; sortField: string; - kql?: string; + kuery?: string; skip?: boolean; isLive?: boolean; } @@ -50,7 +50,7 @@ export const useActionResults = ({ direction, limit, sortField, - kql, + kuery, skip = false, isLive = false, }: UseActionResults) => { @@ -65,7 +65,7 @@ export const useActionResults = ({ { actionId, factoryQueryType: OsqueryQueries.actionResults, - kql, + kuery, pagination: generateTablePaginationOptions(activePage, limit), sort: { direction, diff --git a/x-pack/plugins/osquery/public/actions/actions_table.tsx b/x-pack/plugins/osquery/public/actions/actions_table.tsx index d2bb19d0556c9..cd5e1a685d33b 100644 --- a/x-pack/plugins/osquery/public/actions/actions_table.tsx +++ b/x-pack/plugins/osquery/public/actions/actions_table.tsx @@ -62,7 +62,7 @@ const ActionsTableComponent = () => { const { data: actionsData } = useAllLiveQueries({ activePage: pageIndex, limit: pageSize, - kql: 'user_id: *', + kuery: 'user_id: *', }); const onTableChange = useCallback(({ page = {} }) => { diff --git a/x-pack/plugins/osquery/public/actions/use_all_live_queries.ts b/x-pack/plugins/osquery/public/actions/use_all_live_queries.ts index a90da2e6326a2..cb118d01bc867 100644 --- a/x-pack/plugins/osquery/public/actions/use_all_live_queries.ts +++ b/x-pack/plugins/osquery/public/actions/use_all_live_queries.ts @@ -20,7 +20,7 @@ export interface UseAllLiveQueriesConfig { direction?: Direction; limit?: number; sortField?: string; - kql?: string; + kuery?: string; skip?: boolean; alertId?: string; } @@ -33,7 +33,7 @@ export const useAllLiveQueries = ({ direction = Direction.desc, limit = 100, sortField = '@timestamp', - kql, + kuery, skip = false, alertId, }: UseAllLiveQueriesConfig) => { @@ -51,7 +51,7 @@ export const useAllLiveQueries = ({ { version: API_VERSIONS.public.v1, query: { - kql, + kuery, page: activePage, pageSize: limit, sort: sortField, diff --git a/x-pack/plugins/osquery/public/actions/use_live_query_details.ts b/x-pack/plugins/osquery/public/actions/use_live_query_details.ts index 7f6d4b9231f0a..4437662625b1b 100644 --- a/x-pack/plugins/osquery/public/actions/use_live_query_details.ts +++ b/x-pack/plugins/osquery/public/actions/use_live_query_details.ts @@ -17,7 +17,7 @@ import { useErrorToast } from '../common/hooks/use_error_toast'; interface UseLiveQueryDetails { actionId?: string; isLive?: boolean; - kql?: string; + kuery?: string; skip?: boolean; queryIds?: string[]; } @@ -53,7 +53,7 @@ export interface LiveQueryDetailsItem { export const useLiveQueryDetails = ({ actionId, - kql, + kuery, isLive = false, skip = false, queryIds, // enable finding out specific queries only, eg. in cases @@ -62,7 +62,7 @@ export const useLiveQueryDetails = ({ const setErrorToast = useErrorToast(); return useQuery<{ data: LiveQueryDetailsItem }, Error, LiveQueryDetailsItem>( - ['liveQueries', { actionId, kql, queryIds }], + ['liveQueries', { actionId, kuery, queryIds }], () => http.get(`/api/osquery/live_queries/${actionId}`, { version: API_VERSIONS.public.v1 }), { enabled: !skip && !!actionId, diff --git a/x-pack/plugins/osquery/public/agents/use_agent_groups.ts b/x-pack/plugins/osquery/public/agents/use_agent_groups.ts index ccacdca1734d8..c7b72b489bbec 100644 --- a/x-pack/plugins/osquery/public/agents/use_agent_groups.ts +++ b/x-pack/plugins/osquery/public/agents/use_agent_groups.ts @@ -40,7 +40,7 @@ export const useAgentGroups = () => { const responseData = await lastValueFrom( data.search.search( { - kql: `policy_id: ( ${policiesQuery} )`, + kuery: `policy_id: ( ${policiesQuery} )`, factoryQueryType: OsqueryQueries.agents, pagination: generateTablePaginationOptions(0, 9000), sort: { diff --git a/x-pack/plugins/osquery/public/results/use_all_results.ts b/x-pack/plugins/osquery/public/results/use_all_results.ts index c92cfdd12ff81..09f18cfd2d17c 100644 --- a/x-pack/plugins/osquery/public/results/use_all_results.ts +++ b/x-pack/plugins/osquery/public/results/use_all_results.ts @@ -35,7 +35,7 @@ interface UseAllResults { activePage: number; limit: number; sort: Array<{ field: string; direction: Direction }>; - kql?: string; + kuery?: string; skip?: boolean; isLive?: boolean; } @@ -45,7 +45,7 @@ export const useAllResults = ({ activePage, limit, sort, - kql, + kuery, skip = false, isLive = false, }: UseAllResults) => { @@ -60,7 +60,7 @@ export const useAllResults = ({ { actionId, factoryQueryType: OsqueryQueries.results, - kql, + kuery, pagination: generateTablePaginationOptions(activePage, limit), sort, }, diff --git a/x-pack/plugins/osquery/server/routes/live_query/find_live_query_route.ts b/x-pack/plugins/osquery/server/routes/live_query/find_live_query_route.ts index 8adef8801d6ca..2258843b73227 100644 --- a/x-pack/plugins/osquery/server/routes/live_query/find_live_query_route.ts +++ b/x-pack/plugins/osquery/server/routes/live_query/find_live_query_route.ts @@ -52,7 +52,7 @@ export const findLiveQueryRoute = (router: IRouter) = search.search( { factoryQueryType: OsqueryQueries.actions, - kql: request.query.kql, + kuery: request.query.kuery, pagination: generateTablePaginationOptions( request.query.page ?? 0, request.query.pageSize ?? 100 diff --git a/x-pack/plugins/osquery/server/routes/live_query/get_live_query_details_route.ts b/x-pack/plugins/osquery/server/routes/live_query/get_live_query_details_route.ts index 00fff343bfb08..2b32a3269b693 100644 --- a/x-pack/plugins/osquery/server/routes/live_query/get_live_query_details_route.ts +++ b/x-pack/plugins/osquery/server/routes/live_query/get_live_query_details_route.ts @@ -61,7 +61,6 @@ export const getLiveQueryDetailsRoute = (router: IRouter( { actionId: request.params.id, - kql: request.query.kql, factoryQueryType: OsqueryQueries.actionDetails, }, { abortSignal, strategy: 'osquerySearchStrategy' } diff --git a/x-pack/plugins/osquery/server/routes/live_query/get_live_query_results_route.ts b/x-pack/plugins/osquery/server/routes/live_query/get_live_query_results_route.ts index 87c833b1c678f..a1154b3b8c0d2 100644 --- a/x-pack/plugins/osquery/server/routes/live_query/get_live_query_results_route.ts +++ b/x-pack/plugins/osquery/server/routes/live_query/get_live_query_results_route.ts @@ -61,7 +61,7 @@ export const getLiveQueryResultsRoute = (router: IRouter( { actionId: request.params.id, - kql: request.query.kql, + kuery: request.query.kuery, factoryQueryType: OsqueryQueries.actionDetails, }, { abortSignal, strategy: 'osquerySearchStrategy' } @@ -83,7 +83,7 @@ export const getLiveQueryResultsRoute = (router: IRouter { const { bool: { filter }, - } = getQueryFilter({ filter: kql }); + } = getQueryFilter({ filter: kuery }); const dslQuery = { allow_no_indices: true, diff --git a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/details/query.action_details.dsl.ts b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/details/query.action_details.dsl.ts index 63c26596c0213..3a5fe3db37b7b 100644 --- a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/details/query.action_details.dsl.ts +++ b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/details/query.action_details.dsl.ts @@ -14,13 +14,13 @@ import type { ActionDetailsRequestOptions } from '../../../../../../common/searc export const buildActionDetailsQuery = ({ actionId, - kql, + kuery, componentTemplateExists, }: ActionDetailsRequestOptions): ISearchRequestParams => { const actionIdQuery = `action_id: ${actionId}`; let filter = actionIdQuery; - if (!isEmpty(kql)) { - filter = filter + ` AND ${kql}`; + if (!isEmpty(kuery)) { + filter = filter + ` AND ${kuery}`; } const filterQuery = getQueryFilter({ filter }); diff --git a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/results/query.action_results.dsl.ts b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/results/query.action_results.dsl.ts index f0bc2c7ee6b2f..47901f47b6593 100644 --- a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/results/query.action_results.dsl.ts +++ b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/actions/results/query.action_results.dsl.ts @@ -14,15 +14,15 @@ import { getQueryFilter } from '../../../../../utils/build_query'; export const buildActionResultsQuery = ({ actionId, - kql, + kuery, // pagination: { activePage, querySize }, sort, componentTemplateExists, }: ActionResultsRequestOptions): ISearchRequestParams => { const actionIdQuery = `action_id: ${actionId}`; let filter = actionIdQuery; - if (!isEmpty(kql)) { - filter = filter + ` AND ${kql}`; + if (!isEmpty(kuery)) { + filter = filter + ` AND ${kuery}`; } const filterQuery = getQueryFilter({ filter }); diff --git a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/agents/query.all_agents.dsl.ts b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/agents/query.all_agents.dsl.ts index 56aec2a8bc8a5..20a5f07c5d11d 100644 --- a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/agents/query.all_agents.dsl.ts +++ b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/agents/query.all_agents.dsl.ts @@ -12,14 +12,14 @@ import { getQueryFilter } from '../../../../utils/build_query'; import type { AgentsRequestOptions } from '../../../../../common/search_strategy'; export const buildAgentsQuery = ({ - kql, + kuery, pagination: { cursorStart, querySize }, sort, }: AgentsRequestOptions): ISearchRequestParams => { const activeQuery = `active: true`; let filter = activeQuery; - if (!isEmpty(kql)) { - filter = activeQuery + ` AND ${kql}`; + if (!isEmpty(kuery)) { + filter = activeQuery + ` AND ${kuery}`; } const filterQuery = getQueryFilter({ filter }); diff --git a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/results/query.all_results.dsl.ts b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/results/query.all_results.dsl.ts index b6990ff4a6204..2c8d408672275 100644 --- a/x-pack/plugins/osquery/server/search_strategy/osquery/factory/results/query.all_results.dsl.ts +++ b/x-pack/plugins/osquery/server/search_strategy/osquery/factory/results/query.all_results.dsl.ts @@ -14,15 +14,15 @@ import type { ResultsRequestOptions } from '../../../../../common/search_strateg export const buildResultsQuery = ({ actionId, agentId, - kql, + kuery, sort, pagination: { activePage, querySize }, }: ResultsRequestOptions): ISearchRequestParams => { const actionIdQuery = `action_id: ${actionId}`; const agentQuery = agentId ? ` AND agent.id: ${agentId}` : ''; let filter = actionIdQuery + agentQuery; - if (!isEmpty(kql)) { - filter = filter + ` AND ${kql}`; + if (!isEmpty(kuery)) { + filter = filter + ` AND ${kuery}`; } const filterQuery = getQueryFilter({ filter }); diff --git a/x-pack/plugins/osquery/server/search_strategy/osquery/index.ts b/x-pack/plugins/osquery/server/search_strategy/osquery/index.ts index 4afc6fc484a8d..af14d84fa3637 100644 --- a/x-pack/plugins/osquery/server/search_strategy/osquery/index.ts +++ b/x-pack/plugins/osquery/server/search_strategy/osquery/index.ts @@ -41,7 +41,7 @@ export const osquerySearchStrategyProvider = ( mergeMap((exists) => { const strictRequest = { factoryQueryType: request.factoryQueryType, - kql: request.kql, + kuery: request.kuery, ...('pagination' in request ? { pagination: request.pagination } : {}), ...('sort' in request ? { sort: request.sort } : {}), ...('actionId' in request ? { actionId: request.actionId } : {}), From 867594395c40fa102a96d717724b8bcea5a5a90d Mon Sep 17 00:00:00 2001 From: Tomasz Ciecierski Date: Tue, 22 Aug 2023 13:18:35 +0200 Subject: [PATCH 10/10] fix --- .../public/common/components/event_details/osquery_tab.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/public/common/components/event_details/osquery_tab.tsx b/x-pack/plugins/security_solution/public/common/components/event_details/osquery_tab.tsx index 3f9d9ac777628..289561c0bc4aa 100644 --- a/x-pack/plugins/security_solution/public/common/components/event_details/osquery_tab.tsx +++ b/x-pack/plugins/security_solution/public/common/components/event_details/osquery_tab.tsx @@ -61,7 +61,7 @@ export const useOsqueryTab = ({ const { OsqueryResults, fetchAllLiveQueries } = osquery; const { data: actionsData } = fetchAllLiveQueries({ - kql: `alert_ids: ( ${alertId} )`, + kuery: `alert_ids: ( ${alertId} )`, alertId, skip: shouldEarlyReturn, });