From f056ef39987d9209e120f64539cf8229e1416687 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Sat, 29 Aug 2020 14:11:21 +0200 Subject: [PATCH 1/8] [Security Solution] Refactor Network TLS to use Search Strategy --- .../security_solution/index.ts | 8 +- .../security_solution/network/index.ts | 94 ++++++ .../public/network/containers/tls/index.tsx | 298 ++++++++++-------- .../network/containers/tls/translations.ts | 21 ++ .../pages/ip_details/tls_query_table.tsx | 62 ++-- .../pages/navigation/tls_query_tab_body.tsx | 62 ++-- .../security_solution/factory/index.ts | 2 + .../factory/network/index.ts | 15 + .../factory/network/tls/helpers.ts | 38 +++ .../factory/network/tls/index.ts | 59 ++++ .../network/tls/query.tls_network.dsl.ts | 104 ++++++ 11 files changed, 574 insertions(+), 189 deletions(-) create mode 100644 x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts create mode 100644 x-pack/plugins/security_solution/public/network/containers/tls/translations.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/helpers.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/index.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/query.tls_network.dsl.ts diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts index a188eb7619e6b..264b8d8e0d021 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts @@ -13,10 +13,12 @@ import { HostsRequestOptions, HostsStrategyResponse, } from './hosts'; +import { NetworkQueries, NetworkTlsStrategyResponse, NetworkTlsRequestOptions } from './network'; + export * from './hosts'; export type Maybe = T | null; -export type FactoryQueryTypes = HostsQueries; +export type FactoryQueryTypes = HostsQueries | NetworkQueries; export interface Inspect { dsl: string[]; @@ -100,10 +102,14 @@ export type StrategyResponseType = T extends HostsQ ? HostsStrategyResponse : T extends HostsQueries.hostOverview ? HostOverviewStrategyResponse + : T extends NetworkQueries.tls + ? NetworkTlsStrategyResponse : never; export type StrategyRequestType = T extends HostsQueries.hosts ? HostsRequestOptions : T extends HostsQueries.hostOverview ? HostOverviewRequestOptions + : T extends NetworkQueries.tls + ? NetworkTlsRequestOptions : never; diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts new file mode 100644 index 0000000000000..48f6dc4260989 --- /dev/null +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts @@ -0,0 +1,94 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IEsSearchResponse } from '../../../../../../../src/plugins/data/common'; + +import { CursorType, Inspect, Maybe, PageInfoPaginated, RequestOptionsPaginated } from '..'; + +export enum NetworkQueries { + tls = 'tls', +} + +export interface TlsBuckets { + key: string; + timestamp?: { + value: number; + value_as_string: string; + }; + + subjects: { + buckets: Readonly>; + }; + + ja3: { + buckets: Readonly>; + }; + + issuers: { + buckets: Readonly>; + }; + + not_after: { + buckets: Readonly>; + }; +} + +export interface TlsNode { + _id?: Maybe; + + timestamp?: Maybe; + + notAfter?: Maybe; + + subjects?: Maybe; + + ja3?: Maybe; + + issuers?: Maybe; +} + +export enum FlowTargetSourceDest { + destination = 'destination', + source = 'source', +} + +export enum TlsFields { + _id = '_id', +} + +export interface TlsEdges { + node: TlsNode; + + cursor: CursorType; +} + +export enum Direction { + asc = 'asc', + desc = 'desc', +} + +export interface TlsSortField { + field: TlsFields; + + direction: Direction; +} + +export interface NetworkTlsRequestOptions extends RequestOptionsPaginated { + ip: string; + flowTarget: FlowTargetSourceDest; + sort: TlsSortField; + defaultIndex: string[]; +} + +export interface NetworkTlsStrategyResponse extends IEsSearchResponse { + edges: TlsEdges[]; + + totalCount: number; + + pageInfo: PageInfoPaginated; + + inspect?: Maybe; +} diff --git a/x-pack/plugins/security_solution/public/network/containers/tls/index.tsx b/x-pack/plugins/security_solution/public/network/containers/tls/index.tsx index 17506f9a01cb9..ee5d74a3c0eff 100644 --- a/x-pack/plugins/security_solution/public/network/containers/tls/index.tsx +++ b/x-pack/plugins/security_solution/public/network/containers/tls/index.tsx @@ -4,38 +4,33 @@ * you may not use this file except in compliance with the Elastic License. */ -import { getOr } from 'lodash/fp'; -import React from 'react'; -import { Query } from 'react-apollo'; -import { connect } from 'react-redux'; -import { compose } from 'redux'; +import { noop } from 'lodash/fp'; +import { useState, useEffect, useCallback, useRef } from 'react'; +import { shallowEqual, useSelector } from 'react-redux'; +import deepEqual from 'fast-deep-equal'; +import { ESTermQuery } from '../../../../common/typed_json'; import { DEFAULT_INDEX_KEY } from '../../../../common/constants'; -import { - PageInfoPaginated, - TlsEdges, - TlsSortField, - GetTlsQuery, - FlowTargetSourceDest, -} from '../../../graphql/types'; -import { inputsModel, State, inputsSelectors } from '../../../common/store'; -import { withKibana, WithKibanaProps } from '../../../common/lib/kibana'; -import { createFilter, getDefaultFetchPolicy } from '../../../common/containers/helpers'; +import { inputsModel, State } from '../../../common/store'; +import { useKibana } from '../../../common/lib/kibana'; +import { createFilter } from '../../../common/containers/helpers'; +import { TlsEdges, PageInfoPaginated, FlowTargetSourceDest } from '../../../graphql/types'; import { generateTablePaginationOptions } from '../../../common/components/paginated_table/helpers'; -import { - QueryTemplatePaginated, - QueryTemplatePaginatedProps, -} from '../../../common/containers/query_template_paginated'; import { networkModel, networkSelectors } from '../../store'; -import { tlsQuery } from './index.gql_query'; +import { + NetworkQueries, + NetworkTlsRequestOptions, + NetworkTlsStrategyResponse, +} from '../../../../common/search_strategy/security_solution/network'; +import { AbortError } from '../../../../../../../src/plugins/data/common'; +import * as i18n from './translations'; -const ID = 'tlsQuery'; +const ID = 'networkTlsQuery'; -export interface TlsArgs { +export interface NetworkTlsArgs { id: string; inspect: inputsModel.InspectQuery; isInspected: boolean; - loading: boolean; loadPage: (newActivePage: number) => void; pageInfo: PageInfoPaginated; refetch: inputsModel.Refetch; @@ -43,121 +38,168 @@ export interface TlsArgs { totalCount: number; } -export interface OwnProps extends QueryTemplatePaginatedProps { - children: (args: TlsArgs) => React.ReactNode; +interface UseNetworkTls { flowTarget: FlowTargetSourceDest; ip: string; type: networkModel.NetworkType; + filterQuery?: ESTermQuery | string; + endDate: string; + startDate: string; + skip: boolean; + id?: string; } -export interface TlsComponentReduxProps { - activePage: number; - isInspected: boolean; - limit: number; - sort: TlsSortField; -} +export const useNetworkTls = ({ + endDate, + filterQuery, + flowTarget, + id = ID, + ip, + skip, + startDate, + type, +}: UseNetworkTls): [boolean, NetworkTlsArgs] => { + // const getQuery = inputsSelectors.globalQueryByIdSelector(); + // const { isInspected } = useSelector((state: State) => getQuery(state, id), shallowEqual); + const getTlsSelector = networkSelectors.tlsSelector(); + const { activePage, limit, sort } = useSelector( + (state: State) => getTlsSelector(state, type, flowTarget), + shallowEqual + ); + const { data, notifications, uiSettings } = useKibana().services; + const refetch = useRef(noop); + const abortCtrl = useRef(new AbortController()); + const defaultIndex = uiSettings.get(DEFAULT_INDEX_KEY); + const [loading, setLoading] = useState(false); + + const [networkTlsRequest, setHostRequest] = useState({ + defaultIndex, + factoryQueryType: NetworkQueries.tls, + filterQuery: createFilter(filterQuery), + flowTarget, + // inspect: isInspected, + ip, + pagination: generateTablePaginationOptions(activePage, limit), + sort, + timerange: { + interval: '12h', + from: startDate ? startDate : '', + to: endDate ? endDate : new Date(Date.now()).toISOString(), + }, + }); + + const wrappedLoadMore = useCallback( + (newActivePage: number) => { + setHostRequest((prevRequest) => { + return { + ...prevRequest, + pagination: generateTablePaginationOptions(newActivePage, limit), + }; + }); + }, + [limit] + ); -type TlsProps = OwnProps & TlsComponentReduxProps & WithKibanaProps; + const [networkTlsResponse, setNetworkTlsResponse] = useState({ + tls: [], + id: ID, + inspect: { + dsl: [], + response: [], + }, + isInspected: false, + loadPage: wrappedLoadMore, + pageInfo: { + activePage: 0, + fakeTotalCount: 0, + showMorePagesIndicator: false, + }, + refetch: refetch.current, + totalCount: -1, + }); -class TlsComponentQuery extends QueryTemplatePaginated< - TlsProps, - GetTlsQuery.Query, - GetTlsQuery.Variables -> { - public render() { - const { - activePage, - children, - endDate, - filterQuery, - flowTarget, - id = ID, - ip, - isInspected, - kibana, - limit, - skip, - sourceId, - startDate, - sort, - } = this.props; - const variables: GetTlsQuery.Variables = { - defaultIndex: kibana.services.uiSettings.get(DEFAULT_INDEX_KEY), - filterQuery: createFilter(filterQuery), - flowTarget, - inspect: isInspected, - ip, - pagination: generateTablePaginationOptions(activePage, limit), - sort, - sourceId, - timerange: { - interval: '12h', - from: startDate ? startDate : '', - to: endDate ? endDate : new Date(Date.now()).toISOString(), - }, - }; - return ( - - query={tlsQuery} - fetchPolicy={getDefaultFetchPolicy()} - notifyOnNetworkStatusChange - skip={skip} - variables={variables} - > - {({ data, loading, fetchMore, networkStatus, refetch }) => { - const tls = getOr([], 'source.Tls.edges', data); - this.setFetchMore(fetchMore); - this.setFetchMoreOptions((newActivePage: number) => ({ - variables: { - pagination: generateTablePaginationOptions(newActivePage, limit), + const networkTlsSearch = useCallback( + (request: NetworkTlsRequestOptions) => { + let didCancel = false; + const asyncSearch = async () => { + abortCtrl.current = new AbortController(); + setLoading(true); + + const searchSubscription$ = data.search + .search(request, { + strategy: 'securitySolutionSearchStrategy', + signal: abortCtrl.current.signal, + }) + .subscribe({ + next: (response) => { + if (!response.isPartial && !response.isRunning) { + if (!didCancel) { + setLoading(false); + setNetworkTlsResponse((prevResponse) => ({ + ...prevResponse, + tls: response.edges, + inspect: response.inspect ?? prevResponse.inspect, + pageInfo: response.pageInfo, + refetch: refetch.current, + totalCount: response.totalCount, + })); + } + searchSubscription$.unsubscribe(); + } else if (response.isPartial && !response.isRunning) { + if (!didCancel) { + setLoading(false); + } + // TODO: Make response error status clearer + notifications.toasts.addWarning(i18n.ERROR_NETWORK_TLS); + searchSubscription$.unsubscribe(); + } }, - updateQuery: (prev, { fetchMoreResult }) => { - if (!fetchMoreResult) { - return prev; + error: (msg) => { + if (!(msg instanceof AbortError)) { + notifications.toasts.addDanger({ title: i18n.FAIL_NETWORK_TLS, text: msg.message }); } - return { - ...fetchMoreResult, - source: { - ...fetchMoreResult.source, - Tls: { - ...fetchMoreResult.source.Tls, - edges: [...fetchMoreResult.source.Tls.edges], - }, - }, - }; }, - })); - const isLoading = this.isItAValidLoading(loading, variables, networkStatus); - return children({ - id, - inspect: getOr(null, 'source.Tls.inspect', data), - isInspected, - loading: isLoading, - loadPage: this.wrappedLoadMore, - pageInfo: getOr({}, 'source.Tls.pageInfo', data), - refetch: this.memoizedRefetchQuery(variables, limit, refetch), - tls, - totalCount: getOr(-1, 'source.Tls.totalCount', data), }); - }} - - ); - } -} + }; + abortCtrl.current.abort(); + asyncSearch(); + refetch.current = asyncSearch; + return () => { + didCancel = true; + abortCtrl.current.abort(); + }; + }, + [data.search, notifications.toasts] + ); -const makeMapStateToProps = () => { - const getTlsSelector = networkSelectors.tlsSelector(); - const getQuery = inputsSelectors.globalQueryByIdSelector(); - return (state: State, { flowTarget, id = ID, type }: OwnProps) => { - const { isInspected } = getQuery(state, id); - return { - ...getTlsSelector(state, type, flowTarget), - isInspected, - }; - }; -}; + useEffect(() => { + if (skip) { + return; + } + + setHostRequest((prevRequest) => { + const myRequest = { + ...prevRequest, + defaultIndex, + filterQuery: createFilter(filterQuery), + pagination: generateTablePaginationOptions(activePage, limit), + timerange: { + interval: '12h', + from: startDate, + to: endDate, + }, + sort, + }; + if (!deepEqual(prevRequest, myRequest)) { + return myRequest; + } + return prevRequest; + }); + }, [activePage, defaultIndex, endDate, filterQuery, limit, startDate, sort, skip]); -export const TlsQuery = compose>( - connect(makeMapStateToProps), - withKibana -)(TlsComponentQuery); + useEffect(() => { + networkTlsSearch(networkTlsRequest); + }, [networkTlsRequest, networkTlsSearch]); + + return [loading, networkTlsResponse]; +}; diff --git a/x-pack/plugins/security_solution/public/network/containers/tls/translations.ts b/x-pack/plugins/security_solution/public/network/containers/tls/translations.ts new file mode 100644 index 0000000000000..aafa3ff0a98b0 --- /dev/null +++ b/x-pack/plugins/security_solution/public/network/containers/tls/translations.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; + +export const ERROR_NETWORK_TLS = i18n.translate( + 'xpack.securitySolution.networkTls.errorSearchDescription', + { + defaultMessage: `An error has occurred on network tls search`, + } +); + +export const FAIL_NETWORK_TLS = i18n.translate( + 'xpack.securitySolution.networkTls.failSearchDescription', + { + defaultMessage: `Failed to run search on network tls`, + } +); diff --git a/x-pack/plugins/security_solution/public/network/pages/ip_details/tls_query_table.tsx b/x-pack/plugins/security_solution/public/network/pages/ip_details/tls_query_table.tsx index f0c3628af78d8..5184fccecf07a 100644 --- a/x-pack/plugins/security_solution/public/network/pages/ip_details/tls_query_table.tsx +++ b/x-pack/plugins/security_solution/public/network/pages/ip_details/tls_query_table.tsx @@ -8,7 +8,7 @@ import { getOr } from 'lodash/fp'; import React from 'react'; import { manageQuery } from '../../../common/components/page/manage_query'; import { TlsTable } from '../../components/tls_table'; -import { TlsQuery } from '../../containers/tls'; +import { useNetworkTls } from '../../containers/tls'; import { TlsQueryTableComponentProps } from './types'; const TlsTableManage = manageQuery(TlsTable); @@ -22,34 +22,36 @@ export const TlsQueryTable = ({ skip, startDate, type, -}: TlsQueryTableComponentProps) => ( - - {({ id, inspect, isInspected, tls, totalCount, pageInfo, loading, loadPage, refetch }) => ( - - )} - -); +}: TlsQueryTableComponentProps) => { + const [ + loading, + { id, inspect, isInspected, tls, totalCount, pageInfo, loadPage, refetch }, + ] = useNetworkTls({ + endDate, + filterQuery, + flowTarget, + ip, + skip, + startDate, + type, + }); + + return ( + + ); +}; TlsQueryTable.displayName = 'TlsQueryTable'; diff --git a/x-pack/plugins/security_solution/public/network/pages/navigation/tls_query_tab_body.tsx b/x-pack/plugins/security_solution/public/network/pages/navigation/tls_query_tab_body.tsx index 00da5496e5440..1e49f9ca747c3 100644 --- a/x-pack/plugins/security_solution/public/network/pages/navigation/tls_query_tab_body.tsx +++ b/x-pack/plugins/security_solution/public/network/pages/navigation/tls_query_tab_body.tsx @@ -6,7 +6,7 @@ import React from 'react'; import { getOr } from 'lodash/fp'; import { manageQuery } from '../../../common/components/page/manage_query'; -import { TlsQuery } from '../../../network/containers/tls'; +import { useNetworkTls } from '../../../network/containers/tls'; import { TlsTable } from '../../components/tls_table'; import { TlsQueryTabBodyProps } from './types'; @@ -21,32 +21,34 @@ export const TlsQueryTabBody = ({ skip, startDate, type, -}: TlsQueryTabBodyProps) => ( - - {({ id, inspect, isInspected, tls, totalCount, pageInfo, loading, loadPage, refetch }) => ( - - )} - -); +}: TlsQueryTabBodyProps) => { + const [ + loading, + { id, inspect, isInspected, tls, totalCount, pageInfo, loadPage, refetch }, + ] = useNetworkTls({ + endDate, + filterQuery, + flowTarget, + ip, + skip, + startDate, + type, + }); + + return ( + + ); +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/index.ts index 53433dfc208cb..a50c9e4004856 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/index.ts @@ -7,6 +7,7 @@ import { FactoryQueryTypes } from '../../../../common/search_strategy/security_solution'; import { hostsFactory } from './hosts'; +import { networkFactory } from './network'; import { SecuritySolutionFactory } from './types'; export const securitySolutionFactory: Record< @@ -14,4 +15,5 @@ export const securitySolutionFactory: Record< SecuritySolutionFactory > = { ...hostsFactory, + ...networkFactory, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts new file mode 100644 index 0000000000000..2c21d9741d648 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { FactoryQueryTypes } from '../../../../../common/search_strategy/security_solution'; +import { NetworkQueries } from '../../../../../common/search_strategy/security_solution/network'; + +import { SecuritySolutionFactory } from '../types'; +import { networkTls } from './tls'; + +export const networkFactory: Record> = { + [NetworkQueries.tls]: networkTls, +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/helpers.ts new file mode 100644 index 0000000000000..57c0737cc5852 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/helpers.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getOr } from 'lodash/fp'; + +import { IEsSearchResponse } from '../../../../../../../../../src/plugins/data/common'; +import { + NetworkTlsRequestOptions, + TlsBuckets, + TlsEdges, +} from '../../../../../../common/search_strategy/security_solution/network'; + +export const getTlsEdges = ( + response: IEsSearchResponse, + options: NetworkTlsRequestOptions +): TlsEdges[] => formatTlsEdges(getOr([], 'aggregations.sha1.buckets', response.rawResponse)); + +export const formatTlsEdges = (buckets: TlsBuckets[]): TlsEdges[] => + buckets.map((bucket: TlsBuckets) => { + const edge: TlsEdges = { + node: { + _id: bucket.key, + subjects: bucket.subjects.buckets.map(({ key }) => key), + ja3: bucket.ja3.buckets.map(({ key }) => key), + issuers: bucket.issuers.buckets.map(({ key }) => key), + // eslint-disable-next-line @typescript-eslint/naming-convention + notAfter: bucket.not_after.buckets.map(({ key_as_string }) => key_as_string), + }, + cursor: { + value: bucket.key, + tiebreaker: null, + }, + }; + return edge; + }); diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/index.ts new file mode 100644 index 0000000000000..9286b6a498788 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/index.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getOr } from 'lodash/fp'; + +import { IEsSearchResponse } from '../../../../../../../../../src/plugins/data/common'; + +import { DEFAULT_MAX_TABLE_QUERY_SIZE } from '../../../../../../common/constants'; +import { + NetworkTlsStrategyResponse, + NetworkQueries, + NetworkTlsRequestOptions, + TlsEdges, +} from '../../../../../../common/search_strategy/security_solution/network'; + +import { inspectStringifyObject } from '../../../../../utils/build_query'; +import { SecuritySolutionFactory } from '../../types'; + +import { getTlsEdges } from './helpers'; +import { buildTlsQuery } from './query.tls_network.dsl'; + +export const networkTls: SecuritySolutionFactory = { + buildDsl: (options: NetworkTlsRequestOptions) => { + if (options.pagination && options.pagination.querySize >= DEFAULT_MAX_TABLE_QUERY_SIZE) { + throw new Error(`No query size above ${DEFAULT_MAX_TABLE_QUERY_SIZE}`); + } + return buildTlsQuery(options); + }, + parse: async ( + options: NetworkTlsRequestOptions, + response: IEsSearchResponse + ): Promise => { + const { activePage, cursorStart, fakePossibleCount, querySize } = options.pagination; + const totalCount = getOr(0, 'aggregations.count.value', response.rawResponse); + const tlsEdges: TlsEdges[] = getTlsEdges(response, options); + const fakeTotalCount = fakePossibleCount <= totalCount ? fakePossibleCount : totalCount; + const edges = tlsEdges.splice(cursorStart, querySize - cursorStart); + const inspect = { + dsl: [inspectStringifyObject(buildTlsQuery(options))], + response: [inspectStringifyObject(response)], + }; + const showMorePagesIndicator = totalCount > fakeTotalCount; + + return { + ...response, + edges, + inspect, + pageInfo: { + activePage: activePage ? activePage : 0, + fakeTotalCount, + showMorePagesIndicator, + }, + totalCount, + }; + }, +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/query.tls_network.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/query.tls_network.dsl.ts new file mode 100644 index 0000000000000..407cff126e65d --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/query.tls_network.dsl.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { assertUnreachable } from '../../../../../../common/utility_types'; +import { createQueryFilterClauses } from '../../../../../utils/build_query'; + +import { NetworkTlsRequestOptions } from '../../../../../../common/search_strategy/security_solution/network'; +import { TlsSortField, Direction, TlsFields } from '../../../../../graphql/types'; + +const getAggs = (querySize: number, sort: TlsSortField) => ({ + count: { + cardinality: { + field: 'tls.server.hash.sha1', + }, + }, + sha1: { + terms: { + field: 'tls.server.hash.sha1', + size: querySize, + order: { + ...getQueryOrder(sort), + }, + }, + aggs: { + issuers: { + terms: { + field: 'tls.server.issuer', + }, + }, + subjects: { + terms: { + field: 'tls.server.subject', + }, + }, + not_after: { + terms: { + field: 'tls.server.not_after', + }, + }, + ja3: { + terms: { + field: 'tls.server.ja3s', + }, + }, + }, + }, +}); + +export const buildTlsQuery = ({ + ip, + sort, + filterQuery, + flowTarget, + pagination: { querySize }, + defaultIndex, + timerange: { from, to }, +}: NetworkTlsRequestOptions) => { + const defaultFilter = [ + ...createQueryFilterClauses(filterQuery), + { + range: { + '@timestamp': { gte: from, lte: to, format: 'strict_date_optional_time' }, + }, + }, + ]; + + const filter = ip ? [...defaultFilter, { term: { [`${flowTarget}.ip`]: ip } }] : defaultFilter; + + const dslQuery = { + allowNoIndices: true, + index: defaultIndex, + ignoreUnavailable: true, + body: { + aggs: { + ...getAggs(querySize, sort), + }, + query: { + bool: { + filter, + }, + }, + size: 0, + track_total_hits: false, + }, + }; + + return dslQuery; +}; + +interface QueryOrder { + _key: Direction; +} + +const getQueryOrder = (sort: TlsSortField): QueryOrder => { + switch (sort.field) { + case TlsFields._id: + return { _key: sort.direction }; + default: + return assertUnreachable(sort.field); + } +}; From 04da5a0deb5c9bfcf6415230810c306854c6341b Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Sat, 29 Aug 2020 14:13:39 +0200 Subject: [PATCH 2/8] cleanup --- .../security_solution/factory/network/tls/helpers.ts | 7 ++----- .../security_solution/factory/network/tls/index.ts | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/helpers.ts index 57c0737cc5852..59359fd35a34e 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/helpers.ts @@ -8,15 +8,12 @@ import { getOr } from 'lodash/fp'; import { IEsSearchResponse } from '../../../../../../../../../src/plugins/data/common'; import { - NetworkTlsRequestOptions, TlsBuckets, TlsEdges, } from '../../../../../../common/search_strategy/security_solution/network'; -export const getTlsEdges = ( - response: IEsSearchResponse, - options: NetworkTlsRequestOptions -): TlsEdges[] => formatTlsEdges(getOr([], 'aggregations.sha1.buckets', response.rawResponse)); +export const getTlsEdges = (response: IEsSearchResponse): TlsEdges[] => + formatTlsEdges(getOr([], 'aggregations.sha1.buckets', response.rawResponse)); export const formatTlsEdges = (buckets: TlsBuckets[]): TlsEdges[] => buckets.map((bucket: TlsBuckets) => { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/index.ts index 9286b6a498788..32836c0ef6869 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/tls/index.ts @@ -35,7 +35,7 @@ export const networkTls: SecuritySolutionFactory = { ): Promise => { const { activePage, cursorStart, fakePossibleCount, querySize } = options.pagination; const totalCount = getOr(0, 'aggregations.count.value', response.rawResponse); - const tlsEdges: TlsEdges[] = getTlsEdges(response, options); + const tlsEdges: TlsEdges[] = getTlsEdges(response); const fakeTotalCount = fakePossibleCount <= totalCount ? fakePossibleCount : totalCount; const edges = tlsEdges.splice(cursorStart, querySize - cursorStart); const inspect = { From 898c5c5e6d7ca0689f2a5b0ffc470bda4043c399 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Sat, 29 Aug 2020 16:26:25 +0200 Subject: [PATCH 3/8] [Security Solution] Refactor Network HTTP to use Search Strategy --- .../security_solution/index.ts | 17 +- .../security_solution/network/index.ts | 79 ++++- .../network/containers/network_http/index.tsx | 292 ++++++++++-------- .../containers/network_http/translations.ts | 21 ++ .../ip_details/network_http_query_table.tsx | 70 ++--- .../pages/navigation/http_query_tab_body.tsx | 68 ++-- .../factory/network/http/helpers.ts | 34 ++ .../factory/network/http/index.ts | 59 ++++ .../network/http/query.http_network.dsl.ts | 114 +++++++ .../factory/network/index.ts | 2 + 10 files changed, 554 insertions(+), 202 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/network/containers/network_http/translations.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/helpers.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/index.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/query.http_network.dsl.ts diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts index 264b8d8e0d021..23677e8a09895 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts @@ -13,7 +13,13 @@ import { HostsRequestOptions, HostsStrategyResponse, } from './hosts'; -import { NetworkQueries, NetworkTlsStrategyResponse, NetworkTlsRequestOptions } from './network'; +import { + NetworkQueries, + NetworkTlsStrategyResponse, + NetworkTlsRequestOptions, + NetworkHttpStrategyResponse, + NetworkHttpRequestOptions, +} from './network'; export * from './hosts'; export type Maybe = T | null; @@ -104,6 +110,8 @@ export type StrategyResponseType = T extends HostsQ ? HostOverviewStrategyResponse : T extends NetworkQueries.tls ? NetworkTlsStrategyResponse + : T extends NetworkQueries.http + ? NetworkHttpStrategyResponse : never; export type StrategyRequestType = T extends HostsQueries.hosts @@ -112,4 +120,11 @@ export type StrategyRequestType = T extends HostsQu ? HostOverviewRequestOptions : T extends NetworkQueries.tls ? NetworkTlsRequestOptions + : T extends NetworkQueries.http + ? NetworkHttpRequestOptions : never; + +export interface GenericBuckets { + key: string; + doc_count: number; +} diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts index 48f6dc4260989..2b8be64437bb2 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts @@ -6,9 +6,17 @@ import { IEsSearchResponse } from '../../../../../../../src/plugins/data/common'; -import { CursorType, Inspect, Maybe, PageInfoPaginated, RequestOptionsPaginated } from '..'; +import { + CursorType, + Inspect, + Maybe, + PageInfoPaginated, + RequestOptionsPaginated, + GenericBuckets, +} from '..'; export enum NetworkQueries { + http = 'http', tls = 'tls', } @@ -76,6 +84,10 @@ export interface TlsSortField { direction: Direction; } +export interface NetworkHttpSortField { + direction: Direction; +} + export interface NetworkTlsRequestOptions extends RequestOptionsPaginated { ip: string; flowTarget: FlowTargetSourceDest; @@ -92,3 +104,68 @@ export interface NetworkTlsStrategyResponse extends IEsSearchResponse { inspect?: Maybe; } + +export interface NetworkHttpRequestOptions extends RequestOptionsPaginated { + ip?: string; + networkHttpSort: NetworkHttpSortField; + defaultIndex: string[]; +} + +export interface NetworkHttpStrategyResponse extends IEsSearchResponse { + edges: NetworkHttpEdges[]; + + totalCount: number; + + pageInfo: PageInfoPaginated; + + inspect?: Maybe; +} + +export interface NetworkHttpData { + edges: NetworkHttpEdges[]; + + totalCount: number; + + pageInfo: PageInfoPaginated; + + inspect?: Maybe; +} + +export interface NetworkHttpEdges { + node: NetworkHttpItem; + + cursor: CursorType; +} + +export interface NetworkHttpItem { + _id?: Maybe; + + domains: string[]; + + lastHost?: Maybe; + + lastSourceIp?: Maybe; + + methods: string[]; + + path?: Maybe; + + requestCount?: Maybe; + + statuses: string[]; +} + +export interface NetworkHttpBuckets { + key: string; + doc_count: number; + domains: { + buckets: GenericBuckets[]; + }; + methods: { + buckets: GenericBuckets[]; + }; + source: object; + status: { + buckets: GenericBuckets[]; + }; +} diff --git a/x-pack/plugins/security_solution/public/network/containers/network_http/index.tsx b/x-pack/plugins/security_solution/public/network/containers/network_http/index.tsx index 60845d452d69e..3ca098abe8f17 100644 --- a/x-pack/plugins/security_solution/public/network/containers/network_http/index.tsx +++ b/x-pack/plugins/security_solution/public/network/containers/network_http/index.tsx @@ -4,29 +4,26 @@ * you may not use this file except in compliance with the Elastic License. */ -import { getOr } from 'lodash/fp'; -import React from 'react'; -import { Query } from 'react-apollo'; -import { connect } from 'react-redux'; -import { compose } from 'redux'; +import { noop } from 'lodash/fp'; +import { useState, useEffect, useCallback, useRef } from 'react'; +import { shallowEqual, useSelector } from 'react-redux'; +import deepEqual from 'fast-deep-equal'; +import { ESTermQuery } from '../../../../common/typed_json'; import { DEFAULT_INDEX_KEY } from '../../../../common/constants'; -import { - GetNetworkHttpQuery, - NetworkHttpEdges, - NetworkHttpSortField, - PageInfoPaginated, -} from '../../../graphql/types'; -import { inputsModel, inputsSelectors, State } from '../../../common/store'; -import { withKibana, WithKibanaProps } from '../../../common/lib/kibana'; +import { inputsModel, State } from '../../../common/store'; +import { useKibana } from '../../../common/lib/kibana'; +import { createFilter } from '../../../common/containers/helpers'; +import { NetworkHttpEdges, PageInfoPaginated } from '../../../graphql/types'; import { generateTablePaginationOptions } from '../../../common/components/paginated_table/helpers'; -import { createFilter, getDefaultFetchPolicy } from '../../../common/containers/helpers'; -import { - QueryTemplatePaginated, - QueryTemplatePaginatedProps, -} from '../../../common/containers/query_template_paginated'; import { networkModel, networkSelectors } from '../../store'; -import { networkHttpQuery } from './index.gql_query'; +import { + NetworkQueries, + NetworkHttpRequestOptions, + NetworkHttpStrategyResponse, +} from '../../../../common/search_strategy/security_solution/network'; +import { AbortError } from '../../../../../../../src/plugins/data/common'; +import * as i18n from './translations'; const ID = 'networkHttpQuery'; @@ -35,7 +32,6 @@ export interface NetworkHttpArgs { ip?: string; inspect: inputsModel.InspectQuery; isInspected: boolean; - loading: boolean; loadPage: (newActivePage: number) => void; networkHttp: NetworkHttpEdges[]; pageInfo: PageInfoPaginated; @@ -43,118 +39,168 @@ export interface NetworkHttpArgs { totalCount: number; } -export interface OwnProps extends QueryTemplatePaginatedProps { - children: (args: NetworkHttpArgs) => React.ReactNode; +interface UseNetworkHttp { + id?: string; ip?: string; type: networkModel.NetworkType; + filterQuery?: ESTermQuery | string; + endDate: string; + startDate: string; + skip: boolean; } -export interface NetworkHttpComponentReduxProps { - activePage: number; - isInspected: boolean; - limit: number; - sort: NetworkHttpSortField; -} +export const useNetworkHttp = ({ + endDate, + filterQuery, + id = ID, + ip, + skip, + startDate, + type, +}: UseNetworkHttp): [boolean, NetworkHttpArgs] => { + // const getQuery = inputsSelectors.globalQueryByIdSelector(); + // const { isInspected } = useSelector((state: State) => getQuery(state, id), shallowEqual); + const getHttpSelector = networkSelectors.httpSelector(); + const { activePage, limit, sort } = useSelector( + (state: State) => getHttpSelector(state, type), + shallowEqual + ); + const { data, notifications, uiSettings } = useKibana().services; + const refetch = useRef(noop); + const abortCtrl = useRef(new AbortController()); + const defaultIndex = uiSettings.get(DEFAULT_INDEX_KEY); + const [loading, setLoading] = useState(false); + + const [networkHttpRequest, setHostRequest] = useState({ + defaultIndex, + factoryQueryType: NetworkQueries.http, + filterQuery: createFilter(filterQuery), + // inspect: isInspected, + ip, + pagination: generateTablePaginationOptions(activePage, limit), + networkHttpSort: sort, + timerange: { + interval: '12h', + from: startDate ? startDate : '', + to: endDate ? endDate : new Date(Date.now()).toISOString(), + }, + }); + + const wrappedLoadMore = useCallback( + (newActivePage: number) => { + setHostRequest((prevRequest) => { + return { + ...prevRequest, + pagination: generateTablePaginationOptions(newActivePage, limit), + }; + }); + }, + [limit] + ); -type NetworkHttpProps = OwnProps & NetworkHttpComponentReduxProps & WithKibanaProps; + const [networkHttpResponse, setNetworkHttpResponse] = useState({ + networkHttp: [], + id: ID, + inspect: { + dsl: [], + response: [], + }, + isInspected: false, + loadPage: wrappedLoadMore, + pageInfo: { + activePage: 0, + fakeTotalCount: 0, + showMorePagesIndicator: false, + }, + refetch: refetch.current, + totalCount: -1, + }); -class NetworkHttpComponentQuery extends QueryTemplatePaginated< - NetworkHttpProps, - GetNetworkHttpQuery.Query, - GetNetworkHttpQuery.Variables -> { - public render() { - const { - activePage, - children, - endDate, - filterQuery, - id = ID, - ip, - isInspected, - kibana, - limit, - skip, - sourceId, - sort, - startDate, - } = this.props; - const variables: GetNetworkHttpQuery.Variables = { - defaultIndex: kibana.services.uiSettings.get(DEFAULT_INDEX_KEY), - filterQuery: createFilter(filterQuery), - inspect: isInspected, - ip, - pagination: generateTablePaginationOptions(activePage, limit), - sort, - sourceId, - timerange: { - interval: '12h', - from: startDate!, - to: endDate!, - }, - }; - return ( - - fetchPolicy={getDefaultFetchPolicy()} - notifyOnNetworkStatusChange - query={networkHttpQuery} - skip={skip} - variables={variables} - > - {({ data, loading, fetchMore, networkStatus, refetch }) => { - const networkHttp = getOr([], `source.NetworkHttp.edges`, data); - this.setFetchMore(fetchMore); - this.setFetchMoreOptions((newActivePage: number) => ({ - variables: { - pagination: generateTablePaginationOptions(newActivePage, limit), + const networkHttpSearch = useCallback( + (request: NetworkHttpRequestOptions) => { + let didCancel = false; + const asyncSearch = async () => { + abortCtrl.current = new AbortController(); + setLoading(true); + + const searchSubscription$ = data.search + .search(request, { + strategy: 'securitySolutionSearchStrategy', + signal: abortCtrl.current.signal, + }) + .subscribe({ + next: (response) => { + if (!response.isPartial && !response.isRunning) { + if (!didCancel) { + setLoading(false); + setNetworkHttpResponse((prevResponse) => ({ + ...prevResponse, + networkHttp: response.edges, + inspect: response.inspect ?? prevResponse.inspect, + pageInfo: response.pageInfo, + refetch: refetch.current, + totalCount: response.totalCount, + })); + } + searchSubscription$.unsubscribe(); + } else if (response.isPartial && !response.isRunning) { + if (!didCancel) { + setLoading(false); + } + // TODO: Make response error status clearer + notifications.toasts.addWarning(i18n.ERROR_NETWORK_HTTP); + searchSubscription$.unsubscribe(); + } }, - updateQuery: (prev, { fetchMoreResult }) => { - if (!fetchMoreResult) { - return prev; + error: (msg) => { + if (!(msg instanceof AbortError)) { + notifications.toasts.addDanger({ + title: i18n.FAIL_NETWORK_HTTP, + text: msg.message, + }); } - return { - ...fetchMoreResult, - source: { - ...fetchMoreResult.source, - NetworkHttp: { - ...fetchMoreResult.source.NetworkHttp, - edges: [...fetchMoreResult.source.NetworkHttp.edges], - }, - }, - }; }, - })); - const isLoading = this.isItAValidLoading(loading, variables, networkStatus); - return children({ - id, - inspect: getOr(null, 'source.NetworkHttp.inspect', data), - isInspected, - loading: isLoading, - loadPage: this.wrappedLoadMore, - networkHttp, - pageInfo: getOr({}, 'source.NetworkHttp.pageInfo', data), - refetch: this.memoizedRefetchQuery(variables, limit, refetch), - totalCount: getOr(-1, 'source.NetworkHttp.totalCount', data), }); - }} - - ); - } -} + }; + abortCtrl.current.abort(); + asyncSearch(); + refetch.current = asyncSearch; + return () => { + didCancel = true; + abortCtrl.current.abort(); + }; + }, + [data.search, notifications.toasts] + ); -const makeMapStateToProps = () => { - const getHttpSelector = networkSelectors.httpSelector(); - const getQuery = inputsSelectors.globalQueryByIdSelector(); - return (state: State, { id = ID, type }: OwnProps) => { - const { isInspected } = getQuery(state, id); - return { - ...getHttpSelector(state, type), - isInspected, - }; - }; -}; + useEffect(() => { + if (skip) { + return; + } + + setHostRequest((prevRequest) => { + const myRequest = { + ...prevRequest, + defaultIndex, + filterQuery: createFilter(filterQuery), + pagination: generateTablePaginationOptions(activePage, limit), + timerange: { + interval: '12h', + from: startDate, + to: endDate, + }, + networkHttpSort: sort, + }; + if (!deepEqual(prevRequest, myRequest)) { + return myRequest; + } + return prevRequest; + }); + }, [activePage, defaultIndex, endDate, filterQuery, limit, startDate, sort, skip]); -export const NetworkHttpQuery = compose>( - connect(makeMapStateToProps), - withKibana -)(NetworkHttpComponentQuery); + useEffect(() => { + networkHttpSearch(networkHttpRequest); + }, [networkHttpRequest, networkHttpSearch]); + + return [loading, networkHttpResponse]; +}; diff --git a/x-pack/plugins/security_solution/public/network/containers/network_http/translations.ts b/x-pack/plugins/security_solution/public/network/containers/network_http/translations.ts new file mode 100644 index 0000000000000..7909a5e48b8c4 --- /dev/null +++ b/x-pack/plugins/security_solution/public/network/containers/network_http/translations.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; + +export const ERROR_NETWORK_HTTP = i18n.translate( + 'xpack.securitySolution.networkHttp.errorSearchDescription', + { + defaultMessage: `An error has occurred on network http search`, + } +); + +export const FAIL_NETWORK_HTTP = i18n.translate( + 'xpack.securitySolution.networkHttp.failSearchDescription', + { + defaultMessage: `Failed to run search on network http`, + } +); diff --git a/x-pack/plugins/security_solution/public/network/pages/ip_details/network_http_query_table.tsx b/x-pack/plugins/security_solution/public/network/pages/ip_details/network_http_query_table.tsx index 551de698cfa08..1b1b2b5f4f46e 100644 --- a/x-pack/plugins/security_solution/public/network/pages/ip_details/network_http_query_table.tsx +++ b/x-pack/plugins/security_solution/public/network/pages/ip_details/network_http_query_table.tsx @@ -8,7 +8,7 @@ import React from 'react'; import { getOr } from 'lodash/fp'; import { manageQuery } from '../../../common/components/page/manage_query'; import { OwnProps } from './types'; -import { NetworkHttpQuery } from '../../containers/network_http'; +import { useNetworkHttp } from '../../containers/network_http'; import { NetworkHttpTable } from '../../components/network_http_table'; const NetworkHttpTableManage = manageQuery(NetworkHttpTable); @@ -21,43 +21,35 @@ export const NetworkHttpQueryTable = ({ skip, startDate, type, -}: OwnProps) => ( - - {({ - id, - inspect, - isInspected, - loading, - loadPage, - networkHttp, - pageInfo, - refetch, - totalCount, - }) => ( - - )} - -); +}: OwnProps) => { + const [ + loading, + { id, inspect, isInspected, loadPage, networkHttp, pageInfo, refetch, totalCount }, + ] = useNetworkHttp({ + endDate, + filterQuery, + ip, + skip, + startDate, + type, + }); + + return ( + + ); +}; NetworkHttpQueryTable.displayName = 'NetworkHttpQueryTable'; diff --git a/x-pack/plugins/security_solution/public/network/pages/navigation/http_query_tab_body.tsx b/x-pack/plugins/security_solution/public/network/pages/navigation/http_query_tab_body.tsx index 7e0c4025d6cac..3caff05734c1e 100644 --- a/x-pack/plugins/security_solution/public/network/pages/navigation/http_query_tab_body.tsx +++ b/x-pack/plugins/security_solution/public/network/pages/navigation/http_query_tab_body.tsx @@ -8,7 +8,7 @@ import React from 'react'; import { getOr } from 'lodash/fp'; import { NetworkHttpTable } from '../../components/network_http_table'; -import { NetworkHttpQuery } from '../../containers/network_http'; +import { useNetworkHttp } from '../../containers/network_http'; import { networkModel } from '../../store'; import { manageQuery } from '../../../common/components/page/manage_query'; @@ -22,42 +22,34 @@ export const HttpQueryTabBody = ({ skip, startDate, setQuery, -}: HttpQueryTabBodyProps) => ( - - {({ - id, - inspect, - isInspected, - loading, - loadPage, - networkHttp, - pageInfo, - refetch, - totalCount, - }) => ( - - )} - -); +}: HttpQueryTabBodyProps) => { + const [ + loading, + { id, inspect, isInspected, loadPage, networkHttp, pageInfo, refetch, totalCount }, + ] = useNetworkHttp({ + endDate, + filterQuery, + skip, + startDate, + type: networkModel.NetworkType.page, + }); + + return ( + + ); +}; HttpQueryTabBody.displayName = 'HttpQueryTabBody'; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/helpers.ts new file mode 100644 index 0000000000000..fee1c1c95c244 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/helpers.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { get, getOr } from 'lodash/fp'; + +import { IEsSearchResponse } from '../../../../../../../../../src/plugins/data/common'; +import { + NetworkHttpBuckets, + NetworkHttpEdges, +} from '../../../../../../common/search_strategy/security_solution/network'; + +export const getHttpEdges = (response: IEsSearchResponse): NetworkHttpEdges[] => + formatHttpEdges(getOr([], `aggregations.url.buckets`, response)); + +const formatHttpEdges = (buckets: NetworkHttpBuckets[]): NetworkHttpEdges[] => + buckets.map((bucket: NetworkHttpBuckets) => ({ + node: { + _id: bucket.key, + domains: bucket.domains.buckets.map(({ key }) => key), + methods: bucket.methods.buckets.map(({ key }) => key), + statuses: bucket.status.buckets.map(({ key }) => `${key}`), + lastHost: get('source.hits.hits[0]._source.host.name', bucket), + lastSourceIp: get('source.hits.hits[0]._source.source.ip', bucket), + path: bucket.key, + requestCount: bucket.doc_count, + }, + cursor: { + value: bucket.key, + tiebreaker: null, + }, + })); diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/index.ts new file mode 100644 index 0000000000000..93049616b8172 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/index.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getOr } from 'lodash/fp'; + +import { IEsSearchResponse } from '../../../../../../../../../src/plugins/data/common'; + +import { DEFAULT_MAX_TABLE_QUERY_SIZE } from '../../../../../../common/constants'; +import { + NetworkHttpStrategyResponse, + NetworkQueries, + NetworkHttpRequestOptions, + NetworkHttpEdges, +} from '../../../../../../common/search_strategy/security_solution/network'; + +import { inspectStringifyObject } from '../../../../../utils/build_query'; +import { SecuritySolutionFactory } from '../../types'; + +import { getHttpEdges } from './helpers'; +import { buildHttpQuery } from './query.http_network.dsl'; + +export const networkHttp: SecuritySolutionFactory = { + buildDsl: (options: NetworkHttpRequestOptions) => { + if (options.pagination && options.pagination.querySize >= DEFAULT_MAX_TABLE_QUERY_SIZE) { + throw new Error(`No query size above ${DEFAULT_MAX_TABLE_QUERY_SIZE}`); + } + return buildHttpQuery(options); + }, + parse: async ( + options: NetworkHttpRequestOptions, + response: IEsSearchResponse + ): Promise => { + const { activePage, cursorStart, fakePossibleCount, querySize } = options.pagination; + const totalCount = getOr(0, 'aggregations.count.value', response.rawResponse); + const networkHttpEdges: NetworkHttpEdges[] = getHttpEdges(response); + const fakeTotalCount = fakePossibleCount <= totalCount ? fakePossibleCount : totalCount; + const edges = networkHttpEdges.splice(cursorStart, querySize - cursorStart); + const inspect = { + dsl: [inspectStringifyObject(buildHttpQuery(options))], + response: [inspectStringifyObject(response)], + }; + const showMorePagesIndicator = totalCount > fakeTotalCount; + + return { + ...response, + edges, + inspect, + pageInfo: { + activePage: activePage ? activePage : 0, + fakeTotalCount, + showMorePagesIndicator, + }, + totalCount, + }; + }, +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/query.http_network.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/query.http_network.dsl.ts new file mode 100644 index 0000000000000..9c655ca0f4eaa --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/query.http_network.dsl.ts @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { createQueryFilterClauses } from '../../../../../utils/build_query'; + +import { NetworkHttpRequestOptions } from '../../../../../../common/search_strategy/security_solution/network'; +import { NetworkHttpSortField } from '../../../../../graphql/types'; + +const getCountAgg = () => ({ + http_count: { + cardinality: { + field: 'url.path', + }, + }, +}); + +export const buildHttpQuery = ({ + defaultIndex, + filterQuery, + networkHttpSort, + pagination: { querySize }, + timerange: { from, to }, + ip, +}: NetworkHttpRequestOptions) => { + const filter = [ + ...createQueryFilterClauses(filterQuery), + { + range: { + '@timestamp': { gte: from, lte: to, format: 'strict_date_optional_time' }, + }, + }, + { exists: { field: 'http.request.method' } }, + ]; + + const dslQuery = { + allowNoIndices: true, + index: defaultIndex, + ignoreUnavailable: true, + body: { + aggregations: { + ...getCountAgg(), + ...getHttpAggs(networkHttpSort, querySize), + }, + query: { + bool: ip + ? { + filter, + should: [ + { + term: { + 'source.ip': ip, + }, + }, + { + term: { + 'destination.ip': ip, + }, + }, + ], + minimum_should_match: 1, + } + : { + filter, + }, + }, + }, + size: 0, + track_total_hits: false, + }; + return dslQuery; +}; + +const getHttpAggs = (networkHttpSortField: NetworkHttpSortField, querySize: number) => ({ + url: { + terms: { + field: `url.path`, + size: querySize, + order: { + _count: networkHttpSortField.direction, + }, + }, + aggs: { + methods: { + terms: { + field: 'http.request.method', + size: 4, + }, + }, + domains: { + terms: { + field: 'url.domain', + size: 4, + }, + }, + status: { + terms: { + field: 'http.response.status_code', + size: 4, + }, + }, + source: { + top_hits: { + size: 1, + _source: { + includes: ['host.name', 'source.ip'], + }, + }, + }, + }, + }, +}); diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts index 2c21d9741d648..7d40b034c66bb 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts @@ -8,8 +8,10 @@ import { FactoryQueryTypes } from '../../../../../common/search_strategy/securit import { NetworkQueries } from '../../../../../common/search_strategy/security_solution/network'; import { SecuritySolutionFactory } from '../types'; +import { networkHttp } from './http'; import { networkTls } from './tls'; export const networkFactory: Record> = { + [NetworkQueries.http]: networkHttp, [NetworkQueries.tls]: networkTls, }; From 9e52ca65b2bfde71cd695c35645659eb24bf2184 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Sat, 29 Aug 2020 22:28:17 +0200 Subject: [PATCH 4/8] [Security Solution] Refactor Network Top Countries to use Search Strategy --- .../security_solution/index.ts | 7 + .../security_solution/network/index.ts | 113 +++++++ .../link_to/redirect_to_network.tsx | 5 +- .../public/common/components/links/index.tsx | 5 +- .../public/hosts/containers/hosts/index.tsx | 3 +- .../__snapshots__/index.test.tsx.snap | 4 +- .../network_top_countries_table/columns.tsx | 2 +- .../index.test.tsx | 6 +- .../network_top_countries_table/index.tsx | 232 +++++++------- .../network_top_countries_table/mock.ts | 5 +- .../network_top_countries/index.tsx | 298 ++++++++++-------- .../network_top_countries/translations.ts | 21 ++ .../network_top_countries_query_table.tsx | 76 ++--- .../public/network/pages/ip_details/types.ts | 5 +- .../navigation/countries_query_tab_body.tsx | 74 ++--- .../pages/navigation/network_routes.tsx | 2 +- .../public/network/pages/navigation/types.ts | 2 +- .../public/network/store/selectors.ts | 2 +- .../factory/network/helpers.ts | 18 ++ .../factory/network/http/index.ts | 2 +- .../factory/network/index.ts | 2 + .../factory/network/top_countries/helpers.ts | 53 ++++ .../factory/network/top_countries/index.ts | 62 ++++ .../query.top_countries_network.dsl.ts | 150 +++++++++ 24 files changed, 803 insertions(+), 346 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/network/containers/network_top_countries/translations.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/helpers.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/helpers.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/index.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/query.top_countries_network.dsl.ts diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts index 23677e8a09895..df9099ecabd38 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts @@ -19,9 +19,12 @@ import { NetworkTlsRequestOptions, NetworkHttpStrategyResponse, NetworkHttpRequestOptions, + NetworkTopCountriesStrategyResponse, + NetworkTopCountriesRequestOptions, } from './network'; export * from './hosts'; +export * from './network'; export type Maybe = T | null; export type FactoryQueryTypes = HostsQueries | NetworkQueries; @@ -112,6 +115,8 @@ export type StrategyResponseType = T extends HostsQ ? NetworkTlsStrategyResponse : T extends NetworkQueries.http ? NetworkHttpStrategyResponse + : T extends NetworkQueries.topCountries + ? NetworkTopCountriesStrategyResponse : never; export type StrategyRequestType = T extends HostsQueries.hosts @@ -122,6 +127,8 @@ export type StrategyRequestType = T extends HostsQu ? NetworkTlsRequestOptions : T extends NetworkQueries.http ? NetworkHttpRequestOptions + : T extends NetworkQueries.topCountries + ? NetworkTopCountriesRequestOptions : never; export interface GenericBuckets { diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts index 2b8be64437bb2..347f3c56749d7 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts @@ -5,6 +5,7 @@ */ import { IEsSearchResponse } from '../../../../../../../src/plugins/data/common'; +import { GeoEcs } from '../../../ecs/geo'; import { CursorType, @@ -18,6 +19,23 @@ import { export enum NetworkQueries { http = 'http', tls = 'tls', + topCountries = 'topCountries', +} + +export enum NetworkTopTablesFields { + bytes_in = 'bytes_in', + bytes_out = 'bytes_out', + flows = 'flows', + destination_ips = 'destination_ips', + source_ips = 'source_ips', +} + +export enum NetworkDnsFields { + dnsName = 'dnsName', + queryCount = 'queryCount', + uniqueDomains = 'uniqueDomains', + dnsBytesIn = 'dnsBytesIn', + dnsBytesOut = 'dnsBytesOut', } export interface TlsBuckets { @@ -58,6 +76,13 @@ export interface TlsNode { issuers?: Maybe; } +export enum FlowTarget { + client = 'client', + destination = 'destination', + server = 'server', + source = 'source', +} + export enum FlowTargetSourceDest { destination = 'destination', source = 'source', @@ -121,6 +146,94 @@ export interface NetworkHttpStrategyResponse extends IEsSearchResponse { inspect?: Maybe; } +export interface GeoItem { + geo?: Maybe; + + flowTarget?: Maybe; +} + +export interface TopCountriesItemSource { + country?: Maybe; + + destination_ips?: Maybe; + + flows?: Maybe; + + location?: Maybe; + + source_ips?: Maybe; +} + +export interface NetworkTopCountriesRequestOptions extends RequestOptionsPaginated { + networkTopCountriesSort: NetworkTopTablesSortField; + flowTarget: FlowTargetSourceDest; + ip?: string; +} + +export interface NetworkTopCountriesStrategyResponse extends IEsSearchResponse { + edges: NetworkTopCountriesEdges[]; + + totalCount: number; + + pageInfo: PageInfoPaginated; + + inspect?: Maybe; +} + +export interface NetworkTopCountriesEdges { + node: NetworkTopCountriesItem; + + cursor: CursorType; +} + +export interface NetworkTopCountriesItem { + _id?: Maybe; + + source?: Maybe; + + destination?: Maybe; + + network?: Maybe; +} + +export interface TopCountriesItemDestination { + country?: Maybe; + + destination_ips?: Maybe; + + flows?: Maybe; + + location?: Maybe; + + source_ips?: Maybe; +} + +export interface TopNetworkTablesEcsField { + bytes_in?: Maybe; + + bytes_out?: Maybe; +} + +export interface NetworkTopTablesSortField { + field: NetworkTopTablesFields; + + direction: Direction; +} + +export interface NetworkTopCountriesBuckets { + country: string; + key: string; + bytes_in: { + value: number; + }; + bytes_out: { + value: number; + }; + flows: number; + destination_ips: number; + source_ips: number; +} + export interface NetworkHttpData { edges: NetworkHttpEdges[]; diff --git a/x-pack/plugins/security_solution/public/common/components/link_to/redirect_to_network.tsx b/x-pack/plugins/security_solution/public/common/components/link_to/redirect_to_network.tsx index 8e2b47bd91dbc..100c5e46141a2 100644 --- a/x-pack/plugins/security_solution/public/common/components/link_to/redirect_to_network.tsx +++ b/x-pack/plugins/security_solution/public/common/components/link_to/redirect_to_network.tsx @@ -4,7 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ -import { FlowTarget, FlowTargetSourceDest } from '../../../graphql/types'; +import { + FlowTarget, + FlowTargetSourceDest, +} from '../../../../common/search_strategy/security_solution/network'; import { appendSearch } from './helpers'; diff --git a/x-pack/plugins/security_solution/public/common/components/links/index.tsx b/x-pack/plugins/security_solution/public/common/components/links/index.tsx index 2f7aa1b14cfda..943f2d8336ca7 100644 --- a/x-pack/plugins/security_solution/public/common/components/links/index.tsx +++ b/x-pack/plugins/security_solution/public/common/components/links/index.tsx @@ -32,7 +32,10 @@ import { getCreateCaseUrl, useFormatUrl, } from '../link_to'; -import { FlowTarget, FlowTargetSourceDest } from '../../../graphql/types'; +import { + FlowTarget, + FlowTargetSourceDest, +} from '../../../../common/search_strategy/security_solution/network'; import { useUiSetting$, useKibana } from '../../lib/kibana'; import { isUrlInvalid } from '../../utils/validators'; import { ExternalLinkIcon } from '../external_link_icon'; diff --git a/x-pack/plugins/security_solution/public/hosts/containers/hosts/index.tsx b/x-pack/plugins/security_solution/public/hosts/containers/hosts/index.tsx index 346de9f87313f..0410f630173e1 100644 --- a/x-pack/plugins/security_solution/public/hosts/containers/hosts/index.tsx +++ b/x-pack/plugins/security_solution/public/hosts/containers/hosts/index.tsx @@ -10,13 +10,14 @@ import { useCallback, useEffect, useRef, useState } from 'react'; import { useSelector } from 'react-redux'; import { DEFAULT_INDEX_KEY } from '../../../../common/constants'; -import { HostsEdges, PageInfoPaginated } from '../../../graphql/types'; import { inputsModel, State } from '../../../common/store'; import { createFilter } from '../../../common/containers/helpers'; import { useKibana } from '../../../common/lib/kibana'; import { hostsModel, hostsSelectors } from '../../store'; import { generateTablePaginationOptions } from '../../../common/components/paginated_table/helpers'; import { + HostsEdges, + PageInfoPaginated, DocValueFields, HostsQueries, HostsRequestOptions, diff --git a/x-pack/plugins/security_solution/public/network/components/network_top_countries_table/__snapshots__/index.test.tsx.snap b/x-pack/plugins/security_solution/public/network/components/network_top_countries_table/__snapshots__/index.test.tsx.snap index 1127528c776b7..02a8802bfced1 100644 --- a/x-pack/plugins/security_solution/public/network/components/network_top_countries_table/__snapshots__/index.test.tsx.snap +++ b/x-pack/plugins/security_solution/public/network/components/network_top_countries_table/__snapshots__/index.test.tsx.snap @@ -1,7 +1,7 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP exports[`NetworkTopCountries Table Component rendering it renders the IP Details NetworkTopCountries table 1`] = ` - { ); - expect(wrapper.find('Connect(NetworkTopCountriesTableComponent)')).toMatchSnapshot(); + expect(wrapper.find('Memo(NetworkTopCountriesTableComponent)')).toMatchSnapshot(); }); test('it renders the IP Details NetworkTopCountries table', () => { const wrapper = shallow( @@ -101,7 +101,7 @@ describe('NetworkTopCountries Table Component', () => { ); - expect(wrapper.find('Connect(NetworkTopCountriesTableComponent)')).toMatchSnapshot(); + expect(wrapper.find('Memo(NetworkTopCountriesTableComponent)')).toMatchSnapshot(); }); }); diff --git a/x-pack/plugins/security_solution/public/network/components/network_top_countries_table/index.tsx b/x-pack/plugins/security_solution/public/network/components/network_top_countries_table/index.tsx index 93d3f410ddde4..bbf1c4eeacca7 100644 --- a/x-pack/plugins/security_solution/public/network/components/network_top_countries_table/index.tsx +++ b/x-pack/plugins/security_solution/public/network/components/network_top_countries_table/index.tsx @@ -6,7 +6,7 @@ import { last } from 'lodash/fp'; import React, { useCallback, useMemo } from 'react'; -import { connect, ConnectedProps } from 'react-redux'; +import { useDispatch, useSelector, shallowEqual } from 'react-redux'; import deepEqual from 'fast-deep-equal'; import { IIndexPattern } from 'src/plugins/data/public'; @@ -17,7 +17,7 @@ import { NetworkTopCountriesEdges, NetworkTopTablesFields, NetworkTopTablesSortField, -} from '../../../graphql/types'; +} from '../../../../common/search_strategy/security_solution/network'; import { State } from '../../../common/store'; import { Criteria, ItemsPerRow, PaginatedTable } from '../../../common/components/paginated_table'; @@ -25,7 +25,7 @@ import { Criteria, ItemsPerRow, PaginatedTable } from '../../../common/component import { getCountriesColumnsCurated } from './columns'; import * as i18n from './translations'; -interface OwnProps { +interface NetworkTopCountriesTableProps { data: NetworkTopCountriesEdges[]; fakeTotalCount: number; flowTargeted: FlowTargetSourceDest; @@ -39,8 +39,6 @@ interface OwnProps { type: networkModel.NetworkType; } -type NetworkTopCountriesTableProps = OwnProps & PropsFromRedux; - const rowItems: ItemsPerRow[] = [ { text: i18n.ROWS_5, @@ -54,139 +52,133 @@ const rowItems: ItemsPerRow[] = [ export const NetworkTopCountriesTableId = 'networkTopCountries-top-talkers'; -const NetworkTopCountriesTableComponent = React.memo( - ({ - activePage, - data, - fakeTotalCount, - flowTargeted, - id, - indexPattern, - isInspect, - limit, - loading, - loadPage, - showMorePagesIndicator, - sort, - totalCount, - type, - updateNetworkTable, - }) => { - let tableType: networkModel.TopCountriesTableType; - const headerTitle: string = +const NetworkTopCountriesTableComponent: React.FC = ({ + data, + fakeTotalCount, + flowTargeted, + id, + indexPattern, + isInspect, + loading, + loadPage, + showMorePagesIndicator, + totalCount, + type, +}) => { + const dispatch = useDispatch(); + const getTopCountriesSelector = networkSelectors.topCountriesSelector(); + const { activePage, limit, sort } = useSelector( + (state: State) => getTopCountriesSelector(state, type, flowTargeted), + shallowEqual + ); + + const headerTitle: string = useMemo( + () => flowTargeted === FlowTargetSourceDest.source ? i18n.SOURCE_COUNTRIES - : i18n.DESTINATION_COUNTRIES; + : i18n.DESTINATION_COUNTRIES, + [flowTargeted] + ); + const tableType: networkModel.TopCountriesTableType = useMemo(() => { if (type === networkModel.NetworkType.page) { - tableType = - flowTargeted === FlowTargetSourceDest.source - ? networkModel.NetworkTableType.topCountriesSource - : networkModel.NetworkTableType.topCountriesDestination; - } else { - tableType = - flowTargeted === FlowTargetSourceDest.source - ? networkModel.IpDetailsTableType.topCountriesSource - : networkModel.IpDetailsTableType.topCountriesDestination; + return flowTargeted === FlowTargetSourceDest.source + ? networkModel.NetworkTableType.topCountriesSource + : networkModel.NetworkTableType.topCountriesDestination; } - const field = - sort.field === NetworkTopTablesFields.bytes_out || - sort.field === NetworkTopTablesFields.bytes_in - ? `node.network.${sort.field}` - : `node.${flowTargeted}.${sort.field}`; - - const updateLimitPagination = useCallback( - (newLimit) => - updateNetworkTable({ + return flowTargeted === FlowTargetSourceDest.source + ? networkModel.IpDetailsTableType.topCountriesSource + : networkModel.IpDetailsTableType.topCountriesDestination; + }, [flowTargeted, type]); + + const field = + sort.field === NetworkTopTablesFields.bytes_out || + sort.field === NetworkTopTablesFields.bytes_in + ? `node.network.${sort.field}` + : `node.${flowTargeted}.${sort.field}`; + + const updateLimitPagination = useCallback( + (newLimit) => + dispatch( + networkActions.updateNetworkTable({ networkType: type, tableType, updates: { limit: newLimit }, - }), - [type, updateNetworkTable, tableType] - ); - - const updateActivePage = useCallback( - (newPage) => - updateNetworkTable({ + }) + ), + [dispatch, type, tableType] + ); + + const updateActivePage = useCallback( + (newPage) => + dispatch( + networkActions.updateNetworkTable({ networkType: type, tableType, updates: { activePage: newPage }, - }), - [type, updateNetworkTable, tableType] - ); - - const onChange = useCallback( - (criteria: Criteria) => { - if (criteria.sort != null) { - const splitField = criteria.sort.field.split('.'); - const lastField = last(splitField); - const newSortDirection = - lastField !== sort.field ? Direction.desc : criteria.sort.direction; // sort by desc on init click - const newTopCountriesSort: NetworkTopTablesSortField = { - field: lastField as NetworkTopTablesFields, - direction: newSortDirection as Direction, - }; - if (!deepEqual(newTopCountriesSort, sort)) { - updateNetworkTable({ + }) + ), + [dispatch, type, tableType] + ); + + const onChange = useCallback( + (criteria: Criteria) => { + if (criteria.sort != null) { + const splitField = criteria.sort.field.split('.'); + const lastField = last(splitField); + const newSortDirection = + lastField !== sort.field ? Direction.desc : criteria.sort.direction; // sort by desc on init click + const newTopCountriesSort: NetworkTopTablesSortField = { + field: lastField as NetworkTopTablesFields, + direction: newSortDirection as Direction, + }; + if (!deepEqual(newTopCountriesSort, sort)) { + dispatch( + networkActions.updateNetworkTable({ networkType: type, tableType, updates: { sort: newTopCountriesSort, }, - }); - } + }) + ); } - }, - [type, sort, tableType, updateNetworkTable] - ); - - const columns = useMemo( - () => - getCountriesColumnsCurated(indexPattern, flowTargeted, type, NetworkTopCountriesTableId), - [indexPattern, flowTargeted, type] - ); - - return ( - - ); - } -); - -NetworkTopCountriesTableComponent.displayName = 'NetworkTopCountriesTableComponent'; - -const makeMapStateToProps = () => { - const getTopCountriesSelector = networkSelectors.topCountriesSelector(); - return (state: State, { type, flowTargeted }: OwnProps) => - getTopCountriesSelector(state, type, flowTargeted); -}; - -const mapDispatchToProps = { - updateNetworkTable: networkActions.updateNetworkTable, + } + }, + [sort, dispatch, type, tableType] + ); + + const columns = useMemo( + () => getCountriesColumnsCurated(indexPattern, flowTargeted, type, NetworkTopCountriesTableId), + [indexPattern, flowTargeted, type] + ); + + return ( + + ); }; -const connector = connect(makeMapStateToProps, mapDispatchToProps); - -type PropsFromRedux = ConnectedProps; +NetworkTopCountriesTableComponent.displayName = 'NetworkTopCountriesTableComponent'; -export const NetworkTopCountriesTable = connector(NetworkTopCountriesTableComponent); +export const NetworkTopCountriesTable = React.memo(NetworkTopCountriesTableComponent); diff --git a/x-pack/plugins/security_solution/public/network/components/network_top_countries_table/mock.ts b/x-pack/plugins/security_solution/public/network/components/network_top_countries_table/mock.ts index cee775c93d66f..eb6843647f74a 100644 --- a/x-pack/plugins/security_solution/public/network/components/network_top_countries_table/mock.ts +++ b/x-pack/plugins/security_solution/public/network/components/network_top_countries_table/mock.ts @@ -4,10 +4,11 @@ * you may not use this file except in compliance with the Elastic License. */ -import { NetworkTopCountriesData } from '../../../graphql/types'; +import { NetworkTopCountriesStrategyResponse } from '../../../../common/search_strategy/security_solution/network'; -export const mockData: { NetworkTopCountries: NetworkTopCountriesData } = { +export const mockData: { NetworkTopCountries: NetworkTopCountriesStrategyResponse } = { NetworkTopCountries: { + rawResponse: {} as NetworkTopCountriesStrategyResponse['rawResponse'], totalCount: 524, edges: [ { diff --git a/x-pack/plugins/security_solution/public/network/containers/network_top_countries/index.tsx b/x-pack/plugins/security_solution/public/network/containers/network_top_countries/index.tsx index b167cba460818..14696e640945f 100644 --- a/x-pack/plugins/security_solution/public/network/containers/network_top_countries/index.tsx +++ b/x-pack/plugins/security_solution/public/network/containers/network_top_countries/index.tsx @@ -4,161 +4,205 @@ * you may not use this file except in compliance with the Elastic License. */ -import { getOr } from 'lodash/fp'; -import React from 'react'; -import { Query } from 'react-apollo'; -import { connect } from 'react-redux'; -import { compose } from 'redux'; +import { noop } from 'lodash/fp'; +import { useState, useEffect, useCallback, useRef } from 'react'; +import { shallowEqual, useSelector } from 'react-redux'; +import deepEqual from 'fast-deep-equal'; +import { ESTermQuery } from '../../../../common/typed_json'; import { DEFAULT_INDEX_KEY } from '../../../../common/constants'; +import { inputsModel, State } from '../../../common/store'; +import { useKibana } from '../../../common/lib/kibana'; +import { createFilter } from '../../../common/containers/helpers'; +import { PageInfoPaginated } from '../../../../common/search_strategy/security_solution'; +import { generateTablePaginationOptions } from '../../../common/components/paginated_table/helpers'; +import { networkModel, networkSelectors } from '../../store'; import { FlowTargetSourceDest, - GetNetworkTopCountriesQuery, + NetworkQueries, NetworkTopCountriesEdges, - NetworkTopTablesSortField, - PageInfoPaginated, -} from '../../../graphql/types'; -import { inputsModel, inputsSelectors, State } from '../../../common/store'; -import { withKibana, WithKibanaProps } from '../../../common/lib/kibana'; -import { generateTablePaginationOptions } from '../../../common/components/paginated_table/helpers'; -import { createFilter, getDefaultFetchPolicy } from '../../../common/containers/helpers'; -import { - QueryTemplatePaginated, - QueryTemplatePaginatedProps, -} from '../../../common/containers/query_template_paginated'; -import { networkTopCountriesQuery } from './index.gql_query'; -import { networkModel, networkSelectors } from '../../store'; + NetworkTopCountriesRequestOptions, + NetworkTopCountriesStrategyResponse, +} from '../../../../common/search_strategy/security_solution/network'; +import { AbortError } from '../../../../../../../src/plugins/data/common'; +import * as i18n from './translations'; const ID = 'networkTopCountriesQuery'; export interface NetworkTopCountriesArgs { id: string; - ip?: string; inspect: inputsModel.InspectQuery; isInspected: boolean; - loading: boolean; loadPage: (newActivePage: number) => void; - networkTopCountries: NetworkTopCountriesEdges[]; pageInfo: PageInfoPaginated; refetch: inputsModel.Refetch; + networkTopCountries: NetworkTopCountriesEdges[]; totalCount: number; } -export interface OwnProps extends QueryTemplatePaginatedProps { - children: (args: NetworkTopCountriesArgs) => React.ReactNode; +interface UseNetworkTopCountries { flowTarget: FlowTargetSourceDest; ip?: string; type: networkModel.NetworkType; + filterQuery?: ESTermQuery | string; + endDate: string; + startDate: string; + skip: boolean; + id?: string; } -export interface NetworkTopCountriesComponentReduxProps { - activePage: number; - isInspected: boolean; - limit: number; - sort: NetworkTopTablesSortField; -} +export const useNetworkTopCountries = ({ + endDate, + filterQuery, + flowTarget, + id = ID, + skip, + startDate, + type, +}: UseNetworkTopCountries): [boolean, NetworkTopCountriesArgs] => { + // const getQuery = inputsSelectors.globalQueryByIdSelector(); + // const { isInspected } = useSelector((state: State) => getQuery(state, id), shallowEqual); + const getTopCountriesSelector = networkSelectors.topCountriesSelector(); + const { activePage, limit, sort } = useSelector( + (state: State) => getTopCountriesSelector(state, type, flowTarget), + shallowEqual + ); + const { data, notifications, uiSettings } = useKibana().services; + const refetch = useRef(noop); + const abortCtrl = useRef(new AbortController()); + const defaultIndex = uiSettings.get(DEFAULT_INDEX_KEY); + const [loading, setLoading] = useState(false); -type NetworkTopCountriesProps = OwnProps & NetworkTopCountriesComponentReduxProps & WithKibanaProps; + const [networkTopCountriesRequest, setHostRequest] = useState({ + defaultIndex, + factoryQueryType: NetworkQueries.topCountries, + filterQuery: createFilter(filterQuery), + flowTarget, + // inspect: isInspected, + pagination: generateTablePaginationOptions(activePage, limit), + networkTopCountriesSort: sort, + timerange: { + interval: '12h', + from: startDate ? startDate : '', + to: endDate ? endDate : new Date(Date.now()).toISOString(), + }, + }); -class NetworkTopCountriesComponentQuery extends QueryTemplatePaginated< - NetworkTopCountriesProps, - GetNetworkTopCountriesQuery.Query, - GetNetworkTopCountriesQuery.Variables -> { - public render() { - const { - activePage, - children, - endDate, - flowTarget, - filterQuery, - kibana, - id = `${ID}-${flowTarget}`, - ip, - isInspected, - limit, - skip, - sourceId, - startDate, - sort, - } = this.props; - const variables: GetNetworkTopCountriesQuery.Variables = { - defaultIndex: kibana.services.uiSettings.get(DEFAULT_INDEX_KEY), - filterQuery: createFilter(filterQuery), - flowTarget, - inspect: isInspected, - ip, - pagination: generateTablePaginationOptions(activePage, limit), - sort, - sourceId, - timerange: { - interval: '12h', - from: startDate!, - to: endDate!, - }, - }; - return ( - - fetchPolicy={getDefaultFetchPolicy()} - notifyOnNetworkStatusChange - query={networkTopCountriesQuery} - skip={skip} - variables={variables} - > - {({ data, loading, fetchMore, networkStatus, refetch }) => { - const networkTopCountries = getOr([], `source.NetworkTopCountries.edges`, data); - this.setFetchMore(fetchMore); - this.setFetchMoreOptions((newActivePage: number) => ({ - variables: { - pagination: generateTablePaginationOptions(newActivePage, limit), + const wrappedLoadMore = useCallback( + (newActivePage: number) => { + setHostRequest((prevRequest) => ({ + ...prevRequest, + pagination: generateTablePaginationOptions(newActivePage, limit), + })); + }, + [limit] + ); + + const [networkTopCountriesResponse, setNetworkTopCountriesResponse] = useState< + NetworkTopCountriesArgs + >({ + networkTopCountries: [], + id: ID, + inspect: { + dsl: [], + response: [], + }, + isInspected: false, + loadPage: wrappedLoadMore, + pageInfo: { + activePage: 0, + fakeTotalCount: 0, + showMorePagesIndicator: false, + }, + refetch: refetch.current, + totalCount: -1, + }); + + const networkTopCountriesSearch = useCallback( + (request: NetworkTopCountriesRequestOptions) => { + let didCancel = false; + const asyncSearch = async () => { + abortCtrl.current = new AbortController(); + setLoading(true); + + const searchSubscription$ = data.search + .search(request, { + strategy: 'securitySolutionSearchStrategy', + signal: abortCtrl.current.signal, + }) + .subscribe({ + next: (response) => { + if (!response.isPartial && !response.isRunning) { + if (!didCancel) { + setLoading(false); + setNetworkTopCountriesResponse((prevResponse) => ({ + ...prevResponse, + networkTopCountries: response.edges, + inspect: response.inspect ?? prevResponse.inspect, + pageInfo: response.pageInfo, + refetch: refetch.current, + totalCount: response.totalCount, + })); + } + searchSubscription$.unsubscribe(); + } else if (response.isPartial && !response.isRunning) { + if (!didCancel) { + setLoading(false); + } + // TODO: Make response error status clearer + notifications.toasts.addWarning(i18n.ERROR_NETWORK_TOP_COUNTRIES); + searchSubscription$.unsubscribe(); + } }, - updateQuery: (prev, { fetchMoreResult }) => { - if (!fetchMoreResult) { - return prev; + error: (msg) => { + if (!(msg instanceof AbortError)) { + notifications.toasts.addDanger({ + title: i18n.FAIL_NETWORK_TOP_COUNTRIES, + text: msg.message, + }); } - return { - ...fetchMoreResult, - source: { - ...fetchMoreResult.source, - NetworkTopCountries: { - ...fetchMoreResult.source.NetworkTopCountries, - edges: [...fetchMoreResult.source.NetworkTopCountries.edges], - }, - }, - }; }, - })); - const isLoading = this.isItAValidLoading(loading, variables, networkStatus); - return children({ - id, - inspect: getOr(null, 'source.NetworkTopCountries.inspect', data), - isInspected, - loading: isLoading, - loadPage: this.wrappedLoadMore, - networkTopCountries, - pageInfo: getOr({}, 'source.NetworkTopCountries.pageInfo', data), - refetch: this.memoizedRefetchQuery(variables, limit, refetch), - totalCount: getOr(-1, 'source.NetworkTopCountries.totalCount', data), }); - }} - - ); - } -} + }; + abortCtrl.current.abort(); + asyncSearch(); + refetch.current = asyncSearch; + return () => { + didCancel = true; + abortCtrl.current.abort(); + }; + }, + [data.search, notifications.toasts] + ); -const makeMapStateToProps = () => { - const getTopCountriesSelector = networkSelectors.topCountriesSelector(); - const getQuery = inputsSelectors.globalQueryByIdSelector(); - return (state: State, { flowTarget, id = `${ID}-${flowTarget}`, type }: OwnProps) => { - const { isInspected } = getQuery(state, id); - return { - ...getTopCountriesSelector(state, type, flowTarget), - isInspected, - }; - }; -}; + useEffect(() => { + if (skip) { + return; + } + + setHostRequest((prevRequest) => { + const myRequest = { + ...prevRequest, + defaultIndex, + filterQuery: createFilter(filterQuery), + pagination: generateTablePaginationOptions(activePage, limit), + timerange: { + interval: '12h', + from: startDate, + to: endDate, + }, + networkTopCountriesSort: sort, + }; + if (!deepEqual(prevRequest, myRequest)) { + return myRequest; + } + return prevRequest; + }); + }, [activePage, defaultIndex, endDate, filterQuery, limit, startDate, sort, skip]); -export const NetworkTopCountriesQuery = compose>( - connect(makeMapStateToProps), - withKibana -)(NetworkTopCountriesComponentQuery); + useEffect(() => { + networkTopCountriesSearch(networkTopCountriesRequest); + }, [networkTopCountriesRequest, networkTopCountriesSearch]); + + return [loading, networkTopCountriesResponse]; +}; diff --git a/x-pack/plugins/security_solution/public/network/containers/network_top_countries/translations.ts b/x-pack/plugins/security_solution/public/network/containers/network_top_countries/translations.ts new file mode 100644 index 0000000000000..ff807ee268adf --- /dev/null +++ b/x-pack/plugins/security_solution/public/network/containers/network_top_countries/translations.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; + +export const ERROR_NETWORK_TOP_COUNTRIES = i18n.translate( + 'xpack.securitySolution.networkTopCountries.errorSearchDescription', + { + defaultMessage: `An error has occurred on network top countries search`, + } +); + +export const FAIL_NETWORK_TOP_COUNTRIES = i18n.translate( + 'xpack.securitySolution.networkTopCountries.failSearchDescription', + { + defaultMessage: `Failed to run search on network top countries`, + } +); diff --git a/x-pack/plugins/security_solution/public/network/pages/ip_details/network_top_countries_query_table.tsx b/x-pack/plugins/security_solution/public/network/pages/ip_details/network_top_countries_query_table.tsx index 6bc80ef1a6aae..42ddd3a6bb4a4 100644 --- a/x-pack/plugins/security_solution/public/network/pages/ip_details/network_top_countries_query_table.tsx +++ b/x-pack/plugins/security_solution/public/network/pages/ip_details/network_top_countries_query_table.tsx @@ -8,7 +8,7 @@ import React from 'react'; import { getOr } from 'lodash/fp'; import { manageQuery } from '../../../common/components/page/manage_query'; import { NetworkWithIndexComponentsQueryTableProps } from './types'; -import { NetworkTopCountriesQuery } from '../../containers/network_top_countries'; +import { useNetworkTopCountries } from '../../containers/network_top_countries'; import { NetworkTopCountriesTable } from '../../components/network_top_countries_table'; const NetworkTopCountriesTableManage = manageQuery(NetworkTopCountriesTable); @@ -23,46 +23,38 @@ export const NetworkTopCountriesQueryTable = ({ startDate, type, indexPattern, -}: NetworkWithIndexComponentsQueryTableProps) => ( - - {({ - id, - inspect, - isInspected, - loading, - loadPage, - networkTopCountries, - pageInfo, - refetch, - totalCount, - }) => ( - - )} - -); +}: NetworkWithIndexComponentsQueryTableProps) => { + const [ + loading, + { id, inspect, isInspected, loadPage, networkTopCountries, pageInfo, refetch, totalCount }, + ] = useNetworkTopCountries({ + endDate, + flowTarget, + filterQuery, + ip, + skip, + startDate, + type, + }); + + return ( + + ); +}; NetworkTopCountriesQueryTable.displayName = 'NetworkTopCountriesQueryTable'; diff --git a/x-pack/plugins/security_solution/public/network/pages/ip_details/types.ts b/x-pack/plugins/security_solution/public/network/pages/ip_details/types.ts index 9691214cc2820..d1ee48a9a5d9e 100644 --- a/x-pack/plugins/security_solution/public/network/pages/ip_details/types.ts +++ b/x-pack/plugins/security_solution/public/network/pages/ip_details/types.ts @@ -8,7 +8,10 @@ import { IIndexPattern } from 'src/plugins/data/public'; import { ESTermQuery } from '../../../../common/typed_json'; import { NetworkType } from '../../store/model'; -import { FlowTarget, FlowTargetSourceDest } from '../../../graphql/types'; +import { + FlowTarget, + FlowTargetSourceDest, +} from '../../../../common/search_strategy/security_solution/network'; import { GlobalTimeArgs } from '../../../common/containers/use_global_time'; export const type = NetworkType.details; diff --git a/x-pack/plugins/security_solution/public/network/pages/navigation/countries_query_tab_body.tsx b/x-pack/plugins/security_solution/public/network/pages/navigation/countries_query_tab_body.tsx index 0c569952458e4..1e57ca42257e7 100644 --- a/x-pack/plugins/security_solution/public/network/pages/navigation/countries_query_tab_body.tsx +++ b/x-pack/plugins/security_solution/public/network/pages/navigation/countries_query_tab_body.tsx @@ -8,7 +8,7 @@ import React from 'react'; import { getOr } from 'lodash/fp'; import { NetworkTopCountriesTable } from '../../components/network_top_countries_table'; -import { NetworkTopCountriesQuery } from '../../containers/network_top_countries'; +import { useNetworkTopCountries } from '../../containers/network_top_countries'; import { networkModel } from '../../store'; import { manageQuery } from '../../../common/components/page/manage_query'; @@ -24,45 +24,37 @@ export const CountriesQueryTabBody = ({ setQuery, indexPattern, flowTarget, -}: CountriesQueryTabBodyProps) => ( - - {({ - id, - inspect, - isInspected, - loading, - loadPage, - networkTopCountries, - pageInfo, - refetch, - totalCount, - }) => ( - - )} - -); +}: CountriesQueryTabBodyProps) => { + const [ + loading, + { id, inspect, isInspected, loadPage, networkTopCountries, pageInfo, refetch, totalCount }, + ] = useNetworkTopCountries({ + endDate, + flowTarget, + filterQuery, + skip, + startDate, + type: networkModel.NetworkType.page, + }); + + return ( + + ); +}; CountriesQueryTabBody.displayName = 'CountriesQueryTabBody'; diff --git a/x-pack/plugins/security_solution/public/network/pages/navigation/network_routes.tsx b/x-pack/plugins/security_solution/public/network/pages/navigation/network_routes.tsx index 93582088811dc..2da56a30df7c7 100644 --- a/x-pack/plugins/security_solution/public/network/pages/navigation/network_routes.tsx +++ b/x-pack/plugins/security_solution/public/network/pages/navigation/network_routes.tsx @@ -8,7 +8,7 @@ import React, { useCallback } from 'react'; import { Route, Switch } from 'react-router-dom'; import { EuiFlexItem, EuiSpacer } from '@elastic/eui'; -import { FlowTargetSourceDest } from '../../../graphql/types'; +import { FlowTargetSourceDest } from '../../../../common/search_strategy/security_solution/network'; import { scoreIntervalToDateTime } from '../../../common/components/ml/score/score_interval_to_datetime'; import { IPsQueryTabBody } from './ips_query_tab_body'; diff --git a/x-pack/plugins/security_solution/public/network/pages/navigation/types.ts b/x-pack/plugins/security_solution/public/network/pages/navigation/types.ts index 183c760e40ab1..2ef04d3371c0b 100644 --- a/x-pack/plugins/security_solution/public/network/pages/navigation/types.ts +++ b/x-pack/plugins/security_solution/public/network/pages/navigation/types.ts @@ -8,7 +8,7 @@ import { ESTermQuery } from '../../../../common/typed_json'; import { IIndexPattern } from '../../../../../../../src/plugins/data/common'; import { NavTab } from '../../../common/components/navigation/types'; -import { FlowTargetSourceDest } from '../../../graphql/types'; +import { FlowTargetSourceDest } from '../../../../common/search_strategy/security_solution/network'; import { networkModel } from '../../store'; import { GlobalTimeArgs } from '../../../common/containers/use_global_time'; diff --git a/x-pack/plugins/security_solution/public/network/store/selectors.ts b/x-pack/plugins/security_solution/public/network/store/selectors.ts index cef8b139402ef..0246305092a32 100644 --- a/x-pack/plugins/security_solution/public/network/store/selectors.ts +++ b/x-pack/plugins/security_solution/public/network/store/selectors.ts @@ -7,7 +7,7 @@ import { createSelector } from 'reselect'; import { get } from 'lodash/fp'; -import { FlowTargetSourceDest } from '../../graphql/types'; +import { FlowTargetSourceDest } from '../../../common/search_strategy/security_solution/network'; import { State } from '../../common/store/types'; import { initialNetworkState } from './reducer'; import { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/helpers.ts new file mode 100644 index 0000000000000..a7fba087b87ed --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/helpers.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { assertUnreachable } from '../../../../../common/utility_types'; +import { FlowTargetSourceDest } from '../../../../../common/search_strategy/security_solution/network'; + +export const getOppositeField = (flowTarget: FlowTargetSourceDest): FlowTargetSourceDest => { + switch (flowTarget) { + case FlowTargetSourceDest.source: + return FlowTargetSourceDest.destination; + case FlowTargetSourceDest.destination: + return FlowTargetSourceDest.source; + } + assertUnreachable(flowTarget); +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/index.ts index 93049616b8172..b6c26cd533de2 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/http/index.ts @@ -34,7 +34,7 @@ export const networkHttp: SecuritySolutionFactory = { response: IEsSearchResponse ): Promise => { const { activePage, cursorStart, fakePossibleCount, querySize } = options.pagination; - const totalCount = getOr(0, 'aggregations.count.value', response.rawResponse); + const totalCount = getOr(0, 'aggregations.http_count.value', response.rawResponse); const networkHttpEdges: NetworkHttpEdges[] = getHttpEdges(response); const fakeTotalCount = fakePossibleCount <= totalCount ? fakePossibleCount : totalCount; const edges = networkHttpEdges.splice(cursorStart, querySize - cursorStart); diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts index 7d40b034c66bb..93e5f113197da 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts @@ -10,8 +10,10 @@ import { NetworkQueries } from '../../../../../common/search_strategy/security_s import { SecuritySolutionFactory } from '../types'; import { networkHttp } from './http'; import { networkTls } from './tls'; +import { networkTopCountries } from './top_countries'; export const networkFactory: Record> = { [NetworkQueries.http]: networkHttp, [NetworkQueries.tls]: networkTls, + [NetworkQueries.topCountries]: networkTopCountries, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/helpers.ts new file mode 100644 index 0000000000000..58cc2d6f774fb --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/helpers.ts @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getOr } from 'lodash/fp'; + +import { IEsSearchResponse } from '../../../../../../../../../src/plugins/data/common'; +import { + NetworkTopCountriesBuckets, + NetworkTopCountriesEdges, + NetworkTopCountriesRequestOptions, + FlowTargetSourceDest, +} from '../../../../../../common/search_strategy/security_solution/network'; +import { getOppositeField } from '../helpers'; + +export const getTopCountriesEdges = ( + response: IEsSearchResponse, + options: NetworkTopCountriesRequestOptions +): NetworkTopCountriesEdges[] => + formatTopCountriesEdges( + getOr([], `aggregations.${options.flowTarget}.buckets`, response), + options.flowTarget + ); + +export const formatTopCountriesEdges = ( + buckets: NetworkTopCountriesBuckets[], + flowTarget: FlowTargetSourceDest +): NetworkTopCountriesEdges[] => + buckets.map((bucket: NetworkTopCountriesBuckets) => ({ + node: { + _id: bucket.key, + [flowTarget]: { + country: bucket.key, + flows: getOr(0, 'flows.value', bucket), + [`${getOppositeField(flowTarget)}_ips`]: getOr( + 0, + `${getOppositeField(flowTarget)}_ips.value`, + bucket + ), + [`${flowTarget}_ips`]: getOr(0, `${flowTarget}_ips.value`, bucket), + }, + network: { + bytes_in: getOr(0, 'bytes_in.value', bucket), + bytes_out: getOr(0, 'bytes_out.value', bucket), + }, + }, + cursor: { + value: bucket.key, + tiebreaker: null, + }, + })); diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/index.ts new file mode 100644 index 0000000000000..5b0ced06f2ee9 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/index.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getOr } from 'lodash/fp'; + +import { IEsSearchResponse } from '../../../../../../../../../src/plugins/data/common'; + +import { DEFAULT_MAX_TABLE_QUERY_SIZE } from '../../../../../../common/constants'; +import { + NetworkTopCountriesStrategyResponse, + NetworkQueries, + NetworkTopCountriesRequestOptions, + NetworkTopCountriesEdges, +} from '../../../../../../common/search_strategy/security_solution/network'; + +import { inspectStringifyObject } from '../../../../../utils/build_query'; +import { SecuritySolutionFactory } from '../../types'; + +import { getTopCountriesEdges } from './helpers'; +import { buildTopCountriesQuery } from './query.top_countries_network.dsl'; + +export const networkTopCountries: SecuritySolutionFactory = { + buildDsl: (options: NetworkTopCountriesRequestOptions) => { + if (options.pagination && options.pagination.querySize >= DEFAULT_MAX_TABLE_QUERY_SIZE) { + throw new Error(`No query size above ${DEFAULT_MAX_TABLE_QUERY_SIZE}`); + } + return buildTopCountriesQuery(options); + }, + parse: async ( + options: NetworkTopCountriesRequestOptions, + response: IEsSearchResponse + ): Promise => { + const { activePage, cursorStart, fakePossibleCount, querySize } = options.pagination; + const totalCount = getOr(0, 'aggregations.top_countries_count.value', response.rawResponse); + const networkTopCountriesEdges: NetworkTopCountriesEdges[] = getTopCountriesEdges( + response, + options + ); + const fakeTotalCount = fakePossibleCount <= totalCount ? fakePossibleCount : totalCount; + const edges = networkTopCountriesEdges.splice(cursorStart, querySize - cursorStart); + const inspect = { + dsl: [inspectStringifyObject(buildTopCountriesQuery(options))], + response: [inspectStringifyObject(response)], + }; + const showMorePagesIndicator = totalCount > fakeTotalCount; + + return { + ...response, + edges, + inspect, + pageInfo: { + activePage: activePage ? activePage : 0, + fakeTotalCount, + showMorePagesIndicator, + }, + totalCount, + }; + }, +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/query.top_countries_network.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/query.top_countries_network.dsl.ts new file mode 100644 index 0000000000000..24ecb32df1803 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_countries/query.top_countries_network.dsl.ts @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { createQueryFilterClauses } from '../../../../../utils/build_query'; +import { assertUnreachable } from '../../../../../../common/utility_types'; +import { + Direction, + FlowTargetSourceDest, + NetworkTopTablesSortField, + NetworkTopTablesFields, + NetworkTopCountriesRequestOptions, +} from '../../../../../../common/search_strategy/security_solution/network'; + +const getCountAgg = (flowTarget: FlowTargetSourceDest) => ({ + top_countries_count: { + cardinality: { + field: `${flowTarget}.geo.country_iso_code`, + }, + }, +}); + +export const buildTopCountriesQuery = ({ + defaultIndex, + filterQuery, + flowTarget, + networkTopCountriesSort, + pagination: { querySize }, + timerange: { from, to }, + ip, +}: NetworkTopCountriesRequestOptions) => { + const filter = [ + ...createQueryFilterClauses(filterQuery), + { + range: { + '@timestamp': { gte: from, lte: to, format: 'strict_date_optional_time' }, + }, + }, + ]; + + const dslQuery = { + allowNoIndices: true, + index: defaultIndex, + ignoreUnavailable: true, + body: { + aggregations: { + ...getCountAgg(flowTarget), + ...getFlowTargetAggs(networkTopCountriesSort, flowTarget, querySize), + }, + query: { + bool: ip + ? { + filter, + should: [ + { + term: { + [`${getOppositeField(flowTarget)}.ip`]: ip, + }, + }, + ], + minimum_should_match: 1, + } + : { + filter, + }, + }, + }, + size: 0, + track_total_hits: false, + }; + return dslQuery; +}; + +const getFlowTargetAggs = ( + networkTopCountriesSortField: NetworkTopTablesSortField, + flowTarget: FlowTargetSourceDest, + querySize: number +) => ({ + [flowTarget]: { + terms: { + field: `${flowTarget}.geo.country_iso_code`, + size: querySize, + order: { + ...getQueryOrder(networkTopCountriesSortField), + }, + }, + aggs: { + bytes_in: { + sum: { + field: `${getOppositeField(flowTarget)}.bytes`, + }, + }, + bytes_out: { + sum: { + field: `${flowTarget}.bytes`, + }, + }, + flows: { + cardinality: { + field: 'network.community_id', + }, + }, + source_ips: { + cardinality: { + field: 'source.ip', + }, + }, + destination_ips: { + cardinality: { + field: 'destination.ip', + }, + }, + }, + }, +}); + +export const getOppositeField = (flowTarget: FlowTargetSourceDest): FlowTargetSourceDest => { + switch (flowTarget) { + case FlowTargetSourceDest.source: + return FlowTargetSourceDest.destination; + case FlowTargetSourceDest.destination: + return FlowTargetSourceDest.source; + } + assertUnreachable(flowTarget); +}; + +type QueryOrder = + | { bytes_in: Direction } + | { bytes_out: Direction } + | { flows: Direction } + | { destination_ips: Direction } + | { source_ips: Direction }; + +const getQueryOrder = (networkTopCountriesSortField: NetworkTopTablesSortField): QueryOrder => { + switch (networkTopCountriesSortField.field) { + case NetworkTopTablesFields.bytes_in: + return { bytes_in: networkTopCountriesSortField.direction }; + case NetworkTopTablesFields.bytes_out: + return { bytes_out: networkTopCountriesSortField.direction }; + case NetworkTopTablesFields.flows: + return { flows: networkTopCountriesSortField.direction }; + case NetworkTopTablesFields.destination_ips: + return { destination_ips: networkTopCountriesSortField.direction }; + case NetworkTopTablesFields.source_ips: + return { source_ips: networkTopCountriesSortField.direction }; + } + assertUnreachable(networkTopCountriesSortField.field); +}; From de38d64b3762a09ff1f3e3919700d351d5010576 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Sun, 30 Aug 2020 09:39:25 +0200 Subject: [PATCH 5/8] [Security Solution] Refactor NetworkTopNFlow to use Search Strategy --- .../security_solution/index.ts | 11 + .../security_solution/network/common.ts | 41 +++ .../security_solution/network/http.ts | 71 +++++ .../security_solution/network/index.ts | 181 +---------- .../network/top_countries.ts | 69 ++++ .../security_solution/network/top_n_flow.ts | 124 ++++++++ .../containers/network_top_n_flow/index.tsx | 296 ++++++++++-------- .../network_top_n_flow/translations.ts | 21 ++ .../network_top_n_flow_query_table.tsx | 74 ++--- .../pages/navigation/ips_query_tab_body.tsx | 72 ++--- .../factory/network/helpers.ts | 32 +- .../factory/network/index.ts | 2 + .../factory/network/top_n_flow/helpers.ts | 97 ++++++ .../factory/network/top_n_flow/index.ts | 59 ++++ .../query.top_n_flow_network.dsl.ts | 155 +++++++++ 15 files changed, 922 insertions(+), 383 deletions(-) create mode 100644 x-pack/plugins/security_solution/common/search_strategy/security_solution/network/common.ts create mode 100644 x-pack/plugins/security_solution/common/search_strategy/security_solution/network/http.ts create mode 100644 x-pack/plugins/security_solution/common/search_strategy/security_solution/network/top_countries.ts create mode 100644 x-pack/plugins/security_solution/common/search_strategy/security_solution/network/top_n_flow.ts create mode 100644 x-pack/plugins/security_solution/public/network/containers/network_top_n_flow/translations.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/helpers.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/index.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/query.top_n_flow_network.dsl.ts diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts index df9099ecabd38..ca444e8b6e12f 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts @@ -21,6 +21,8 @@ import { NetworkHttpRequestOptions, NetworkTopCountriesStrategyResponse, NetworkTopCountriesRequestOptions, + NetworkTopNFlowStrategyResponse, + NetworkTopNFlowRequestOptions, } from './network'; export * from './hosts'; @@ -117,6 +119,8 @@ export type StrategyResponseType = T extends HostsQ ? NetworkHttpStrategyResponse : T extends NetworkQueries.topCountries ? NetworkTopCountriesStrategyResponse + : T extends NetworkQueries.topNFlow + ? NetworkTopNFlowStrategyResponse : never; export type StrategyRequestType = T extends HostsQueries.hosts @@ -129,9 +133,16 @@ export type StrategyRequestType = T extends HostsQu ? NetworkHttpRequestOptions : T extends NetworkQueries.topCountries ? NetworkTopCountriesRequestOptions + : T extends NetworkQueries.topNFlow + ? NetworkTopNFlowRequestOptions : never; export interface GenericBuckets { key: string; doc_count: number; } + +export interface TotalValue { + value: number; + relation: string; +} diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/common.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/common.ts new file mode 100644 index 0000000000000..043be03972ed6 --- /dev/null +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/common.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { GeoEcs } from '../../../ecs/geo'; +import { Maybe } from '..'; + +export enum NetworkTopTablesFields { + bytes_in = 'bytes_in', + bytes_out = 'bytes_out', + flows = 'flows', + destination_ips = 'destination_ips', + source_ips = 'source_ips', +} + +export interface NetworkTopTablesSortField { + field: NetworkTopTablesFields; + direction: Direction; +} + +export enum FlowTargetSourceDest { + destination = 'destination', + source = 'source', +} + +export enum Direction { + asc = 'asc', + desc = 'desc', +} + +export interface GeoItem { + geo?: Maybe; + flowTarget?: Maybe; +} + +export interface TopNetworkTablesEcsField { + bytes_in?: Maybe; + bytes_out?: Maybe; +} diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/http.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/http.ts new file mode 100644 index 0000000000000..568444eaa36de --- /dev/null +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/http.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IEsSearchResponse } from '../../../../../../../src/plugins/data/common'; +import { Direction } from './common'; +import { + CursorType, + GenericBuckets, + Inspect, + Maybe, + PageInfoPaginated, + RequestOptionsPaginated, +} from '..'; + +export interface NetworkHttpRequestOptions extends RequestOptionsPaginated { + ip?: string; + networkHttpSort: NetworkHttpSortField; + defaultIndex: string[]; +} + +export interface NetworkHttpStrategyResponse extends IEsSearchResponse { + edges: NetworkHttpEdges[]; + totalCount: number; + pageInfo: PageInfoPaginated; + inspect?: Maybe; +} + +export interface NetworkHttpSortField { + direction: Direction; +} + +export interface NetworkHttpData { + edges: NetworkHttpEdges[]; + totalCount: number; + pageInfo: PageInfoPaginated; + inspect?: Maybe; +} + +export interface NetworkHttpEdges { + node: NetworkHttpItem; + cursor: CursorType; +} + +export interface NetworkHttpItem { + _id?: Maybe; + domains: string[]; + lastHost?: Maybe; + lastSourceIp?: Maybe; + methods: string[]; + path?: Maybe; + requestCount?: Maybe; + statuses: string[]; +} + +export interface NetworkHttpBuckets { + key: string; + doc_count: number; + domains: { + buckets: GenericBuckets[]; + }; + methods: { + buckets: GenericBuckets[]; + }; + source: object; + status: { + buckets: GenericBuckets[]; + }; +} diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts index 347f3c56749d7..bfb8f31f4b300 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts @@ -5,29 +5,19 @@ */ import { IEsSearchResponse } from '../../../../../../../src/plugins/data/common'; -import { GeoEcs } from '../../../ecs/geo'; -import { - CursorType, - Inspect, - Maybe, - PageInfoPaginated, - RequestOptionsPaginated, - GenericBuckets, -} from '..'; +import { CursorType, Inspect, Maybe, PageInfoPaginated, RequestOptionsPaginated } from '..'; + +export * from './common'; +export * from './http'; +export * from './top_countries'; +export * from './top_n_flow'; export enum NetworkQueries { http = 'http', tls = 'tls', topCountries = 'topCountries', -} - -export enum NetworkTopTablesFields { - bytes_in = 'bytes_in', - bytes_out = 'bytes_out', - flows = 'flows', - destination_ips = 'destination_ips', - source_ips = 'source_ips', + topNFlow = 'topNFlow', } export enum NetworkDnsFields { @@ -109,10 +99,6 @@ export interface TlsSortField { direction: Direction; } -export interface NetworkHttpSortField { - direction: Direction; -} - export interface NetworkTlsRequestOptions extends RequestOptionsPaginated { ip: string; flowTarget: FlowTargetSourceDest; @@ -129,156 +115,3 @@ export interface NetworkTlsStrategyResponse extends IEsSearchResponse { inspect?: Maybe; } - -export interface NetworkHttpRequestOptions extends RequestOptionsPaginated { - ip?: string; - networkHttpSort: NetworkHttpSortField; - defaultIndex: string[]; -} - -export interface NetworkHttpStrategyResponse extends IEsSearchResponse { - edges: NetworkHttpEdges[]; - - totalCount: number; - - pageInfo: PageInfoPaginated; - - inspect?: Maybe; -} - -export interface GeoItem { - geo?: Maybe; - - flowTarget?: Maybe; -} - -export interface TopCountriesItemSource { - country?: Maybe; - - destination_ips?: Maybe; - - flows?: Maybe; - - location?: Maybe; - - source_ips?: Maybe; -} - -export interface NetworkTopCountriesRequestOptions extends RequestOptionsPaginated { - networkTopCountriesSort: NetworkTopTablesSortField; - flowTarget: FlowTargetSourceDest; - ip?: string; -} - -export interface NetworkTopCountriesStrategyResponse extends IEsSearchResponse { - edges: NetworkTopCountriesEdges[]; - - totalCount: number; - - pageInfo: PageInfoPaginated; - - inspect?: Maybe; -} - -export interface NetworkTopCountriesEdges { - node: NetworkTopCountriesItem; - - cursor: CursorType; -} - -export interface NetworkTopCountriesItem { - _id?: Maybe; - - source?: Maybe; - - destination?: Maybe; - - network?: Maybe; -} - -export interface TopCountriesItemDestination { - country?: Maybe; - - destination_ips?: Maybe; - - flows?: Maybe; - - location?: Maybe; - - source_ips?: Maybe; -} - -export interface TopNetworkTablesEcsField { - bytes_in?: Maybe; - - bytes_out?: Maybe; -} - -export interface NetworkTopTablesSortField { - field: NetworkTopTablesFields; - - direction: Direction; -} - -export interface NetworkTopCountriesBuckets { - country: string; - key: string; - bytes_in: { - value: number; - }; - bytes_out: { - value: number; - }; - flows: number; - destination_ips: number; - source_ips: number; -} - -export interface NetworkHttpData { - edges: NetworkHttpEdges[]; - - totalCount: number; - - pageInfo: PageInfoPaginated; - - inspect?: Maybe; -} - -export interface NetworkHttpEdges { - node: NetworkHttpItem; - - cursor: CursorType; -} - -export interface NetworkHttpItem { - _id?: Maybe; - - domains: string[]; - - lastHost?: Maybe; - - lastSourceIp?: Maybe; - - methods: string[]; - - path?: Maybe; - - requestCount?: Maybe; - - statuses: string[]; -} - -export interface NetworkHttpBuckets { - key: string; - doc_count: number; - domains: { - buckets: GenericBuckets[]; - }; - methods: { - buckets: GenericBuckets[]; - }; - source: object; - status: { - buckets: GenericBuckets[]; - }; -} diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/top_countries.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/top_countries.ts new file mode 100644 index 0000000000000..9d1895447ecff --- /dev/null +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/top_countries.ts @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IEsSearchResponse } from '../../../../../../../src/plugins/data/common'; +import { + GeoItem, + NetworkTopTablesSortField, + FlowTargetSourceDest, + TopNetworkTablesEcsField, +} from './common'; +import { CursorType, Inspect, Maybe, PageInfoPaginated, RequestOptionsPaginated } from '..'; + +export interface TopCountriesItemSource { + country?: Maybe; + destination_ips?: Maybe; + flows?: Maybe; + location?: Maybe; + source_ips?: Maybe; +} + +export interface NetworkTopCountriesRequestOptions extends RequestOptionsPaginated { + networkTopCountriesSort: NetworkTopTablesSortField; + flowTarget: FlowTargetSourceDest; + ip?: string; +} + +export interface NetworkTopCountriesStrategyResponse extends IEsSearchResponse { + edges: NetworkTopCountriesEdges[]; + totalCount: number; + pageInfo: PageInfoPaginated; + inspect?: Maybe; +} + +export interface NetworkTopCountriesEdges { + node: NetworkTopCountriesItem; + cursor: CursorType; +} + +export interface NetworkTopCountriesItem { + _id?: Maybe; + source?: Maybe; + destination?: Maybe; + network?: Maybe; +} + +export interface TopCountriesItemDestination { + country?: Maybe; + destination_ips?: Maybe; + flows?: Maybe; + location?: Maybe; + source_ips?: Maybe; +} + +export interface NetworkTopCountriesBuckets { + country: string; + key: string; + bytes_in: { + value: number; + }; + bytes_out: { + value: number; + }; + flows: number; + destination_ips: number; + source_ips: number; +} diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/top_n_flow.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/top_n_flow.ts new file mode 100644 index 0000000000000..4dbf6a47ae01f --- /dev/null +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/top_n_flow.ts @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IEsSearchResponse } from '../../../../../../../src/plugins/data/common'; +import { + GeoItem, + NetworkTopTablesSortField, + FlowTargetSourceDest, + TopNetworkTablesEcsField, +} from './common'; +import { + CursorType, + GenericBuckets, + Inspect, + Maybe, + PageInfoPaginated, + RequestOptionsPaginated, + TotalValue, +} from '..'; + +export interface NetworkTopNFlowRequestOptions extends RequestOptionsPaginated { + networkTopNFlowSort: NetworkTopTablesSortField; + flowTarget: FlowTargetSourceDest; + ip?: Maybe; +} + +export interface NetworkTopNFlowStrategyResponse extends IEsSearchResponse { + edges: NetworkTopNFlowEdges[]; + totalCount: number; + pageInfo: PageInfoPaginated; + inspect?: Maybe; +} + +export interface NetworkTopNFlowEdges { + node: NetworkTopNFlowItem; + cursor: CursorType; +} + +export interface NetworkTopNFlowItem { + _id?: Maybe; + source?: Maybe; + destination?: Maybe; + network?: Maybe; +} + +export interface TopNFlowItemSource { + autonomous_system?: Maybe; + domain?: Maybe; + ip?: Maybe; + location?: Maybe; + flows?: Maybe; + destination_ips?: Maybe; +} + +export interface AutonomousSystemItem { + name?: Maybe; + number?: Maybe; +} + +export interface TopNFlowItemDestination { + autonomous_system?: Maybe; + domain?: Maybe; + ip?: Maybe; + location?: Maybe; + flows?: Maybe; + source_ips?: Maybe; +} + +export interface AutonomousSystemHit { + doc_count: number; + top_as: { + hits: { + total: TotalValue | number; + max_score: number | null; + hits: Array<{ + _source: T; + sort?: [number]; + _index?: string; + _type?: string; + _id?: string; + _score?: number | null; + }>; + }; + }; +} + +export interface NetworkTopNFlowBuckets { + key: string; + autonomous_system: AutonomousSystemHit; + bytes_in: { + value: number; + }; + bytes_out: { + value: number; + }; + domain: { + buckets: GenericBuckets[]; + }; + location: LocationHit; + flows: number; + destination_ips?: number; + source_ips?: number; +} + +export interface LocationHit { + doc_count: number; + top_geo: { + hits: { + total: TotalValue | number; + max_score: number | null; + hits: Array<{ + _source: T; + sort?: [number]; + _index?: string; + _type?: string; + _id?: string; + _score?: number | null; + }>; + }; + }; +} diff --git a/x-pack/plugins/security_solution/public/network/containers/network_top_n_flow/index.tsx b/x-pack/plugins/security_solution/public/network/containers/network_top_n_flow/index.tsx index 770574b0813c1..86e366b253dbe 100644 --- a/x-pack/plugins/security_solution/public/network/containers/network_top_n_flow/index.tsx +++ b/x-pack/plugins/security_solution/public/network/containers/network_top_n_flow/index.tsx @@ -4,161 +4,203 @@ * you may not use this file except in compliance with the Elastic License. */ -import { getOr } from 'lodash/fp'; -import React from 'react'; -import { Query } from 'react-apollo'; -import { connect } from 'react-redux'; -import { compose } from 'redux'; +import { noop } from 'lodash/fp'; +import { useState, useEffect, useCallback, useRef } from 'react'; +import { shallowEqual, useSelector } from 'react-redux'; +import deepEqual from 'fast-deep-equal'; +import { ESTermQuery } from '../../../../common/typed_json'; import { DEFAULT_INDEX_KEY } from '../../../../common/constants'; +import { inputsModel, State } from '../../../common/store'; +import { useKibana } from '../../../common/lib/kibana'; +import { createFilter } from '../../../common/containers/helpers'; +import { PageInfoPaginated } from '../../../../common/search_strategy/security_solution'; +import { generateTablePaginationOptions } from '../../../common/components/paginated_table/helpers'; +import { networkModel, networkSelectors } from '../../store'; import { FlowTargetSourceDest, - GetNetworkTopNFlowQuery, + NetworkQueries, NetworkTopNFlowEdges, - NetworkTopTablesSortField, - PageInfoPaginated, -} from '../../../graphql/types'; -import { withKibana, WithKibanaProps } from '../../../common/lib/kibana'; -import { inputsModel, inputsSelectors, State } from '../../../common/store'; -import { generateTablePaginationOptions } from '../../../common/components/paginated_table/helpers'; -import { createFilter, getDefaultFetchPolicy } from '../../../common/containers/helpers'; -import { - QueryTemplatePaginated, - QueryTemplatePaginatedProps, -} from '../../../common/containers/query_template_paginated'; -import { networkTopNFlowQuery } from './index.gql_query'; -import { networkModel, networkSelectors } from '../../store'; + NetworkTopNFlowRequestOptions, + NetworkTopNFlowStrategyResponse, +} from '../../../../common/search_strategy/security_solution/network'; +import { AbortError } from '../../../../../../../src/plugins/data/common'; +import * as i18n from './translations'; const ID = 'networkTopNFlowQuery'; export interface NetworkTopNFlowArgs { id: string; - ip?: string; inspect: inputsModel.InspectQuery; isInspected: boolean; - loading: boolean; loadPage: (newActivePage: number) => void; - networkTopNFlow: NetworkTopNFlowEdges[]; pageInfo: PageInfoPaginated; refetch: inputsModel.Refetch; + networkTopNFlow: NetworkTopNFlowEdges[]; totalCount: number; } -export interface OwnProps extends QueryTemplatePaginatedProps { - children: (args: NetworkTopNFlowArgs) => React.ReactNode; +interface UseNetworkTopNFlow { flowTarget: FlowTargetSourceDest; ip?: string; type: networkModel.NetworkType; + filterQuery?: ESTermQuery | string; + endDate: string; + startDate: string; + skip: boolean; + id?: string; } -export interface NetworkTopNFlowComponentReduxProps { - activePage: number; - isInspected: boolean; - limit: number; - sort: NetworkTopTablesSortField; -} +export const useNetworkTopNFlow = ({ + endDate, + filterQuery, + flowTarget, + id = ID, + skip, + startDate, + type, +}: UseNetworkTopNFlow): [boolean, NetworkTopNFlowArgs] => { + // const getQuery = inputsSelectors.globalQueryByIdSelector(); + // const { isInspected } = useSelector((state: State) => getQuery(state, id), shallowEqual); + const getTopNFlowSelector = networkSelectors.topNFlowSelector(); + const { activePage, limit, sort } = useSelector( + (state: State) => getTopNFlowSelector(state, type, flowTarget), + shallowEqual + ); + const { data, notifications, uiSettings } = useKibana().services; + const refetch = useRef(noop); + const abortCtrl = useRef(new AbortController()); + const defaultIndex = uiSettings.get(DEFAULT_INDEX_KEY); + const [loading, setLoading] = useState(false); -type NetworkTopNFlowProps = OwnProps & NetworkTopNFlowComponentReduxProps & WithKibanaProps; + const [networkTopNFlowRequest, setTopNFlowRequest] = useState({ + defaultIndex, + factoryQueryType: NetworkQueries.topNFlow, + filterQuery: createFilter(filterQuery), + flowTarget, + // inspect: isInspected, + pagination: generateTablePaginationOptions(activePage, limit), + networkTopNFlowSort: sort, + timerange: { + interval: '12h', + from: startDate ? startDate : '', + to: endDate ? endDate : new Date(Date.now()).toISOString(), + }, + }); -class NetworkTopNFlowComponentQuery extends QueryTemplatePaginated< - NetworkTopNFlowProps, - GetNetworkTopNFlowQuery.Query, - GetNetworkTopNFlowQuery.Variables -> { - public render() { - const { - activePage, - children, - endDate, - flowTarget, - filterQuery, - kibana, - id = `${ID}-${flowTarget}`, - ip, - isInspected, - limit, - skip, - sourceId, - startDate, - sort, - } = this.props; - const variables: GetNetworkTopNFlowQuery.Variables = { - defaultIndex: kibana.services.uiSettings.get(DEFAULT_INDEX_KEY), - filterQuery: createFilter(filterQuery), - flowTarget, - inspect: isInspected, - ip, - pagination: generateTablePaginationOptions(activePage, limit), - sort, - sourceId, - timerange: { - interval: '12h', - from: startDate!, - to: endDate!, - }, - }; - return ( - - fetchPolicy={getDefaultFetchPolicy()} - notifyOnNetworkStatusChange - query={networkTopNFlowQuery} - skip={skip} - variables={variables} - > - {({ data, loading, fetchMore, networkStatus, refetch }) => { - const networkTopNFlow = getOr([], `source.NetworkTopNFlow.edges`, data); - this.setFetchMore(fetchMore); - this.setFetchMoreOptions((newActivePage: number) => ({ - variables: { - pagination: generateTablePaginationOptions(newActivePage, limit), + const wrappedLoadMore = useCallback( + (newActivePage: number) => { + setTopNFlowRequest((prevRequest) => ({ + ...prevRequest, + pagination: generateTablePaginationOptions(newActivePage, limit), + })); + }, + [limit] + ); + + const [networkTopNFlowResponse, setNetworkTopNFlowResponse] = useState({ + networkTopNFlow: [], + id: ID, + inspect: { + dsl: [], + response: [], + }, + isInspected: false, + loadPage: wrappedLoadMore, + pageInfo: { + activePage: 0, + fakeTotalCount: 0, + showMorePagesIndicator: false, + }, + refetch: refetch.current, + totalCount: -1, + }); + + const networkTopNFlowSearch = useCallback( + (request: NetworkTopNFlowRequestOptions) => { + let didCancel = false; + const asyncSearch = async () => { + abortCtrl.current = new AbortController(); + setLoading(true); + + const searchSubscription$ = data.search + .search(request, { + strategy: 'securitySolutionSearchStrategy', + signal: abortCtrl.current.signal, + }) + .subscribe({ + next: (response) => { + if (!response.isPartial && !response.isRunning) { + if (!didCancel) { + setLoading(false); + setNetworkTopNFlowResponse((prevResponse) => ({ + ...prevResponse, + networkTopNFlow: response.edges, + inspect: response.inspect ?? prevResponse.inspect, + pageInfo: response.pageInfo, + refetch: refetch.current, + totalCount: response.totalCount, + })); + } + searchSubscription$.unsubscribe(); + } else if (response.isPartial && !response.isRunning) { + if (!didCancel) { + setLoading(false); + } + // TODO: Make response error status clearer + notifications.toasts.addWarning(i18n.ERROR_NETWORK_TOP_N_FLOW); + searchSubscription$.unsubscribe(); + } }, - updateQuery: (prev, { fetchMoreResult }) => { - if (!fetchMoreResult) { - return prev; + error: (msg) => { + if (!(msg instanceof AbortError)) { + notifications.toasts.addDanger({ + title: i18n.FAIL_NETWORK_TOP_N_FLOW, + text: msg.message, + }); } - return { - ...fetchMoreResult, - source: { - ...fetchMoreResult.source, - NetworkTopNFlow: { - ...fetchMoreResult.source.NetworkTopNFlow, - edges: [...fetchMoreResult.source.NetworkTopNFlow.edges], - }, - }, - }; }, - })); - const isLoading = this.isItAValidLoading(loading, variables, networkStatus); - return children({ - id, - inspect: getOr(null, 'source.NetworkTopNFlow.inspect', data), - isInspected, - loading: isLoading, - loadPage: this.wrappedLoadMore, - networkTopNFlow, - pageInfo: getOr({}, 'source.NetworkTopNFlow.pageInfo', data), - refetch: this.memoizedRefetchQuery(variables, limit, refetch), - totalCount: getOr(-1, 'source.NetworkTopNFlow.totalCount', data), }); - }} - - ); - } -} + }; + abortCtrl.current.abort(); + asyncSearch(); + refetch.current = asyncSearch; + return () => { + didCancel = true; + abortCtrl.current.abort(); + }; + }, + [data.search, notifications.toasts] + ); -const makeMapStateToProps = () => { - const getTopNFlowSelector = networkSelectors.topNFlowSelector(); - const getQuery = inputsSelectors.globalQueryByIdSelector(); - return (state: State, { flowTarget, id = `${ID}-${flowTarget}`, type }: OwnProps) => { - const { isInspected } = getQuery(state, id); - return { - ...getTopNFlowSelector(state, type, flowTarget), - isInspected, - }; - }; -}; + useEffect(() => { + if (skip) { + return; + } + + setTopNFlowRequest((prevRequest) => { + const myRequest = { + ...prevRequest, + defaultIndex, + filterQuery: createFilter(filterQuery), + pagination: generateTablePaginationOptions(activePage, limit), + timerange: { + interval: '12h', + from: startDate, + to: endDate, + }, + networkTopNFlowSort: sort, + }; + if (!deepEqual(prevRequest, myRequest)) { + return myRequest; + } + return prevRequest; + }); + }, [activePage, defaultIndex, endDate, filterQuery, limit, startDate, sort, skip]); -export const NetworkTopNFlowQuery = compose>( - connect(makeMapStateToProps), - withKibana -)(NetworkTopNFlowComponentQuery); + useEffect(() => { + networkTopNFlowSearch(networkTopNFlowRequest); + }, [networkTopNFlowRequest, networkTopNFlowSearch]); + + return [loading, networkTopNFlowResponse]; +}; diff --git a/x-pack/plugins/security_solution/public/network/containers/network_top_n_flow/translations.ts b/x-pack/plugins/security_solution/public/network/containers/network_top_n_flow/translations.ts new file mode 100644 index 0000000000000..4ea704571cf2e --- /dev/null +++ b/x-pack/plugins/security_solution/public/network/containers/network_top_n_flow/translations.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; + +export const ERROR_NETWORK_TOP_N_FLOW = i18n.translate( + 'xpack.securitySolution.networkTopNFlow.errorSearchDescription', + { + defaultMessage: `An error has occurred on network top n flow search`, + } +); + +export const FAIL_NETWORK_TOP_N_FLOW = i18n.translate( + 'xpack.securitySolution.networkTopNFlow.failSearchDescription', + { + defaultMessage: `Failed to run search on network top n flow`, + } +); diff --git a/x-pack/plugins/security_solution/public/network/pages/ip_details/network_top_n_flow_query_table.tsx b/x-pack/plugins/security_solution/public/network/pages/ip_details/network_top_n_flow_query_table.tsx index 158b4057a7d5e..821452201b78b 100644 --- a/x-pack/plugins/security_solution/public/network/pages/ip_details/network_top_n_flow_query_table.tsx +++ b/x-pack/plugins/security_solution/public/network/pages/ip_details/network_top_n_flow_query_table.tsx @@ -8,7 +8,7 @@ import { getOr } from 'lodash/fp'; import React from 'react'; import { manageQuery } from '../../../common/components/page/manage_query'; import { NetworkTopNFlowTable } from '../../components/network_top_n_flow_table'; -import { NetworkTopNFlowQuery } from '../../containers/network_top_n_flow'; +import { useNetworkTopNFlow } from '../../containers/network_top_n_flow'; import { NetworkWithIndexComponentsQueryTableProps } from './types'; const NetworkTopNFlowTableManage = manageQuery(NetworkTopNFlowTable); @@ -22,45 +22,37 @@ export const NetworkTopNFlowQueryTable = ({ skip, startDate, type, -}: NetworkWithIndexComponentsQueryTableProps) => ( - - {({ - id, - inspect, - isInspected, - loading, - loadPage, - networkTopNFlow, - pageInfo, - refetch, - totalCount, - }) => ( - - )} - -); +}: NetworkWithIndexComponentsQueryTableProps) => { + const [ + loading, + { id, inspect, isInspected, loadPage, networkTopNFlow, pageInfo, refetch, totalCount }, + ] = useNetworkTopNFlow({ + endDate, + filterQuery, + flowTarget, + ip, + skip, + startDate, + type, + }); + + return ( + + ); +}; NetworkTopNFlowQueryTable.displayName = 'NetworkTopNFlowQueryTable'; diff --git a/x-pack/plugins/security_solution/public/network/pages/navigation/ips_query_tab_body.tsx b/x-pack/plugins/security_solution/public/network/pages/navigation/ips_query_tab_body.tsx index a9f4d504847a0..c83bf6ff80901 100644 --- a/x-pack/plugins/security_solution/public/network/pages/navigation/ips_query_tab_body.tsx +++ b/x-pack/plugins/security_solution/public/network/pages/navigation/ips_query_tab_body.tsx @@ -8,7 +8,7 @@ import React from 'react'; import { getOr } from 'lodash/fp'; import { NetworkTopNFlowTable } from '../../components/network_top_n_flow_table'; -import { NetworkTopNFlowQuery } from '../../containers/network_top_n_flow'; +import { useNetworkTopNFlow } from '../../containers/network_top_n_flow'; import { networkModel } from '../../store'; import { manageQuery } from '../../../common/components/page/manage_query'; @@ -23,44 +23,36 @@ export const IPsQueryTabBody = ({ startDate, setQuery, flowTarget, -}: IPsQueryTabBodyProps) => ( - - {({ - id, - inspect, - isInspected, - loading, - loadPage, - networkTopNFlow, - pageInfo, - refetch, - totalCount, - }) => ( - - )} - -); +}: IPsQueryTabBodyProps) => { + const [ + loading, + { id, inspect, isInspected, loadPage, networkTopNFlow, pageInfo, refetch, totalCount }, + ] = useNetworkTopNFlow({ + endDate, + flowTarget, + filterQuery, + skip, + startDate, + type: networkModel.NetworkType.page, + }); + + return ( + + ); +}; IPsQueryTabBody.displayName = 'IPsQueryTabBody'; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/helpers.ts index a7fba087b87ed..c6393e6a289d7 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/helpers.ts @@ -5,7 +5,12 @@ */ import { assertUnreachable } from '../../../../../common/utility_types'; -import { FlowTargetSourceDest } from '../../../../../common/search_strategy/security_solution/network'; +import { + Direction, + NetworkTopTablesSortField, + FlowTargetSourceDest, + NetworkTopTablesFields, +} from '../../../../../common/search_strategy/security_solution/network'; export const getOppositeField = (flowTarget: FlowTargetSourceDest): FlowTargetSourceDest => { switch (flowTarget) { @@ -16,3 +21,28 @@ export const getOppositeField = (flowTarget: FlowTargetSourceDest): FlowTargetSo } assertUnreachable(flowTarget); }; + +type QueryOrder = + | { bytes_in: Direction } + | { bytes_out: Direction } + | { flows: Direction } + | { destination_ips: Direction } + | { source_ips: Direction }; + +export const getQueryOrder = ( + networkTopCountriesSortField: NetworkTopTablesSortField +): QueryOrder => { + switch (networkTopCountriesSortField.field) { + case NetworkTopTablesFields.bytes_in: + return { bytes_in: networkTopCountriesSortField.direction }; + case NetworkTopTablesFields.bytes_out: + return { bytes_out: networkTopCountriesSortField.direction }; + case NetworkTopTablesFields.flows: + return { flows: networkTopCountriesSortField.direction }; + case NetworkTopTablesFields.destination_ips: + return { destination_ips: networkTopCountriesSortField.direction }; + case NetworkTopTablesFields.source_ips: + return { source_ips: networkTopCountriesSortField.direction }; + } + assertUnreachable(networkTopCountriesSortField.field); +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts index 93e5f113197da..2561ec1de48e2 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts @@ -11,9 +11,11 @@ import { SecuritySolutionFactory } from '../types'; import { networkHttp } from './http'; import { networkTls } from './tls'; import { networkTopCountries } from './top_countries'; +import { networkTopNFlow } from './top_n_flow'; export const networkFactory: Record> = { [NetworkQueries.http]: networkHttp, [NetworkQueries.tls]: networkTls, [NetworkQueries.topCountries]: networkTopCountries, + [NetworkQueries.topNFlow]: networkTopNFlow, }; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/helpers.ts new file mode 100644 index 0000000000000..098fd43cd03fa --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/helpers.ts @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getOr } from 'lodash/fp'; + +import { IEsSearchResponse } from '../../../../../../../../../src/plugins/data/common'; +import { + GeoItem, + NetworkTopNFlowBuckets, + NetworkTopNFlowEdges, + NetworkTopNFlowRequestOptions, + FlowTargetSourceDest, + AutonomousSystemItem, +} from '../../../../../../common/search_strategy/security_solution/network'; +import { getOppositeField } from '../helpers'; + +export const getTopNFlowEdges = ( + response: IEsSearchResponse, + options: NetworkTopNFlowRequestOptions +): NetworkTopNFlowEdges[] => + formatTopNFlowEdges( + getOr([], `aggregations.${options.flowTarget}.buckets`, response.rawResponse), + options.flowTarget + ); + +const formatTopNFlowEdges = ( + buckets: NetworkTopNFlowBuckets[], + flowTarget: FlowTargetSourceDest +): NetworkTopNFlowEdges[] => + buckets.map((bucket: NetworkTopNFlowBuckets) => ({ + node: { + _id: bucket.key, + [flowTarget]: { + domain: bucket.domain.buckets.map((bucketDomain) => bucketDomain.key), + ip: bucket.key, + location: getGeoItem(bucket), + autonomous_system: getAsItem(bucket), + flows: getOr(0, 'flows.value', bucket), + [`${getOppositeField(flowTarget)}_ips`]: getOr( + 0, + `${getOppositeField(flowTarget)}_ips.value`, + bucket + ), + }, + network: { + bytes_in: getOr(0, 'bytes_in.value', bucket), + bytes_out: getOr(0, 'bytes_out.value', bucket), + }, + }, + cursor: { + value: bucket.key, + tiebreaker: null, + }, + })); + +const getFlowTargetFromString = (flowAsString: string) => + flowAsString === 'source' ? FlowTargetSourceDest.source : FlowTargetSourceDest.destination; + +const getGeoItem = (result: NetworkTopNFlowBuckets): GeoItem | null => + result.location.top_geo.hits.hits.length > 0 && result.location.top_geo.hits.hits[0]._source + ? { + geo: getOr( + '', + `location.top_geo.hits.hits[0]._source.${ + Object.keys(result.location.top_geo.hits.hits[0]._source)[0] + }.geo`, + result + ), + flowTarget: getFlowTargetFromString( + Object.keys(result.location.top_geo.hits.hits[0]._source)[0] + ), + } + : null; + +const getAsItem = (result: NetworkTopNFlowBuckets): AutonomousSystemItem | null => + result.autonomous_system.top_as.hits.hits.length > 0 && + result.autonomous_system.top_as.hits.hits[0]._source + ? { + number: getOr( + null, + `autonomous_system.top_as.hits.hits[0]._source.${ + Object.keys(result.autonomous_system.top_as.hits.hits[0]._source)[0] + }.as.number`, + result + ), + name: getOr( + '', + `autonomous_system.top_as.hits.hits[0]._source.${ + Object.keys(result.autonomous_system.top_as.hits.hits[0]._source)[0] + }.as.organization.name`, + result + ), + } + : null; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/index.ts new file mode 100644 index 0000000000000..b9014f76b8def --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/index.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getOr } from 'lodash/fp'; + +import { IEsSearchResponse } from '../../../../../../../../../src/plugins/data/common'; + +import { DEFAULT_MAX_TABLE_QUERY_SIZE } from '../../../../../../common/constants'; +import { + NetworkTopNFlowStrategyResponse, + NetworkQueries, + NetworkTopNFlowRequestOptions, + NetworkTopNFlowEdges, +} from '../../../../../../common/search_strategy/security_solution/network'; + +import { inspectStringifyObject } from '../../../../../utils/build_query'; +import { SecuritySolutionFactory } from '../../types'; + +import { getTopNFlowEdges } from './helpers'; +import { buildTopNFlowQuery } from './query.top_n_flow_network.dsl'; + +export const networkTopNFlow: SecuritySolutionFactory = { + buildDsl: (options: NetworkTopNFlowRequestOptions) => { + if (options.pagination && options.pagination.querySize >= DEFAULT_MAX_TABLE_QUERY_SIZE) { + throw new Error(`No query size above ${DEFAULT_MAX_TABLE_QUERY_SIZE}`); + } + return buildTopNFlowQuery(options); + }, + parse: async ( + options: NetworkTopNFlowRequestOptions, + response: IEsSearchResponse + ): Promise => { + const { activePage, cursorStart, fakePossibleCount, querySize } = options.pagination; + const totalCount = getOr(0, 'aggregations.top_n_flow_count.value', response.rawResponse); + const networkTopNFlowEdges: NetworkTopNFlowEdges[] = getTopNFlowEdges(response, options); + const fakeTotalCount = fakePossibleCount <= totalCount ? fakePossibleCount : totalCount; + const edges = networkTopNFlowEdges.splice(cursorStart, querySize - cursorStart); + const inspect = { + dsl: [inspectStringifyObject(buildTopNFlowQuery(options))], + response: [inspectStringifyObject(response)], + }; + const showMorePagesIndicator = totalCount > fakeTotalCount; + + return { + ...response, + edges, + inspect, + pageInfo: { + activePage: activePage ? activePage : 0, + fakeTotalCount, + showMorePagesIndicator, + }, + totalCount, + }; + }, +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/query.top_n_flow_network.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/query.top_n_flow_network.dsl.ts new file mode 100644 index 0000000000000..49c51117d2b05 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/top_n_flow/query.top_n_flow_network.dsl.ts @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + FlowTargetSourceDest, + NetworkTopTablesSortField, + NetworkTopNFlowRequestOptions, +} from '../../../../../../common/search_strategy/security_solution/network'; +import { createQueryFilterClauses } from '../../../../../utils/build_query'; +import { getOppositeField, getQueryOrder } from '../helpers'; + +const getCountAgg = (flowTarget: FlowTargetSourceDest) => ({ + top_n_flow_count: { + cardinality: { + field: `${flowTarget}.ip`, + }, + }, +}); + +export const buildTopNFlowQuery = ({ + defaultIndex, + filterQuery, + flowTarget, + networkTopNFlowSort, + pagination: { querySize }, + timerange: { from, to }, + ip, +}: NetworkTopNFlowRequestOptions) => { + const filter = [ + ...createQueryFilterClauses(filterQuery), + { + range: { + '@timestamp': { gte: from, lte: to, format: 'strict_date_optional_time' }, + }, + }, + ]; + + const dslQuery = { + allowNoIndices: true, + index: defaultIndex, + ignoreUnavailable: true, + body: { + aggregations: { + ...getCountAgg(flowTarget), + ...getFlowTargetAggs(networkTopNFlowSort, flowTarget, querySize), + }, + query: { + bool: ip + ? { + filter, + should: [ + { + term: { + [`${getOppositeField(flowTarget)}.ip`]: ip, + }, + }, + ], + minimum_should_match: 1, + } + : { + filter, + }, + }, + }, + size: 0, + track_total_hits: false, + }; + return dslQuery; +}; + +const getFlowTargetAggs = ( + networkTopNFlowSortField: NetworkTopTablesSortField, + flowTarget: FlowTargetSourceDest, + querySize: number +) => ({ + [flowTarget]: { + terms: { + field: `${flowTarget}.ip`, + size: querySize, + order: { + ...getQueryOrder(networkTopNFlowSortField), + }, + }, + aggs: { + bytes_in: { + sum: { + field: `${getOppositeField(flowTarget)}.bytes`, + }, + }, + bytes_out: { + sum: { + field: `${flowTarget}.bytes`, + }, + }, + domain: { + terms: { + field: `${flowTarget}.domain`, + order: { + timestamp: 'desc', + }, + }, + aggs: { + timestamp: { + max: { + field: '@timestamp', + }, + }, + }, + }, + location: { + filter: { + exists: { + field: `${flowTarget}.geo`, + }, + }, + aggs: { + top_geo: { + top_hits: { + _source: `${flowTarget}.geo.*`, + size: 1, + }, + }, + }, + }, + autonomous_system: { + filter: { + exists: { + field: `${flowTarget}.as`, + }, + }, + aggs: { + top_as: { + top_hits: { + _source: `${flowTarget}.as.*`, + size: 1, + }, + }, + }, + }, + flows: { + cardinality: { + field: 'network.community_id', + }, + }, + [`${getOppositeField(flowTarget)}_ips`]: { + cardinality: { + field: `${getOppositeField(flowTarget)}.ip`, + }, + }, + }, + }, +}); From 18c0a2efba06417463a3e05eaa8226ba4eaac094 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Sun, 30 Aug 2020 11:38:46 +0200 Subject: [PATCH 6/8] [Security Solution] Refactor NetworkDns to use Search Strategy --- .../security_solution/index.ts | 6 + .../security_solution/network/dns.ts | 71 ++++ .../security_solution/network/index.ts | 10 +- .../network/containers/network_dns/_index.tsx | 168 +++++++++ .../containers/network_dns/histogram.ts | 65 ++++ .../network/containers/network_dns/index.tsx | 351 +++++++++--------- .../containers/network_dns/translations.ts | 21 ++ .../pages/navigation/dns_query_tab_body.tsx | 68 ++-- .../factory/network/dns/helpers.ts | 40 ++ .../factory/network/dns/index.ts | 59 +++ .../network/dns/query.dns_network.dsl.ts | 134 +++++++ .../factory/network/index.ts | 2 + 12 files changed, 770 insertions(+), 225 deletions(-) create mode 100644 x-pack/plugins/security_solution/common/search_strategy/security_solution/network/dns.ts create mode 100644 x-pack/plugins/security_solution/public/network/containers/network_dns/_index.tsx create mode 100644 x-pack/plugins/security_solution/public/network/containers/network_dns/histogram.ts create mode 100644 x-pack/plugins/security_solution/public/network/containers/network_dns/translations.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/helpers.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/index.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/query.dns_network.dsl.ts diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts index ca444e8b6e12f..748cc63a3773a 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts @@ -15,6 +15,8 @@ import { } from './hosts'; import { NetworkQueries, + NetworkDnsStrategyResponse, + NetworkDnsRequestOptions, NetworkTlsStrategyResponse, NetworkTlsRequestOptions, NetworkHttpStrategyResponse, @@ -113,6 +115,8 @@ export type StrategyResponseType = T extends HostsQ ? HostsStrategyResponse : T extends HostsQueries.hostOverview ? HostOverviewStrategyResponse + : T extends NetworkQueries.dns + ? NetworkDnsStrategyResponse : T extends NetworkQueries.tls ? NetworkTlsStrategyResponse : T extends NetworkQueries.http @@ -127,6 +131,8 @@ export type StrategyRequestType = T extends HostsQu ? HostsRequestOptions : T extends HostsQueries.hostOverview ? HostOverviewRequestOptions + : T extends NetworkQueries.dns + ? NetworkDnsRequestOptions : T extends NetworkQueries.tls ? NetworkTlsRequestOptions : T extends NetworkQueries.http diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/dns.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/dns.ts new file mode 100644 index 0000000000000..09e5c09600511 --- /dev/null +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/dns.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IEsSearchResponse } from '../../../../../../../src/plugins/data/common'; +import { Direction } from './common'; +import { CursorType, Inspect, Maybe, PageInfoPaginated, RequestOptionsPaginated } from '..'; + +export interface NetworkDnsSortField { + field: NetworkDnsFields; + + direction: Direction; +} + +export enum NetworkDnsFields { + dnsName = 'dnsName', + queryCount = 'queryCount', + uniqueDomains = 'uniqueDomains', + dnsBytesIn = 'dnsBytesIn', + dnsBytesOut = 'dnsBytesOut', +} + +export interface NetworkDnsRequestOptions extends RequestOptionsPaginated { + isPtrIncluded: boolean; + networkDnsSortField: NetworkDnsSortField; + stackByField?: Maybe; +} + +export interface NetworkDnsStrategyResponse extends IEsSearchResponse { + edges: NetworkDnsEdges[]; + totalCount: number; + pageInfo: PageInfoPaginated; + inspect?: Maybe; + histogram?: Maybe; +} + +export interface NetworkDnsEdges { + node: NetworkDnsItem; + cursor: CursorType; +} + +export interface NetworkDnsItem { + _id?: Maybe; + dnsBytesIn?: Maybe; + dnsBytesOut?: Maybe; + dnsName?: Maybe; + queryCount?: Maybe; + uniqueDomains?: Maybe; +} + +export interface MatrixOverOrdinalHistogramData { + x: string; + y: number; + g: string; +} + +export interface NetworkDnsBuckets { + key: string; + doc_count: number; + unique_domains: { + value: number; + }; + dns_bytes_in: { + value: number; + }; + dns_bytes_out: { + value: number; + }; +} diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts index bfb8f31f4b300..14d7c32f46635 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/network/index.ts @@ -9,25 +9,19 @@ import { IEsSearchResponse } from '../../../../../../../src/plugins/data/common' import { CursorType, Inspect, Maybe, PageInfoPaginated, RequestOptionsPaginated } from '..'; export * from './common'; +export * from './dns'; export * from './http'; export * from './top_countries'; export * from './top_n_flow'; export enum NetworkQueries { + dns = 'dns', http = 'http', tls = 'tls', topCountries = 'topCountries', topNFlow = 'topNFlow', } -export enum NetworkDnsFields { - dnsName = 'dnsName', - queryCount = 'queryCount', - uniqueDomains = 'uniqueDomains', - dnsBytesIn = 'dnsBytesIn', - dnsBytesOut = 'dnsBytesOut', -} - export interface TlsBuckets { key: string; timestamp?: { diff --git a/x-pack/plugins/security_solution/public/network/containers/network_dns/_index.tsx b/x-pack/plugins/security_solution/public/network/containers/network_dns/_index.tsx new file mode 100644 index 0000000000000..be792d0647fa6 --- /dev/null +++ b/x-pack/plugins/security_solution/public/network/containers/network_dns/_index.tsx @@ -0,0 +1,168 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getOr } from 'lodash/fp'; +import React from 'react'; +import { Query } from 'react-apollo'; +import { connect } from 'react-redux'; +import { compose } from 'redux'; + +import { DEFAULT_INDEX_KEY } from '../../../../common/constants'; +import { + GetNetworkDnsQuery, + NetworkDnsEdges, + NetworkDnsSortField, + PageInfoPaginated, + MatrixOverOrdinalHistogramData, +} from '../../../graphql/types'; +import { inputsModel, State, inputsSelectors } from '../../../common/store'; +import { withKibana, WithKibanaProps } from '../../../common/lib/kibana'; +import { generateTablePaginationOptions } from '../../../common/components/paginated_table/helpers'; +import { createFilter, getDefaultFetchPolicy } from '../../../common/containers/helpers'; +import { + QueryTemplatePaginated, + QueryTemplatePaginatedProps, +} from '../../../common/containers/query_template_paginated'; +import { networkDnsQuery } from './index.gql_query'; + +import { networkModel, networkSelectors } from '../../store'; + +const ID = 'networkDnsQuery'; +export const HISTOGRAM_ID = 'networkDnsHistogramQuery'; +export interface NetworkDnsArgs { + id: string; + inspect: inputsModel.InspectQuery; + isInspected: boolean; + loading: boolean; + loadPage: (newActivePage: number) => void; + networkDns: NetworkDnsEdges[]; + pageInfo: PageInfoPaginated; + refetch: inputsModel.Refetch; + stackByField?: string; + totalCount: number; + histogram: MatrixOverOrdinalHistogramData[]; +} + +export interface OwnProps extends QueryTemplatePaginatedProps { + children: (args: NetworkDnsArgs) => React.ReactNode; + type: networkModel.NetworkType; +} + +export interface NetworkDnsComponentReduxProps { + activePage: number; + sort: NetworkDnsSortField; + isInspected: boolean; + isPtrIncluded: boolean; + limit: number; +} + +type NetworkDnsProps = OwnProps & NetworkDnsComponentReduxProps & WithKibanaProps; + +export class NetworkDnsComponentQuery extends QueryTemplatePaginated< + NetworkDnsProps, + GetNetworkDnsQuery.Query, + GetNetworkDnsQuery.Variables +> { + public render() { + const { + activePage, + children, + sort, + endDate, + filterQuery, + id = ID, + isInspected, + isPtrIncluded, + kibana, + limit, + skip, + sourceId, + startDate, + } = this.props; + const variables: GetNetworkDnsQuery.Variables = { + defaultIndex: kibana.services.uiSettings.get(DEFAULT_INDEX_KEY), + filterQuery: createFilter(filterQuery), + inspect: isInspected, + isPtrIncluded, + pagination: generateTablePaginationOptions(activePage, limit), + sort, + sourceId, + timerange: { + interval: '12h', + from: startDate!, + to: endDate!, + }, + }; + + return ( + + fetchPolicy={getDefaultFetchPolicy()} + notifyOnNetworkStatusChange + query={networkDnsQuery} + skip={skip} + variables={variables} + > + {({ data, loading, fetchMore, networkStatus, refetch }) => { + const networkDns = getOr([], `source.NetworkDns.edges`, data); + this.setFetchMore(fetchMore); + this.setFetchMoreOptions((newActivePage: number) => ({ + variables: { + pagination: generateTablePaginationOptions(newActivePage, limit), + }, + updateQuery: (prev, { fetchMoreResult }) => { + if (!fetchMoreResult) { + return prev; + } + return { + ...fetchMoreResult, + source: { + ...fetchMoreResult.source, + NetworkDns: { + ...fetchMoreResult.source.NetworkDns, + edges: [...fetchMoreResult.source.NetworkDns.edges], + }, + }, + }; + }, + })); + const isLoading = this.isItAValidLoading(loading, variables, networkStatus); + return children({ + id, + inspect: getOr(null, 'source.NetworkDns.inspect', data), + isInspected, + loading: isLoading, + loadPage: this.wrappedLoadMore, + networkDns, + pageInfo: getOr({}, 'source.NetworkDns.pageInfo', data), + refetch: this.memoizedRefetchQuery(variables, limit, refetch), + totalCount: getOr(-1, 'source.NetworkDns.totalCount', data), + histogram: getOr(null, 'source.NetworkDns.histogram', data), + }); + }} + + ); + } +} + +const makeMapStateToProps = () => { + const getNetworkDnsSelector = networkSelectors.dnsSelector(); + const getQuery = inputsSelectors.globalQueryByIdSelector(); + const mapStateToProps = (state: State, { id = ID }: OwnProps) => { + const { isInspected } = getQuery(state, id); + return { + ...getNetworkDnsSelector(state), + isInspected, + id, + }; + }; + + return mapStateToProps; +}; + +export const NetworkDnsQuery = compose>( + connect(makeMapStateToProps), + withKibana +)(NetworkDnsComponentQuery); diff --git a/x-pack/plugins/security_solution/public/network/containers/network_dns/histogram.ts b/x-pack/plugins/security_solution/public/network/containers/network_dns/histogram.ts new file mode 100644 index 0000000000000..dce0c3bd2b30d --- /dev/null +++ b/x-pack/plugins/security_solution/public/network/containers/network_dns/histogram.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React from 'react'; +import { connect } from 'react-redux'; +import { compose } from 'redux'; +import { DocumentNode } from 'graphql'; +import { ScaleType } from '@elastic/charts'; + +import { MatrixHistogram } from '../../../common/components/matrix_histogram'; +import { + MatrixHistogramOption, + GetSubTitle, +} from '../../../common/components/matrix_histogram/types'; +import { UpdateDateRange } from '../../../common/components/charts/common'; +import { GlobalTimeArgs } from '../../../common/containers/use_global_time'; +import { withKibana } from '../../../common/lib/kibana'; +import { QueryTemplatePaginatedProps } from '../../../common/containers/query_template_paginated'; +import { DEFAULT_TABLE_ACTIVE_PAGE, DEFAULT_TABLE_LIMIT } from '../../../common/store/constants'; +import { networkModel, networkSelectors } from '../../store'; +import { State, inputsSelectors } from '../../../common/store'; + +export const HISTOGRAM_ID = 'networkDnsHistogramQuery'; + +interface DnsHistogramOwnProps extends QueryTemplatePaginatedProps { + dataKey: string | string[]; + defaultStackByOption: MatrixHistogramOption; + errorMessage: string; + isDnsHistogram?: boolean; + query: DocumentNode; + scaleType: ScaleType; + setQuery: GlobalTimeArgs['setQuery']; + showLegend?: boolean; + stackByOptions: MatrixHistogramOption[]; + subtitle?: string | GetSubTitle; + title: string; + type: networkModel.NetworkType; + updateDateRange: UpdateDateRange; + yTickFormatter?: (value: number) => string; +} + +const makeMapHistogramStateToProps = () => { + const getNetworkDnsSelector = networkSelectors.dnsSelector(); + const getQuery = inputsSelectors.globalQueryByIdSelector(); + const mapStateToProps = (state: State, { id = HISTOGRAM_ID }: DnsHistogramOwnProps) => { + const { isInspected } = getQuery(state, id); + return { + ...getNetworkDnsSelector(state), + activePage: DEFAULT_TABLE_ACTIVE_PAGE, + limit: DEFAULT_TABLE_LIMIT, + isInspected, + id, + }; + }; + + return mapStateToProps; +}; + +export const NetworkDnsHistogramQuery = compose>( + connect(makeMapHistogramStateToProps), + withKibana +)(MatrixHistogram); diff --git a/x-pack/plugins/security_solution/public/network/containers/network_dns/index.tsx b/x-pack/plugins/security_solution/public/network/containers/network_dns/index.tsx index 72e3161de5373..7963f866f7599 100644 --- a/x-pack/plugins/security_solution/public/network/containers/network_dns/index.tsx +++ b/x-pack/plugins/security_solution/public/network/containers/network_dns/index.tsx @@ -4,48 +4,36 @@ * you may not use this file except in compliance with the Elastic License. */ -import { getOr } from 'lodash/fp'; -import React from 'react'; -import { Query } from 'react-apollo'; -import { connect } from 'react-redux'; -import { compose } from 'redux'; - -import { DocumentNode } from 'graphql'; -import { ScaleType } from '@elastic/charts'; +import { noop } from 'lodash/fp'; +import { useState, useEffect, useCallback, useRef } from 'react'; +import { shallowEqual, useSelector } from 'react-redux'; +import deepEqual from 'fast-deep-equal'; + +import { ESTermQuery } from '../../../../common/typed_json'; import { DEFAULT_INDEX_KEY } from '../../../../common/constants'; -import { - GetNetworkDnsQuery, - NetworkDnsEdges, - NetworkDnsSortField, - PageInfoPaginated, - MatrixOverOrdinalHistogramData, -} from '../../../graphql/types'; -import { inputsModel, State, inputsSelectors } from '../../../common/store'; -import { withKibana, WithKibanaProps } from '../../../common/lib/kibana'; +import { inputsModel, State } from '../../../common/store'; +import { useKibana } from '../../../common/lib/kibana'; +import { createFilter } from '../../../common/containers/helpers'; +import { NetworkDnsEdges, PageInfoPaginated } from '../../../graphql/types'; import { generateTablePaginationOptions } from '../../../common/components/paginated_table/helpers'; -import { createFilter, getDefaultFetchPolicy } from '../../../common/containers/helpers'; -import { - QueryTemplatePaginated, - QueryTemplatePaginatedProps, -} from '../../../common/containers/query_template_paginated'; -import { networkDnsQuery } from './index.gql_query'; -import { DEFAULT_TABLE_ACTIVE_PAGE, DEFAULT_TABLE_LIMIT } from '../../../common/store/constants'; -import { MatrixHistogram } from '../../../common/components/matrix_histogram'; -import { - MatrixHistogramOption, - GetSubTitle, -} from '../../../common/components/matrix_histogram/types'; -import { UpdateDateRange } from '../../../common/components/charts/common'; -import { GlobalTimeArgs } from '../../../common/containers/use_global_time'; import { networkModel, networkSelectors } from '../../store'; +import { + NetworkQueries, + NetworkDnsRequestOptions, + NetworkDnsStrategyResponse, + MatrixOverOrdinalHistogramData, +} from '../../../../common/search_strategy/security_solution/network'; +import { AbortError } from '../../../../../../../src/plugins/data/common'; +import * as i18n from './translations'; + +export * from './histogram'; const ID = 'networkDnsQuery'; -export const HISTOGRAM_ID = 'networkDnsHistogramQuery'; + export interface NetworkDnsArgs { id: string; inspect: inputsModel.InspectQuery; isInspected: boolean; - loading: boolean; loadPage: (newActivePage: number) => void; networkDns: NetworkDnsEdges[]; pageInfo: PageInfoPaginated; @@ -55,162 +43,167 @@ export interface NetworkDnsArgs { histogram: MatrixOverOrdinalHistogramData[]; } -export interface OwnProps extends QueryTemplatePaginatedProps { - children: (args: NetworkDnsArgs) => React.ReactNode; +interface UseNetworkDns { + id?: string; type: networkModel.NetworkType; + filterQuery?: ESTermQuery | string; + endDate: string; + startDate: string; + skip: boolean; } -interface DnsHistogramOwnProps extends QueryTemplatePaginatedProps { - dataKey: string | string[]; - defaultStackByOption: MatrixHistogramOption; - errorMessage: string; - isDnsHistogram?: boolean; - query: DocumentNode; - scaleType: ScaleType; - setQuery: GlobalTimeArgs['setQuery']; - showLegend?: boolean; - stackByOptions: MatrixHistogramOption[]; - subtitle?: string | GetSubTitle; - title: string; - type: networkModel.NetworkType; - updateDateRange: UpdateDateRange; - yTickFormatter?: (value: number) => string; -} +export const useNetworkDns = ({ + endDate, + filterQuery, + id = ID, + skip, + startDate, + type, +}: UseNetworkDns): [boolean, NetworkDnsArgs] => { + // const getQuery = inputsSelectors.globalQueryByIdSelector(); + // const { isInspected } = useSelector((state: State) => getQuery(state, id), shallowEqual); + const getNetworkDnsSelector = networkSelectors.dnsSelector(); + const { activePage, sort, isPtrIncluded, limit } = useSelector( + (state: State) => getNetworkDnsSelector(state), + shallowEqual + ); + const { data, notifications, uiSettings } = useKibana().services; + const refetch = useRef(noop); + const abortCtrl = useRef(new AbortController()); + const defaultIndex = uiSettings.get(DEFAULT_INDEX_KEY); + const [loading, setLoading] = useState(false); -export interface NetworkDnsComponentReduxProps { - activePage: number; - sort: NetworkDnsSortField; - isInspected: boolean; - isPtrIncluded: boolean; - limit: number; -} + const [networkDnsRequest, setNetworkDnsRequest] = useState({ + defaultIndex, + factoryQueryType: NetworkQueries.dns, + filterQuery: createFilter(filterQuery), + isPtrIncluded, + // inspect: isInspected, + pagination: generateTablePaginationOptions(activePage, limit), + networkDnsSortField: sort, + timerange: { + interval: '12h', + from: startDate ? startDate : '', + to: endDate ? endDate : new Date(Date.now()).toISOString(), + }, + }); + + const wrappedLoadMore = useCallback( + (newActivePage: number) => { + setNetworkDnsRequest((prevRequest) => ({ + ...prevRequest, + pagination: generateTablePaginationOptions(newActivePage, limit), + })); + }, + [limit] + ); -type NetworkDnsProps = OwnProps & NetworkDnsComponentReduxProps & WithKibanaProps; - -export class NetworkDnsComponentQuery extends QueryTemplatePaginated< - NetworkDnsProps, - GetNetworkDnsQuery.Query, - GetNetworkDnsQuery.Variables -> { - public render() { - const { - activePage, - children, - sort, - endDate, - filterQuery, - id = ID, - isInspected, - isPtrIncluded, - kibana, - limit, - skip, - sourceId, - startDate, - } = this.props; - const variables: GetNetworkDnsQuery.Variables = { - defaultIndex: kibana.services.uiSettings.get(DEFAULT_INDEX_KEY), - filterQuery: createFilter(filterQuery), - inspect: isInspected, - isPtrIncluded, - pagination: generateTablePaginationOptions(activePage, limit), - sort, - sourceId, - timerange: { - interval: '12h', - from: startDate!, - to: endDate!, - }, - }; - - return ( - - fetchPolicy={getDefaultFetchPolicy()} - notifyOnNetworkStatusChange - query={networkDnsQuery} - skip={skip} - variables={variables} - > - {({ data, loading, fetchMore, networkStatus, refetch }) => { - const networkDns = getOr([], `source.NetworkDns.edges`, data); - this.setFetchMore(fetchMore); - this.setFetchMoreOptions((newActivePage: number) => ({ - variables: { - pagination: generateTablePaginationOptions(newActivePage, limit), + const [networkDnsResponse, setNetworkDnsResponse] = useState({ + networkDns: [], + histogram: [], + id: ID, + inspect: { + dsl: [], + response: [], + }, + isInspected: false, + loadPage: wrappedLoadMore, + pageInfo: { + activePage: 0, + fakeTotalCount: 0, + showMorePagesIndicator: false, + }, + refetch: refetch.current, + totalCount: -1, + }); + + const networkDnsSearch = useCallback( + (request: NetworkDnsRequestOptions) => { + let didCancel = false; + const asyncSearch = async () => { + abortCtrl.current = new AbortController(); + setLoading(true); + + const searchSubscription$ = data.search + .search(request, { + strategy: 'securitySolutionSearchStrategy', + signal: abortCtrl.current.signal, + }) + .subscribe({ + next: (response) => { + if (!response.isPartial && !response.isRunning) { + if (!didCancel) { + setLoading(false); + setNetworkDnsResponse((prevResponse) => ({ + ...prevResponse, + networkDns: response.edges, + inspect: response.inspect ?? prevResponse.inspect, + pageInfo: response.pageInfo, + refetch: refetch.current, + totalCount: response.totalCount, + histogram: response.histogram ?? prevResponse.histogram, + })); + } + searchSubscription$.unsubscribe(); + } else if (response.isPartial && !response.isRunning) { + if (!didCancel) { + setLoading(false); + } + // TODO: Make response error status clearer + notifications.toasts.addWarning(i18n.ERROR_NETWORK_DNS); + searchSubscription$.unsubscribe(); + } }, - updateQuery: (prev, { fetchMoreResult }) => { - if (!fetchMoreResult) { - return prev; + error: (msg) => { + if (!(msg instanceof AbortError)) { + notifications.toasts.addDanger({ + title: i18n.FAIL_NETWORK_DNS, + text: msg.message, + }); } - return { - ...fetchMoreResult, - source: { - ...fetchMoreResult.source, - NetworkDns: { - ...fetchMoreResult.source.NetworkDns, - edges: [...fetchMoreResult.source.NetworkDns.edges], - }, - }, - }; }, - })); - const isLoading = this.isItAValidLoading(loading, variables, networkStatus); - return children({ - id, - inspect: getOr(null, 'source.NetworkDns.inspect', data), - isInspected, - loading: isLoading, - loadPage: this.wrappedLoadMore, - networkDns, - pageInfo: getOr({}, 'source.NetworkDns.pageInfo', data), - refetch: this.memoizedRefetchQuery(variables, limit, refetch), - totalCount: getOr(-1, 'source.NetworkDns.totalCount', data), - histogram: getOr(null, 'source.NetworkDns.histogram', data), }); - }} - - ); - } -} + }; + abortCtrl.current.abort(); + asyncSearch(); + refetch.current = asyncSearch; + return () => { + didCancel = true; + abortCtrl.current.abort(); + }; + }, + [data.search, notifications.toasts] + ); -const makeMapStateToProps = () => { - const getNetworkDnsSelector = networkSelectors.dnsSelector(); - const getQuery = inputsSelectors.globalQueryByIdSelector(); - const mapStateToProps = (state: State, { id = ID }: OwnProps) => { - const { isInspected } = getQuery(state, id); - return { - ...getNetworkDnsSelector(state), - isInspected, - id, - }; - }; - - return mapStateToProps; -}; + useEffect(() => { + if (skip) { + return; + } -const makeMapHistogramStateToProps = () => { - const getNetworkDnsSelector = networkSelectors.dnsSelector(); - const getQuery = inputsSelectors.globalQueryByIdSelector(); - const mapStateToProps = (state: State, { id = HISTOGRAM_ID }: DnsHistogramOwnProps) => { - const { isInspected } = getQuery(state, id); - return { - ...getNetworkDnsSelector(state), - activePage: DEFAULT_TABLE_ACTIVE_PAGE, - limit: DEFAULT_TABLE_LIMIT, - isInspected, - id, - }; - }; - - return mapStateToProps; -}; + setNetworkDnsRequest((prevRequest) => { + const myRequest = { + ...prevRequest, + defaultIndex, + isPtrIncluded, + filterQuery: createFilter(filterQuery), + pagination: generateTablePaginationOptions(activePage, limit), + timerange: { + interval: '12h', + from: startDate, + to: endDate, + }, + networkDnsSortField: sort, + }; + if (!deepEqual(prevRequest, myRequest)) { + return myRequest; + } + return prevRequest; + }); + }, [activePage, defaultIndex, endDate, filterQuery, limit, startDate, sort, skip, isPtrIncluded]); -export const NetworkDnsQuery = compose>( - connect(makeMapStateToProps), - withKibana -)(NetworkDnsComponentQuery); + useEffect(() => { + networkDnsSearch(networkDnsRequest); + }, [networkDnsRequest, networkDnsSearch]); -export const NetworkDnsHistogramQuery = compose>( - connect(makeMapHistogramStateToProps), - withKibana -)(MatrixHistogram); + return [loading, networkDnsResponse]; +}; diff --git a/x-pack/plugins/security_solution/public/network/containers/network_dns/translations.ts b/x-pack/plugins/security_solution/public/network/containers/network_dns/translations.ts new file mode 100644 index 0000000000000..54c36dd1536f1 --- /dev/null +++ b/x-pack/plugins/security_solution/public/network/containers/network_dns/translations.ts @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; + +export const ERROR_NETWORK_DNS = i18n.translate( + 'xpack.securitySolution.networkDns.errorSearchDescription', + { + defaultMessage: `An error has occurred on network dns search`, + } +); + +export const FAIL_NETWORK_DNS = i18n.translate( + 'xpack.securitySolution.networkDns.failSearchDescription', + { + defaultMessage: `Failed to run search on network dns`, + } +); diff --git a/x-pack/plugins/security_solution/public/network/pages/navigation/dns_query_tab_body.tsx b/x-pack/plugins/security_solution/public/network/pages/navigation/dns_query_tab_body.tsx index 77283dc330257..10cc11258cbff 100644 --- a/x-pack/plugins/security_solution/public/network/pages/navigation/dns_query_tab_body.tsx +++ b/x-pack/plugins/security_solution/public/network/pages/navigation/dns_query_tab_body.tsx @@ -8,7 +8,7 @@ import React, { useEffect, useCallback, useMemo } from 'react'; import { getOr } from 'lodash/fp'; import { NetworkDnsTable } from '../../components/network_dns_table'; -import { NetworkDnsQuery, HISTOGRAM_ID } from '../../containers/network_dns'; +import { useNetworkDns, HISTOGRAM_ID } from '../../containers/network_dns'; import { manageQuery } from '../../../common/components/page/manage_query'; import { NetworkComponentQueryProps } from './types'; @@ -42,7 +42,7 @@ export const histogramConfigs: Omit = { subtitle: undefined, }; -export const DnsQueryTabBody = ({ +const DnsQueryTabBodyComponent: React.FC = ({ deleteQuery, endDate, filterQuery, @@ -50,7 +50,7 @@ export const DnsQueryTabBody = ({ startDate, setQuery, type, -}: NetworkComponentQueryProps) => { +}) => { useEffect(() => { return () => { if (deleteQuery) { @@ -59,6 +59,17 @@ export const DnsQueryTabBody = ({ }; }, [deleteQuery]); + const [ + loading, + { totalCount, networkDns, pageInfo, loadPage, id, inspect, isInspected, refetch }, + ] = useNetworkDns({ + endDate, + filterQuery, + skip, + startDate, + type, + }); + const getTitle = useCallback( (option: MatrixHistogramOption) => i18n.DOMAINS_COUNT_BY(option.text), [] @@ -85,43 +96,24 @@ export const DnsQueryTabBody = ({ type={networkModel.NetworkType.page} {...dnsHistogramConfigs} /> - - {({ - totalCount, - loading, - networkDns, - pageInfo, - loadPage, - id, - inspect, - isInspected, - refetch, - }) => ( - - )} - + /> ); }; -DnsQueryTabBody.displayName = 'DNSQueryTabBody'; +DnsQueryTabBodyComponent.displayName = 'DnsQueryTabBodyComponent'; + +export const DnsQueryTabBody = React.memo(DnsQueryTabBodyComponent); diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/helpers.ts new file mode 100644 index 0000000000000..b756e3ea6d3f8 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/helpers.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { get, getOr } from 'lodash/fp'; + +import { IEsSearchResponse } from '../../../../../../../../../src/plugins/data/common'; +import { + NetworkDnsBuckets, + NetworkDnsEdges, +} from '../../../../../../common/search_strategy/security_solution/network'; + +export const getDnsEdges = (response: IEsSearchResponse): NetworkDnsEdges[] => + formatDnsEdges(getOr([], `aggregations.dns_name_query_count.buckets`, response)); + +export const formatDnsEdges = (buckets: NetworkDnsBuckets[]): NetworkDnsEdges[] => + buckets.map((bucket: NetworkDnsBuckets) => ({ + node: { + _id: bucket.key, + dnsBytesIn: getOrNumber('dns_bytes_in.value', bucket), + dnsBytesOut: getOrNumber('dns_bytes_out.value', bucket), + dnsName: bucket.key, + queryCount: bucket.doc_count, + uniqueDomains: getOrNumber('unique_domains.value', bucket), + }, + cursor: { + value: bucket.key, + tiebreaker: null, + }, + })); + +const getOrNumber = (path: string, bucket: NetworkDnsBuckets) => { + const numb = get(path, bucket); + if (numb == null) { + return null; + } + return numb; +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/index.ts new file mode 100644 index 0000000000000..90522e6b09443 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/index.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getOr } from 'lodash/fp'; + +import { IEsSearchResponse } from '../../../../../../../../../src/plugins/data/common'; + +import { DEFAULT_MAX_TABLE_QUERY_SIZE } from '../../../../../../common/constants'; +import { + NetworkDnsStrategyResponse, + NetworkQueries, + NetworkDnsRequestOptions, + NetworkDnsEdges, +} from '../../../../../../common/search_strategy/security_solution/network'; + +import { inspectStringifyObject } from '../../../../../utils/build_query'; +import { SecuritySolutionFactory } from '../../types'; + +import { getDnsEdges } from './helpers'; +import { buildDnsQuery } from './query.dns_network.dsl'; + +export const networkDns: SecuritySolutionFactory = { + buildDsl: (options: NetworkDnsRequestOptions) => { + if (options.pagination && options.pagination.querySize >= DEFAULT_MAX_TABLE_QUERY_SIZE) { + throw new Error(`No query size above ${DEFAULT_MAX_TABLE_QUERY_SIZE}`); + } + return buildDnsQuery(options); + }, + parse: async ( + options: NetworkDnsRequestOptions, + response: IEsSearchResponse + ): Promise => { + const { activePage, cursorStart, fakePossibleCount, querySize } = options.pagination; + const totalCount = getOr(0, 'aggregations.dns_count.value', response.rawResponse); + const networkDnsEdges: NetworkDnsEdges[] = getDnsEdges(response); + const fakeTotalCount = fakePossibleCount <= totalCount ? fakePossibleCount : totalCount; + const edges = networkDnsEdges.splice(cursorStart, querySize - cursorStart); + const inspect = { + dsl: [inspectStringifyObject(buildDnsQuery(options))], + response: [inspectStringifyObject(response)], + }; + const showMorePagesIndicator = totalCount > fakeTotalCount; + + return { + ...response, + edges, + inspect, + pageInfo: { + activePage: activePage ? activePage : 0, + fakeTotalCount, + showMorePagesIndicator, + }, + totalCount, + }; + }, +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/query.dns_network.dsl.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/query.dns_network.dsl.ts new file mode 100644 index 0000000000000..1ff4880e29050 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/query.dns_network.dsl.ts @@ -0,0 +1,134 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { isEmpty } from 'lodash/fp'; + +import { assertUnreachable } from '../../../../../../common/utility_types'; +import { + Direction, + NetworkDnsRequestOptions, + NetworkDnsFields, + NetworkDnsSortField, +} from '../../../../../../common/search_strategy/security_solution/network'; +import { createQueryFilterClauses } from '../../../../../utils/build_query'; + +type QueryOrder = + | { _count: Direction } + | { _key: Direction } + | { unique_domains: Direction } + | { dns_bytes_in: Direction } + | { dns_bytes_out: Direction }; + +const getQueryOrder = (networkDnsSortField: NetworkDnsSortField): QueryOrder => { + switch (networkDnsSortField.field) { + case NetworkDnsFields.queryCount: + return { _count: networkDnsSortField.direction }; + case NetworkDnsFields.dnsName: + return { _key: networkDnsSortField.direction }; + case NetworkDnsFields.uniqueDomains: + return { unique_domains: networkDnsSortField.direction }; + case NetworkDnsFields.dnsBytesIn: + return { dns_bytes_in: networkDnsSortField.direction }; + case NetworkDnsFields.dnsBytesOut: + return { dns_bytes_out: networkDnsSortField.direction }; + } + assertUnreachable(networkDnsSortField.field); +}; + +const getCountAgg = () => ({ + dns_count: { + cardinality: { + field: 'dns.question.registered_domain', + }, + }, +}); + +const createIncludePTRFilter = (isPtrIncluded: boolean) => + isPtrIncluded + ? {} + : { + must_not: [ + { + term: { + 'dns.question.type': { + value: 'PTR', + }, + }, + }, + ], + }; + +export const buildDnsQuery = ({ + defaultIndex, + docValueFields, + filterQuery, + isPtrIncluded, + networkDnsSortField, + pagination: { querySize }, + stackByField = 'dns.question.registered_domain', + timerange: { from, to }, +}: NetworkDnsRequestOptions) => { + const filter = [ + ...createQueryFilterClauses(filterQuery), + { + range: { + '@timestamp': { + gte: from, + lte: to, + format: 'strict_date_optional_time', + }, + }, + }, + ]; + + const dslQuery = { + allowNoIndices: true, + index: defaultIndex, + ignoreUnavailable: true, + body: { + ...(isEmpty(docValueFields) ? { docvalue_fields: docValueFields } : {}), + aggregations: { + ...getCountAgg(), + dns_name_query_count: { + terms: { + field: stackByField, + size: querySize, + order: { + ...getQueryOrder(networkDnsSortField), + }, + }, + aggs: { + unique_domains: { + cardinality: { + field: 'dns.question.name', + }, + }, + dns_bytes_in: { + sum: { + field: 'source.bytes', + }, + }, + dns_bytes_out: { + sum: { + field: 'destination.bytes', + }, + }, + }, + }, + }, + query: { + bool: { + filter, + ...createIncludePTRFilter(isPtrIncluded), + }, + }, + }, + size: 0, + track_total_hits: false, + }; + + return dslQuery; +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts index 2561ec1de48e2..51e12db625042 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/index.ts @@ -8,12 +8,14 @@ import { FactoryQueryTypes } from '../../../../../common/search_strategy/securit import { NetworkQueries } from '../../../../../common/search_strategy/security_solution/network'; import { SecuritySolutionFactory } from '../types'; +import { networkDns } from './dns'; import { networkHttp } from './http'; import { networkTls } from './tls'; import { networkTopCountries } from './top_countries'; import { networkTopNFlow } from './top_n_flow'; export const networkFactory: Record> = { + [NetworkQueries.dns]: networkDns, [NetworkQueries.http]: networkHttp, [NetworkQueries.tls]: networkTls, [NetworkQueries.topCountries]: networkTopCountries, From 4023e0a07fa71ac009a599714ba2d36e0e29f1b5 Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Sun, 6 Sep 2020 11:06:26 +0200 Subject: [PATCH 7/8] cleanup --- .../security_solution/factory/network/dns/helpers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/helpers.ts index b756e3ea6d3f8..aa242e6ece7bf 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/helpers.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/helpers.ts @@ -13,7 +13,7 @@ import { } from '../../../../../../common/search_strategy/security_solution/network'; export const getDnsEdges = (response: IEsSearchResponse): NetworkDnsEdges[] => - formatDnsEdges(getOr([], `aggregations.dns_name_query_count.buckets`, response)); + formatDnsEdges(getOr([], `aggregations.dns_name_query_count.buckets`, response.rawResponse)); export const formatDnsEdges = (buckets: NetworkDnsBuckets[]): NetworkDnsEdges[] => buckets.map((bucket: NetworkDnsBuckets) => ({ From ebb2ea9e34fc1e82caa9195d4033077d2f07509b Mon Sep 17 00:00:00 2001 From: Patryk Kopycinski Date: Mon, 7 Sep 2020 12:23:00 +0200 Subject: [PATCH 8/8] cleanup --- .../security_solution/factory/network/dns/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/index.ts index 90522e6b09443..8e734ca9d1179 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/network/dns/index.ts @@ -40,7 +40,6 @@ export const networkDns: SecuritySolutionFactory = { const edges = networkDnsEdges.splice(cursorStart, querySize - cursorStart); const inspect = { dsl: [inspectStringifyObject(buildDnsQuery(options))], - response: [inspectStringifyObject(response)], }; const showMorePagesIndicator = totalCount > fakeTotalCount;