Skip to content

Commit

Permalink
Merge branch 'master' into fix/migrate-timedefaults
Browse files Browse the repository at this point in the history
  • Loading branch information
@elasticmachine
elasticmachine committed Aug 3, 2020
2 parents 71bc2e9 + 34d9c58 commit 33a34aa
Show file tree
Hide file tree
Showing 102 changed files with 2,388 additions and 676 deletions.
55 changes: 53 additions & 2 deletions ...ugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ describe('get_data_telemetry', () => {
{ name: 'logs-endpoint.1234', docCount: 0 }, // Matching pattern with a dot in the name
// New Indexing strategy: everything can be inferred from the constant_keyword values
{
name: 'logs-nginx.access-default-000001',
name: '.ds-logs-nginx.access-default-000001',
datasetName: 'nginx.access',
datasetType: 'logs',
shipper: 'filebeat',
Expand All @@ -84,14 +84,50 @@ describe('get_data_telemetry', () => {
sizeInBytes: 1000,
},
{
name: 'logs-nginx.access-default-000002',
name: '.ds-logs-nginx.access-default-000002',
datasetName: 'nginx.access',
datasetType: 'logs',
shipper: 'filebeat',
isECS: true,
docCount: 1000,
sizeInBytes: 60,
},
{
name: '.ds-traces-something-default-000002',
datasetName: 'something',
datasetType: 'traces',
packageName: 'some-package',
isECS: true,
docCount: 1000,
sizeInBytes: 60,
},
{
name: '.ds-metrics-something.else-default-000002',
datasetName: 'something.else',
datasetType: 'metrics',
managedBy: 'ingest-manager',
isECS: true,
docCount: 1000,
sizeInBytes: 60,
},
// Filter out if it has datasetName and datasetType but none of the shipper, packageName or managedBy === 'ingest-manager'
{
name: 'some-index-that-should-not-show',
datasetName: 'should-not-show',
datasetType: 'logs',
isECS: true,
docCount: 1000,
sizeInBytes: 60,
},
{
name: 'other-index-that-should-not-show',
datasetName: 'should-not-show-either',
datasetType: 'metrics',
managedBy: 'me',
isECS: true,
docCount: 1000,
sizeInBytes: 60,
},
])
).toStrictEqual([
{
Expand Down Expand Up @@ -138,6 +174,21 @@ describe('get_data_telemetry', () => {
doc_count: 2000,
size_in_bytes: 1060,
},
{
dataset: { name: 'something', type: 'traces' },
package: { name: 'some-package' },
index_count: 1,
ecs_index_count: 1,
doc_count: 1000,
size_in_bytes: 60,
},
{
dataset: { name: 'something.else', type: 'metrics' },
index_count: 1,
ecs_index_count: 1,
doc_count: 1000,
size_in_bytes: 60,
},
]);
});
});
Expand Down
77 changes: 55 additions & 22 deletions src/plugins/telemetry/server/telemetry_collection/get_data_telemetry/get_data_telemetry.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ export interface DataTelemetryDocument extends DataTelemetryBasePayload {
name?: string;
type?: DataTelemetryType | 'unknown' | string; // The union of types is to help autocompletion with some known `dataset.type`s
};
package?: {
name: string;
};
shipper?: string;
pattern_name?: DataPatternName;
}
Expand All @@ -44,6 +47,8 @@ export type DataTelemetryPayload = DataTelemetryDocument[];

export interface DataTelemetryIndex {
name: string;
packageName?: string; // Populated by Ingest Manager at `_meta.package.name`
managedBy?: string; // Populated by Ingest Manager at `_meta.managed_by`
datasetName?: string; // To be obtained from `mappings.dataset.name` if it's a constant keyword
datasetType?: string; // To be obtained from `mappings.dataset.type` if it's a constant keyword
shipper?: string; // To be obtained from `_meta.beat` if it's set
Expand All @@ -58,6 +63,7 @@ export interface DataTelemetryIndex {
type AtLeastOne<T, U = { [K in keyof T]: Pick<T, K> }> = Partial<T> & U[keyof U];

type DataDescriptor = AtLeastOne<{
packageName: string;
datasetName: string;
datasetType: string;
shipper: string;
Expand All @@ -67,17 +73,28 @@ type DataDescriptor = AtLeastOne<{
function findMatchingDescriptors({
name,
shipper,
packageName,
managedBy,
datasetName,
datasetType,
}: DataTelemetryIndex): DataDescriptor[] {
// If we already have the data from the indices' mappings...
if ([shipper, datasetName, datasetType].some(Boolean)) {
if (
[shipper, packageName].some(Boolean) ||
(managedBy === 'ingest-manager' && [datasetType, datasetName].some(Boolean))
) {
return [
{
...(shipper && { shipper }),
...(packageName && { packageName }),
...(datasetName && { datasetName }),
...(datasetType && { datasetType }),
} as AtLeastOne<{ datasetName: string; datasetType: string; shipper: string }>, // Using casting here because TS doesn't infer at least one exists from the if clause
} as AtLeastOne<{
packageName: string;
datasetName: string;
datasetType: string;
shipper: string;
}>, // Using casting here because TS doesn't infer at least one exists from the if clause
];
}

Expand Down Expand Up @@ -122,6 +139,7 @@ export function buildDataTelemetryPayload(indices: DataTelemetryIndex[]): DataTe
({ name }) =>
!(
name.startsWith('.') &&
!name.startsWith('.ds-') && // data_stream-related indices can be included
!startingDotPatternsUntilTheFirstAsterisk.find((pattern) => name.startsWith(pattern))
)
);
Expand All @@ -130,10 +148,17 @@ export function buildDataTelemetryPayload(indices: DataTelemetryIndex[]): DataTe

for (const indexCandidate of indexCandidates) {
const matchingDescriptors = findMatchingDescriptors(indexCandidate);
for (const { datasetName, datasetType, shipper, patternName } of matchingDescriptors) {
const key = `${datasetName}-${datasetType}-${shipper}-${patternName}`;
for (const {
datasetName,
datasetType,
packageName,
shipper,
patternName,
} of matchingDescriptors) {
const key = `${datasetName}-${datasetType}-${packageName}-${shipper}-${patternName}`;
acc.set(key, {
...((datasetName || datasetType) && { dataset: { name: datasetName, type: datasetType } }),
...(packageName && { package: { name: packageName } }),
...(shipper && { shipper }),
...(patternName && { pattern_name: patternName }),
...increaseCounters(acc.get(key), indexCandidate),
Expand Down Expand Up @@ -165,6 +190,12 @@ interface IndexMappings {
mappings: {
_meta?: {
beat?: string;

// Ingest Manager provided metadata
package?: {
name?: string;
};
managed_by?: string; // Typically "ingest-manager"
};
properties: {
dataset?: {
Expand Down Expand Up @@ -195,7 +226,7 @@ export async function getDataTelemetry(callCluster: LegacyAPICaller) {
try {
const index = [
...DATA_DATASETS_INDEX_PATTERNS_UNIQUE.map(({ pattern }) => pattern),
'*-*-*-*', // Include new indexing strategy indices {type}-{dataset}-{namespace}-{rollover_counter}
'*-*-*', // Include data-streams aliases `{type}-{dataset}-{namespace}`
];
const [indexMappings, indexStats]: [IndexMappings, IndexStats] = await Promise.all([
// GET */_mapping?filter_path=*.mappings._meta.beat,*.mappings.properties.ecs.properties.version.type,*.mappings.properties.dataset.properties.type.value,*.mappings.properties.dataset.properties.name.value
Expand All @@ -204,16 +235,17 @@ export async function getDataTelemetry(callCluster: LegacyAPICaller) {
filterPath: [
// _meta.beat tells the shipper
'*.mappings._meta.beat',
// _meta.package.name tells the Ingest Manager's package
'*.mappings._meta.package.name',
// _meta.managed_by is usually populated by Ingest Manager for the UI to identify it
'*.mappings._meta.managed_by',
// Does it have `ecs.version` in the mappings? => It follows the ECS conventions
'*.mappings.properties.ecs.properties.version.type',

// Disable the fields below because they are still pending to be confirmed:
// https://github.com/elastic/ecs/pull/845
// TODO: Re-enable when the final fields are confirmed
// // If `dataset.type` is a `constant_keyword`, it can be reported as a type
// '*.mappings.properties.dataset.properties.type.value',
// // If `dataset.name` is a `constant_keyword`, it can be reported as the dataset
// '*.mappings.properties.dataset.properties.name.value',
// If `dataset.type` is a `constant_keyword`, it can be reported as a type
'*.mappings.properties.dataset.properties.type.value',
// If `dataset.name` is a `constant_keyword`, it can be reported as the dataset
'*.mappings.properties.dataset.properties.name.value',
],
}),
// GET <index>/_stats/docs,store?level=indices&filter_path=indices.*.total
Expand All @@ -227,24 +259,25 @@ export async function getDataTelemetry(callCluster: LegacyAPICaller) {

const indexNames = Object.keys({ ...indexMappings, ...indexStats?.indices });
const indices = indexNames.map((name) => {
const isECS = !!indexMappings[name]?.mappings?.properties.ecs?.properties.version?.type;
const shipper = indexMappings[name]?.mappings?._meta?.beat;
const datasetName = indexMappings[name]?.mappings?.properties.dataset?.properties.name?.value;
const datasetType = indexMappings[name]?.mappings?.properties.dataset?.properties.type?.value;
const baseIndexInfo = {
name,
isECS: !!indexMappings[name]?.mappings?.properties.ecs?.properties.version?.type,
shipper: indexMappings[name]?.mappings?._meta?.beat,
packageName: indexMappings[name]?.mappings?._meta?.package?.name,
managedBy: indexMappings[name]?.mappings?._meta?.managed_by,
datasetName: indexMappings[name]?.mappings?.properties.dataset?.properties.name?.value,
datasetType: indexMappings[name]?.mappings?.properties.dataset?.properties.type?.value,
};

const stats = (indexStats?.indices || {})[name];
if (stats) {
return {
name,
datasetName,
datasetType,
shipper,
isECS,
...baseIndexInfo,
docCount: stats.total?.docs?.count,
sizeInBytes: stats.total?.store?.size_in_bytes,
};
}
return { name, datasetName, datasetType, shipper, isECS };
return baseIndexInfo;
});
return buildDataTelemetryPayload(indices);
} catch (e) {
Expand Down
23 changes: 10 additions & 13 deletions x-pack/plugins/apm/public/components/shared/DatePicker/__test__/DatePicker.test.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,35 +64,33 @@ describe('DatePicker', () => {
});

beforeEach(() => {
jest.clearAllMocks();
jest.resetAllMocks();
});

it('should set default query params in the URL', () => {
it('sets default query params in the URL', () => {
mountDatePicker();
expect(mockHistoryPush).toHaveBeenCalledTimes(1);
expect(mockHistoryPush).toHaveBeenCalledWith(
expect.objectContaining({
search:
'rangeFrom=now-15m&rangeTo=now&refreshPaused=false&refreshInterval=10000',
search: 'rangeFrom=now-15m&rangeTo=now',
})
);
});

it('should add missing default value', () => {
it('adds missing default value', () => {
mountDatePicker({
rangeTo: 'now',
refreshInterval: 5000,
});
expect(mockHistoryPush).toHaveBeenCalledTimes(1);
expect(mockHistoryPush).toHaveBeenCalledWith(
expect.objectContaining({
search:
'rangeFrom=now-15m&rangeTo=now&refreshInterval=5000&refreshPaused=false',
search: 'rangeFrom=now-15m&rangeTo=now&refreshInterval=5000',
})
);
});

it('should not set default query params in the URL when values already defined', () => {
it('does not set default query params in the URL when values already defined', () => {
mountDatePicker({
rangeFrom: 'now-1d',
rangeTo: 'now',
Expand All @@ -102,7 +100,7 @@ describe('DatePicker', () => {
expect(mockHistoryPush).toHaveBeenCalledTimes(0);
});

it('should update the URL when the date range changes', () => {
it('updates the URL when the date range changes', () => {
const datePicker = mountDatePicker();
datePicker.find(EuiSuperDatePicker).props().onTimeChange({
start: 'updated-start',
Expand All @@ -113,13 +111,12 @@ describe('DatePicker', () => {
expect(mockHistoryPush).toHaveBeenCalledTimes(2);
expect(mockHistoryPush).toHaveBeenLastCalledWith(
expect.objectContaining({
search:
'rangeFrom=updated-start&rangeTo=updated-end&refreshInterval=5000&refreshPaused=false',
search: 'rangeFrom=updated-start&rangeTo=updated-end',
})
);
});

it('should auto-refresh when refreshPaused is false', async () => {
it('enables auto-refresh when refreshPaused is false', async () => {
jest.useFakeTimers();
const wrapper = mountDatePicker({
refreshPaused: false,
Expand All @@ -132,7 +129,7 @@ describe('DatePicker', () => {
wrapper.unmount();
});

it('should NOT auto-refresh when refreshPaused is true', async () => {
it('disables auto-refresh when refreshPaused is true', async () => {
jest.useFakeTimers();
mountDatePicker({ refreshPaused: true, refreshInterval: 1000 });
expect(mockRefreshTimeRange).not.toHaveBeenCalled();
Expand Down
16 changes: 1 addition & 15 deletions x-pack/plugins/apm/public/components/shared/DatePicker/index.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,7 @@ import { useUrlParams } from '../../../hooks/useUrlParams';
import { clearCache } from '../../../services/rest/callApi';
import { useApmPluginContext } from '../../../hooks/useApmPluginContext';
import { UI_SETTINGS } from '../../../../../../../src/plugins/data/common';
import {
TimePickerQuickRange,
TimePickerTimeDefaults,
TimePickerRefreshInterval,
} from './typings';
import { TimePickerQuickRange, TimePickerTimeDefaults } from './typings';

function removeUndefinedAndEmptyProps<T extends object>(obj: T): Partial<T> {
return pickBy(obj, (value) => value !== undefined && !isEmpty(String(value)));
Expand All @@ -36,19 +32,9 @@ export function DatePicker() {
UI_SETTINGS.TIMEPICKER_TIME_DEFAULTS
);

const timePickerRefreshIntervalDefaults = core.uiSettings.get<
TimePickerRefreshInterval
>(UI_SETTINGS.TIMEPICKER_REFRESH_INTERVAL_DEFAULTS);

const DEFAULT_VALUES = {
rangeFrom: timePickerTimeDefaults.from,
rangeTo: timePickerTimeDefaults.to,
refreshPaused: timePickerRefreshIntervalDefaults.pause,
/*
* Must be replaced by timePickerRefreshIntervalDefaults.value when this issue is fixed.
* https://github.com/elastic/kibana/issues/70562
*/
refreshInterval: 10000,
};

const commonlyUsedRanges = timePickerQuickRanges.map(
Expand Down
4 changes: 2 additions & 2 deletions x-pack/plugins/canvas/i18n/components.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ export const ComponentStrings = {
}),
getTitleText: () =>
i18n.translate('xpack.canvas.embedObject.titleText', {
defaultMessage: 'Add from Visualize library',
defaultMessage: 'Add from Kibana',
}),
},
AdvancedFilter: {
Expand Down Expand Up @@ -1308,7 +1308,7 @@ export const ComponentStrings = {
}),
getEmbedObjectMenuItemLabel: () =>
i18n.translate('xpack.canvas.workpadHeaderElementMenu.embedObjectMenuItemLabel', {
defaultMessage: 'Add from Visualize library',
defaultMessage: 'Add from Kibana',
}),
getFilterMenuItemLabel: () =>
i18n.translate('xpack.canvas.workpadHeaderElementMenu.filterMenuItemLabel', {
Expand Down
15 changes: 15 additions & 0 deletions x-pack/plugins/data_enhanced/server/search/es_search_strategy.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,4 +113,19 @@ describe('ES search strategy', () => {
expect(method).toBe('POST');
expect(path).toBe('/foo-%E7%A8%8B/_rollup_search');
});

it('sets wait_for_completion_timeout and keep_alive in the request', async () => {
mockApiCaller.mockResolvedValueOnce(mockAsyncResponse);

const params = { index: 'foo-*', body: {} };
const esSearch = await enhancedEsSearchStrategyProvider(mockConfig$, mockLogger);

await esSearch.search((mockContext as unknown) as RequestHandlerContext, { params });

expect(mockApiCaller).toBeCalled();
expect(mockApiCaller.mock.calls[0][0]).toBe('transport.request');
const { query } = mockApiCaller.mock.calls[0][1];
expect(query).toHaveProperty('wait_for_completion_timeout');
expect(query).toHaveProperty('keep_alive');
});
});
Loading

0 comments on commit 33a34aa

Please sign in to comment.