Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[7.x] [data views] allow fields that start with an underscore in the field list (#111238) #111976

Merged
merged 1 commit into from
Sep 13, 2021
Merged

[7.x] [data views] allow fields that start with an underscore in the field list (#111238) #111976

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/plugins/data/common/index_patterns/index_patterns/flatten_hit.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,10 @@ describe('flattenHit', () => {
fields: {
date: ['1'],
zzz: ['z'],
_abc: ['a'],
},
});
const expectedOrder = ['date', 'name', 'zzz', '_id', '_routing', '_score', '_type'];
const expectedOrder = ['_abc', 'date', 'name', 'zzz', '_id', '_routing', '_score', '_type'];
expect(Object.keys(response)).toEqual(expectedOrder);
expect(Object.entries(response).map(([key]) => key)).toEqual(expectedOrder);
});
Expand Down
1 change: 0 additions & 1 deletion src/plugins/data/common/index_patterns/index_patterns/flatten_hit.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,6 @@ function decorateFlattenedWrapper(hit: Record<string, any>, metaFields: Record<s

// unwrap computed fields
_.forOwn(hit.fields, function (val, key: any) {
if (key[0] === '_' && !_.includes(metaFields, key)) return;
// Flatten an array with 0 or 1 elements to a single value.
if (Array.isArray(val) && val.length <= 1) {
flattened[key] = val[0];
Expand Down
1 change: 1 addition & 0 deletions src/plugins/data/server/index_patterns/fetcher/index_patterns_fetcher.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ export interface FieldDescriptor {
type: string;
esTypes: string[];
subType?: FieldSubType;
metadata_field?: boolean;
}

interface FieldSubType {
Expand Down
11 changes: 11 additions & 0 deletions ...gins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,17 @@ describe('index_patterns/field_capabilities/field_capabilities', () => {
});

describe('response order', () => {
it('supports fields that start with an underscore', async () => {
const fields = ['_field_a', '_field_b'];

stubDeps({
fieldsFromFieldCaps: fields.map((name) => ({ name })),
});

const fieldNames = (await getFieldCapabilities()).map((field) => field.name);
expect(fieldNames).toEqual(fields);
});

it('always returns fields in alphabetical order', async () => {
const letters = 'ambcdfjopngihkel'.split('');
const sortedLetters = sortBy(letters);
Expand Down
14 changes: 8 additions & 6 deletions src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_capabilities.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,20 +34,21 @@ export async function getFieldCapabilities(
const fieldsFromFieldCapsByName = keyBy(readFieldCapsResponse(esFieldCaps.body), 'name');

const allFieldsUnsorted = Object.keys(fieldsFromFieldCapsByName)
.filter((name) => !name.startsWith('_'))
// not all meta fields are provided, so remove and manually add
.filter((name) => !fieldsFromFieldCapsByName[name].metadata_field)
.concat(metaFields)
.reduce<{ names: string[]; hash: Record<string, string> }>(
.reduce<{ names: string[]; map: Map<string, string> }>(
(agg, value) => {
// This is intentionally using a "hash" and a "push" to be highly optimized with very large indexes
if (agg.hash[value] != null) {
// This is intentionally using a Map to be highly optimized with very large indexes AND be safe for user provided data
if (agg.map.get(value) != null) {
return agg;
} else {
agg.hash[value] = value;
agg.map.set(value, value);
agg.names.push(value);
return agg;
}
},
{ names: [], hash: {} }
{ names: [], map: new Map<string, string>() }
)
.names.map<FieldDescriptor>((name) =>
defaults({}, fieldsFromFieldCapsByName[name], {
Expand All @@ -56,6 +57,7 @@ export async function getFieldCapabilities(
searchable: false,
aggregatable: false,
readFromDocValues: false,
metadata_field: metaFields.includes(name),
})
)
.map(mergeOverrides);
Expand Down
1 change: 1 addition & 0 deletions ...ins/data/server/index_patterns/fetcher/lib/field_capabilities/field_caps_response.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ describe('index_patterns/field_capabilities/field_caps_response', () => {
'searchable',
'aggregatable',
'readFromDocValues',
'metadata_field',
]);
});
}
Expand Down
4 changes: 4 additions & 0 deletions src/plugins/data/server/index_patterns/fetcher/lib/field_capabilities/field_caps_response.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,8 @@ export function readFieldCapsResponse(
}),
{}
),
// @ts-expect-error
metadata_field: capsByType[types[0]].metadata_field,
};
// This is intentionally using a "hash" and a "push" to be highly optimized with very large indexes
agg.array.push(field);
Expand All @@ -131,6 +133,8 @@ export function readFieldCapsResponse(
searchable: isSearchable,
aggregatable: isAggregatable,
readFromDocValues: shouldReadFieldFromDocValues(isAggregatable, esType),
// @ts-expect-error
metadata_field: capsByType[types[0]].metadata_field,
};
// This is intentionally using a "hash" and a "push" to be highly optimized with very large indexes
agg.array.push(field);
Expand Down
9 changes: 9 additions & 0 deletions test/api_integration/apis/index_patterns/fields_for_time_pattern_route/pattern.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ export default function ({ getService }) {
aggregatable: true,
searchable: true,
readFromDocValues: true,
metadata_field: false,
},
{
name: 'Jan01',
Expand All @@ -46,6 +47,7 @@ export default function ({ getService }) {
aggregatable: true,
searchable: true,
readFromDocValues: true,
metadata_field: false,
},
{
name: 'Jan02',
Expand All @@ -54,6 +56,7 @@ export default function ({ getService }) {
aggregatable: true,
searchable: true,
readFromDocValues: true,
metadata_field: false,
},
],
});
Expand All @@ -77,6 +80,7 @@ export default function ({ getService }) {
aggregatable: true,
searchable: true,
readFromDocValues: true,
metadata_field: false,
},
{
name: 'Jan02',
Expand All @@ -85,6 +89,7 @@ export default function ({ getService }) {
aggregatable: true,
searchable: true,
readFromDocValues: true,
metadata_field: false,
},
],
});
Expand All @@ -109,6 +114,7 @@ export default function ({ getService }) {
aggregatable: true,
searchable: true,
readFromDocValues: true,
metadata_field: false,
},
{
name: 'Jan02',
Expand All @@ -117,20 +123,23 @@ export default function ({ getService }) {
aggregatable: true,
searchable: true,
readFromDocValues: true,
metadata_field: false,
},
{
name: 'meta1',
type: 'string',
aggregatable: false,
searchable: false,
readFromDocValues: false,
metadata_field: true,
},
{
name: 'meta2',
type: 'string',
aggregatable: false,
searchable: false,
readFromDocValues: false,
metadata_field: true,
},
],
});
Expand Down
4 changes: 4 additions & 0 deletions test/api_integration/apis/index_patterns/fields_for_wildcard_route/conflicts.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ export default function ({ getService }) {
aggregatable: true,
searchable: true,
readFromDocValues: true,
metadata_field: false,
},
{
name: 'number_conflict',
Expand All @@ -43,6 +44,7 @@ export default function ({ getService }) {
aggregatable: true,
searchable: true,
readFromDocValues: true,
metadata_field: false,
},
{
name: 'string_conflict',
Expand All @@ -51,6 +53,7 @@ export default function ({ getService }) {
aggregatable: true,
searchable: true,
readFromDocValues: false,
metadata_field: false,
},
{
name: 'success',
Expand All @@ -63,6 +66,7 @@ export default function ({ getService }) {
boolean: ['logs-2017.01.02'],
keyword: ['logs-2017.01.01'],
},
metadata_field: false,
},
],
});
Expand Down
13 changes: 13 additions & 0 deletions test/api_integration/apis/index_patterns/fields_for_wildcard_route/response.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ export default function ({ getService }) {
aggregatable: true,
name: 'bar',
readFromDocValues: true,
metadata_field: false,
},
{
type: 'string',
Expand All @@ -33,6 +34,7 @@ export default function ({ getService }) {
aggregatable: false,
name: 'baz',
readFromDocValues: false,
metadata_field: false,
},
{
type: 'string',
Expand All @@ -42,6 +44,7 @@ export default function ({ getService }) {
name: 'baz.keyword',
readFromDocValues: true,
subType: { multi: { parent: 'baz' } },
metadata_field: false,
},
{
type: 'number',
Expand All @@ -50,6 +53,7 @@ export default function ({ getService }) {
aggregatable: true,
name: 'foo',
readFromDocValues: true,
metadata_field: false,
},
{
aggregatable: true,
Expand All @@ -63,6 +67,7 @@ export default function ({ getService }) {
},
},
type: 'string',
metadata_field: false,
},
];

Expand Down Expand Up @@ -100,6 +105,7 @@ export default function ({ getService }) {
readFromDocValues: false,
searchable: true,
type: 'string',
metadata_field: true,
},
{
aggregatable: false,
Expand All @@ -108,6 +114,7 @@ export default function ({ getService }) {
readFromDocValues: false,
searchable: false,
type: '_source',
metadata_field: true,
},
{
type: 'boolean',
Expand All @@ -116,6 +123,7 @@ export default function ({ getService }) {
aggregatable: true,
name: 'bar',
readFromDocValues: true,
metadata_field: false,
},
{
aggregatable: false,
Expand All @@ -124,6 +132,7 @@ export default function ({ getService }) {
readFromDocValues: false,
searchable: true,
type: 'string',
metadata_field: false,
},
{
type: 'string',
Expand All @@ -133,13 +142,15 @@ export default function ({ getService }) {
name: 'baz.keyword',
readFromDocValues: true,
subType: { multi: { parent: 'baz' } },
metadata_field: false,
},
{
aggregatable: false,
name: 'crazy_meta_field',
readFromDocValues: false,
searchable: false,
type: 'string',
metadata_field: true,
},
{
type: 'number',
Expand All @@ -148,6 +159,7 @@ export default function ({ getService }) {
aggregatable: true,
name: 'foo',
readFromDocValues: true,
metadata_field: false,
},
{
aggregatable: true,
Expand All @@ -161,6 +173,7 @@ export default function ({ getService }) {
},
},
type: 'string',
metadata_field: false,
},
],
})
Expand Down