Skip to content

Commit

Permalink
fix: use aggregation search instead of suggest query (#13861)
Browse files Browse the repository at this point in the history 
* fix: use searchAggregates instead of suggest query

* fix: aggregation fields
  • Loading branch information
@karanh37
karanh37 committed Nov 8, 2023
1 parent ee4616a commit 1ea2e68
Show file tree
Hide file tree
Showing 5 changed files with 32 additions and 385 deletions.
99 changes: 28 additions & 71 deletions openmetadata-ui/src/main/resources/ui/src/constants/AdvancedSearch.constants.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
*/

import { t } from 'i18next';
import { isUndefined, uniq } from 'lodash';
import {
BasicConfig,
Fields,
Expand All @@ -24,7 +23,6 @@ import AntdConfig from 'react-awesome-query-builder/lib/config/antd';
import { EntityFields, SuggestionField } from '../enums/AdvancedSearch.enum';
import { SearchIndex } from '../enums/search.enum';
import { getAggregateFieldOptions } from '../rest/miscAPI';
import { suggestQuery } from '../rest/searchAPI';
import { renderAdvanceSearchButtons } from '../utils/AdvancedSearchUtils';
import { getCombinedQueryFilterObject } from '../utils/ExplorePage/ExplorePageUtils';

Expand Down Expand Up @@ -220,64 +218,28 @@ export const emptyJsonTree: JsonTree = {
*/
export const autocomplete: (args: {
searchIndex: SearchIndex | SearchIndex[];
entitySearchIndex: SearchIndex | SearchIndex[];
entityField: EntityFields;
suggestField?: SuggestionField;
}) => SelectFieldSettings['asyncFetch'] = ({
searchIndex,
suggestField,
entitySearchIndex,
entityField,
}) => {
const isUserAndTeamSearchIndex =
searchIndex.includes(SearchIndex.USER) ||
searchIndex.includes(SearchIndex.TEAM);

}) => SelectFieldSettings['asyncFetch'] = ({ searchIndex, entityField }) => {
return (search) => {
if (search) {
return suggestQuery({
query: search ?? '*',
searchIndex: searchIndex,
field: suggestField,
// fetch source if index is type of user or team and both
fetchSource: isUserAndTeamSearchIndex,
}).then((resp) => {
return {
values: uniq(resp).map(({ text, _source }) => {
// set displayName or name if index is type of user or team and both.
// else set the text
const name =
isUserAndTeamSearchIndex && !isUndefined(_source)
? _source?.displayName || _source.name
: text;

return {
value: name,
title: name,
};
}),
hasMore: false,
};
});
} else {
return getAggregateFieldOptions(
entitySearchIndex,
entityField,
'',
JSON.stringify(getCombinedQueryFilterObject())
).then((response) => {
const buckets =
response.data.aggregations[`sterms#${entityField}`].buckets;

return {
values: buckets.map((bucket) => ({
value: bucket.key,
title: bucket.label ?? bucket.key,
})),
hasMore: false,
};
});
}
return getAggregateFieldOptions(
searchIndex,
entityField,
search ?? '',
JSON.stringify(getCombinedQueryFilterObject())
).then((response) => {
const buckets =
response.data.aggregations[`sterms#${entityField}`].buckets;

return {
values: buckets.map((bucket) => ({
value: bucket.key,
title: bucket.label ?? bucket.key,
})),
hasMore: false,
};
});
// }
};
};

Expand Down Expand Up @@ -306,8 +268,10 @@ const getCommonQueryBuilderFields = (

fieldSettings: {
asyncFetch: autocomplete({
searchIndex: [SearchIndex.USER, SearchIndex.TEAM],
entitySearchIndex: [SearchIndex.USER, SearchIndex.TEAM],
searchIndex: entitySearchIndex ?? [
SearchIndex.USER,
SearchIndex.TEAM,
],
entityField: EntityFields.OWNER,
}),
useAsyncSearch: true,
Expand All @@ -320,8 +284,10 @@ const getCommonQueryBuilderFields = (
mainWidgetProps,
fieldSettings: {
asyncFetch: autocomplete({
searchIndex: [SearchIndex.TAG, SearchIndex.GLOSSARY],
entitySearchIndex,
searchIndex: entitySearchIndex ?? [
SearchIndex.TAG,
SearchIndex.GLOSSARY,
],
entityField: EntityFields.TAG,
}),
useAsyncSearch: true,
Expand All @@ -334,8 +300,7 @@ const getCommonQueryBuilderFields = (
mainWidgetProps,
fieldSettings: {
asyncFetch: autocomplete({
searchIndex: [SearchIndex.TAG, SearchIndex.GLOSSARY],
entitySearchIndex,
searchIndex: entitySearchIndex ?? [SearchIndex.TAG],
entityField: EntityFields.TIER,
}),
useAsyncSearch: true,
Expand Down Expand Up @@ -364,9 +329,7 @@ const getServiceQueryBuilderFields = (index: SearchIndex) => {
fieldSettings: {
asyncFetch: autocomplete({
searchIndex: index,
entitySearchIndex: index,
entityField: EntityFields.SERVICE,
suggestField: SuggestionField.SERVICE,
}),
useAsyncSearch: true,
},
Expand All @@ -387,9 +350,7 @@ const tableQueryBuilderFields: Fields = {
fieldSettings: {
asyncFetch: autocomplete({
searchIndex: SearchIndex.TABLE,
entitySearchIndex: SearchIndex.TABLE,
entityField: EntityFields.DATABASE,
suggestField: SuggestionField.DATABASE,
}),
useAsyncSearch: true,
},
Expand All @@ -402,9 +363,7 @@ const tableQueryBuilderFields: Fields = {
fieldSettings: {
asyncFetch: autocomplete({
searchIndex: SearchIndex.TABLE,
entitySearchIndex: SearchIndex.TABLE,
entityField: EntityFields.DATABASE_SCHEMA,
suggestField: SuggestionField.SCHEMA,
}),
useAsyncSearch: true,
},
Expand All @@ -417,9 +376,7 @@ const tableQueryBuilderFields: Fields = {
fieldSettings: {
asyncFetch: autocomplete({
searchIndex: SearchIndex.TABLE,
entitySearchIndex: SearchIndex.TABLE,
entityField: EntityFields.COLUMN,
suggestField: SuggestionField.COLUMN,
}),
useAsyncSearch: true,
},
Expand Down
8 changes: 4 additions & 4 deletions openmetadata-ui/src/main/resources/ui/src/enums/AdvancedSearch.enum.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,10 @@ export enum EntityFields {
OWNER = 'displayName.keyword',
TAG = 'tags.tagFQN',
TIER = 'tier.tagFQN',
SERVICE = 'service.name',
DATABASE = 'database.name',
DATABASE_SCHEMA = 'databaseSchema.name',
COLUMN = 'columns.name',
SERVICE = 'service.name.keyword',
DATABASE = 'database.name.keyword',
DATABASE_SCHEMA = 'databaseSchema.name.keyword',
COLUMN = 'columns.name.keyword',
CHART = 'charts.displayName.keyword',
TASK = 'tasks.displayName.keyword',
}
1 change: 0 additions & 1 deletion openmetadata-ui/src/main/resources/ui/src/pages/DataInsightPage/DataInsightProvider.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@ export const DataInsightContext = createContext<DataInsightContextType>(
);
const fetchTeamSuggestions = autocomplete({
searchIndex: SearchIndex.TEAM,
entitySearchIndex: SearchIndex.TEAM,
entityField: EntityFields.OWNER,
});

Expand Down
170 changes: 0 additions & 170 deletions openmetadata-ui/src/main/resources/ui/src/rest/searchAPI.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,113 +83,9 @@ const mockTableSearchResponse = {
},
};

const mockSuggestUserResponse = {
suggest: {
'metadata-suggest': [
{
text: 'a',
offset: 0,
length: 1,
options: [
{
text: 'Aaron Johnson',
_index: 'user_search_index',
_type: '_doc',
_id: '2cae227c-e2c4-487c-b52c-a96ae242d90d',
_score: 10.0,
_source: {
id: '2cae227c-e2c4-487c-b52c-a96ae242d90d',
name: 'aaron_johnson0',
fullyQualifiedName: 'aaron_johnson0',
displayName: 'Aaron Johnson',
version: 0.1,
updatedAt: 1661336540995,
updatedBy: 'anonymous',
email: 'aaron_johnson0@gmail.com',
href: 'http://localhost:8585/api/v1/users/2cae227c-e2c4-487c-b52c-a96ae242d90d',
isAdmin: false,
deleted: false,
roles: [],
inheritedRoles: [],
entityType: 'user',
teams: null,
some: {
nested: {
nullValue: null,
},
},
},
},
],
},
],
},
};

describe('searchAPI tests', () => {
beforeEach(() => jest.resetModules());

it('suggestQuery should return object and aggregations', async () => {
jest.mock('./index', () => ({
get: jest
.fn()
.mockImplementation(() =>
Promise.resolve({ data: mockTableSearchResponse })
),
}));
// eslint-disable-next-line @typescript-eslint/no-var-requires
const { searchQuery } = require('./searchAPI');
const res = await searchQuery({ searchIndex: SearchIndex.TABLE });

expect(res.hits.total.value).toBe(10_000);

expect(res.hits.hits).toHaveLength(1);
expect(res.hits.hits[0]._index).toEqual(SearchIndex.TABLE);
expect(res.hits.hits[0]._source).toEqual(
expect.objectContaining({
id: '9b30a945-239a-4cb7-93b0-f1b7425aed41',
name: 'raw_product_catalog',
fullyQualifiedName:
'sample_data.ecommerce_db.shopify.raw_product_catalog',
description:
'This is a raw product catalog table contains the product listing, price, seller etc.. represented in our online DB. ',
version: 0.1,
updatedAt: 1661336543968,
updatedBy: 'anonymous',
href: 'http://localhost:8585/api/v1/tables/9b30a945-239a-4cb7-93b0-f1b7425aed41',
tableType: 'Regular',
})
);

expect(res.aggregations).toEqual(
expect.objectContaining({
EntityType: {
buckets: expect.arrayContaining([
{
key: 'table',
doc_count: 10960,
},
]),
},
ServiceName: {
buckets: expect.arrayContaining([
{
key: 'trino',
doc_count: 10924,
},
{
key: 'sample_data',
doc_count: 36,
},
]),
},
Tags: {
buckets: [],
},
})
);
});

it('searchQuery should not return nulls', async () => {
jest.mock('./index', () => ({
get: jest
Expand Down Expand Up @@ -224,70 +120,4 @@ describe('searchAPI tests', () => {

expect(res.hits.hits[0]._source.type).toBe('table');
});

it('suggestQuery should return object and text', async () => {
jest.mock('./index', () => ({
get: jest
.fn()
.mockImplementation(() =>
Promise.resolve({ data: mockSuggestUserResponse })
),
}));
// eslint-disable-next-line @typescript-eslint/no-var-requires
const { suggestQuery } = require('./searchAPI');
const res = await suggestQuery({ searchIndex: SearchIndex.USER });

expect(res).toEqual([
expect.objectContaining({
_index: SearchIndex.USER,
_source: expect.objectContaining({
id: '2cae227c-e2c4-487c-b52c-a96ae242d90d',
name: 'aaron_johnson0',
fullyQualifiedName: 'aaron_johnson0',
displayName: 'Aaron Johnson',
version: 0.1,
updatedAt: 1661336540995,
updatedBy: 'anonymous',
email: 'aaron_johnson0@gmail.com',
href: 'http://localhost:8585/api/v1/users/2cae227c-e2c4-487c-b52c-a96ae242d90d',
isAdmin: false,
deleted: false,
roles: [],
inheritedRoles: [],
entityType: 'user',
}),
}),
]);
});

it('suggestQuery should not return nulls', async () => {
jest.mock('./index', () => ({
get: jest
.fn()
.mockImplementation(() =>
Promise.resolve({ data: mockSuggestUserResponse })
),
}));
// eslint-disable-next-line @typescript-eslint/no-var-requires
const { suggestQuery } = require('./searchAPI');
const res = await suggestQuery({ searchIndex: SearchIndex.USER });

// Deep checking for null values
expect(flatten(res[0]._source).filter(isNull)).toHaveLength(0);
});

it('suggestQuery should have type field', async () => {
jest.mock('./index', () => ({
get: jest
.fn()
.mockImplementation(() =>
Promise.resolve({ data: mockSuggestUserResponse })
),
}));
// eslint-disable-next-line @typescript-eslint/no-var-requires
const { suggestQuery } = require('./searchAPI');
const res = await suggestQuery({ searchIndex: SearchIndex.USER });

expect(res[0]._source.type).toBe('user');
});
});
Loading

0 comments on commit 1ea2e68

Please sign in to comment.