From 4086a8dafab03260085e12b27cd167897aff60ee Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 19 Mar 2024 20:43:36 -0700 Subject: [PATCH] [MDS] Add Vega support for importing saved objects (#6123) (#6214) (#6222) * Add MDS support for Vega * Refactor field to data_source_id * Add to CHANGELOG.md * Added test cases and renamed field to use data_source_name * Add prefix datasource name test case and add example in default hjson * Move CHANGELOG to appropriate section * Increased test coverage of search() method * Add test cases for util function * Add util function to modify Vega Spec * Add method to verify Vega saved object type * Add import saved object support for Vega * Add unit tests for Vega objects in create and conflict modes * Refactored utils test file * Add to CHANGELOG * Use bulkget instead of single get * Add datasource references to the specs * Fix bootstrap errors * Add edge case where title is potentially undefined * Address PR comments * Add more test coverage for checking conflict * Fix unit test --------- (cherry picked from commit de978d4516a715eedf56b4abaa16a7aa11a2a654) # Conflicts: # CHANGELOG.md (cherry picked from commit d14463759f908231cf1dc932d8fdb89a893a82d6) Signed-off-by: github-actions[bot] Co-authored-by: github-actions[bot] Co-authored-by: Ashwin P Chandran --- .../check_conflict_for_data_source.test.ts | 161 +++++++++++- .../import/check_conflict_for_data_source.ts | 47 +++- .../import/create_saved_objects.test.ts | 116 +++++++- .../import/create_saved_objects.ts | 162 +++++++----- .../import/import_saved_objects.test.ts | 1 + .../import/import_saved_objects.ts | 1 + .../vega_spec_with_multiple_urls.hjson | 147 +++++++++++ .../vega_spec_with_multiple_urls.json | 95 +++++++ .../vega_spec_with_multiple_urls_mds.hjson | 148 +++++++++++ .../vega_spec_with_multiple_urls_mds.json | 96 +++++++ .../vega_spec_with_opensearch_query.hjson | 61 +++++ .../vega_spec_with_opensearch_query.json | 36 +++ .../vega_spec_without_opensearch_query.hjson | 117 +++++++++ .../vega_spec_without_opensearch_query.json | 66 +++++ .../server/saved_objects/import/utils.test.ts | 247 ++++++++++++++++++ src/core/server/saved_objects/import/utils.ts | 103 ++++++++ src/plugins/vis_type_vega/server/plugin.ts | 16 +- src/plugins/vis_type_vega/server/services.ts | 10 + .../vega_outdated_references_mds.hjson | 223 ++++++++++++++++ .../vega_spec_up_to_date_urls_mds.hjson | 185 +++++++++++++ .../vega_spec_with_multiple_urls.hjson | 165 ++++++++++++ .../vega_spec_with_multiple_urls.json | 110 ++++++++ .../vega_spec_with_multiple_urls_mds.hjson | 185 +++++++++++++ .../vega_spec_with_multiple_urls_mds.json | 127 +++++++++ .../vega_spec_with_opensearch_query.hjson | 61 +++++ .../vega_spec_with_opensearch_query.json | 36 +++ .../vega_spec_with_opensearch_query_mds.hjson | 62 +++++ .../vega_spec_with_opensearch_query_mds.json | 37 +++ .../vega_spec_without_opensearch_query.hjson | 117 +++++++++ .../vega_spec_without_opensearch_query.json | 66 +++++ src/plugins/vis_type_vega/server/types.ts | 2 + .../vis_type_vega/server/utils.test.ts | 181 +++++++++++++ src/plugins/vis_type_vega/server/utils.ts | 98 +++++++ .../vega_visualization_client_wrapper.test.ts | 243 +++++++++++++++++ .../vega_visualization_client_wrapper.ts | 110 ++++++++ 35 files changed, 3567 insertions(+), 71 deletions(-) create mode 100644 src/core/server/saved_objects/import/test_utils/vega_spec_with_multiple_urls.hjson create mode 100644 src/core/server/saved_objects/import/test_utils/vega_spec_with_multiple_urls.json create mode 100644 src/core/server/saved_objects/import/test_utils/vega_spec_with_multiple_urls_mds.hjson create mode 100644 src/core/server/saved_objects/import/test_utils/vega_spec_with_multiple_urls_mds.json create mode 100644 src/core/server/saved_objects/import/test_utils/vega_spec_with_opensearch_query.hjson create mode 100644 src/core/server/saved_objects/import/test_utils/vega_spec_with_opensearch_query.json create mode 100644 src/core/server/saved_objects/import/test_utils/vega_spec_without_opensearch_query.hjson create mode 100644 src/core/server/saved_objects/import/test_utils/vega_spec_without_opensearch_query.json create mode 100644 src/core/server/saved_objects/import/utils.test.ts create mode 100644 src/core/server/saved_objects/import/utils.ts create mode 100644 src/plugins/vis_type_vega/server/services.ts create mode 100644 src/plugins/vis_type_vega/server/test_utils/vega_outdated_references_mds.hjson create mode 100644 src/plugins/vis_type_vega/server/test_utils/vega_spec_up_to_date_urls_mds.hjson create mode 100644 src/plugins/vis_type_vega/server/test_utils/vega_spec_with_multiple_urls.hjson create mode 100644 src/plugins/vis_type_vega/server/test_utils/vega_spec_with_multiple_urls.json create mode 100644 src/plugins/vis_type_vega/server/test_utils/vega_spec_with_multiple_urls_mds.hjson create mode 100644 src/plugins/vis_type_vega/server/test_utils/vega_spec_with_multiple_urls_mds.json create mode 100644 src/plugins/vis_type_vega/server/test_utils/vega_spec_with_opensearch_query.hjson create mode 100644 src/plugins/vis_type_vega/server/test_utils/vega_spec_with_opensearch_query.json create mode 100644 src/plugins/vis_type_vega/server/test_utils/vega_spec_with_opensearch_query_mds.hjson create mode 100644 src/plugins/vis_type_vega/server/test_utils/vega_spec_with_opensearch_query_mds.json create mode 100644 src/plugins/vis_type_vega/server/test_utils/vega_spec_without_opensearch_query.hjson create mode 100644 src/plugins/vis_type_vega/server/test_utils/vega_spec_without_opensearch_query.json create mode 100644 src/plugins/vis_type_vega/server/utils.test.ts create mode 100644 src/plugins/vis_type_vega/server/utils.ts create mode 100644 src/plugins/vis_type_vega/server/vega_visualization_client_wrapper.test.ts create mode 100644 src/plugins/vis_type_vega/server/vega_visualization_client_wrapper.ts diff --git a/src/core/server/saved_objects/import/check_conflict_for_data_source.test.ts b/src/core/server/saved_objects/import/check_conflict_for_data_source.test.ts index 2b50ba8e9b3..b2a6ae6fda6 100644 --- a/src/core/server/saved_objects/import/check_conflict_for_data_source.test.ts +++ b/src/core/server/saved_objects/import/check_conflict_for_data_source.test.ts @@ -5,7 +5,7 @@ import { mockUuidv4 } from './__mocks__'; import { SavedObjectReference, SavedObjectsImportRetry } from 'opensearch-dashboards/public'; -import { SavedObject } from '../types'; +import { SavedObject, SavedObjectsClientContract } from '../types'; import { SavedObjectsErrorHelpers } from '..'; import { checkConflictsForDataSource, @@ -24,6 +24,45 @@ const createObject = (type: string, id: string): SavedObjectType => ({ references: (Symbol() as unknown) as SavedObjectReference[], }); +const createVegaVisualizationObject = (id: string): SavedObjectType => { + const visState = + id.split('_').length > 1 + ? '{"title":"some-title","type":"vega","aggs":[],"params":{"spec":"{\\n data: {\\n url: {\\n index: example_index\\n data_source_name: old-datasource-title\\n }\\n }\\n}"}}' + : '{"title":"some-title","type":"vega","aggs":[],"params":{"spec":"{\\n data: {\\n url: {\\n index: example_index\\n }\\n }\\n}"}}'; + return { + type: 'visualization', + id, + attributes: { title: 'some-title', visState }, + references: + id.split('_').length > 1 + ? [{ id: id.split('_')[0], type: 'data-source', name: 'dataSource' }] + : [], + } as SavedObjectType; +}; + +const getSavedObjectClient = (): SavedObjectsClientContract => { + const savedObject = {} as SavedObjectsClientContract; + savedObject.get = jest.fn().mockImplementation((type, id) => { + if (type === 'data-source' && id === 'old-datasource-id') { + return Promise.resolve({ + attributes: { + title: 'old-datasource-title', + }, + }); + } else if (type === 'data-source') { + return Promise.resolve({ + attributes: { + title: 'some-datasource-title', + }, + }); + } + + return Promise.resolve(undefined); + }); + + return savedObject; +}; + const getResultMock = { conflict: (type: string, id: string) => { const error = SavedObjectsErrorHelpers.createConflictError(type, id).output.payload; @@ -56,6 +95,7 @@ describe('#checkConflictsForDataSource', () => { retries?: SavedObjectsImportRetry[]; createNewCopies?: boolean; dataSourceId?: string; + savedObjectsClient?: SavedObjectsClientContract; }): ConflictsForDataSourceParams => { return { ...partial }; }; @@ -140,4 +180,123 @@ describe('#checkConflictsForDataSource', () => { importIdMap: new Map(), }); }); + + /* + Vega test cases + */ + it('will attach datasource name to Vega spec when importing from local to datasource', async () => { + const vegaSavedObject = createVegaVisualizationObject('some-object-id'); + const params = setupParams({ + objects: [vegaSavedObject], + ignoreRegularConflicts: true, + dataSourceId: 'some-datasource-id', + savedObjectsClient: getSavedObjectClient(), + }); + const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params); + + expect(params.savedObjectsClient?.get).toHaveBeenCalledWith( + 'data-source', + 'some-datasource-id' + ); + expect(checkConflictsForDataSourceResult).toEqual( + expect.objectContaining({ + filteredObjects: [ + { + ...vegaSavedObject, + attributes: { + title: 'some-title', + visState: + '{"title":"some-title","type":"vega","aggs":[],"params":{"spec":"{\\n data: {\\n url: {\\n index: example_index\\n data_source_name: some-datasource-title\\n }\\n }\\n}"}}', + }, + id: 'some-datasource-id_some-object-id', + references: [ + { + id: 'some-datasource-id', + type: 'data-source', + name: 'dataSource', + }, + ], + }, + ], + errors: [], + importIdMap: new Map([ + [ + `visualization:some-object-id`, + { id: 'some-datasource-id_some-object-id', omitOriginId: true }, + ], + ]), + }) + ); + }); + + it('will not change Vega spec when importing from datasource to different datasource', async () => { + const vegaSavedObject = createVegaVisualizationObject('old-datasource-id_some-object-id'); + const params = setupParams({ + objects: [vegaSavedObject], + ignoreRegularConflicts: true, + dataSourceId: 'some-datasource-id', + savedObjectsClient: getSavedObjectClient(), + }); + const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params); + + expect(params.savedObjectsClient?.get).toHaveBeenCalledWith( + 'data-source', + 'some-datasource-id' + ); + expect(checkConflictsForDataSourceResult).toEqual( + expect.objectContaining({ + filteredObjects: [ + { + ...vegaSavedObject, + attributes: { + title: 'some-title', + visState: + '{"title":"some-title","type":"vega","aggs":[],"params":{"spec":"{\\n data: {\\n url: {\\n index: example_index\\n data_source_name: old-datasource-title\\n }\\n }\\n}"}}', + }, + id: 'some-datasource-id_some-object-id', + }, + ], + errors: [], + importIdMap: new Map([ + [ + `visualization:some-object-id`, + { id: 'some-datasource-id_some-object-id', omitOriginId: true }, + ], + ]), + }) + ); + }); + + it('will not change Vega spec when dataSourceTitle is undefined', async () => { + const vegaSavedObject = createVegaVisualizationObject('old-datasource-id_some-object-id'); + const params = setupParams({ + objects: [vegaSavedObject], + ignoreRegularConflicts: true, + dataSourceId: 'nonexistent-datasource-title-id', + savedObjectsClient: getSavedObjectClient(), + }); + const checkConflictsForDataSourceResult = await checkConflictsForDataSource(params); + + expect(params.savedObjectsClient?.get).toHaveBeenCalledWith( + 'data-source', + 'nonexistent-datasource-title-id' + ); + expect(checkConflictsForDataSourceResult).toEqual( + expect.objectContaining({ + filteredObjects: [ + { + ...vegaSavedObject, + id: 'nonexistent-datasource-title-id_some-object-id', + }, + ], + errors: [], + importIdMap: new Map([ + [ + `visualization:some-object-id`, + { id: 'nonexistent-datasource-title-id_some-object-id', omitOriginId: true }, + ], + ]), + }) + ); + }); }); diff --git a/src/core/server/saved_objects/import/check_conflict_for_data_source.ts b/src/core/server/saved_objects/import/check_conflict_for_data_source.ts index 6611b01dfb2..a0400c57d02 100644 --- a/src/core/server/saved_objects/import/check_conflict_for_data_source.ts +++ b/src/core/server/saved_objects/import/check_conflict_for_data_source.ts @@ -3,13 +3,24 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { SavedObject, SavedObjectsImportError, SavedObjectsImportRetry } from '../types'; +import { + SavedObject, + SavedObjectsClientContract, + SavedObjectsImportError, + SavedObjectsImportRetry, +} from '../types'; +import { + extractVegaSpecFromSavedObject, + getDataSourceTitleFromId, + updateDataSourceNameInVegaSpec, +} from './utils'; export interface ConflictsForDataSourceParams { objects: Array>; ignoreRegularConflicts?: boolean; retries?: SavedObjectsImportRetry[]; dataSourceId?: string; + savedObjectsClient?: SavedObjectsClientContract; } interface ImportIdMapEntry { @@ -31,6 +42,7 @@ export async function checkConflictsForDataSource({ ignoreRegularConflicts, retries = [], dataSourceId, + savedObjectsClient, }: ConflictsForDataSourceParams) { const filteredObjects: Array> = []; const errors: SavedObjectsImportError[] = []; @@ -43,6 +55,12 @@ export async function checkConflictsForDataSource({ (acc, cur) => acc.set(`${cur.type}:${cur.id}`, cur), new Map() ); + + const dataSourceTitle = + !!dataSourceId && !!savedObjectsClient + ? await getDataSourceTitleFromId(dataSourceId, savedObjectsClient) + : undefined; + objects.forEach((object) => { const { type, @@ -74,6 +92,33 @@ export async function checkConflictsForDataSource({ /** * Only update importIdMap and filtered objects */ + + // Some visualization types will need special modifications, like Vega visualizations + if (object.type === 'visualization') { + const vegaSpec = extractVegaSpecFromSavedObject(object); + + if (!!vegaSpec && !!dataSourceTitle) { + const updatedVegaSpec = updateDataSourceNameInVegaSpec({ + spec: vegaSpec, + newDataSourceName: dataSourceTitle, + }); + + // @ts-expect-error + const visStateObject = JSON.parse(object.attributes?.visState); + visStateObject.params.spec = updatedVegaSpec; + + // @ts-expect-error + object.attributes.visState = JSON.stringify(visStateObject); + if (!!dataSourceId) { + object.references.push({ + id: dataSourceId, + name: 'dataSource', + type: 'data-source', + }); + } + } + } + const omitOriginId = ignoreRegularConflicts; importIdMap.set(`${type}:${id}`, { id: `${dataSourceId}_${rawId}`, omitOriginId }); filteredObjects.push({ ...object, id: `${dataSourceId}_${rawId}` }); diff --git a/src/core/server/saved_objects/import/create_saved_objects.test.ts b/src/core/server/saved_objects/import/create_saved_objects.test.ts index f1118842c96..1a9e218f169 100644 --- a/src/core/server/saved_objects/import/create_saved_objects.test.ts +++ b/src/core/server/saved_objects/import/create_saved_objects.test.ts @@ -115,6 +115,37 @@ const visualizationObj = { }, }, }; + +const getVegaVisualizationObj = (id: string) => ({ + type: 'visualization', + id, + attributes: { + title: 'some-title', + visState: + '{"title":"some-title","type":"vega","aggs":[],"params":{"spec":"{\\n data: {\\n url: {\\n index: example_index\\n }\\n }\\n}"}}', + }, + references: [], + namespaces: ['default'], + version: 'some-version', + updated_at: 'some-date', +}); + +const getVegaMDSVisualizationObj = (id: string, dataSourceId: string) => ({ + type: 'visualization', + id: dataSourceId ? `${dataSourceId}_${id}` : id, + attributes: { + title: 'some-other-title', + visState: + '{"title":"some-other-title","type":"vega","aggs":[],"params":{"spec":"{\\n data: {\\n url: {\\n index: example_index\\n data_source_name: old-datasource-title\\n }\\n }\\n}"}}', + }, + references: [ + { + id: dataSourceId, + name: 'dataSource', + type: 'data-source', + }, + ], +}); // non-multi-namespace types shouldn't have origin IDs, but we include test cases to ensure it's handled gracefully // non-multi-namespace types by definition cannot result in an unresolvable conflict, so we don't include test cases for those const importId3 = 'id-foo'; @@ -142,8 +173,11 @@ describe('#createSavedObjects', () => { overwrite?: boolean; dataSourceId?: string; dataSourceTitle?: string; + savedObjectsCustomClient?: jest.Mocked; }): CreateSavedObjectsParams => { - savedObjectsClient = savedObjectsClientMock.create(); + savedObjectsClient = !!partial.savedObjectsCustomClient + ? partial.savedObjectsCustomClient + : savedObjectsClientMock.create(); bulkCreate = savedObjectsClient.bulkCreate; return { accumulatedErrors: [], ...partial, savedObjectsClient, importIdMap }; }; @@ -490,6 +524,29 @@ describe('#createSavedObjects', () => { expect(results).toEqual(expectedResultsWithDataSource); }; + const testVegaVisualizationsWithDataSources = async (params: { + objects: SavedObject[]; + expectedFilteredObjects: Array>; + dataSourceId?: string; + dataSourceTitle?: string; + }) => { + const savedObjectsCustomClient = savedObjectsClientMock.create(); + + const options = setupParams({ + ...params, + savedObjectsCustomClient, + }); + savedObjectsCustomClient.bulkCreate = jest.fn().mockResolvedValue({ + saved_objects: params.objects.map((obj) => { + return getResultMock.success(obj, options); + }), + }); + + const results = await createSavedObjects(options); + + expect(results.createdObjects).toMatchObject(params.expectedFilteredObjects); + }; + describe('with an undefined namespace', () => { test('calls bulkCreate once with input objects', async () => { await testBulkCreateObjects(); @@ -546,4 +603,61 @@ describe('#createSavedObjects', () => { ); }); }); + + describe('with a data source for Vega saved objects', () => { + test('can attach a data source name to the Vega spec if there is a local query', async () => { + const objects = [getVegaVisualizationObj('some-vega-id')]; + const expectedObject = getVegaVisualizationObj('some-vega-id'); + const expectedFilteredObjects = [ + { + ...expectedObject, + attributes: { + title: 'some-title_dataSourceName', + visState: + '{"title":"some-title","type":"vega","aggs":[],"params":{"spec":"{\\n data: {\\n url: {\\n index: example_index\\n data_source_name: dataSourceName\\n }\\n }\\n}"}}', + }, + id: 'some-vega-id', + references: [ + { + id: 'some-datasource-id', + type: 'data-source', + name: 'dataSource', + }, + ], + }, + ]; + await testVegaVisualizationsWithDataSources({ + objects, + expectedFilteredObjects, + dataSourceId: 'some-datasource-id', + dataSourceTitle: 'dataSourceName', + }); + }); + + test('will not update the data source name in the Vega spec if no local cluster queries', async () => { + const objects = [getVegaMDSVisualizationObj('some-vega-id', 'old-datasource-id')]; + const expectedObject = getVegaMDSVisualizationObj('some-vega-id', 'old-datasource-id'); + expectedObject.references.push({ + id: 'some-datasource-id', + name: 'dataSource', + type: 'data-source', + }); + const expectedFilteredObjects = [ + { + ...expectedObject, + attributes: { + title: 'some-other-title_dataSourceName', + visState: + '{"title":"some-other-title","type":"vega","aggs":[],"params":{"spec":"{\\n data: {\\n url: {\\n index: example_index\\n data_source_name: old-datasource-title\\n }\\n }\\n}"}}', + }, + }, + ]; + await testVegaVisualizationsWithDataSources({ + objects, + expectedFilteredObjects, + dataSourceId: 'some-datasource-id', + dataSourceTitle: 'dataSourceName', + }); + }); + }); }); diff --git a/src/core/server/saved_objects/import/create_saved_objects.ts b/src/core/server/saved_objects/import/create_saved_objects.ts index 6fd08520281..eab33d6c19b 100644 --- a/src/core/server/saved_objects/import/create_saved_objects.ts +++ b/src/core/server/saved_objects/import/create_saved_objects.ts @@ -31,6 +31,7 @@ import { SavedObject, SavedObjectsClientContract, SavedObjectsImportError } from '../types'; import { extractErrors } from './extract_errors'; import { CreatedObject } from './types'; +import { extractVegaSpecFromSavedObject, updateDataSourceNameInVegaSpec } from './utils'; interface CreateSavedObjectsParams { objects: Array>; @@ -80,89 +81,116 @@ export const createSavedObjects = async ({ ); // filter out the 'version' field of each object, if it exists - - const objectsToCreate = filteredObjects.map(({ version, ...object }) => { - if (dataSourceId) { - // @ts-expect-error - if (dataSourceTitle && object.attributes.title) { - if ( - object.type === 'dashboard' || - object.type === 'visualization' || - object.type === 'search' - ) { - // @ts-expect-error - object.attributes.title = object.attributes.title + `_${dataSourceTitle}`; + const objectsToCreate = await Promise.all( + filteredObjects.map(({ version, ...object }) => { + if (dataSourceId) { + // @ts-expect-error + if (dataSourceTitle && object.attributes.title) { + if ( + object.type === 'dashboard' || + object.type === 'visualization' || + object.type === 'search' + ) { + // @ts-expect-error + object.attributes.title = object.attributes.title + `_${dataSourceTitle}`; + } } - } - if (object.type === 'index-pattern') { - object.references = [ - { - id: `${dataSourceId}`, - type: 'data-source', - name: 'dataSource', - }, - ]; - } + // Some visualization types will need special modifications, like Vega visualizations + if (object.type === 'visualization') { + const vegaSpec = extractVegaSpecFromSavedObject(object); - if (object.type === 'visualization' || object.type === 'search') { - // @ts-expect-error - const searchSourceString = object.attributes?.kibanaSavedObjectMeta?.searchSourceJSON; - // @ts-expect-error - const visStateString = object.attributes?.visState; + if (!!vegaSpec && !!dataSourceTitle) { + const updatedVegaSpec = updateDataSourceNameInVegaSpec({ + spec: vegaSpec, + newDataSourceName: dataSourceTitle, + }); - if (searchSourceString) { - const searchSource = JSON.parse(searchSourceString); - if (searchSource.index) { - const searchSourceIndex = searchSource.index.includes('_') - ? searchSource.index.split('_')[searchSource.index.split('_').length - 1] - : searchSource.index; - searchSource.index = `${dataSourceId}_` + searchSourceIndex; + // @ts-expect-error + const visStateObject = JSON.parse(object.attributes?.visState); + visStateObject.params.spec = updatedVegaSpec; // @ts-expect-error - object.attributes.kibanaSavedObjectMeta.searchSourceJSON = JSON.stringify(searchSource); + object.attributes.visState = JSON.stringify(visStateObject); + object.references.push({ + id: `${dataSourceId}`, + type: 'data-source', + name: 'dataSource', + }); } } - if (visStateString) { - const visState = JSON.parse(visStateString); - const controlList = visState.params?.controls; - if (controlList) { + if (object.type === 'index-pattern') { + object.references = [ + { + id: `${dataSourceId}`, + type: 'data-source', + name: 'dataSource', + }, + ]; + } + + if (object.type === 'visualization' || object.type === 'search') { + // @ts-expect-error + const searchSourceString = object.attributes?.kibanaSavedObjectMeta?.searchSourceJSON; + // @ts-expect-error + const visStateString = object.attributes?.visState; + + if (searchSourceString) { + const searchSource = JSON.parse(searchSourceString); + if (searchSource.index) { + const searchSourceIndex = searchSource.index.includes('_') + ? searchSource.index.split('_')[searchSource.index.split('_').length - 1] + : searchSource.index; + searchSource.index = `${dataSourceId}_` + searchSourceIndex; + + // @ts-expect-error + object.attributes.kibanaSavedObjectMeta.searchSourceJSON = JSON.stringify( + searchSource + ); + } + } + + if (visStateString) { + const visState = JSON.parse(visStateString); + const controlList = visState.params?.controls; + if (controlList) { + // @ts-expect-error + controlList.map((control) => { + if (control.indexPattern) { + const controlIndexPattern = control.indexPattern.includes('_') + ? control.indexPattern.split('_')[control.indexPattern.split('_').length - 1] + : control.indexPattern; + control.indexPattern = `${dataSourceId}_` + controlIndexPattern; + } + }); + } // @ts-expect-error - controlList.map((control) => { - if (control.indexPattern) { - const controlIndexPattern = control.indexPattern.includes('_') - ? control.indexPattern.split('_')[control.indexPattern.split('_').length - 1] - : control.indexPattern; - control.indexPattern = `${dataSourceId}_` + controlIndexPattern; - } - }); + object.attributes.visState = JSON.stringify(visState); } - // @ts-expect-error - object.attributes.visState = JSON.stringify(visState); } } - } - // use the import ID map to ensure that each reference is being created with the correct ID - const references = object.references?.map((reference) => { - const { type, id } = reference; - const importIdEntry = importIdMap.get(`${type}:${id}`); + // use the import ID map to ensure that each reference is being created with the correct ID + const references = object.references?.map((reference) => { + const { type, id } = reference; + const importIdEntry = importIdMap.get(`${type}:${id}`); + if (importIdEntry?.id) { + return { ...reference, id: importIdEntry.id }; + } + return reference; + }); + // use the import ID map to ensure that each object is being created with the correct ID, also ensure that the `originId` is set on + // the created object if it did not have one (or is omitted if specified) + const importIdEntry = importIdMap.get(`${object.type}:${object.id}`); if (importIdEntry?.id) { - return { ...reference, id: importIdEntry.id }; + objectIdMap.set(`${object.type}:${importIdEntry.id}`, object); + const originId = importIdEntry.omitOriginId ? undefined : object.originId ?? object.id; + return { ...object, id: importIdEntry.id, originId, ...(references && { references }) }; } - return reference; - }); - // use the import ID map to ensure that each object is being created with the correct ID, also ensure that the `originId` is set on - // the created object if it did not have one (or is omitted if specified) - const importIdEntry = importIdMap.get(`${object.type}:${object.id}`); - if (importIdEntry?.id) { - objectIdMap.set(`${object.type}:${importIdEntry.id}`, object); - const originId = importIdEntry.omitOriginId ? undefined : object.originId ?? object.id; - return { ...object, id: importIdEntry.id, originId, ...(references && { references }) }; - } - return { ...object, ...(references && { references }) }; - }); + return { ...object, ...(references && { references }) }; + }) + ); const resolvableErrors = ['conflict', 'ambiguous_conflict', 'missing_references']; let expectedResults = objectsToCreate; if (!accumulatedErrors.some(({ error: { type } }) => resolvableErrors.includes(type))) { diff --git a/src/core/server/saved_objects/import/import_saved_objects.test.ts b/src/core/server/saved_objects/import/import_saved_objects.test.ts index 3dda6931bd1..fff5b60c89c 100644 --- a/src/core/server/saved_objects/import/import_saved_objects.test.ts +++ b/src/core/server/saved_objects/import/import_saved_objects.test.ts @@ -259,6 +259,7 @@ describe('#importSavedObjectsFromStream', () => { objects: collectedObjects, ignoreRegularConflicts: overwrite, dataSourceId: testDataSourceId, + savedObjectsClient, }; expect(checkConflictsForDataSource).toHaveBeenCalledWith(checkConflictsForDataSourceParams); }); diff --git a/src/core/server/saved_objects/import/import_saved_objects.ts b/src/core/server/saved_objects/import/import_saved_objects.ts index a5744478fd7..fac10acc0f9 100644 --- a/src/core/server/saved_objects/import/import_saved_objects.ts +++ b/src/core/server/saved_objects/import/import_saved_objects.ts @@ -117,6 +117,7 @@ export async function importSavedObjectsFromStream({ objects: checkConflictsResult.filteredObjects, ignoreRegularConflicts: overwrite, dataSourceId, + savedObjectsClient, }); checkOriginConflictsParams.objects = checkConflictsForDataSourceResult.filteredObjects; } diff --git a/src/core/server/saved_objects/import/test_utils/vega_spec_with_multiple_urls.hjson b/src/core/server/saved_objects/import/test_utils/vega_spec_with_multiple_urls.hjson new file mode 100644 index 00000000000..98e791db851 --- /dev/null +++ b/src/core/server/saved_objects/import/test_utils/vega_spec_with_multiple_urls.hjson @@ -0,0 +1,147 @@ +{ + $schema: https://vega.github.io/schema/vega/v5.json + width: 800 + height: 600 + padding: 5 + signals: [ + { + name: mapType + value: topojson + } + ] + // Every data source type that Dashboards supports + data: [ + { + name: exampleIndexSource + url: { + %context%: true + %timefield%: @timestamp + index: your_index_name + body: { + size: 1000 + query: { + match_all: { + } + } + } + } + format: { + property: hits.hits + } + } + { + name: urlData + url: https://example.com/data.json + format: { + type: json + } + } + { + name: topojsonData + url: https://example.com/map.topojson + format: { + type: topojson + feature: your_feature_name + } + } + { + name: geojsonData + url: https://example.com/map.geojson + format: { + type: json + } + } + ] + projections: [ + { + name: projection + type: { + signal: mapType + } + } + ] + marks: [ + { + type: symbol + from: { + data: exampleIndexSource + } + encode: { + enter: { + x: { + field: _source.location.lon + } + y: { + field: _source.location.lat + } + size: { + value: 50 + } + fill: { + value: steelblue + } + stroke: { + value: white + } + tooltip: { + signal: datum._source.name + } + } + } + } + { + type: symbol + from: { + data: urlData + } + encode: { + enter: { + x: { + field: longitude + } + y: { + field: latitude + } + size: { + value: 50 + } + fill: { + value: green + } + stroke: { + value: white + } + tooltip: { + field: name + } + } + } + } + { + type: shape + from: { + data: topojsonData + } + encode: { + enter: { + fill: { + value: lightgray + } + } + } + } + { + type: shape + from: { + data: geojsonData + } + encode: { + enter: { + fill: { + value: lightblue + } + } + } + } + ] +} diff --git a/src/core/server/saved_objects/import/test_utils/vega_spec_with_multiple_urls.json b/src/core/server/saved_objects/import/test_utils/vega_spec_with_multiple_urls.json new file mode 100644 index 00000000000..8ec22019e82 --- /dev/null +++ b/src/core/server/saved_objects/import/test_utils/vega_spec_with_multiple_urls.json @@ -0,0 +1,95 @@ +{ + "$schema": "https://vega.github.io/schema/vega/v5.json", + "width": 800, + "height": 600, + "padding": 5, + "signals": [ + {"name": "mapType", "value": "topojson"} + ], + "data": [ + { + "name": "exampleIndexSource", + "url": { + "%context%": true, + "%timefield%": "@timestamp", + "index": "your_index_name", + "body": { + "size": 1000, + "query": { + "match_all": {} + } + } + }, + "format": {"property": "hits.hits"} + }, + { + "name": "urlData", + "url": "https://example.com/data.json", + "format": {"type": "json"} + }, + { + "name": "topojsonData", + "url": "https://example.com/map.topojson", + "format": {"type": "topojson", "feature": "your_feature_name"} + }, + { + "name": "geojsonData", + "url": "https://example.com/map.geojson", + "format": {"type": "json"} + } + ], + "projections": [ + { + "name": "projection", + "type": {"signal": "mapType"} + } + ], + "marks": [ + { + "type": "symbol", + "from": {"data": "exampleIndexSource"}, + "encode": { + "enter": { + "x": {"field": "_source.location.lon"}, + "y": {"field": "_source.location.lat"}, + "size": {"value": 50}, + "fill": {"value": "steelblue"}, + "stroke": {"value": "white"}, + "tooltip": {"signal": "datum._source.name"} + } + } + }, + { + "type": "symbol", + "from": {"data": "urlData"}, + "encode": { + "enter": { + "x": {"field": "longitude"}, + "y": {"field": "latitude"}, + "size": {"value": 50}, + "fill": {"value": "green"}, + "stroke": {"value": "white"}, + "tooltip": {"field": "name"} + } + } + }, + { + "type": "shape", + "from": {"data": "topojsonData"}, + "encode": { + "enter": { + "fill": {"value": "lightgray"} + } + } + }, + { + "type": "shape", + "from": {"data": "geojsonData"}, + "encode": { + "enter": { + "fill": {"value": "lightblue"} + } + } + } + ] +} diff --git a/src/core/server/saved_objects/import/test_utils/vega_spec_with_multiple_urls_mds.hjson b/src/core/server/saved_objects/import/test_utils/vega_spec_with_multiple_urls_mds.hjson new file mode 100644 index 00000000000..6cf4dcb16db --- /dev/null +++ b/src/core/server/saved_objects/import/test_utils/vega_spec_with_multiple_urls_mds.hjson @@ -0,0 +1,148 @@ +{ + $schema: https://vega.github.io/schema/vega/v5.json + width: 800 + height: 600 + padding: 5 + signals: [ + { + name: mapType + value: topojson + } + ] + // Every data source type that Dashboards supports + data: [ + { + name: exampleIndexSource + url: { + %context%: true + %timefield%: @timestamp + index: your_index_name + data_source_name: some datasource name + body: { + size: 1000 + query: { + match_all: { + } + } + } + } + format: { + property: hits.hits + } + } + { + name: urlData + url: https://example.com/data.json + format: { + type: json + } + } + { + name: topojsonData + url: https://example.com/map.topojson + format: { + type: topojson + feature: your_feature_name + } + } + { + name: geojsonData + url: https://example.com/map.geojson + format: { + type: json + } + } + ] + projections: [ + { + name: projection + type: { + signal: mapType + } + } + ] + marks: [ + { + type: symbol + from: { + data: exampleIndexSource + } + encode: { + enter: { + x: { + field: _source.location.lon + } + y: { + field: _source.location.lat + } + size: { + value: 50 + } + fill: { + value: steelblue + } + stroke: { + value: white + } + tooltip: { + signal: datum._source.name + } + } + } + } + { + type: symbol + from: { + data: urlData + } + encode: { + enter: { + x: { + field: longitude + } + y: { + field: latitude + } + size: { + value: 50 + } + fill: { + value: green + } + stroke: { + value: white + } + tooltip: { + field: name + } + } + } + } + { + type: shape + from: { + data: topojsonData + } + encode: { + enter: { + fill: { + value: lightgray + } + } + } + } + { + type: shape + from: { + data: geojsonData + } + encode: { + enter: { + fill: { + value: lightblue + } + } + } + } + ] +} diff --git a/src/core/server/saved_objects/import/test_utils/vega_spec_with_multiple_urls_mds.json b/src/core/server/saved_objects/import/test_utils/vega_spec_with_multiple_urls_mds.json new file mode 100644 index 00000000000..41c14b07991 --- /dev/null +++ b/src/core/server/saved_objects/import/test_utils/vega_spec_with_multiple_urls_mds.json @@ -0,0 +1,96 @@ +{ + "$schema": "https://vega.github.io/schema/vega/v5.json", + "width": 800, + "height": 600, + "padding": 5, + "signals": [ + {"name": "mapType", "value": "topojson"} + ], + "data": [ + { + "name": "exampleIndexSource", + "url": { + "%context%": true, + "%timefield%": "@timestamp", + "index": "your_index_name", + "data_source_name": "some datasource name", + "body": { + "size": 1000, + "query": { + "match_all": {} + } + } + }, + "format": {"property": "hits.hits"} + }, + { + "name": "urlData", + "url": "https://example.com/data.json", + "format": {"type": "json"} + }, + { + "name": "topojsonData", + "url": "https://example.com/map.topojson", + "format": {"type": "topojson", "feature": "your_feature_name"} + }, + { + "name": "geojsonData", + "url": "https://example.com/map.geojson", + "format": {"type": "json"} + } + ], + "projections": [ + { + "name": "projection", + "type": {"signal": "mapType"} + } + ], + "marks": [ + { + "type": "symbol", + "from": {"data": "exampleIndexSource"}, + "encode": { + "enter": { + "x": {"field": "_source.location.lon"}, + "y": {"field": "_source.location.lat"}, + "size": {"value": 50}, + "fill": {"value": "steelblue"}, + "stroke": {"value": "white"}, + "tooltip": {"signal": "datum._source.name"} + } + } + }, + { + "type": "symbol", + "from": {"data": "urlData"}, + "encode": { + "enter": { + "x": {"field": "longitude"}, + "y": {"field": "latitude"}, + "size": {"value": 50}, + "fill": {"value": "green"}, + "stroke": {"value": "white"}, + "tooltip": {"field": "name"} + } + } + }, + { + "type": "shape", + "from": {"data": "topojsonData"}, + "encode": { + "enter": { + "fill": {"value": "lightgray"} + } + } + }, + { + "type": "shape", + "from": {"data": "geojsonData"}, + "encode": { + "enter": { + "fill": {"value": "lightblue"} + } + } + } + ] +} diff --git a/src/core/server/saved_objects/import/test_utils/vega_spec_with_opensearch_query.hjson b/src/core/server/saved_objects/import/test_utils/vega_spec_with_opensearch_query.hjson new file mode 100644 index 00000000000..17f3f2e482e --- /dev/null +++ b/src/core/server/saved_objects/import/test_utils/vega_spec_with_opensearch_query.hjson @@ -0,0 +1,61 @@ +{ + $schema: https://vega.github.io/schema/vega-lite/v5.json + data: { + url: { + %context%: true + %timefield%: @timestamp + index: opensearch_dashboards_sample_data_logs + body: { + aggs: { + 1: { + terms: { + field: geo.dest + order: { + _count: desc + } + size: 5 + } + } + } + } + } + format: { + property: aggregations.1.buckets + } + } + transform: [ + { + calculate: datum.key + as: dest + } + { + calculate: datum.doc_count + as: count + } + ] + layer: [ + { + mark: { + type: bar + tooltip: true + } + } + ] + encoding: { + x: { + field: count + type: quantitative + axis: { + title: Count + } + } + y: { + field: dest + type: nominal + axis: { + title: Dest + } + sort: -x + } + } +} diff --git a/src/core/server/saved_objects/import/test_utils/vega_spec_with_opensearch_query.json b/src/core/server/saved_objects/import/test_utils/vega_spec_with_opensearch_query.json new file mode 100644 index 00000000000..49392f5de16 --- /dev/null +++ b/src/core/server/saved_objects/import/test_utils/vega_spec_with_opensearch_query.json @@ -0,0 +1,36 @@ +{ + "$schema": "https://vega.github.io/schema/vega-lite/v5.json", + "data": { + "url": { + "%context%": true, + "%timefield%": "@timestamp", + "index": "opensearch_dashboards_sample_data_logs", + "body": { + "aggs": { + "1": { + "terms": { + "field": "geo.dest", + "order": {"_count": "desc"}, + "size": 5 + } + } + } + } + }, + "format": {"property": "aggregations.1.buckets"} + }, + "transform": [ + {"calculate": "datum.key", "as": "dest"}, + {"calculate": "datum.doc_count", "as": "count"} + ], + "layer": [{"mark": {"type": "bar", "tooltip": true}}], + "encoding": { + "x": {"field": "count", "type": "quantitative", "axis": {"title": "Count"}}, + "y": { + "field": "dest", + "type": "nominal", + "axis": {"title": "Dest"}, + "sort": "-x" + } + } +} diff --git a/src/core/server/saved_objects/import/test_utils/vega_spec_without_opensearch_query.hjson b/src/core/server/saved_objects/import/test_utils/vega_spec_without_opensearch_query.hjson new file mode 100644 index 00000000000..8c4a0193ba9 --- /dev/null +++ b/src/core/server/saved_objects/import/test_utils/vega_spec_without_opensearch_query.hjson @@ -0,0 +1,117 @@ +{ + $schema: https://vega.github.io/schema/vega/v5.json + width: 400 + height: 200 + padding: 5 + // Data contained entirely within the spec + data: [ + { + name: table + values: [ + { + category: A + count: 28 + } + { + category: B + count: 55 + } + { + category: C + count: 43 + } + { + category: D + count: 91 + } + { + category: E + count: 81 + } + { + category: F + count: 53 + } + { + category: G + count: 19 + } + { + category: H + count: 87 + } + ] + } + ] + scales: [ + { + name: xscale + type: band + domain: { + data: table + field: category + } + range: width + padding: 0.05 + round: true + } + { + name: yscale + type: linear + domain: { + data: table + field: count + } + range: height + nice: true + } + ] + axes: [ + { + orient: bottom + scale: xscale + } + { + orient: left + scale: yscale + } + ] + marks: [ + { + type: rect + from: { + data: table + } + encode: { + enter: { + x: { + scale: xscale + field: category + } + width: { + scale: xscale + band: 1 + } + y: { + scale: yscale + field: count + } + y2: { + scale: yscale + value: 0 + } + } + update: { + fill: { + value: steelblue + } + } + hover: { + fill: { + value: red + } + } + } + } + ] +} diff --git a/src/core/server/saved_objects/import/test_utils/vega_spec_without_opensearch_query.json b/src/core/server/saved_objects/import/test_utils/vega_spec_without_opensearch_query.json new file mode 100644 index 00000000000..d24b9b20737 --- /dev/null +++ b/src/core/server/saved_objects/import/test_utils/vega_spec_without_opensearch_query.json @@ -0,0 +1,66 @@ +{ + "$schema": "https://vega.github.io/schema/vega/v5.json", + "width": 400, + "height": 200, + "padding": 5, + + "data": [ + { + "name": "table", + "values": [ + {"category": "A", "count": 28}, + {"category": "B", "count": 55}, + {"category": "C", "count": 43}, + {"category": "D", "count": 91}, + {"category": "E", "count": 81}, + {"category": "F", "count": 53}, + {"category": "G", "count": 19}, + {"category": "H", "count": 87} + ] + } + ], + + "scales": [ + { + "name": "xscale", + "type": "band", + "domain": {"data": "table", "field": "category"}, + "range": "width", + "padding": 0.05, + "round": true + }, + { + "name": "yscale", + "type": "linear", + "domain": {"data": "table", "field": "count"}, + "range": "height", + "nice": true + } + ], + + "axes": [ + { "orient": "bottom", "scale": "xscale" }, + { "orient": "left", "scale": "yscale" } + ], + + "marks": [ + { + "type": "rect", + "from": {"data": "table"}, + "encode": { + "enter": { + "x": {"scale": "xscale", "field": "category"}, + "width": {"scale": "xscale", "band": 1}, + "y": {"scale": "yscale", "field": "count"}, + "y2": {"scale": "yscale", "value": 0} + }, + "update": { + "fill": {"value": "steelblue"} + }, + "hover": { + "fill": {"value": "red"} + } + } + } + ] + } diff --git a/src/core/server/saved_objects/import/utils.test.ts b/src/core/server/saved_objects/import/utils.test.ts new file mode 100644 index 00000000000..604b6f6d473 --- /dev/null +++ b/src/core/server/saved_objects/import/utils.test.ts @@ -0,0 +1,247 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { readFileSync } from 'fs'; +import { + extractVegaSpecFromSavedObject, + getDataSourceTitleFromId, + updateDataSourceNameInVegaSpec, +} from './utils'; +import { parse } from 'hjson'; +import { isEqual } from 'lodash'; +import { join } from 'path'; +import { SavedObject, SavedObjectsClientContract } from '../types'; + +describe('updateDataSourceNameInVegaSpec()', () => { + const loadHJSONStringFromFile = (filepath: string) => { + return readFileSync(join(__dirname, filepath)).toString(); + }; + + const loadJSONFromFile = (filepath: string) => { + return JSON.parse(readFileSync(join(__dirname, filepath)).toString()); + }; + + /* + JSON Test cases + */ + test('(JSON) When data has only one url body and it is an opensearch query, add data_source_name field to the spec', () => { + const openSearchQueryJSON = loadJSONFromFile( + './test_utils/vega_spec_with_opensearch_query.json' + ); + const jsonString = JSON.stringify(openSearchQueryJSON); + const modifiedString = JSON.parse( + updateDataSourceNameInVegaSpec({ spec: jsonString, newDataSourceName: 'newDataSource' }) + ); + + expect(modifiedString.data.url.hasOwnProperty('data_source_name')).toBe(true); + expect(modifiedString.data.url.data_source_name).toBe('newDataSource'); + + // These fields should be unchanged + Object.keys(openSearchQueryJSON).forEach((field) => { + if (field !== 'data') { + expect( + isEqual( + modifiedString[field as keyof typeof openSearchQueryJSON], + openSearchQueryJSON[field as keyof typeof openSearchQueryJSON] + ) + ).toBe(true); + } + }); + }); + + test('(JSON) When data has only one url body and it is not an opensearch query, change nothing', () => { + const nonOpenSearchQueryJSON = loadJSONFromFile( + './test_utils/vega_spec_without_opensearch_query.json' + ); + const jsonString = JSON.stringify(nonOpenSearchQueryJSON); + const modifiedJSON = JSON.parse( + updateDataSourceNameInVegaSpec({ spec: jsonString, newDataSourceName: 'noDataSource' }) + ); + expect(isEqual(modifiedJSON, nonOpenSearchQueryJSON)).toBe(true); + }); + + test('(JSON) When data has multiple url bodies, make sure only opensearch queries are updated with data_source_names', () => { + const multipleDataSourcesJSON = loadJSONFromFile( + './test_utils/vega_spec_with_multiple_urls.json' + ); + const jsonString = JSON.stringify(multipleDataSourcesJSON); + const modifiedString = JSON.parse( + updateDataSourceNameInVegaSpec({ spec: jsonString, newDataSourceName: 'newDataSource' }) + ); + + expect(modifiedString.data.length).toBe(multipleDataSourcesJSON.data.length); + for (let i = 0; i < modifiedString.data.length; i++) { + const originalUrlBody = multipleDataSourcesJSON.data[i]; + const urlBody = modifiedString.data[i]; + + if (urlBody.name !== 'exampleIndexSource') { + expect(isEqual(originalUrlBody, urlBody)).toBe(true); + } else { + expect(urlBody.url.hasOwnProperty('data_source_name')).toBe(true); + expect(urlBody.url.data_source_name).toBe('newDataSource'); + } + } + + // These fields should be unchanged + Object.keys(multipleDataSourcesJSON).forEach((field) => { + if (field !== 'data') { + expect( + isEqual( + modifiedString[field as keyof typeof multipleDataSourcesJSON], + multipleDataSourcesJSON[field as keyof typeof multipleDataSourcesJSON] + ) + ).toBe(true); + } + }); + }); + + test('(JSON) When an MDS object does not reference local queries, return the same spec', () => { + const multipleDataSourcesJSONMds = loadJSONFromFile( + './test_utils/vega_spec_with_multiple_urls_mds.json' + ); + const jsonString = JSON.stringify(multipleDataSourcesJSONMds); + const modifiedJSON = JSON.parse( + updateDataSourceNameInVegaSpec({ spec: jsonString, newDataSourceName: 'noDataSource' }) + ); + expect(isEqual(modifiedJSON, multipleDataSourcesJSONMds)).toBe(true); + }); + + /* + HJSON Test cases + */ + test('(HJSON) When data has only one url body and it is an opensearch query, add data_source_name field to the spec', () => { + const hjsonString = loadHJSONStringFromFile( + '/test_utils/vega_spec_with_opensearch_query.hjson' + ); + + const originalHJSONParse = parse(hjsonString, { keepWsc: true }); + const hjsonParse = parse( + updateDataSourceNameInVegaSpec({ spec: hjsonString, newDataSourceName: 'newDataSource' }), + { + keepWsc: true, + } + ); + + expect(hjsonParse.data.url.hasOwnProperty('data_source_name')).toBe(true); + expect(hjsonParse.data.url.data_source_name).toBe('newDataSource'); + + // These fields should be unchanged + Object.keys(originalHJSONParse).forEach((field) => { + if (field !== 'data') { + expect( + isEqual( + originalHJSONParse[field as keyof typeof originalHJSONParse], + hjsonParse[field as keyof typeof originalHJSONParse] + ) + ).toBe(true); + } + }); + }); + + test('(HJSON) When data has only one url body and it is not an opensearch query, change nothing', () => { + const hjsonString = loadHJSONStringFromFile( + '/test_utils/vega_spec_without_opensearch_query.hjson' + ); + const originalHJSONParse = parse(hjsonString, { keepWsc: true }); + const hjsonParse = parse( + updateDataSourceNameInVegaSpec({ spec: hjsonString, newDataSourceName: 'noDataSource' }) + ); + + expect(isEqual(originalHJSONParse, hjsonParse)).toBe(true); + }); + + test('(HJSON) When data has multiple url bodies, make sure only opensearch queries are updated with data_source_names', () => { + const hjsonString = loadHJSONStringFromFile('/test_utils/vega_spec_with_multiple_urls.hjson'); + const originalHJSONParse = parse(hjsonString, { keepWsc: true }); + const hjsonParse = parse( + updateDataSourceNameInVegaSpec({ spec: hjsonString, newDataSourceName: 'newDataSource' }) + ); + + expect(hjsonParse.data.length).toBe(originalHJSONParse.data.length); + for (let i = 0; i < hjsonParse.data.length; i++) { + const originalUrlBody = originalHJSONParse.data[i]; + const urlBody = hjsonParse.data[i]; + + if (urlBody.name !== 'exampleIndexSource') { + expect(isEqual(originalUrlBody, urlBody)).toBe(true); + } else { + expect(urlBody.url.hasOwnProperty('data_source_name')).toBe(true); + expect(urlBody.url.data_source_name).toBe('newDataSource'); + } + } + + // These fields should be unchanged + Object.keys(originalHJSONParse).forEach((field) => { + if (field !== 'data') { + expect( + isEqual( + originalHJSONParse[field as keyof typeof originalHJSONParse], + hjsonParse[field as keyof typeof originalHJSONParse] + ) + ).toBe(true); + } + }); + }); + + test('(HJSON) When an MDS object does not reference local queries, return the same spec', () => { + const hjsonString = loadHJSONStringFromFile( + '/test_utils/vega_spec_with_multiple_urls_mds.hjson' + ); + const originalHJSONParse = parse(hjsonString, { keepWsc: true }); + const hjsonParse = parse( + updateDataSourceNameInVegaSpec({ spec: hjsonString, newDataSourceName: 'newDataSource' }) + ); + + expect(isEqual(originalHJSONParse, hjsonParse)).toBe(true); + }); +}); + +describe('extractVegaSpecFromSavedObject()', () => { + test('For a Vega visualization saved object, return its spec', () => { + const spec = 'some-vega-spec'; + const vegaSavedObject = { + attributes: { + visState: `{"type": "vega", "params": {"spec": "${spec}"}}`, + }, + } as SavedObject; + + expect(extractVegaSpecFromSavedObject(vegaSavedObject)).toBe(spec); + }); + + test('For another saved object type, return undefined', () => { + const nonVegaSavedObject = { + attributes: { + visState: `{"type": "area", "params": {"spec": "some-spec"}}`, + }, + } as SavedObject; + + expect(extractVegaSpecFromSavedObject(nonVegaSavedObject)).toBe(undefined); + }); +}); + +describe('getDataSourceTitleFromId()', () => { + const savedObjectsClient = {} as SavedObjectsClientContract; + savedObjectsClient.get = jest.fn().mockImplementation((type, id) => { + if (type === 'data-source' && id === 'valid-id') { + return Promise.resolve({ + attributes: { + title: 'some-datasource-title', + }, + }); + } + + return Promise.resolve({}); + }); + + test('When a valid id is passed, return the correct title', async () => { + expect(await getDataSourceTitleFromId('valid-id', savedObjectsClient)).toBe( + 'some-datasource-title' + ); + }); + + test('When a nonexistent id is passed, return nothing', async () => { + expect(await getDataSourceTitleFromId('nonexistent-id', savedObjectsClient)).toBe(undefined); + }); +}); diff --git a/src/core/server/saved_objects/import/utils.ts b/src/core/server/saved_objects/import/utils.ts new file mode 100644 index 00000000000..9bb1d10cd0e --- /dev/null +++ b/src/core/server/saved_objects/import/utils.ts @@ -0,0 +1,103 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { parse, stringify } from 'hjson'; +import { SavedObject, SavedObjectsClientContract } from '../types'; + +export interface UpdateDataSourceNameInVegaSpecProps { + spec: string; + newDataSourceName: string; +} + +export const updateDataSourceNameInVegaSpec = ( + props: UpdateDataSourceNameInVegaSpecProps +): string => { + const { spec } = props; + + let parsedSpec = parseJSONSpec(spec); + const isJSONString = !!parsedSpec; + if (!parsedSpec) { + parsedSpec = parse(spec, { keepWsc: true }); + } + + const dataField = parsedSpec.data; + + if (dataField instanceof Array) { + parsedSpec.data = dataField.map((dataObject) => { + return updateDataSourceNameForDataObject(dataObject, props); + }); + } else if (dataField instanceof Object) { + parsedSpec.data = updateDataSourceNameForDataObject(dataField, props); + } else { + throw new Error(`"data" field should be an object or an array of objects`); + } + + return isJSONString + ? JSON.stringify(parsedSpec) + : stringify(parsedSpec, { + bracesSameLine: true, + keepWsc: true, + }); +}; + +export const getDataSourceTitleFromId = async ( + dataSourceId: string, + savedObjectsClient: SavedObjectsClientContract +) => { + return await savedObjectsClient.get('data-source', dataSourceId).then((response) => { + // @ts-expect-error + return response?.attributes?.title ?? undefined; + }); +}; + +export const extractVegaSpecFromSavedObject = (savedObject: SavedObject) => { + if (isVegaVisualization(savedObject)) { + // @ts-expect-error + const visStateObject = JSON.parse(savedObject.attributes?.visState); + return visStateObject.params.spec; + } + + return undefined; +}; + +const isVegaVisualization = (savedObject: SavedObject) => { + // @ts-expect-error + const visState = savedObject.attributes?.visState; + if (!!visState) { + const visStateObject = JSON.parse(visState); + return !!visStateObject.type && visStateObject.type === 'vega'; + } + return false; +}; + +const updateDataSourceNameForDataObject = ( + dataObject: any, + props: UpdateDataSourceNameInVegaSpecProps +) => { + const { newDataSourceName } = props; + if ( + dataObject.hasOwnProperty('url') && + dataObject.url.hasOwnProperty('index') && + !dataObject.url.hasOwnProperty('data_source_name') + ) { + dataObject.url.data_source_name = newDataSourceName; + } + + return dataObject; +}; + +const parseJSONSpec = (spec: string) => { + try { + const jsonSpec = JSON.parse(spec); + + if (jsonSpec && typeof jsonSpec === 'object') { + return jsonSpec; + } + } catch (e) { + return undefined; + } + + return undefined; +}; diff --git a/src/plugins/vis_type_vega/server/plugin.ts b/src/plugins/vis_type_vega/server/plugin.ts index cf333921169..4451cb70a28 100644 --- a/src/plugins/vis_type_vega/server/plugin.ts +++ b/src/plugins/vis_type_vega/server/plugin.ts @@ -36,6 +36,11 @@ import { VisTypeVegaPluginSetup, VisTypeVegaPluginStart, } from './types'; +import { + VEGA_VISUALIZATION_CLIENT_WRAPPER_ID, + vegaVisualizationClientWrapper, +} from './vega_visualization_client_wrapper'; +import { setDataSourceEnabled } from './services'; export class VisTypeVegaPlugin implements Plugin { private readonly config: ConfigObservable; @@ -44,10 +49,19 @@ export class VisTypeVegaPlugin implements Plugin('DataSource'); diff --git a/src/plugins/vis_type_vega/server/test_utils/vega_outdated_references_mds.hjson b/src/plugins/vis_type_vega/server/test_utils/vega_outdated_references_mds.hjson new file mode 100644 index 00000000000..5b23c66e67f --- /dev/null +++ b/src/plugins/vis_type_vega/server/test_utils/vega_outdated_references_mds.hjson @@ -0,0 +1,223 @@ +{ + $schema: https://vega.github.io/schema/vega/v5.json + width: 800 + height: 600 + padding: 5 + signals: [ + { + name: mapType + value: topojson + } + ] + // Every data source type that Dashboards supports + data: [ + { + name: localExampleIndexSource + url: { + %context%: true + %timefield%: @timestamp + index: local_index_name + body: { + size: 1000 + query: { + match_all: { + } + } + } + } + format: { + property: hits.hits + } + } + { + name: otherExampleIndexSource + url: { + %context%: true + %timefield%: @timestamp + index: your_other_index_name + data_source_name: a-title + body: { + size: 1000 + query: { + match_all: { + } + } + } + } + format: { + property: hits.hits + } + } + { + name: exampleIndexSource + url: { + %context%: true + %timefield%: @timestamp + index: your_index_name + data_source_name: b-title + body: { + size: 1000 + query: { + match_all: { + } + } + } + } + format: { + property: hits.hits + } + } + { + name: exampleIndexSourceC + url: { + %context%: true + %timefield%: @timestamp + index: your_index_name_c + data_source_name: c-title + body: { + size: 1000 + query: { + match_all: { + } + } + } + } + format: { + property: hits.hits + } + } + { + name: exampleIndexSourceD + url: { + %context%: true + %timefield%: @timestamp + index: your_index_name_d + data_source_name: d-title + body: { + size: 1000 + query: { + match_all: { + } + } + } + } + format: { + property: hits.hits + } + } + { + name: urlData + url: https://example.com/data.json + format: { + type: json + } + } + { + name: topojsonData + url: https://example.com/map.topojson + format: { + type: topojson + feature: your_feature_name + } + } + { + name: geojsonData + url: https://example.com/map.geojson + format: { + type: json + } + } + ] + projections: [ + { + name: projection + type: { + signal: mapType + } + } + ] + marks: [ + { + type: symbol + from: { + data: exampleIndexSource + } + encode: { + enter: { + x: { + field: _source.location.lon + } + y: { + field: _source.location.lat + } + size: { + value: 50 + } + fill: { + value: steelblue + } + stroke: { + value: white + } + tooltip: { + signal: datum._source.name + } + } + } + } + { + type: symbol + from: { + data: urlData + } + encode: { + enter: { + x: { + field: longitude + } + y: { + field: latitude + } + size: { + value: 50 + } + fill: { + value: green + } + stroke: { + value: white + } + tooltip: { + field: name + } + } + } + } + { + type: shape + from: { + data: topojsonData + } + encode: { + enter: { + fill: { + value: lightgray + } + } + } + } + { + type: shape + from: { + data: geojsonData + } + encode: { + enter: { + fill: { + value: lightblue + } + } + } + } + ] +} diff --git a/src/plugins/vis_type_vega/server/test_utils/vega_spec_up_to_date_urls_mds.hjson b/src/plugins/vis_type_vega/server/test_utils/vega_spec_up_to_date_urls_mds.hjson new file mode 100644 index 00000000000..8336fe9ac7d --- /dev/null +++ b/src/plugins/vis_type_vega/server/test_utils/vega_spec_up_to_date_urls_mds.hjson @@ -0,0 +1,185 @@ +{ + $schema: https://vega.github.io/schema/vega/v5.json + width: 800 + height: 600 + padding: 5 + signals: [ + { + name: mapType + value: topojson + } + ] + // Every data source type that Dashboards supports + data: [ + { + name: localExampleIndexSource + url: { + %context%: true + %timefield%: @timestamp + index: local_index_name + body: { + size: 1000 + query: { + match_all: { + } + } + } + } + format: { + property: hits.hits + } + } + { + name: otherExampleIndexSource + url: { + %context%: true + %timefield%: @timestamp + index: your_other_index_name + data_source_name: a-title + body: { + size: 1000 + query: { + match_all: { + } + } + } + } + format: { + property: hits.hits + } + } + { + name: exampleIndexSource + url: { + %context%: true + %timefield%: @timestamp + index: your_index_name + data_source_name: b-title + body: { + size: 1000 + query: { + match_all: { + } + } + } + } + format: { + property: hits.hits + } + } + { + name: urlData + url: https://example.com/data.json + format: { + type: json + } + } + { + name: topojsonData + url: https://example.com/map.topojson + format: { + type: topojson + feature: your_feature_name + } + } + { + name: geojsonData + url: https://example.com/map.geojson + format: { + type: json + } + } + ] + projections: [ + { + name: projection + type: { + signal: mapType + } + } + ] + marks: [ + { + type: symbol + from: { + data: exampleIndexSource + } + encode: { + enter: { + x: { + field: _source.location.lon + } + y: { + field: _source.location.lat + } + size: { + value: 50 + } + fill: { + value: steelblue + } + stroke: { + value: white + } + tooltip: { + signal: datum._source.name + } + } + } + } + { + type: symbol + from: { + data: urlData + } + encode: { + enter: { + x: { + field: longitude + } + y: { + field: latitude + } + size: { + value: 50 + } + fill: { + value: green + } + stroke: { + value: white + } + tooltip: { + field: name + } + } + } + } + { + type: shape + from: { + data: topojsonData + } + encode: { + enter: { + fill: { + value: lightgray + } + } + } + } + { + type: shape + from: { + data: geojsonData + } + encode: { + enter: { + fill: { + value: lightblue + } + } + } + } + ] +} diff --git a/src/plugins/vis_type_vega/server/test_utils/vega_spec_with_multiple_urls.hjson b/src/plugins/vis_type_vega/server/test_utils/vega_spec_with_multiple_urls.hjson new file mode 100644 index 00000000000..d8085c5923f --- /dev/null +++ b/src/plugins/vis_type_vega/server/test_utils/vega_spec_with_multiple_urls.hjson @@ -0,0 +1,165 @@ +{ + $schema: https://vega.github.io/schema/vega/v5.json + width: 800 + height: 600 + padding: 5 + signals: [ + { + name: mapType + value: topojson + } + ] + // Every data source type that Dashboards supports + data: [ + { + name: otherExampleIndexSource + url: { + %context%: true + %timefield%: @timestamp + index: your_other_index_name + body: { + size: 1000 + query: { + match_all: { + } + } + } + } + format: { + property: hits.hits + } + } + { + name: exampleIndexSource + url: { + %context%: true + %timefield%: @timestamp + index: your_index_name + body: { + size: 1000 + query: { + match_all: { + } + } + } + } + format: { + property: hits.hits + } + } + { + name: urlData + url: https://example.com/data.json + format: { + type: json + } + } + { + name: topojsonData + url: https://example.com/map.topojson + format: { + type: topojson + feature: your_feature_name + } + } + { + name: geojsonData + url: https://example.com/map.geojson + format: { + type: json + } + } + ] + projections: [ + { + name: projection + type: { + signal: mapType + } + } + ] + marks: [ + { + type: symbol + from: { + data: exampleIndexSource + } + encode: { + enter: { + x: { + field: _source.location.lon + } + y: { + field: _source.location.lat + } + size: { + value: 50 + } + fill: { + value: steelblue + } + stroke: { + value: white + } + tooltip: { + signal: datum._source.name + } + } + } + } + { + type: symbol + from: { + data: urlData + } + encode: { + enter: { + x: { + field: longitude + } + y: { + field: latitude + } + size: { + value: 50 + } + fill: { + value: green + } + stroke: { + value: white + } + tooltip: { + field: name + } + } + } + } + { + type: shape + from: { + data: topojsonData + } + encode: { + enter: { + fill: { + value: lightgray + } + } + } + } + { + type: shape + from: { + data: geojsonData + } + encode: { + enter: { + fill: { + value: lightblue + } + } + } + } + ] +} diff --git a/src/plugins/vis_type_vega/server/test_utils/vega_spec_with_multiple_urls.json b/src/plugins/vis_type_vega/server/test_utils/vega_spec_with_multiple_urls.json new file mode 100644 index 00000000000..440fc26784e --- /dev/null +++ b/src/plugins/vis_type_vega/server/test_utils/vega_spec_with_multiple_urls.json @@ -0,0 +1,110 @@ +{ + "$schema": "https://vega.github.io/schema/vega/v5.json", + "width": 800, + "height": 600, + "padding": 5, + "signals": [ + {"name": "mapType", "value": "topojson"} + ], + "data": [ + { + "name": "otherExampleIndexSource", + "url": { + "%context%": true, + "%timefield%": "@timestamp", + "index": "your_other_index_name", + "body": { + "size": 1000, + "query": { + "match_all": {} + } + } + }, + "format": {"property": "hits.hits"} + }, + { + "name": "exampleIndexSource", + "url": { + "%context%": true, + "%timefield%": "@timestamp", + "index": "your_index_name", + "body": { + "size": 1000, + "query": { + "match_all": {} + } + } + }, + "format": {"property": "hits.hits"} + }, + { + "name": "urlData", + "url": "https://example.com/data.json", + "format": {"type": "json"} + }, + { + "name": "topojsonData", + "url": "https://example.com/map.topojson", + "format": {"type": "topojson", "feature": "your_feature_name"} + }, + { + "name": "geojsonData", + "url": "https://example.com/map.geojson", + "format": {"type": "json"} + } + ], + "projections": [ + { + "name": "projection", + "type": {"signal": "mapType"} + } + ], + "marks": [ + { + "type": "symbol", + "from": {"data": "exampleIndexSource"}, + "encode": { + "enter": { + "x": {"field": "_source.location.lon"}, + "y": {"field": "_source.location.lat"}, + "size": {"value": 50}, + "fill": {"value": "steelblue"}, + "stroke": {"value": "white"}, + "tooltip": {"signal": "datum._source.name"} + } + } + }, + { + "type": "symbol", + "from": {"data": "urlData"}, + "encode": { + "enter": { + "x": {"field": "longitude"}, + "y": {"field": "latitude"}, + "size": {"value": 50}, + "fill": {"value": "green"}, + "stroke": {"value": "white"}, + "tooltip": {"field": "name"} + } + } + }, + { + "type": "shape", + "from": {"data": "topojsonData"}, + "encode": { + "enter": { + "fill": {"value": "lightgray"} + } + } + }, + { + "type": "shape", + "from": {"data": "geojsonData"}, + "encode": { + "enter": { + "fill": {"value": "lightblue"} + } + } + } + ] +} diff --git a/src/plugins/vis_type_vega/server/test_utils/vega_spec_with_multiple_urls_mds.hjson b/src/plugins/vis_type_vega/server/test_utils/vega_spec_with_multiple_urls_mds.hjson new file mode 100644 index 00000000000..b92cdfca988 --- /dev/null +++ b/src/plugins/vis_type_vega/server/test_utils/vega_spec_with_multiple_urls_mds.hjson @@ -0,0 +1,185 @@ +{ + $schema: https://vega.github.io/schema/vega/v5.json + width: 800 + height: 600 + padding: 5 + signals: [ + { + name: mapType + value: topojson + } + ] + // Every data source type that Dashboards supports + data: [ + { + name: localExampleIndexSource + url: { + %context%: true + %timefield%: @timestamp + index: local_index_name + body: { + size: 1000 + query: { + match_all: { + } + } + } + } + format: { + property: hits.hits + } + } + { + name: otherExampleIndexSource + url: { + %context%: true + %timefield%: @timestamp + index: your_other_index_name + data_source_name: some other datasource name + body: { + size: 1000 + query: { + match_all: { + } + } + } + } + format: { + property: hits.hits + } + } + { + name: exampleIndexSource + url: { + %context%: true + %timefield%: @timestamp + index: your_index_name + data_source_name: some datasource name + body: { + size: 1000 + query: { + match_all: { + } + } + } + } + format: { + property: hits.hits + } + } + { + name: urlData + url: https://example.com/data.json + format: { + type: json + } + } + { + name: topojsonData + url: https://example.com/map.topojson + format: { + type: topojson + feature: your_feature_name + } + } + { + name: geojsonData + url: https://example.com/map.geojson + format: { + type: json + } + } + ] + projections: [ + { + name: projection + type: { + signal: mapType + } + } + ] + marks: [ + { + type: symbol + from: { + data: exampleIndexSource + } + encode: { + enter: { + x: { + field: _source.location.lon + } + y: { + field: _source.location.lat + } + size: { + value: 50 + } + fill: { + value: steelblue + } + stroke: { + value: white + } + tooltip: { + signal: datum._source.name + } + } + } + } + { + type: symbol + from: { + data: urlData + } + encode: { + enter: { + x: { + field: longitude + } + y: { + field: latitude + } + size: { + value: 50 + } + fill: { + value: green + } + stroke: { + value: white + } + tooltip: { + field: name + } + } + } + } + { + type: shape + from: { + data: topojsonData + } + encode: { + enter: { + fill: { + value: lightgray + } + } + } + } + { + type: shape + from: { + data: geojsonData + } + encode: { + enter: { + fill: { + value: lightblue + } + } + } + } + ] +} diff --git a/src/plugins/vis_type_vega/server/test_utils/vega_spec_with_multiple_urls_mds.json b/src/plugins/vis_type_vega/server/test_utils/vega_spec_with_multiple_urls_mds.json new file mode 100644 index 00000000000..3e883388bc5 --- /dev/null +++ b/src/plugins/vis_type_vega/server/test_utils/vega_spec_with_multiple_urls_mds.json @@ -0,0 +1,127 @@ +{ + "$schema": "https://vega.github.io/schema/vega/v5.json", + "width": 800, + "height": 600, + "padding": 5, + "signals": [ + {"name": "mapType", "value": "topojson"} + ], + "data": [ + { + "name": "localExampleIndexSource", + "url": { + "%context%": true, + "%timefield%": "@timestamp", + "index": "local_index_name", + "body": { + "size": 1000, + "query": { + "match_all": {} + } + } + }, + "format": {"property": "hits.hits"} + }, + { + "name": "otherExampleIndexSource", + "url": { + "%context%": true, + "%timefield%": "@timestamp", + "index": "your_other_index_name", + "data_source_name": "some other datasource name", + "body": { + "size": 1000, + "query": { + "match_all": {} + } + } + }, + "format": {"property": "hits.hits"} + }, + { + "name": "exampleIndexSource", + "url": { + "%context%": true, + "%timefield%": "@timestamp", + "index": "your_index_name", + "data_source_name": "some datasource name", + "body": { + "size": 1000, + "query": { + "match_all": {} + } + } + }, + "format": {"property": "hits.hits"} + }, + { + "name": "urlData", + "url": "https://example.com/data.json", + "format": {"type": "json"} + }, + { + "name": "topojsonData", + "url": "https://example.com/map.topojson", + "format": {"type": "topojson", "feature": "your_feature_name"} + }, + { + "name": "geojsonData", + "url": "https://example.com/map.geojson", + "format": {"type": "json"} + } + ], + "projections": [ + { + "name": "projection", + "type": {"signal": "mapType"} + } + ], + "marks": [ + { + "type": "symbol", + "from": {"data": "exampleIndexSource"}, + "encode": { + "enter": { + "x": {"field": "_source.location.lon"}, + "y": {"field": "_source.location.lat"}, + "size": {"value": 50}, + "fill": {"value": "steelblue"}, + "stroke": {"value": "white"}, + "tooltip": {"signal": "datum._source.name"} + } + } + }, + { + "type": "symbol", + "from": {"data": "urlData"}, + "encode": { + "enter": { + "x": {"field": "longitude"}, + "y": {"field": "latitude"}, + "size": {"value": 50}, + "fill": {"value": "green"}, + "stroke": {"value": "white"}, + "tooltip": {"field": "name"} + } + } + }, + { + "type": "shape", + "from": {"data": "topojsonData"}, + "encode": { + "enter": { + "fill": {"value": "lightgray"} + } + } + }, + { + "type": "shape", + "from": {"data": "geojsonData"}, + "encode": { + "enter": { + "fill": {"value": "lightblue"} + } + } + } + ] +} diff --git a/src/plugins/vis_type_vega/server/test_utils/vega_spec_with_opensearch_query.hjson b/src/plugins/vis_type_vega/server/test_utils/vega_spec_with_opensearch_query.hjson new file mode 100644 index 00000000000..17f3f2e482e --- /dev/null +++ b/src/plugins/vis_type_vega/server/test_utils/vega_spec_with_opensearch_query.hjson @@ -0,0 +1,61 @@ +{ + $schema: https://vega.github.io/schema/vega-lite/v5.json + data: { + url: { + %context%: true + %timefield%: @timestamp + index: opensearch_dashboards_sample_data_logs + body: { + aggs: { + 1: { + terms: { + field: geo.dest + order: { + _count: desc + } + size: 5 + } + } + } + } + } + format: { + property: aggregations.1.buckets + } + } + transform: [ + { + calculate: datum.key + as: dest + } + { + calculate: datum.doc_count + as: count + } + ] + layer: [ + { + mark: { + type: bar + tooltip: true + } + } + ] + encoding: { + x: { + field: count + type: quantitative + axis: { + title: Count + } + } + y: { + field: dest + type: nominal + axis: { + title: Dest + } + sort: -x + } + } +} diff --git a/src/plugins/vis_type_vega/server/test_utils/vega_spec_with_opensearch_query.json b/src/plugins/vis_type_vega/server/test_utils/vega_spec_with_opensearch_query.json new file mode 100644 index 00000000000..49392f5de16 --- /dev/null +++ b/src/plugins/vis_type_vega/server/test_utils/vega_spec_with_opensearch_query.json @@ -0,0 +1,36 @@ +{ + "$schema": "https://vega.github.io/schema/vega-lite/v5.json", + "data": { + "url": { + "%context%": true, + "%timefield%": "@timestamp", + "index": "opensearch_dashboards_sample_data_logs", + "body": { + "aggs": { + "1": { + "terms": { + "field": "geo.dest", + "order": {"_count": "desc"}, + "size": 5 + } + } + } + } + }, + "format": {"property": "aggregations.1.buckets"} + }, + "transform": [ + {"calculate": "datum.key", "as": "dest"}, + {"calculate": "datum.doc_count", "as": "count"} + ], + "layer": [{"mark": {"type": "bar", "tooltip": true}}], + "encoding": { + "x": {"field": "count", "type": "quantitative", "axis": {"title": "Count"}}, + "y": { + "field": "dest", + "type": "nominal", + "axis": {"title": "Dest"}, + "sort": "-x" + } + } +} diff --git a/src/plugins/vis_type_vega/server/test_utils/vega_spec_with_opensearch_query_mds.hjson b/src/plugins/vis_type_vega/server/test_utils/vega_spec_with_opensearch_query_mds.hjson new file mode 100644 index 00000000000..7f307e84b0a --- /dev/null +++ b/src/plugins/vis_type_vega/server/test_utils/vega_spec_with_opensearch_query_mds.hjson @@ -0,0 +1,62 @@ +{ + $schema: https://vega.github.io/schema/vega-lite/v5.json + data: { + url: { + %context%: true + %timefield%: @timestamp + index: opensearch_dashboards_sample_data_logs + data_source_name: example data source + body: { + aggs: { + 1: { + terms: { + field: geo.dest + order: { + _count: desc + } + size: 5 + } + } + } + } + } + format: { + property: aggregations.1.buckets + } + } + transform: [ + { + calculate: datum.key + as: dest + } + { + calculate: datum.doc_count + as: count + } + ] + layer: [ + { + mark: { + type: bar + tooltip: true + } + } + ] + encoding: { + x: { + field: count + type: quantitative + axis: { + title: Count + } + } + y: { + field: dest + type: nominal + axis: { + title: Dest + } + sort: -x + } + } +} diff --git a/src/plugins/vis_type_vega/server/test_utils/vega_spec_with_opensearch_query_mds.json b/src/plugins/vis_type_vega/server/test_utils/vega_spec_with_opensearch_query_mds.json new file mode 100644 index 00000000000..7b90845be17 --- /dev/null +++ b/src/plugins/vis_type_vega/server/test_utils/vega_spec_with_opensearch_query_mds.json @@ -0,0 +1,37 @@ +{ + "$schema": "https://vega.github.io/schema/vega-lite/v5.json", + "data": { + "url": { + "%context%": true, + "%timefield%": "@timestamp", + "index": "opensearch_dashboards_sample_data_logs", + "data_source_name": "example data source", + "body": { + "aggs": { + "1": { + "terms": { + "field": "geo.dest", + "order": {"_count": "desc"}, + "size": 5 + } + } + } + } + }, + "format": {"property": "aggregations.1.buckets"} + }, + "transform": [ + {"calculate": "datum.key", "as": "dest"}, + {"calculate": "datum.doc_count", "as": "count"} + ], + "layer": [{"mark": {"type": "bar", "tooltip": true}}], + "encoding": { + "x": {"field": "count", "type": "quantitative", "axis": {"title": "Count"}}, + "y": { + "field": "dest", + "type": "nominal", + "axis": {"title": "Dest"}, + "sort": "-x" + } + } +} diff --git a/src/plugins/vis_type_vega/server/test_utils/vega_spec_without_opensearch_query.hjson b/src/plugins/vis_type_vega/server/test_utils/vega_spec_without_opensearch_query.hjson new file mode 100644 index 00000000000..8c4a0193ba9 --- /dev/null +++ b/src/plugins/vis_type_vega/server/test_utils/vega_spec_without_opensearch_query.hjson @@ -0,0 +1,117 @@ +{ + $schema: https://vega.github.io/schema/vega/v5.json + width: 400 + height: 200 + padding: 5 + // Data contained entirely within the spec + data: [ + { + name: table + values: [ + { + category: A + count: 28 + } + { + category: B + count: 55 + } + { + category: C + count: 43 + } + { + category: D + count: 91 + } + { + category: E + count: 81 + } + { + category: F + count: 53 + } + { + category: G + count: 19 + } + { + category: H + count: 87 + } + ] + } + ] + scales: [ + { + name: xscale + type: band + domain: { + data: table + field: category + } + range: width + padding: 0.05 + round: true + } + { + name: yscale + type: linear + domain: { + data: table + field: count + } + range: height + nice: true + } + ] + axes: [ + { + orient: bottom + scale: xscale + } + { + orient: left + scale: yscale + } + ] + marks: [ + { + type: rect + from: { + data: table + } + encode: { + enter: { + x: { + scale: xscale + field: category + } + width: { + scale: xscale + band: 1 + } + y: { + scale: yscale + field: count + } + y2: { + scale: yscale + value: 0 + } + } + update: { + fill: { + value: steelblue + } + } + hover: { + fill: { + value: red + } + } + } + } + ] +} diff --git a/src/plugins/vis_type_vega/server/test_utils/vega_spec_without_opensearch_query.json b/src/plugins/vis_type_vega/server/test_utils/vega_spec_without_opensearch_query.json new file mode 100644 index 00000000000..d24b9b20737 --- /dev/null +++ b/src/plugins/vis_type_vega/server/test_utils/vega_spec_without_opensearch_query.json @@ -0,0 +1,66 @@ +{ + "$schema": "https://vega.github.io/schema/vega/v5.json", + "width": 400, + "height": 200, + "padding": 5, + + "data": [ + { + "name": "table", + "values": [ + {"category": "A", "count": 28}, + {"category": "B", "count": 55}, + {"category": "C", "count": 43}, + {"category": "D", "count": 91}, + {"category": "E", "count": 81}, + {"category": "F", "count": 53}, + {"category": "G", "count": 19}, + {"category": "H", "count": 87} + ] + } + ], + + "scales": [ + { + "name": "xscale", + "type": "band", + "domain": {"data": "table", "field": "category"}, + "range": "width", + "padding": 0.05, + "round": true + }, + { + "name": "yscale", + "type": "linear", + "domain": {"data": "table", "field": "count"}, + "range": "height", + "nice": true + } + ], + + "axes": [ + { "orient": "bottom", "scale": "xscale" }, + { "orient": "left", "scale": "yscale" } + ], + + "marks": [ + { + "type": "rect", + "from": {"data": "table"}, + "encode": { + "enter": { + "x": {"scale": "xscale", "field": "category"}, + "width": {"scale": "xscale", "band": 1}, + "y": {"scale": "yscale", "field": "count"}, + "y2": {"scale": "yscale", "value": 0} + }, + "update": { + "fill": {"value": "steelblue"} + }, + "hover": { + "fill": {"value": "red"} + } + } + } + ] + } diff --git a/src/plugins/vis_type_vega/server/types.ts b/src/plugins/vis_type_vega/server/types.ts index 9695f6dc23d..bcf4120577a 100644 --- a/src/plugins/vis_type_vega/server/types.ts +++ b/src/plugins/vis_type_vega/server/types.ts @@ -29,6 +29,7 @@ */ import { Observable } from 'rxjs'; +import { DataSourcePluginSetup } from 'src/plugins/data_source/server'; import { HomeServerPluginSetup } from '../../home/server'; import { UsageCollectionSetup } from '../../usage_collection/server'; @@ -45,6 +46,7 @@ export interface VegaSavedObjectAttributes { export interface VisTypeVegaPluginSetupDependencies { usageCollection?: UsageCollectionSetup; home?: HomeServerPluginSetup; + dataSource?: DataSourcePluginSetup; } // eslint-disable-next-line @typescript-eslint/no-empty-interface diff --git a/src/plugins/vis_type_vega/server/utils.test.ts b/src/plugins/vis_type_vega/server/utils.test.ts new file mode 100644 index 00000000000..73d0f82954c --- /dev/null +++ b/src/plugins/vis_type_vega/server/utils.test.ts @@ -0,0 +1,181 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + extractDataSourceNamesInVegaSpec, + extractVegaSpecFromAttributes, + findDataSourceIdbyName, +} from './utils'; +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { SavedObjectsClientContract, SavedObjectsFindOptions } from 'src/core/server'; + +describe('findDataSourceIdbyName()', () => { + const savedObjectsClient = {} as SavedObjectsClientContract; + savedObjectsClient.find = jest.fn().mockImplementation((query: SavedObjectsFindOptions) => { + if (query.search === `"uniqueDataSource"`) { + return Promise.resolve({ + total: 1, + saved_objects: [{ id: 'some-datasource-id', attributes: { title: 'uniqueDataSource' } }], + }); + } else if (query.search === `"duplicateDataSource"`) { + return Promise.resolve({ + total: 2, + saved_objects: [ + { id: 'some-datasource-id', attributes: { title: 'duplicateDataSource' } }, + { id: 'some-other-datasource-id', attributes: { title: 'duplicateDataSource' } }, + ], + }); + } else if (query.search === `"DataSource"`) { + return Promise.resolve({ + total: 2, + saved_objects: [ + { id: 'some-datasource-id', attributes: { title: 'DataSource' } }, + { id: 'some-other-datasource-id', attributes: { title: 'DataSource Copy' } }, + ], + }); + } else { + return Promise.resolve({ + total: 0, + saved_objects: [], + }); + } + }); + + test('If no matching dataSourceName, then throw error', () => { + expect( + findDataSourceIdbyName({ dataSourceName: 'nonexistentDataSource', savedObjectsClient }) + ).rejects.toThrowError( + 'Expected exactly 1 result for data_source_name "nonexistentDataSource" but got 0 results' + ); + }); + + test('If duplicate dataSourceNames, then throw error', () => { + expect( + findDataSourceIdbyName({ dataSourceName: 'duplicateDataSource', savedObjectsClient }) + ).rejects.toThrowError( + 'Expected exactly 1 result for data_source_name "duplicateDataSource" but got 2 results' + ); + }); + + test('If dataSource is enabled but only one dataSourceName, then return id', async () => { + expect( + await findDataSourceIdbyName({ dataSourceName: 'uniqueDataSource', savedObjectsClient }) + ).toBe('some-datasource-id'); + }); + + test('If dataSource is enabled and the dataSourceName is a prefix of another, ensure the prefix is only returned', async () => { + expect(await findDataSourceIdbyName({ dataSourceName: 'DataSource', savedObjectsClient })).toBe( + 'some-datasource-id' + ); + }); +}); + +describe('extractDataSourceNamesInVegaSpec()', () => { + const loadHJSONStringFromFile = (filepath: string) => { + return readFileSync(join(__dirname, filepath)).toString(); + }; + + const loadJSONFromFile = (filepath: string) => { + return JSON.parse(readFileSync(join(__dirname, filepath)).toString()); + }; + + // JSON test cases + test('(JSON) Set should be empty when no queries are in the Vega spec', () => { + const noQueryJSON = loadJSONFromFile('/test_utils/vega_spec_without_opensearch_query.json'); + expect(extractDataSourceNamesInVegaSpec(JSON.stringify(noQueryJSON))).toMatchObject(new Set()); + }); + + test('(JSON) Set should be empty when one local cluster query is in the Vega spec', () => { + const oneLocalQueryJSON = loadJSONFromFile('/test_utils/vega_spec_with_opensearch_query.json'); + expect(extractDataSourceNamesInVegaSpec(JSON.stringify(oneLocalQueryJSON))).toMatchObject( + new Set() + ); + }); + + test('(JSON) Set should have exactly one data_source_name when one data source query is in the Vega spec', () => { + const oneDataSourceQueryJSON = loadJSONFromFile( + '/test_utils/vega_spec_with_opensearch_query_mds.json' + ); + expect(extractDataSourceNamesInVegaSpec(JSON.stringify(oneDataSourceQueryJSON))).toMatchObject( + new Set(['example data source']) + ); + }); + + test('(JSON) Set should be empty when many local cluster queries are in the Vega spec', () => { + const manyLocalQueriesJSON = loadJSONFromFile('/test_utils/vega_spec_with_multiple_urls.json'); + expect(extractDataSourceNamesInVegaSpec(JSON.stringify(manyLocalQueriesJSON))).toMatchObject( + new Set() + ); + }); + + test('(JSON) Set have multiple data_source_name fields when the Vega spec has a mix of local cluster and data source queries', () => { + const manyDataSourceQueriesJSON = loadJSONFromFile( + '/test_utils/vega_spec_with_multiple_urls_mds.json' + ); + expect( + extractDataSourceNamesInVegaSpec(JSON.stringify(manyDataSourceQueriesJSON)) + ).toMatchObject(new Set(['some other datasource name', 'some datasource name'])); + }); + + // HJSON test cases + test('(HJSON) Set should be empty when no queries are in the Vega spec', () => { + const noQueryHJSON = loadHJSONStringFromFile( + '/test_utils/vega_spec_without_opensearch_query.hjson' + ); + expect(extractDataSourceNamesInVegaSpec(noQueryHJSON)).toMatchObject(new Set()); + }); + + test('(HJSON) Set should be empty when one local cluster query is in the Vega spec', () => { + const oneLocalQueryHJSON = loadHJSONStringFromFile( + '/test_utils/vega_spec_with_opensearch_query.hjson' + ); + expect(extractDataSourceNamesInVegaSpec(oneLocalQueryHJSON)).toMatchObject(new Set()); + }); + + test('(HJSON) Set should have exactly one data_source_name when one data source query is in the Vega spec', () => { + const oneDataSourceQueryHJSON = loadHJSONStringFromFile( + '/test_utils/vega_spec_with_opensearch_query_mds.hjson' + ); + expect(extractDataSourceNamesInVegaSpec(oneDataSourceQueryHJSON)).toMatchObject( + new Set(['example data source']) + ); + }); + + test('(HJSON) Set should be empty when many local cluster queries are in the Vega spec', () => { + const manyLocalQueriesHJSON = loadHJSONStringFromFile( + '/test_utils/vega_spec_with_multiple_urls.hjson' + ); + expect(extractDataSourceNamesInVegaSpec(manyLocalQueriesHJSON)).toMatchObject(new Set()); + }); + + test('(HJSON) Set have multiple data_source_name fields when the Vega spec has a mix of local cluster and data source queries', () => { + const manyDataSourceQueriesHJSON = loadHJSONStringFromFile( + '/test_utils/vega_spec_with_multiple_urls_mds.hjson' + ); + expect(extractDataSourceNamesInVegaSpec(manyDataSourceQueriesHJSON)).toMatchObject( + new Set(['some other datasource name', 'some datasource name']) + ); + }); +}); + +describe('extractVegaSpecFromSavedObject()', () => { + test('For a Vega visualization saved object, return its spec', () => { + const spec = 'some-vega-spec'; + const vegaAttributes = { + visState: `{"type": "vega", "params": {"spec": "${spec}"}}`, + }; + + expect(extractVegaSpecFromAttributes(vegaAttributes)).toBe(spec); + }); + + test('For another saved object type, return undefined', () => { + const nonVegaAttributes = { + visState: `{"type": "area", "params": {"spec": "some-spec"}}`, + }; + + expect(extractVegaSpecFromAttributes(nonVegaAttributes)).toBe(undefined); + }); +}); diff --git a/src/plugins/vis_type_vega/server/utils.ts b/src/plugins/vis_type_vega/server/utils.ts new file mode 100644 index 00000000000..f8c83dce531 --- /dev/null +++ b/src/plugins/vis_type_vega/server/utils.ts @@ -0,0 +1,98 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { parse } from 'hjson'; +import { SavedObjectsClientContract } from 'src/core/server'; +import { DataSourceAttributes } from 'src/plugins/data_source/common/data_sources'; + +export interface FindDataSourceByTitleQueryProps { + dataSourceName: string; + savedObjectsClient: SavedObjectsClientContract; +} + +export const findDataSourceIdbyName = async (props: FindDataSourceByTitleQueryProps) => { + const { dataSourceName } = props; + const dataSources = await dataSourceFindQuery(props); + + // In the case that data_source_name is a prefix of another name, match exact data_source_name + const possibleDataSourceObjects = dataSources.saved_objects.filter( + (obj) => obj.attributes.title === dataSourceName + ); + + if (possibleDataSourceObjects.length !== 1) { + throw new Error( + `Expected exactly 1 result for data_source_name "${dataSourceName}" but got ${possibleDataSourceObjects.length} results` + ); + } + + return possibleDataSourceObjects.pop()?.id; +}; + +export const extractVegaSpecFromAttributes = (attributes: unknown) => { + if (isVegaVisualization(attributes)) { + // @ts-expect-error + const visStateObject = JSON.parse(attributes?.visState); + return visStateObject.params.spec; + } + + return undefined; +}; + +export const extractDataSourceNamesInVegaSpec = (spec: string) => { + const parsedSpec = parse(spec, { keepWsc: true }); + const dataField = parsedSpec.data; + const dataSourceNameSet = new Set(); + + if (dataField instanceof Array) { + dataField.forEach((dataObject) => { + const dataSourceName = getDataSourceNameFromObject(dataObject); + if (!!dataSourceName) { + dataSourceNameSet.add(dataSourceName); + } + }); + } else if (dataField instanceof Object) { + const dataSourceName = getDataSourceNameFromObject(dataField); + if (!!dataSourceName) { + dataSourceNameSet.add(dataSourceName); + } + } else { + throw new Error(`"data" field should be an object or an array of objects`); + } + + return dataSourceNameSet; +}; + +const getDataSourceNameFromObject = (dataObject: any) => { + if ( + dataObject.hasOwnProperty('url') && + dataObject.url.hasOwnProperty('index') && + dataObject.url.hasOwnProperty('data_source_name') + ) { + return dataObject.url.data_source_name; + } + + return undefined; +}; + +const isVegaVisualization = (attributes: unknown) => { + // @ts-expect-error + const visState = attributes?.visState; + if (!!visState) { + const visStateObject = JSON.parse(visState); + return !!visStateObject.type && visStateObject.type === 'vega'; + } + return false; +}; + +const dataSourceFindQuery = async (props: FindDataSourceByTitleQueryProps) => { + const { savedObjectsClient, dataSourceName } = props; + return await savedObjectsClient.find({ + type: 'data-source', + perPage: 10, + search: `"${dataSourceName}"`, + searchFields: ['title'], + fields: ['id', 'title'], + }); +}; diff --git a/src/plugins/vis_type_vega/server/vega_visualization_client_wrapper.test.ts b/src/plugins/vis_type_vega/server/vega_visualization_client_wrapper.test.ts new file mode 100644 index 00000000000..09af5459cf1 --- /dev/null +++ b/src/plugins/vis_type_vega/server/vega_visualization_client_wrapper.test.ts @@ -0,0 +1,243 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { SavedObjectsClientWrapperOptions, SavedObjectsFindOptions } from 'src/core/server'; +import { savedObjectsClientMock } from '../../../core/server/mocks'; +import { vegaVisualizationClientWrapper } from './vega_visualization_client_wrapper'; + +jest.mock('./services', () => ({ + getDataSourceEnabled: jest + .fn() + .mockReturnValueOnce({ enabled: false }) + .mockReturnValue({ enabled: true }), +})); + +describe('vegaVisualizationClientWrapper()', () => { + const loadHJSONStringFromFile = (filepath: string) => { + return readFileSync(join(__dirname, filepath)).toString(); + }; + + const getAttributesGivenSpec = (spec: string) => { + return { + title: 'Some Spec', + visState: JSON.stringify({ + title: 'Some Spec', + type: 'vega', + aggs: [], + params: { + spec, + }, + }), + }; + }; + + const client = savedObjectsClientMock.create(); + client.bulkGet = jest + .fn() + .mockImplementation((dataSourceIds: Array<{ id: string; type: string }>) => { + return Promise.resolve({ + saved_objects: dataSourceIds.map((request) => { + if (request.type === 'data-source' && request.id === 'id-a') { + return { + id: 'id-a', + attributes: { + title: 'a-title', + }, + }; + } else if (request.type === 'data-source' && request.id === 'id-b') { + return { + id: 'id-b', + attributes: { + title: 'b-title', + }, + }; + } else if (request.type === 'data-source' && request.id === 'id-z') { + return { + id: 'id-z', + attributes: { + title: 'z-title', + }, + }; + } + + return { + id: request.id, + attributes: undefined, + }; + }), + }); + }); + client.find = jest.fn().mockImplementation((query: SavedObjectsFindOptions) => { + if (query.search === `"c-title"`) { + return Promise.resolve({ + total: 1, + saved_objects: [{ id: 'id-c', attributes: { title: 'c-title' } }], + }); + } else if (query.search === `"d-title"`) { + return Promise.resolve({ + total: 1, + saved_objects: [{ id: 'id-d', attributes: { title: 'd-title' } }], + }); + } else { + return Promise.resolve({ + total: 0, + saved_objects: [], + }); + } + }); + const mockedWrapperOptions = {} as SavedObjectsClientWrapperOptions; + mockedWrapperOptions.client = client; + + beforeEach(() => { + client.create.mockClear(); + }); + + test('Should just call create as usual if MDS is disabled', async () => { + const wrapper = vegaVisualizationClientWrapper(mockedWrapperOptions); + await wrapper.create('visualization', {}, { references: [] }); + expect(client.create).toBeCalledWith( + 'visualization', + {}, + expect.objectContaining({ references: [] }) + ); + }); + + test('Should just call create as usual if object type is not visualization type', async () => { + const wrapper = vegaVisualizationClientWrapper(mockedWrapperOptions); + await wrapper.create('dashboard', {}, { references: [] }); + expect(client.create).toBeCalledWith( + 'dashboard', + {}, + expect.objectContaining({ references: [] }) + ); + }); + + test('Should just call create as usual if object type is not vega type', async () => { + const wrapper = vegaVisualizationClientWrapper(mockedWrapperOptions); + // Avoids whitespacing issues by letting stringify format the string + const visState = JSON.stringify( + JSON.parse('{"type": "area", "params": {"spec": "no-spec-here"}}') + ); + await wrapper.create('visualization', { visState }, { references: [] }); + expect(client.create).toBeCalledWith( + 'visualization', + { visState }, + expect.objectContaining({ references: [] }) + ); + }); + + test('Should not update anything if the spec does not specify any data_source_name', async () => { + const spec = loadHJSONStringFromFile('/test_utils/vega_spec_with_multiple_urls.hjson'); + const attributes = getAttributesGivenSpec(spec); + const wrapper = vegaVisualizationClientWrapper(mockedWrapperOptions); + await wrapper.create('visualization', attributes, { references: [] }); + expect(client.create).toBeCalledWith( + 'visualization', + attributes, + expect.objectContaining({ references: [] }) + ); + }); + + test('Should not update anything if the references is still up-to-date', async () => { + const spec = loadHJSONStringFromFile('/test_utils/vega_spec_up_to_date_urls_mds.hjson'); + const attributes = getAttributesGivenSpec(spec); + const references = [ + { + id: 'id-a', + type: 'data-source', + name: 'dataSource', + }, + { + id: 'id-b', + type: 'data-source', + name: 'dataSource', + }, + ]; + const wrapper = vegaVisualizationClientWrapper(mockedWrapperOptions); + await wrapper.create('visualization', attributes, { references }); + expect(client.create).toBeCalledWith( + 'visualization', + attributes, + expect.objectContaining({ references }) + ); + }); + + test('Should throw an error if the Vega spec has invalid data_source_name field(s)', () => { + const spec = loadHJSONStringFromFile('/test_utils/vega_spec_with_multiple_urls_mds.hjson'); + const visState = { + title: 'Some Spec', + type: 'vega', + aggs: [], + params: { + spec, + }, + }; + const attributes = { + title: 'Some Spec', + visState: JSON.stringify(visState), + }; + const wrapper = vegaVisualizationClientWrapper(mockedWrapperOptions); + expect(wrapper.create('visualization', attributes, { references: [] })).rejects.toThrowError( + `Expected exactly 1 result for data_source_name` + ); + }); + + test('Should update only the references section', async () => { + const spec = loadHJSONStringFromFile('/test_utils/vega_outdated_references_mds.hjson'); + const attributes = getAttributesGivenSpec(spec); + const commonReferences = [ + { + id: 'some-dashboard', + type: 'dashboard', + name: 'someDashboard', + }, + { + id: 'id-a', + type: 'data-source', + name: 'dataSource', + }, + { + id: 'id-b', + type: 'data-source', + name: 'dataSource', + }, + ]; + const oldReferences = [ + ...commonReferences, + { + id: 'id-z', + type: 'data-source', + name: 'dataSource', + }, + { + id: 'non-existent-id', + type: 'data-source', + name: 'dataSource', + }, + ]; + const newReferences = [ + ...commonReferences, + { + id: 'id-c', + type: 'data-source', + name: 'dataSource', + }, + { + id: 'id-d', + type: 'data-source', + name: 'dataSource', + }, + ]; + const wrapper = vegaVisualizationClientWrapper(mockedWrapperOptions); + await wrapper.create('visualization', attributes, { references: oldReferences }); + expect(client.create).toBeCalledWith( + 'visualization', + attributes, + expect.objectContaining({ references: newReferences }) + ); + }); +}); diff --git a/src/plugins/vis_type_vega/server/vega_visualization_client_wrapper.ts b/src/plugins/vis_type_vega/server/vega_visualization_client_wrapper.ts new file mode 100644 index 00000000000..4deada346c3 --- /dev/null +++ b/src/plugins/vis_type_vega/server/vega_visualization_client_wrapper.ts @@ -0,0 +1,110 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + SavedObjectsClientWrapperFactory, + SavedObjectsClientWrapperOptions, + SavedObjectsCreateOptions, + SavedObjectsErrorHelpers, +} from '../../../core/server'; +import { + extractDataSourceNamesInVegaSpec, + extractVegaSpecFromAttributes, + findDataSourceIdbyName, +} from './utils'; +import { getDataSourceEnabled } from './services'; + +export const VEGA_VISUALIZATION_CLIENT_WRAPPER_ID = 'vega-visualization-client-wrapper'; + +export const vegaVisualizationClientWrapper: SavedObjectsClientWrapperFactory = ( + wrapperOptions: SavedObjectsClientWrapperOptions +) => { + const createForVega = async ( + type: string, + attributes: T, + options?: SavedObjectsCreateOptions + ) => { + const vegaSpec = extractVegaSpecFromAttributes(attributes); + if (type !== 'visualization' || vegaSpec === undefined || !getDataSourceEnabled().enabled) { + return await wrapperOptions.client.create(type, attributes, options); + } + const dataSourceNamesSet = extractDataSourceNamesInVegaSpec(vegaSpec); + + const existingDataSourceReferences = options?.references + ?.filter((reference) => reference.type === 'data-source') + .map((dataSourceReference) => { + return { + id: dataSourceReference.id, + type: dataSourceReference.type, + }; + }); + + const existingDataSourceIdToNameMap = new Map(); + if (!!existingDataSourceReferences && existingDataSourceReferences.length > 0) { + (await wrapperOptions.client.bulkGet(existingDataSourceReferences)).saved_objects.forEach( + (object) => { + // @ts-expect-error + if (!!object.attributes && !!object.attributes.title) { + // @ts-expect-error + existingDataSourceIdToNameMap.set(object.id, object.attributes.title); + } + } + ); + } + + // Filters out outdated datasource references + const newReferences = options?.references?.filter((reference) => { + if (reference.type !== 'data-source') { + return true; + } + const dataSourceName = existingDataSourceIdToNameMap.get(reference.id); + if (dataSourceNamesSet.has(dataSourceName)) { + dataSourceNamesSet.delete(dataSourceName); + return true; + } + + return false; + }); + + for await (const dataSourceName of dataSourceNamesSet) { + const dataSourceId = await findDataSourceIdbyName({ + dataSourceName, + savedObjectsClient: wrapperOptions.client, + }); + if (dataSourceId) { + newReferences?.push({ + id: dataSourceId, + name: 'dataSource', + type: 'data-source', + }); + } else { + throw SavedObjectsErrorHelpers.createBadRequestError( + `data_source_name "${dataSourceName}" cannot be found in saved objects` + ); + } + } + + return await wrapperOptions.client.create(type, attributes, { + ...options, + references: newReferences, + }); + }; + + return { + ...wrapperOptions.client, + create: createForVega, + bulkCreate: wrapperOptions.client.bulkCreate, + checkConflicts: wrapperOptions.client.checkConflicts, + delete: wrapperOptions.client.delete, + find: wrapperOptions.client.find, + bulkGet: wrapperOptions.client.bulkGet, + get: wrapperOptions.client.get, + update: wrapperOptions.client.update, + bulkUpdate: wrapperOptions.client.bulkUpdate, + errors: wrapperOptions.client.errors, + addToNamespaces: wrapperOptions.client.addToNamespaces, + deleteFromNamespaces: wrapperOptions.client.deleteFromNamespaces, + }; +};