[Workspace] Add optional workspaces parameter to all saved objects A…

…PI (opensearch-project#185)

* [Workspace] Add workspaces parameters to all saved objects API

Signed-off-by: gaobinlong <gbinlong@amazon.com>

* feat: update snapshot

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: optimize logic when checkConflict and bulkCreate (opensearch-project#189)

* feat: optimize logic when checkConflict and bulkCreate

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: add options.workspace check

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: throw error when workspace check error in repository create

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: modify judgement

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: always get objects from DB when create-with-override

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

---------

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: call get when create with override

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: update test according to count

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: add integration test

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* fix: unit test

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: regenerate ids when import

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: add more unit test

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: minor changes logic on repository

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: update unit test

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: update test

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: optimization according to comments

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: update test

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

* feat: optimize code

Signed-off-by: SuZhou-Joe <suzhou@amazon.com>

---------

Signed-off-by: gaobinlong <gbinlong@amazon.com>
Signed-off-by: SuZhou-Joe <suzhou@amazon.com>
Co-authored-by: SuZhou-Joe <suzhou@amazon.com>

src/core/public/saved_objects/saved_objects_client.ts

@@ -345,6 +345,7 @@ export class SavedObjectsClient {
  filter: 'filter',
  namespaces: 'namespaces',
  preference: 'preference',
+ workspaces: 'workspaces',
  };
 
  const renamedQuery = renameKeys<SavedObjectsFindOptions, any>(renameMap, options);

src/core/server/saved_objects/export/get_sorted_objects_for_export.ts

@@ -60,6 +60,8 @@ export interface SavedObjectsExportOptions {
  excludeExportDetails?: boolean;
  /** optional namespace to override the namespace used by the savedObjectsClient. */
  namespace?: string;
+ /** optional workspaces to override the workspaces used by the savedObjectsClient. */
+ workspaces?: string[];
 }
 
 /**
@@ -87,13 +89,15 @@ async function fetchObjectsToExport({
  exportSizeLimit,
  savedObjectsClient,
  namespace,
+ workspaces,
 }: {
  objects?: SavedObjectsExportOptions['objects'];
  types?: string[];
  search?: string;
  exportSizeLimit: number;
  savedObjectsClient: SavedObjectsClientContract;
  namespace?: string;
+ workspaces?: string[];
 }) {
  if ((types?.length ?? 0) > 0 && (objects?.length ?? 0) > 0) {
  throw Boom.badRequest(`Can't specify both "types" and "objects" properties when exporting`);
@@ -105,7 +109,9 @@ async function fetchObjectsToExport({
  if (typeof search === 'string') {
  throw Boom.badRequest(`Can't specify both "search" and "objects" properties when exporting`);
  }
- const bulkGetResult = await savedObjectsClient.bulkGet(objects, { namespace });
+ const bulkGetResult = await savedObjectsClient.bulkGet(objects, {
+ namespace,
+ });
  const erroredObjects = bulkGetResult.saved_objects.filter((obj) => !!obj.error);
  if (erroredObjects.length) {
  const err = Boom.badRequest();
@@ -121,6 +127,7 @@ async function fetchObjectsToExport({
  search,
  perPage: exportSizeLimit,
  namespaces: namespace ? [namespace] : undefined,
+ ...(workspaces ? { workspaces } : {}),
  });
  if (findResponse.total > exportSizeLimit) {
  throw Boom.badRequest(`Can't export more than ${exportSizeLimit} objects`);
@@ -153,6 +160,7 @@ export async function exportSavedObjectsToStream({
  includeReferencesDeep = false,
  excludeExportDetails = false,
  namespace,
+ workspaces,
 }: SavedObjectsExportOptions) {
  const rootObjects = await fetchObjectsToExport({
  types,
@@ -161,6 +169,7 @@ export async function exportSavedObjectsToStream({
  savedObjectsClient,
  exportSizeLimit,
  namespace,
+ workspaces,
  });
  let exportedObjects: Array<SavedObject<unknown>> = [];
  let missingReferences: SavedObjectsExportResultDetails['missingReferences'] = [];

src/core/server/saved_objects/import/check_conflicts.ts

@@ -44,6 +44,7 @@ interface CheckConflictsParams {
  ignoreRegularConflicts?: boolean;
  retries?: SavedObjectsImportRetry[];
  createNewCopies?: boolean;
+ workspaces?: string[];
 }
 
 const isUnresolvableConflict = (error: SavedObjectError) =>
@@ -56,6 +57,7 @@ export async function checkConflicts({
  ignoreRegularConflicts,
  retries = [],
  createNewCopies,
+ workspaces,
 }: CheckConflictsParams) {
  const filteredObjects: Array<SavedObject<{ title?: string }>> = [];
  const errors: SavedObjectsImportError[] = [];
@@ -77,6 +79,7 @@ export async function checkConflicts({
  });
  const checkConflictsResult = await savedObjectsClient.checkConflicts(objectsToCheck, {
  namespace,
+ workspaces,
  });
  const errorMap = checkConflictsResult.errors.reduce(
  (acc, { type, id, error }) => acc.set(`${type}:${id}`, error),

src/core/server/saved_objects/import/create_saved_objects.ts

@@ -41,6 +41,7 @@ interface CreateSavedObjectsParams<T> {
  overwrite?: boolean;
  dataSourceId?: string;
  dataSourceTitle?: string;
+ workspaces?: string[];
 }
 interface CreateSavedObjectsResult<T> {
  createdObjects: Array<CreatedObject<T>>;
@@ -60,6 +61,7 @@ export const createSavedObjects = async <T>({
  overwrite,
  dataSourceId,
  dataSourceTitle,
+ workspaces,
 }: CreateSavedObjectsParams<T>): Promise<CreateSavedObjectsResult<T>> => {
  // filter out any objects that resulted in errors
  const errorSet = accumulatedErrors.reduce(
@@ -169,6 +171,7 @@ export const createSavedObjects = async <T>({
  const bulkCreateResponse = await savedObjectsClient.bulkCreate(objectsToCreate, {
  namespace,
  overwrite,
+ workspaces,
  });
  expectedResults = bulkCreateResponse.saved_objects;
  }

src/core/server/saved_objects/import/import_saved_objects.test.ts

@@ -42,7 +42,7 @@ import { typeRegistryMock } from '../saved_objects_type_registry.mock';
 import { importSavedObjectsFromStream } from './import_saved_objects';
 
 import { collectSavedObjects } from './collect_saved_objects';
-import { regenerateIds } from './regenerate_ids';
+import { regenerateIds, regenerateIdsWithReference } from './regenerate_ids';
 import { validateReferences } from './validate_references';
 import { checkConflicts } from './check_conflicts';
 import { checkOriginConflicts } from './check_origin_conflicts';
@@ -70,6 +70,7 @@ describe('#importSavedObjectsFromStream', () => {
  importIdMap: new Map(),
  });
  getMockFn(regenerateIds).mockReturnValue(new Map());
+ getMockFn(regenerateIdsWithReference).mockReturnValue(Promise.resolve(new Map()));
  getMockFn(validateReferences).mockResolvedValue([]);
  getMockFn(checkConflicts).mockResolvedValue({
  errors: [],
@@ -278,6 +279,15 @@ describe('#importSavedObjectsFromStream', () => {
  ]),
  });
  getMockFn(validateReferences).mockResolvedValue([errors[1]]);
+ getMockFn(regenerateIdsWithReference).mockResolvedValue(
+ Promise.resolve(
+ new Map([
+ ['foo', {}],
+ ['bar', {}],
+ ['baz', {}],
+ ])
+ )
+ );
  getMockFn(checkConflicts).mockResolvedValue({
  errors: [errors[2]],
  filteredObjects,

src/core/server/saved_objects/import/import_saved_objects.ts

@@ -38,7 +38,7 @@ import { validateReferences } from './validate_references';
 import { checkOriginConflicts } from './check_origin_conflicts';
 import { createSavedObjects } from './create_saved_objects';
 import { checkConflicts } from './check_conflicts';
-import { regenerateIds } from './regenerate_ids';
+import { regenerateIds, regenerateIdsWithReference } from './regenerate_ids';
 import { checkConflictsForDataSource } from './check_conflict_for_data_source';
 
 /**
@@ -57,6 +57,7 @@ export async function importSavedObjectsFromStream({
  namespace,
  dataSourceId,
  dataSourceTitle,
+ workspaces,
 }: SavedObjectsImportOptions): Promise<SavedObjectsImportResponse> {
  let errorAccumulator: SavedObjectsImportError[] = [];
  const supportedTypes = typeRegistry.getImportableAndExportableTypes().map((type) => type.name);
@@ -86,12 +87,23 @@ export async function importSavedObjectsFromStream({
  importIdMap = regenerateIds(collectSavedObjectsResult.collectedObjects, dataSourceId);
  } else {
  // in check conclict and override mode
+
+ if (workspaces) {
+ importIdMap = await regenerateIdsWithReference({
+ savedObjects: collectSavedObjectsResult.collectedObjects,
+ savedObjectsClient,
+ workspaces,
+ objectLimit,
+ importIdMap,
+ });
+ }
  // Check single-namespace objects for conflicts in this namespace, and check multi-namespace objects for conflicts across all namespaces
  const checkConflictsParams = {
  objects: collectSavedObjectsResult.collectedObjects,
  savedObjectsClient,
  namespace,
  ignoreRegularConflicts: overwrite,
+ workspaces,
  };
 
  const checkConflictsResult = await checkConflicts(checkConflictsParams);
@@ -142,6 +154,7 @@ export async function importSavedObjectsFromStream({
  namespace,
  dataSourceId,
  dataSourceTitle,
+ ...(workspaces ? { workspaces } : {}),
  };
  const createSavedObjectsResult = await createSavedObjects(createSavedObjectsParams);
  errorAccumulator = [...errorAccumulator, ...createSavedObjectsResult.errors];

src/core/server/saved_objects/import/regenerate_ids.test.ts

@@ -29,8 +29,10 @@
  */
 
 import { mockUuidv4 } from './__mocks__';
-import { regenerateIds } from './regenerate_ids';
+import { regenerateIds, regenerateIdsWithReference } from './regenerate_ids';
 import { SavedObject } from '../types';
+import { savedObjectsClientMock } from '../service/saved_objects_client.mock';
+import { SavedObjectsBulkResponse } from '../service';
 
 describe('#regenerateIds', () => {
  const objects = ([
@@ -68,3 +70,70 @@ describe('#regenerateIds', () => {
  `);
  });
 });
+
+describe('#regenerateIdsWithReference', () => {
+ const objects = ([
+ { type: 'foo', id: '1' },
+ { type: 'bar', id: '2' },
+ { type: 'baz', id: '3' },
+ ] as any) as SavedObject[];
+
+ test('returns expected values', async () => {
+ const mockedSavedObjectsClient = savedObjectsClientMock.create();
+ mockUuidv4.mockReturnValueOnce('uuidv4 #1');
+ const result: SavedObjectsBulkResponse<unknown> = {
+ saved_objects: [
+ {
+ error: {
+ statusCode: 404,
+ error: '',
+ message: '',
+ },
+ id: '1',
+ type: 'foo',
+ attributes: {},
+ references: [],
+ },
+ {
+ id: '2',
+ type: 'bar',
+ attributes: {},
+ references: [],
+ workspaces: ['bar'],
+ },
+ {
+ id: '3',
+ type: 'baz',
+ attributes: {},
+ references: [],
+ workspaces: ['foo'],
+ },
+ ],
+ };
+ mockedSavedObjectsClient.bulkGet.mockResolvedValue(result);
+ expect(
+ await regenerateIdsWithReference({
+ savedObjects: objects,
+ savedObjectsClient: mockedSavedObjectsClient,
+ workspaces: ['bar'],
+ objectLimit: 1000,
+ importIdMap: new Map(),
+ })
+ ).toMatchInlineSnapshot(`
+ Map {
+ "foo:1" => Object {
+ "id": "1",
+ "omitOriginId": true,
+ },
+ "bar:2" => Object {
+ "id": "2",
+ "omitOriginId": false,
+ },
+ "baz:3" => Object {
+ "id": "uuidv4 #1",
+ "omitOriginId": true,
+ },
+ }
+ `);
+ });
+});

src/core/server/saved_objects/import/regenerate_ids.ts

@@ -29,7 +29,8 @@
  */
 
 import { v4 as uuidv4 } from 'uuid';
-import { SavedObject } from '../types';
+import { SavedObject, SavedObjectsClientContract } from '../types';
+import { SavedObjectsUtils } from '../service';
 
 /**
  * Takes an array of saved objects and returns an importIdMap of randomly-generated new IDs.
@@ -47,3 +48,35 @@ export const regenerateIds = (objects: SavedObject[], dataSourceId: string | und
  }, new Map<string, { id: string; omitOriginId?: boolean }>());
  return importIdMap;
 };
+
+export const regenerateIdsWithReference = async (props: {
+ savedObjects: SavedObject[];
+ savedObjectsClient: SavedObjectsClientContract;
+ workspaces: string[];
+ objectLimit: number;
+ importIdMap: Map<string, { id?: string; omitOriginId?: boolean }>;
+}): Promise<Map<string, { id?: string; omitOriginId?: boolean }>> => {
+ const { savedObjects, savedObjectsClient, workspaces, importIdMap } = props;
+
+ const bulkGetResult = await savedObjectsClient.bulkGet(
+ savedObjects.map((item) => ({ type: item.type, id: item.id }))
+ );
+
+ return bulkGetResult.saved_objects.reduce((acc, object) => {
+ if (object.error?.statusCode === 404) {
+ acc.set(`${object.type}:${object.id}`, { id: object.id, omitOriginId: true });
+ return acc;
+ }
+
+ const filteredWorkspaces = SavedObjectsUtils.filterWorkspacesAccordingToBaseWorkspaces(
+ workspaces,
+ object.workspaces
+ );
+ if (filteredWorkspaces.length) {
+ acc.set(`${object.type}:${object.id}`, { id: uuidv4(), omitOriginId: true });
+ } else {
+ acc.set(`${object.type}:${object.id}`, { id: object.id, omitOriginId: false });
+ }
+ return acc;
+ }, importIdMap);
+};

src/core/server/saved_objects/import/resolve_import_errors.ts

@@ -61,6 +61,7 @@ export async function resolveSavedObjectsImportErrors({
  createNewCopies,
  dataSourceId,
  dataSourceTitle,
+ workspaces,
 }: SavedObjectsResolveImportErrorsOptions): Promise<SavedObjectsImportResponse> {
  // throw a BadRequest error if we see invalid retries
  validateRetries(retries);
@@ -163,6 +164,7 @@ export async function resolveSavedObjectsImportErrors({
  overwrite,
  dataSourceId,
  dataSourceTitle,
+ workspaces,
  };
  const { createdObjects, errors: bulkCreateErrors } = await createSavedObjects(
  createSavedObjectsParams

src/core/server/saved_objects/import/types.ts

@@ -189,6 +189,8 @@ export interface SavedObjectsImportOptions {
  createNewCopies: boolean;
  dataSourceId?: string;
  dataSourceTitle?: string;
+ /** if specified, will import in given workspaces, else will import as global object */
+ workspaces?: string[];
 }
 
 /**
@@ -212,6 +214,8 @@ export interface SavedObjectsResolveImportErrorsOptions {
  createNewCopies: boolean;
  dataSourceId?: string;
  dataSourceTitle?: string;
+ /** if specified, will import in given workspaces, else will import as global object */
+ workspaces?: string[];
 }
 
 export type CreatedObject<T> = SavedObject<T> & { destinationId?: string };