Skip to content

Commit

Permalink
[Feature] Complied saved_objects create/find (opensearch-project#18)
Browse files Browse the repository at this point in the history
* temp: save

Signed-off-by: SuZhoue-Joe <suzhou@amazon.com>

* feat: make create/find support workspaces

Signed-off-by: SuZhoue-Joe <suzhou@amazon.com>

* feat: extract management code

Signed-off-by: SuZhoue-Joe <suzhou@amazon.com>

* fix: type check

Signed-off-by: SuZhoue-Joe <suzhou@amazon.com>

* fix: build error

Signed-off-by: SuZhoue-Joe <suzhou@amazon.com>

* feat: enable workspaces on saved client server side

Signed-off-by: SuZhoue-Joe <suzhou@amazon.com>

* feat: some optimization

Signed-off-by: SuZhoue-Joe <suzhou@amazon.com>

* feat: extract management code

Signed-off-by: SuZhoue-Joe <suzhou@amazon.com>

* feat: merge fix

Signed-off-by: SuZhoue-Joe <suzhou@amazon.com>

* feat: optimize code

Signed-off-by: SuZhoue-Joe <suzhou@amazon.com>

* feat: reuse common function

Signed-off-by: SuZhoue-Joe <suzhou@amazon.com>

* feat: optimize code when create

Signed-off-by: SuZhoue-Joe <suzhou@amazon.com>

* feat: remove useless test code

Signed-off-by: SuZhoue-Joe <suzhou@amazon.com>

---------

Signed-off-by: SuZhoue-Joe <suzhou@amazon.com>
  • Loading branch information
SuZhou-Joe authored and Hailong-am committed Jul 18, 2023
1 parent fa45d8d commit e8cfc77
Show file tree
Hide file tree
Showing 19 changed files with 189 additions and 16 deletions.
33 changes: 30 additions & 3 deletions src/core/public/saved_objects/saved_objects_client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ import {

import { SimpleSavedObject } from './simple_saved_object';
import { HttpFetchOptions, HttpSetup } from '../http';
import { WorkspacesStart } from '../workspace';

type SavedObjectsFindOptions = Omit<
SavedObjectFindOptionsServer,
Expand All @@ -61,6 +62,7 @@ export interface SavedObjectsCreateOptions {
/** {@inheritDoc SavedObjectsMigrationVersion} */
migrationVersion?: SavedObjectsMigrationVersion;
references?: SavedObjectReference[];
workspaces?: string[];
}

/**
Expand Down Expand Up @@ -183,6 +185,7 @@ const getObjectsToFetch = (queue: BatchQueueEntry[]): ObjectTypeAndId[] => {
export class SavedObjectsClient {
private http: HttpSetup;
private batchQueue: BatchQueueEntry[];
private currentWorkspaceId?: string;

/**
* Throttled processing of get requests into bulk requests at 100ms interval
Expand Down Expand Up @@ -227,6 +230,15 @@ export class SavedObjectsClient {
this.batchQueue = [];
}

private async _getCurrentWorkspace(): Promise<string | null> {
return this.currentWorkspaceId || null;
}

public async setCurrentWorkspace(workspaceId: string): Promise<boolean> {
this.currentWorkspaceId = workspaceId;
return true;
}

/**
* Persists an object
*
Expand All @@ -235,7 +247,7 @@ export class SavedObjectsClient {
* @param options
* @returns
*/
public create = <T = unknown>(
public create = async <T = unknown>(
type: string,
attributes: T,
options: SavedObjectsCreateOptions = {}
Expand All @@ -248,6 +260,7 @@ export class SavedObjectsClient {
const query = {
overwrite: options.overwrite,
};
const currentWorkspaceId = await this._getCurrentWorkspace();

const createRequest: Promise<SavedObject<T>> = this.savedObjectsFetch(path, {
method: 'POST',
Expand All @@ -256,6 +269,11 @@ export class SavedObjectsClient {
attributes,
migrationVersion: options.migrationVersion,
references: options.references,
...(options.workspaces || currentWorkspaceId
? {
workspaces: options.workspaces || [currentWorkspaceId],
}
: {}),
}),
});

Expand Down Expand Up @@ -328,7 +346,7 @@ export class SavedObjectsClient {
* @property {object} [options.hasReference] - { type, id }
* @returns A find result with objects matching the specified search.
*/
public find = <T = unknown>(
public find = async <T = unknown>(
options: SavedObjectsFindOptions
): Promise<SavedObjectsFindResponsePublic<T>> => {
const path = this.getPath(['_find']);
Expand All @@ -345,9 +363,18 @@ export class SavedObjectsClient {
filter: 'filter',
namespaces: 'namespaces',
preference: 'preference',
workspaces: 'workspaces',
};

const renamedQuery = renameKeys<SavedObjectsFindOptions, any>(renameMap, options);
const workspaces = [
...(options.workspaces || [await this._getCurrentWorkspace()]),
'public',
].filter((item) => item);

const renamedQuery = renameKeys<SavedObjectsFindOptions, any>(renameMap, {
...options,
workspaces,
});
const query = pick.apply(null, [renamedQuery, ...Object.values<string>(renameMap)]);

const request: ReturnType<SavedObjectsApi['find']> = this.savedObjectsFetch(path, {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,8 @@ export interface SavedObjectsExportOptions {
excludeExportDetails?: boolean;
/** optional namespace to override the namespace used by the savedObjectsClient. */
namespace?: string;
/** optional workspaces to override the workspaces used by the savedObjectsClient. */
workspaces?: string[];
}

/**
Expand Down Expand Up @@ -87,13 +89,15 @@ async function fetchObjectsToExport({
exportSizeLimit,
savedObjectsClient,
namespace,
workspaces,
}: {
objects?: SavedObjectsExportOptions['objects'];
types?: string[];
search?: string;
exportSizeLimit: number;
savedObjectsClient: SavedObjectsClientContract;
namespace?: string;
workspaces?: string[];
}) {
if ((types?.length ?? 0) > 0 && (objects?.length ?? 0) > 0) {
throw Boom.badRequest(`Can't specify both "types" and "objects" properties when exporting`);
Expand All @@ -105,7 +109,7 @@ async function fetchObjectsToExport({
if (typeof search === 'string') {
throw Boom.badRequest(`Can't specify both "search" and "objects" properties when exporting`);
}
const bulkGetResult = await savedObjectsClient.bulkGet(objects, { namespace });
const bulkGetResult = await savedObjectsClient.bulkGet(objects, { namespace, workspaces });
const erroredObjects = bulkGetResult.saved_objects.filter((obj) => !!obj.error);
if (erroredObjects.length) {
const err = Boom.badRequest();
Expand All @@ -121,6 +125,7 @@ async function fetchObjectsToExport({
search,
perPage: exportSizeLimit,
namespaces: namespace ? [namespace] : undefined,
workspaces,
});
if (findResponse.total > exportSizeLimit) {
throw Boom.badRequest(`Can't export more than ${exportSizeLimit} objects`);
Expand Down Expand Up @@ -153,6 +158,7 @@ export async function exportSavedObjectsToStream({
includeReferencesDeep = false,
excludeExportDetails = false,
namespace,
workspaces,
}: SavedObjectsExportOptions) {
const rootObjects = await fetchObjectsToExport({
types,
Expand All @@ -161,6 +167,7 @@ export async function exportSavedObjectsToStream({
savedObjectsClient,
exportSizeLimit,
namespace,
workspaces,
});
let exportedObjects: Array<SavedObject<unknown>> = [];
let missingReferences: SavedObjectsExportResultDetails['missingReferences'] = [];
Expand Down
5 changes: 4 additions & 1 deletion src/core/server/saved_objects/import/create_saved_objects.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ interface CreateSavedObjectsParams<T> {
importIdMap: Map<string, { id?: string; omitOriginId?: boolean }>;
namespace?: string;
overwrite?: boolean;
workspaces?: string[];
}
interface CreateSavedObjectsResult<T> {
createdObjects: Array<CreatedObject<T>>;
Expand All @@ -56,6 +57,7 @@ export const createSavedObjects = async <T>({
importIdMap,
namespace,
overwrite,
workspaces,
}: CreateSavedObjectsParams<T>): Promise<CreateSavedObjectsResult<T>> => {
// filter out any objects that resulted in errors
const errorSet = accumulatedErrors.reduce(
Expand Down Expand Up @@ -103,14 +105,15 @@ export const createSavedObjects = async <T>({
const bulkCreateResponse = await savedObjectsClient.bulkCreate(objectsToCreate, {
namespace,
overwrite,
workspaces,
});
expectedResults = bulkCreateResponse.saved_objects;
}

// remap results to reflect the object IDs that were submitted for import
// this ensures that consumers understand the results
const remappedResults = expectedResults.map<CreatedObject<T>>((result) => {
const { id } = objectIdMap.get(`${result.type}:${result.id}`)!;
const { id } = objectIdMap.get(`${result.type}:${result.id}`) || ({} as SavedObject<T>);
// also, include a `destinationId` field if the object create attempt was made with a different ID
return { ...result, id, ...(id !== result.id && { destinationId: result.id }) };
});
Expand Down
2 changes: 2 additions & 0 deletions src/core/server/saved_objects/import/import_saved_objects.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ export async function importSavedObjectsFromStream({
savedObjectsClient,
typeRegistry,
namespace,
workspaces,
}: SavedObjectsImportOptions): Promise<SavedObjectsImportResponse> {
let errorAccumulator: SavedObjectsImportError[] = [];
const supportedTypes = typeRegistry.getImportableAndExportableTypes().map((type) => type.name);
Expand Down Expand Up @@ -118,6 +119,7 @@ export async function importSavedObjectsFromStream({
importIdMap,
overwrite,
namespace,
workspaces,
};
const createSavedObjectsResult = await createSavedObjects(createSavedObjectsParams);
errorAccumulator = [...errorAccumulator, ...createSavedObjectsResult.errors];
Expand Down
2 changes: 2 additions & 0 deletions src/core/server/saved_objects/import/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,8 @@ export interface SavedObjectsImportOptions {
namespace?: string;
/** If true, will create new copies of import objects, each with a random `id` and undefined `originId`. */
createNewCopies: boolean;
/** if specified, will import in given workspaces, else will import as global object */
workspaces?: string[];
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,9 @@ function defaultMapping(): IndexMapping {
},
},
},
workspaces: {
type: 'keyword',
},
},
};
}
Expand Down
8 changes: 7 additions & 1 deletion src/core/server/saved_objects/routes/bulk_create.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@

import { schema } from '@osd/config-schema';
import { IRouter } from '../../http';
import { formatWorkspaces, workspacesValidator } from './utils';

export const registerBulkCreateRoute = (router: IRouter) => {
router.post(
Expand All @@ -38,6 +39,7 @@ export const registerBulkCreateRoute = (router: IRouter) => {
validate: {
query: schema.object({
overwrite: schema.boolean({ defaultValue: false }),
workspaces: workspacesValidator,
}),
body: schema.arrayOf(
schema.object({
Expand All @@ -62,7 +64,11 @@ export const registerBulkCreateRoute = (router: IRouter) => {
},
router.handleLegacyErrors(async (context, req, res) => {
const { overwrite } = req.query;
const result = await context.core.savedObjects.client.bulkCreate(req.body, { overwrite });
const workspaces = formatWorkspaces(req.query.workspaces);
const result = await context.core.savedObjects.client.bulkCreate(req.body, {
overwrite,
workspaces,
});
return res.ok({ body: result });
})
);
Expand Down
12 changes: 10 additions & 2 deletions src/core/server/saved_objects/routes/create.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,15 +56,23 @@ export const registerCreateRoute = (router: IRouter) => {
)
),
initialNamespaces: schema.maybe(schema.arrayOf(schema.string(), { minSize: 1 })),
workspaces: schema.maybe(schema.arrayOf(schema.string(), { minSize: 1 })),
}),
},
},
router.handleLegacyErrors(async (context, req, res) => {
const { type, id } = req.params;
const { overwrite } = req.query;
const { attributes, migrationVersion, references, initialNamespaces } = req.body;
const { attributes, migrationVersion, references, initialNamespaces, workspaces } = req.body;

const options = { id, overwrite, migrationVersion, references, initialNamespaces };
const options = {
id,
overwrite,
migrationVersion,
references,
initialNamespaces,
workspaces,
};
const result = await context.core.savedObjects.client.create(type, attributes, options);
return res.ok({ body: result });
})
Expand Down
11 changes: 10 additions & 1 deletion src/core/server/saved_objects/routes/export.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,12 +57,20 @@ export const registerExportRoute = (router: IRouter, config: SavedObjectConfig)
search: schema.maybe(schema.string()),
includeReferencesDeep: schema.boolean({ defaultValue: false }),
excludeExportDetails: schema.boolean({ defaultValue: false }),
workspaces: schema.maybe(schema.arrayOf(schema.string())),
}),
},
},
router.handleLegacyErrors(async (context, req, res) => {
const savedObjectsClient = context.core.savedObjects.client;
const { type, objects, search, excludeExportDetails, includeReferencesDeep } = req.body;
const {
type,
objects,
search,
excludeExportDetails,
includeReferencesDeep,
workspaces,
} = req.body;
const types = typeof type === 'string' ? [type] : type;

// need to access the registry for type validation, can't use the schema for this
Expand Down Expand Up @@ -98,6 +106,7 @@ export const registerExportRoute = (router: IRouter, config: SavedObjectConfig)
exportSizeLimit: maxImportExportSize,
includeReferencesDeep,
excludeExportDetails,
workspaces,
});

const docsToExport: string[] = await createPromiseFromStreams([
Expand Down
4 changes: 4 additions & 0 deletions src/core/server/saved_objects/routes/find.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@

import { schema } from '@osd/config-schema';
import { IRouter } from '../../http';
import { formatWorkspaces, workspacesValidator } from './utils';

export const registerFindRoute = (router: IRouter) => {
router.get(
Expand Down Expand Up @@ -59,6 +60,7 @@ export const registerFindRoute = (router: IRouter) => {
namespaces: schema.maybe(
schema.oneOf([schema.string(), schema.arrayOf(schema.string())])
),
workspaces: workspacesValidator,
}),
},
},
Expand All @@ -67,6 +69,7 @@ export const registerFindRoute = (router: IRouter) => {

const namespaces =
typeof req.query.namespaces === 'string' ? [req.query.namespaces] : req.query.namespaces;
const workspaces = formatWorkspaces(query.workspaces);

const result = await context.core.savedObjects.client.find({
perPage: query.per_page,
Expand All @@ -81,6 +84,7 @@ export const registerFindRoute = (router: IRouter) => {
fields: typeof query.fields === 'string' ? [query.fields] : query.fields,
filter: query.filter,
namespaces,
workspaces,
});

return res.ok({ body: result });
Expand Down
6 changes: 5 additions & 1 deletion src/core/server/saved_objects/routes/import.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ import { schema } from '@osd/config-schema';
import { IRouter } from '../../http';
import { importSavedObjectsFromStream } from '../import';
import { SavedObjectConfig } from '../saved_objects_config';
import { createSavedObjectsStreamFromNdJson } from './utils';
import { createSavedObjectsStreamFromNdJson, formatWorkspaces, workspacesValidator } from './utils';

interface FileStream extends Readable {
hapi: {
Expand All @@ -60,6 +60,7 @@ export const registerImportRoute = (router: IRouter, config: SavedObjectConfig)
{
overwrite: schema.boolean({ defaultValue: false }),
createNewCopies: schema.boolean({ defaultValue: false }),
workspaces: workspacesValidator,
},
{
validate: (object) => {
Expand Down Expand Up @@ -91,13 +92,16 @@ export const registerImportRoute = (router: IRouter, config: SavedObjectConfig)
});
}

const workspaces = formatWorkspaces(req.query.workspaces);

const result = await importSavedObjectsFromStream({
savedObjectsClient: context.core.savedObjects.client,
typeRegistry: context.core.savedObjects.typeRegistry,
readStream,
objectLimit: maxImportExportSize,
overwrite,
createNewCopies,
workspaces,
});

return res.ok({ body: result });
Expand Down
Loading

0 comments on commit e8cfc77

Please sign in to comment.