Skip to content

Commit

Permalink
dirty code hack removes conditional flows around only fetching some d…
Browse files Browse the repository at this point in the history 
…ocs, starts including migrating up and down and naive merge update
  • Loading branch information
@TinaHeiligers
TinaHeiligers committed Nov 9, 2023
1 parent eb627b9 commit 0531e2b
Show file tree
Hide file tree
Showing 2 changed files with 120 additions and 60 deletions.
156 changes: 111 additions & 45 deletions ...ges/core/saved-objects/core-saved-objects-api-server-internal/src/lib/apis/bulk_update.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import {
DecoratedError,
AuthorizeUpdateObject,
SavedObjectsRawDoc,
SavedObjectSanitizedDoc,
} from '@kbn/core-saved-objects-server';
import { SavedObjectsUtils } from '@kbn/core-saved-objects-utils-server';
import { encodeVersion } from '@kbn/core-saved-objects-base-server-internal';
Expand All @@ -21,6 +22,7 @@ import {
SavedObjectsBulkUpdateOptions,
SavedObjectsBulkUpdateResponse,
} from '@kbn/core-saved-objects-api-server';
import { MgetResponseItem } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import { DEFAULT_REFRESH_SETTING } from '../constants';
import {
type Either,
Expand All @@ -35,8 +37,11 @@ import {
rawDocExistsInNamespace,
isValidRequest,
isRight,
getSavedObjectFromSource,
mergeForUpdate,
} from './utils';
import { ApiExecutionContext } from './types';
import { MigrationHelper } from './helpers';

export interface PerformUpdateParams<T = unknown> {
objects: Array<SavedObjectsBulkUpdateObject<T>>;
Expand All @@ -61,21 +66,31 @@ export const performBulkUpdate = async <T>(

// START PREFLIGHT CHECKS: start of preflight check helpers calls and virtual hash maps creation(s) to keep track of objects.
let bulkGetRequestIndexCounter = 0;
type DocumentToSave = Record<string, unknown>;
type DocumentUpdates = Record<string, unknown>;
type ExpectedBulkGetResult = Either<
{ type: string; id: string; error: Payload },
{
type: string;
id: string;
version?: string;
documentToSave: DocumentToSave;
documentUpdates: DocumentUpdates;
objectNamespace?: string;
esRequestIndex: number;
migrationVersionCompatibility?: 'compatible' | 'raw';
rawDocSource?: MgetResponseItem<unknown>;
}
>;
// create the expected results from fetching all docs
const expectedBulkGetResults = objects.map<ExpectedBulkGetResult>((object) => {
const { type, id, attributes, references, version, namespace: objectNamespace } = object;
const {
type,
id,
attributes,
references,
version,
namespace: objectNamespace,
migrationVersionCompatibility,
} = object;
let error: DecoratedError | undefined;
// initial check on request validity
try {
Expand All @@ -94,31 +109,21 @@ export const performBulkUpdate = async <T>(
if (error) {
return left({ id, type, error: errorContent(error) });
}
// the update to the doc as Kibana Client sees it (possibly v1 when server is v2)
// we need to move this to after fetching and migrating the docs
const documentToSave = {
// at this point, these are partial updates to a doc, as requested
const documentUpdates = {
[type]: attributes,
updated_at: time,
...(Array.isArray(references) && { references }),
};
// we want to fetch all the docs, regardless of namespace type
const requiresNamespacesCheck = registry.isMultiNamespace(object.type);

return right({
type,
id,
version,
documentToSave,
documentUpdates,
objectNamespace,
// keep all objects, regardless of namespace checking requirements
...(!requiresNamespacesCheck && {
skipNamespaceCheck: true,
}),
...(requiresNamespacesCheck && {
skipNamespaceCheck: false,
}),
esRequestIndex: bulkGetRequestIndexCounter++,
migrationVersionCompatibility,
rawDocSource: {} as MgetResponseItem<unknown>,
});
});

Expand All @@ -137,8 +142,9 @@ export const performBulkUpdate = async <T>(
// `objectNamespace` is a namespace string, while `namespace` is a namespace ID.
// The object namespace string, if defined, will supersede the operation's namespace ID
// (converted here to the namespaceString for the namespace ID).
const namespaceString = SavedObjectsUtils.namespaceIdToString(namespace);
const getNamespaceString = (objectNamespace?: string) => objectNamespace ?? namespaceString;
const getNamespaceString = (objectNamespace?: string) =>
objectNamespace ?? SavedObjectsUtils.namespaceIdToString(namespace);

const getNamespaceId = (objectNamespace?: string) =>
objectNamespace !== undefined
? SavedObjectsUtils.namespaceStringToId(objectNamespace)
Expand Down Expand Up @@ -178,7 +184,7 @@ export const performBulkUpdate = async <T>(
type: string;
id: string;
namespaces: string[];
documentToSave: DocumentToSave;
documentUpdates: DocumentUpdates;
esRequestIndex: number;
}
>;
Expand All @@ -188,29 +194,39 @@ export const performBulkUpdate = async <T>(
return expectedBulkGetResult;
}

const { esRequestIndex, id, type, version, documentToSave, objectNamespace } =
expectedBulkGetResult.value;
const {
esRequestIndex,
id,
type,
version,
documentUpdates,
objectNamespace,
migrationVersionCompatibility,
} = expectedBulkGetResult.value;

let namespaces;
let versionProperties;
if (esRequestIndex !== undefined) {
const indexFound = bulkGetResponse?.statusCode !== 404;
const actualResult = indexFound ? bulkGetResponse?.body.docs[esRequestIndex] : undefined;
const docFound = indexFound && isMgetDoc(actualResult) && actualResult.found;
if (
!docFound ||
!rawDocExistsInNamespace(
registry,
actualResult as SavedObjectsRawDoc,
getNamespaceId(objectNamespace)
)
) {
return left({
id,
type,
error: errorContent(SavedObjectsErrorHelpers.createGenericNotFoundError(type, id)),
});
}

const indexFound = bulkGetResponse?.statusCode !== 404;
const actualResult = indexFound ? bulkGetResponse?.body.docs[esRequestIndex] : undefined;
const docFound = indexFound && isMgetDoc(actualResult) && actualResult.found;

if (
!docFound ||
!rawDocExistsInNamespace(
registry,
actualResult as SavedObjectsRawDoc,
getNamespaceId(objectNamespace)
)
) {
return left({
id,
type,
error: errorContent(SavedObjectsErrorHelpers.createGenericNotFoundError(type, id)),
});
}

if (registry.isMultiNamespace(type)) {
// @ts-expect-error MultiGetHit is incorrectly missing _id, _source
namespaces = actualResult!._source.namespaces ?? [
// @ts-expect-error MultiGetHit is incorrectly missing _id, _source
Expand All @@ -224,14 +240,64 @@ export const performBulkUpdate = async <T>(
}
versionProperties = getExpectedVersionProperties(version);
}
let migrated;
const documentFromSource = getSavedObjectFromSource<T>(
registry,
type,
id,
actualResult as SavedObjectsRawDoc,
{ migrationVersionCompatibility }
);
try {
migrated = migratorHelper.migrateStorageDocument(documentFromSource) as SavedObject<T>;
} catch (migrateStorageDocError) {
return left({
id,
type,
error: errorContent(
SavedObjectsErrorHelpers.decorateGeneralError(
migrateStorageDocError,
'Failed to migrate document to the latest version'
)
),
});
}
const updatedAttributes = mergeForUpdate({
targetAttributes: {
...migrated!.attributes,
},
updatedAttributes: await encryptionHelper.optionallyEncryptAttributes(
type,
id,
namespaces,
documentUpdates
),
typeMappings: registry.getType(type)!.mappings,
});
const migratedUpdatedSavedObjectDoc = migratorHelper.migrateInputDocument({
...migrated!,
id,
type,
// need to override the redacted NS values from the decrypted/migrated document
namespace: objectNamespace,
namespaces,
attributes: updatedAttributes,
...documentUpdates,
});

const docToSend = serializer.savedObjectToRaw(
migratedUpdatedSavedObjectDoc as SavedObjectSanitizedDoc
);
// @TINA: types problem, a lot above is wrong and a lot below needs to change.
const expectedResult = {
type,
id,
namespaces,
esRequestIndex: bulkUpdateRequestIndexCounter++,
documentToSave: expectedBulkGetResult.value.documentToSave,
documentUpdates: docToSend,
// documentUpdates: expectedBulkGetResult.value.documentUpdates, // merge raw updates with existing doc migrated to the client version
};
// migrate down, merge update, migrate back up.

bulkUpdateParams.push(
{
Expand All @@ -243,12 +309,12 @@ export const performBulkUpdate = async <T>(
},
{
doc: {
...documentToSave,
...documentUpdates,
[type]: await encryptionHelper.optionallyEncryptAttributes(
type,
id,
objectNamespace || namespace,
documentToSave[type]
documentUpdates[type]
),
},
}
Expand All @@ -274,7 +340,7 @@ export const performBulkUpdate = async <T>(
return expectedResult.value as any;
}

const { type, id, namespaces, documentToSave, esRequestIndex } = expectedResult.value;
const { type, id, namespaces, documentUpdates, esRequestIndex } = expectedResult.value;
const response = bulkUpdateResponse?.items[esRequestIndex] ?? {};
const rawResponse = Object.values(response)[0] as any;

Expand All @@ -288,7 +354,7 @@ export const performBulkUpdate = async <T>(
const { _seq_no: seqNo, _primary_term: primaryTerm, get } = rawResponse;

// eslint-disable-next-line @typescript-eslint/naming-convention
const { [type]: attributes, references, updated_at } = documentToSave;
const { [type]: attributes, references, updated_at } = documentUpdates;

const { originId } = get._source;
return {
Expand Down
24 changes: 9 additions & 15 deletions ...ed-objects/core-saved-objects-api-server-internal/src/lib/apis/helpers/preflight_check.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import type {
} from '@kbn/core-saved-objects-server';
import { SavedObjectsUtils } from '@kbn/core-saved-objects-utils-server';
import { SavedObjectsErrorHelpers, SavedObjectsRawDocSource } from '@kbn/core-saved-objects-server';
import { MgetResponseItem } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import type { RepositoryEsClient } from '../../repository_es_client';
import type { PreflightCheckForBulkDeleteParams } from '../internals/repository_bulk_delete_internal_types';
import type { CreatePointInTimeFinderFn } from '../../point_in_time_finder';
Expand Down Expand Up @@ -263,20 +264,11 @@ export class PreflightCheckHelper {
}: PreflightGetDocsForBulkUpdateParams) {
// const { expectedBulkGetResults, namespace } = params;
const validObjectsAsRight = validObjects as ExpectedBulkGetResultRight[];
const bulkGetDocs = validObjectsAsRight.map(({ value: { type, id, objectNamespace } }) => ({
_id: this.serializer.generateRawId(this.getNamespaceId(objectNamespace, namespace), type, id),
_index: this.getIndexForType(type),
}));

// `objectNamespace` is a namespace string, while `namespace` is a namespace ID.
// each of the validObjects in the map might have it's own objectNamespace, we get that using a custom function

// @TINA note: only using type, id and namespace to get the docs, not searching by attributes
const bulkGetDocs = validObjectsAsRight
.filter(({ value }) => value.esRequestIndex !== undefined) // no-op, should be every doc anyway
.map(({ value: { type, id, objectNamespace } }) => ({
_id: this.serializer.generateRawId(this.getNamespaceId(objectNamespace), type, id),
_index: this.getIndexForType(type), // the index in which to get the object
_source: true, // we need all the fields and not only type and namespaces
// _source: ['type', 'namespaces'],
}));
// @TINA note: initial call to fetch all docs, seems to be issued for single and multinamespace types
const bulkGetResponse = bulkGetDocs.length
? await this.client.mget({ body: { docs: bulkGetDocs } }, { ignore: [404], meta: true })
: undefined;
Expand Down Expand Up @@ -407,7 +399,7 @@ export type ExpectedBulkGetResult = Either<
type: string;
id: string;
version?: string;
documentToSave: DocumentToSave;
documentUpdates: DocumentToSave;
objectNamespace?: string;
esRequestIndex?: number;
}
Expand All @@ -427,8 +419,10 @@ interface ExpectedBulkGetResultRight {
type: string;
id: string;
version?: string;
documentToSave: DocumentToSave;
documentUpdates: DocumentToSave;
objectNamespace?: string;
esRequestIndex?: number;
migrationVersionCompatibility?: 'compatible' | 'raw';
rawDocSource?: MgetResponseItem<unknown>;
};
}

0 comments on commit 0531e2b

Please sign in to comment.