From 79df02f43245994e538c1ae0eb5f44b9f312d30f Mon Sep 17 00:00:00 2001 From: "FAREAST\\vikassingh" Date: Fri, 18 Nov 2022 18:44:14 +0530 Subject: [PATCH] Added Hierarchical Partition Key Support --- sdk/cosmosdb/cosmos/src/ClientContext.ts | 4 +- sdk/cosmosdb/cosmos/src/client/Item/Item.ts | 8 +- sdk/cosmosdb/cosmos/src/client/Item/Items.ts | 51 ++++--- .../cosmos/src/documents/PartitionKey.ts | 44 +++++- .../src/documents/PartitionKeyDefinition.ts | 9 +- .../PartitionKeyDefinitionVersion.ts | 6 + .../cosmos/src/documents/PartitionKind.ts | 6 + sdk/cosmosdb/cosmos/src/documents/index.ts | 2 + .../cosmos/src/extractPartitionKey.ts | 46 ++++-- sdk/cosmosdb/cosmos/src/request/request.ts | 7 +- sdk/cosmosdb/cosmos/src/utils/batch.ts | 132 ++++++++---------- sdk/cosmosdb/cosmos/src/utils/hashing/hash.ts | 18 +++ .../cosmos/src/utils/hashing/multiHash.ts | 8 ++ sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts | 17 +-- sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts | 10 +- sdk/cosmosdb/cosmos/src/utils/typeChecks.ts | 18 +++ .../test/internal/unit/hashing/v1.spec.ts | 26 ++-- .../test/internal/unit/hashing/v2.spec.ts | 24 ++-- .../test/internal/unit/utils/batch.spec.ts | 28 ---- .../cosmos/test/public/common/TestHelpers.ts | 45 +++--- 20 files changed, 301 insertions(+), 208 deletions(-) create mode 100644 sdk/cosmosdb/cosmos/src/documents/PartitionKeyDefinitionVersion.ts create mode 100644 sdk/cosmosdb/cosmos/src/documents/PartitionKind.ts create mode 100644 sdk/cosmosdb/cosmos/src/utils/hashing/hash.ts create mode 100644 sdk/cosmosdb/cosmos/src/utils/hashing/multiHash.ts create mode 100644 sdk/cosmosdb/cosmos/src/utils/typeChecks.ts delete mode 100644 sdk/cosmosdb/cosmos/test/internal/unit/utils/batch.spec.ts diff --git a/sdk/cosmosdb/cosmos/src/ClientContext.ts b/sdk/cosmosdb/cosmos/src/ClientContext.ts index f72c13940d2b..0810f1cbe9a1 100644 --- a/sdk/cosmosdb/cosmos/src/ClientContext.ts +++ b/sdk/cosmosdb/cosmos/src/ClientContext.ts @@ -13,7 +13,7 @@ import { Constants, HTTPMethod, OperationType, ResourceType } from "./common/con import { getIdFromLink, getPathFromLink, parseLink } from "./common/helper"; import { StatusCodes, SubStatusCodes } from "./common/statusCodes"; import { Agent, CosmosClientOptions } from "./CosmosClientOptions"; -import { ConnectionPolicy, ConsistencyLevel, DatabaseAccount, PartitionKey } from "./documents"; +import { ConnectionPolicy, ConsistencyLevel, DatabaseAccount, PartitionKey, mapPartitionToInternal } from "./documents"; import { GlobalEndpointManager } from "./globalEndpointManager"; import { PluginConfig, PluginOn, executePlugins } from "./plugins/Plugin"; import { FetchFunctionCallback, SqlQuerySpec } from "./queryExecutionContext"; @@ -757,7 +757,7 @@ export class ClientContext { options: requestContext.options, partitionKeyRangeId: requestContext.partitionKeyRangeId, useMultipleWriteLocations: this.connectionPolicy.useMultipleWriteLocations, - partitionKey: requestContext.partitionKey, + partitionKey: requestContext.partitionKey !== undefined ? mapPartitionToInternal(requestContext.partitionKey) : undefined, }); } diff --git a/sdk/cosmosdb/cosmos/src/client/Item/Item.ts b/sdk/cosmosdb/cosmos/src/client/Item/Item.ts index d402f3687cc1..9231bfb212ee 100644 --- a/sdk/cosmosdb/cosmos/src/client/Item/Item.ts +++ b/sdk/cosmosdb/cosmos/src/client/Item/Item.ts @@ -9,7 +9,7 @@ import { ResourceType, StatusCodes, } from "../../common"; -import { PartitionKey } from "../../documents"; +import { PartitionKey, PartitionKeyInternal, mapPartitionToInternal } from "../../documents"; import { extractPartitionKey, undefinedPartitionKey } from "../../extractPartitionKey"; import { RequestOptions, Response } from "../../request"; import { PatchRequestBody } from "../../utils/patch"; @@ -24,7 +24,7 @@ import { ItemResponse } from "./ItemResponse"; * @see {@link Items} for operations on all items; see `container.items`. */ export class Item { - private partitionKey: PartitionKey; + private partitionKey: PartitionKeyInternal; /** * Returns a reference URL to the resource. Used for linking in Permissions. */ @@ -41,10 +41,10 @@ export class Item { constructor( public readonly container: Container, public readonly id: string, - partitionKey: PartitionKey, + partitionKey: PartitionKey | undefined, private readonly clientContext: ClientContext ) { - this.partitionKey = partitionKey; + this.partitionKey = partitionKey === undefined ? undefined : mapPartitionToInternal(partitionKey); } /** diff --git a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts index 371e40f7a8e1..a61a45e6a68e 100644 --- a/sdk/cosmosdb/cosmos/src/client/Item/Items.ts +++ b/sdk/cosmosdb/cosmos/src/client/Item/Items.ts @@ -17,16 +17,16 @@ import { ItemResponse } from "./ItemResponse"; import { Batch, isKeyInRange, - Operation, - getPartitionKeyToHash, - decorateOperation, + prepareOperations, OperationResponse, OperationInput, BulkOptions, decorateBatchOperation, + hasResource, } from "../../utils/batch"; -import { hashV1PartitionKey } from "../../utils/hashing/v1"; -import { hashV2PartitionKey } from "../../utils/hashing/v2"; +import { stripUndefined } from "../../utils/typeChecks"; +import { hashPartitionKey } from "../../utils/hashing/hash"; +import { PartitionKeyDefinition } from "../../documents"; /** * @hidden @@ -408,7 +408,8 @@ export class Items { const { resources: partitionKeyRanges } = await this.container .readPartitionKeyRanges() .fetchAll(); - const { resource: definition } = await this.container.getPartitionKeyDefinition(); + let { resource } = await this.container.readPartitionKeyDefinition(); + const partitionDefinition = stripUndefined(resource, "PartitionKeyDefinition."); const batches: Batch[] = partitionKeyRanges.map((keyRange: PartitionKeyRange) => { return { min: keyRange.minInclusive, @@ -418,19 +419,8 @@ export class Items { operations: [], }; }); - operations - .map((operation) => decorateOperation(operation, definition, options)) - .forEach((operation: Operation, index: number) => { - const partitionProp = definition.paths[0].replace("/", ""); - const isV2 = definition.version && definition.version === 2; - const toHashKey = getPartitionKeyToHash(operation, partitionProp); - const hashed = isV2 ? hashV2PartitionKey(toHashKey) : hashV1PartitionKey(toHashKey); - const batchForKey = batches.find((batch: Batch) => { - return isKeyInRange(batch.min, batch.max, hashed); - }); - batchForKey.operations.push(operation); - batchForKey.indexes.push(index); - }); + + this.groupOperationsBasedOnPartitionKey(operations, partitionDefinition, options, batches); const path = getPathFromLink(this.container.url, ResourceType.item); @@ -460,7 +450,7 @@ export class Items { // partition key types as well since we don't support them, so for now we throw if (err.code === 410) { throw new Error( - "Partition key error. Either the partitions have split or an operation has an unsupported partitionKey type" + "Partition key error. Either the partitions have split or an operation has an unsupported partitionKey type" + err.message ); } throw new Error(`Bulk request errored with: ${err.message}`); @@ -470,6 +460,27 @@ export class Items { return orderedResponses; } + /** + * Function to create batches based of partition key Ranges. + * 1. if {@link operation} has {@link hasResource} then {@link PartitionKey} is extracted from there. + * @param operations + * @param partitionDefinition + * @param options + * @param batches + */ + private groupOperationsBasedOnPartitionKey(operations: OperationInput[], partitionDefinition: PartitionKeyDefinition, options: RequestOptions | undefined, batches: Batch[]) { + operations + .forEach((operationInput, index: number) => { + const { operation, partitionKey } = prepareOperations(operationInput, partitionDefinition, options); + const hashed = hashPartitionKey(stripUndefined(partitionKey, "PartitionKey"), partitionDefinition); + const batchForKey = stripUndefined(batches.find((batch: Batch) => { + return isKeyInRange(batch.min, batch.max, hashed); + }), "No suitable Batch found."); + batchForKey.operations.push(operation); + batchForKey.indexes.push(index); + }); + } + /** * Execute transactional batch operations on items. * diff --git a/sdk/cosmosdb/cosmos/src/documents/PartitionKey.ts b/sdk/cosmosdb/cosmos/src/documents/PartitionKey.ts index be19cd7d496d..374e4217d038 100644 --- a/sdk/cosmosdb/cosmos/src/documents/PartitionKey.ts +++ b/sdk/cosmosdb/cosmos/src/documents/PartitionKey.ts @@ -1,5 +1,45 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -import { PartitionKeyDefinition } from "./PartitionKeyDefinition"; -export type PartitionKey = PartitionKeyDefinition | string | number | unknown; +export type PartitionKey = PrimitivePartitionKeyValue | PrimitivePartitionKeyValue[]; + +/** + * Internal Representation Of Partition Key. + */ +export type PartitionKeyInternal = PrimitivePartitionKeyValue[]; + +/** + * A primitive Partition Key value. + */ +export type PrimitivePartitionKeyValue = + | string + | number + | boolean + | NullPartitionType + | NonePartitionKey; + +/** + * The returned object represents a partition key value that allows creating and accessing items + * with a null value for the partition key. + */ +export type NullPartitionType = null; +export const NullPartitionKeyLiteral: NullPartitionType = null; + +/** + * The returned object represents a partition key value that allows creating and accessing items + * without a value for partition key + */ +export type NonePartitionKey = { + [K in any]: never; +}; +export const NonePartitionKeyLiteral: NonePartitionKey = {}; + +/** + * Maps PartitionKey to InternalPartitionKey. + * @param partitionKey + * @returns + */ +export function mapPartitionToInternal(partitionKey: PartitionKey): PartitionKeyInternal { + if (Array.isArray(partitionKey)) return partitionKey; + else return [partitionKey]; +} \ No newline at end of file diff --git a/sdk/cosmosdb/cosmos/src/documents/PartitionKeyDefinition.ts b/sdk/cosmosdb/cosmos/src/documents/PartitionKeyDefinition.ts index 983bb837236c..98dbf0408a19 100644 --- a/sdk/cosmosdb/cosmos/src/documents/PartitionKeyDefinition.ts +++ b/sdk/cosmosdb/cosmos/src/documents/PartitionKeyDefinition.ts @@ -1,5 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +import { PartitionKeyDefinitionVersion } from "./PartitionKeyDefinitionVersion"; +import { PartitionKeyKind } from "./PartitionKind"; + export interface PartitionKeyDefinition { /** * An array of paths for which data within the collection can be partitioned. Paths must not contain a wildcard or @@ -11,6 +14,10 @@ export interface PartitionKeyDefinition { * An optional field, if not specified the default value is 1. To use the large partition key set the version to 2. * To learn about large partition keys, see [how to create containers with large partition key](https://docs.microsoft.com/en-us/azure/cosmos-db/large-partition-keys) article. */ - version?: number; + version?: PartitionKeyDefinitionVersion; systemKey?: boolean; + /** + * What kind of partition key is being defined (default: "Hash") + */ + kind?: PartitionKeyKind; } diff --git a/sdk/cosmosdb/cosmos/src/documents/PartitionKeyDefinitionVersion.ts b/sdk/cosmosdb/cosmos/src/documents/PartitionKeyDefinitionVersion.ts new file mode 100644 index 000000000000..cbea94767fed --- /dev/null +++ b/sdk/cosmosdb/cosmos/src/documents/PartitionKeyDefinitionVersion.ts @@ -0,0 +1,6 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +export enum PartitionKeyDefinitionVersion { + V1 = 1, + V2 = 2 + } \ No newline at end of file diff --git a/sdk/cosmosdb/cosmos/src/documents/PartitionKind.ts b/sdk/cosmosdb/cosmos/src/documents/PartitionKind.ts new file mode 100644 index 000000000000..65946c3ddc90 --- /dev/null +++ b/sdk/cosmosdb/cosmos/src/documents/PartitionKind.ts @@ -0,0 +1,6 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +export enum PartitionKeyKind { + Hash = "Hash", + MultiHash = "MultiHash" +} \ No newline at end of file diff --git a/sdk/cosmosdb/cosmos/src/documents/index.ts b/sdk/cosmosdb/cosmos/src/documents/index.ts index 0c6afe336af8..5c86e4370c10 100644 --- a/sdk/cosmosdb/cosmos/src/documents/index.ts +++ b/sdk/cosmosdb/cosmos/src/documents/index.ts @@ -10,6 +10,8 @@ export * from "./IndexingMode"; export * from "./IndexingPolicy"; export * from "./IndexKind"; export * from "./PartitionKey"; +export * from "./PartitionKeyDefinitionVersion" +export * from "./PartitionKind" export * from "./PartitionKeyDefinition"; export * from "./PermissionMode"; export * from "./TriggerOperation"; diff --git a/sdk/cosmosdb/cosmos/src/extractPartitionKey.ts b/sdk/cosmosdb/cosmos/src/extractPartitionKey.ts index e4ef2c425742..145dc128a5ba 100644 --- a/sdk/cosmosdb/cosmos/src/extractPartitionKey.ts +++ b/sdk/cosmosdb/cosmos/src/extractPartitionKey.ts @@ -1,47 +1,67 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +import { AzureLogger, createClientLogger } from "@azure/logger"; import { parsePath } from "./common"; -import { PartitionKey, PartitionKeyDefinition } from "./documents"; +import { NonePartitionKeyLiteral, NullPartitionKeyLiteral, PartitionKeyDefinition, PartitionKeyInternal, PrimitivePartitionKeyValue } from "./documents"; + +const logger: AzureLogger = createClientLogger("extractPartitionKey"); /** + * Function to extract PartitionKey based on {@link PartitionKeyDefinition} + * from an object. + * Retuns + * 1. {@link PartitionKeyInternal[]} - if extraction is successful. + * 2. {@link undefined} - if either {@link partitionKeyDefinition} is not well formed + * or an unsupported partitionkey type is encountered. * @hidden */ export function extractPartitionKey( document: unknown, - partitionKeyDefinition: PartitionKeyDefinition -): PartitionKey[] { + partitionKeyDefinition?: PartitionKeyDefinition +): PartitionKeyInternal | undefined { if ( partitionKeyDefinition && partitionKeyDefinition.paths && partitionKeyDefinition.paths.length > 0 ) { - const partitionKey: PartitionKey[] = []; + const partitionKeys: PrimitivePartitionKeyValue[] = []; partitionKeyDefinition.paths.forEach((path: string) => { - const pathParts = parsePath(path); + const pathParts: string[] = parsePath(path); let obj = document; for (const part of pathParts) { - if (typeof obj === "object" && part in obj) { + if (typeof obj === "object" && obj !==null && part in obj) { obj = (obj as Record)[part]; } else { obj = undefined; break; } } - partitionKey.push(obj); + if(typeof obj === "string" || typeof obj === "number" || typeof obj === "boolean" ) { + partitionKeys.push(obj); + } else if(obj === NullPartitionKeyLiteral) { + partitionKeys.push(NullPartitionKeyLiteral); + } else if(obj === undefined || JSON.stringify(obj) === JSON.stringify(NonePartitionKeyLiteral)) { + if(partitionKeyDefinition.systemKey === true) { + return [] + } + partitionKeys.push(NonePartitionKeyLiteral); + } else { + logger.warning('Unsupported PartitionKey found.'); + return undefined; + } }); - if (partitionKey.length === 1 && partitionKey[0] === undefined) { - return undefinedPartitionKey(partitionKeyDefinition); - } - return partitionKey; + return partitionKeys; } + logger.warning("Unexpected Partition Key Definition Found."); + return undefined; } /** * @hidden */ -export function undefinedPartitionKey(partitionKeyDefinition: PartitionKeyDefinition): unknown[] { +export function undefinedPartitionKey(partitionKeyDefinition: PartitionKeyDefinition): PartitionKeyInternal { if (partitionKeyDefinition.systemKey === true) { return []; } else { - return [{}]; + return partitionKeyDefinition.paths.map(() => NonePartitionKeyLiteral); } } diff --git a/sdk/cosmosdb/cosmos/src/request/request.ts b/sdk/cosmosdb/cosmos/src/request/request.ts index 65a1e42daf74..b680398f8821 100644 --- a/sdk/cosmosdb/cosmos/src/request/request.ts +++ b/sdk/cosmosdb/cosmos/src/request/request.ts @@ -3,7 +3,7 @@ import { setAuthorizationHeader } from "../auth"; import { Constants, HTTPMethod, jsonStringifyAndEscapeNonASCII, ResourceType } from "../common"; import { CosmosClientOptions } from "../CosmosClientOptions"; -import { PartitionKey } from "../documents"; +import { PartitionKeyInternal } from "../documents"; import { CosmosHeaders } from "../queryExecutionContext"; import { FeedOptions, RequestOptions } from "./index"; import { defaultLogger } from "../common/logger"; @@ -41,7 +41,7 @@ interface GetHeadersOptions { options: RequestOptions & FeedOptions; partitionKeyRangeId?: string; useMultipleWriteLocations?: boolean; - partitionKey?: PartitionKey; + partitionKey?: PartitionKeyInternal; } const JsonContentType = "application/json"; @@ -168,9 +168,6 @@ export async function getHeaders({ } if (partitionKey !== undefined && !headers[Constants.HttpHeaders.PartitionKey]) { - if (partitionKey === null || !Array.isArray(partitionKey)) { - partitionKey = [partitionKey as string]; - } headers[Constants.HttpHeaders.PartitionKey] = jsonStringifyAndEscapeNonASCII(partitionKey); } diff --git a/sdk/cosmosdb/cosmos/src/utils/batch.ts b/sdk/cosmosdb/cosmos/src/utils/batch.ts index 537d2f2f3bad..e647c00f8a2b 100644 --- a/sdk/cosmosdb/cosmos/src/utils/batch.ts +++ b/sdk/cosmosdb/cosmos/src/utils/batch.ts @@ -3,10 +3,12 @@ import { JSONObject } from "../queryExecutionContext"; import { extractPartitionKey } from "../extractPartitionKey"; -import { PartitionKeyDefinition } from "../documents"; +import { NonePartitionKeyLiteral, PartitionKey, PartitionKeyDefinition, PrimitivePartitionKeyValue, mapPartitionToInternal } from "../documents"; import { RequestOptions } from ".."; import { PatchRequestBody } from "./patch"; import { v4 } from "uuid"; +import { parsePath } from "../common"; +import { stripUndefined } from "./typeChecks"; const uuid = v4; export type Operation = @@ -17,6 +19,7 @@ export type Operation = | ReplaceOperation | BulkPatchOperation; + export interface Batch { min: string; max: string; @@ -70,7 +73,7 @@ export type OperationInput = | PatchOperationInput; export interface CreateOperationInput { - partitionKey?: string | number | null | Record | undefined; + partitionKey?: PartitionKey; ifMatch?: string; ifNoneMatch?: string; operationType: typeof BulkOperationType.Create; @@ -78,7 +81,7 @@ export interface CreateOperationInput { } export interface UpsertOperationInput { - partitionKey?: string | number | null | Record | undefined; + partitionKey?: PartitionKey; ifMatch?: string; ifNoneMatch?: string; operationType: typeof BulkOperationType.Upsert; @@ -86,19 +89,19 @@ export interface UpsertOperationInput { } export interface ReadOperationInput { - partitionKey?: string | number | boolean | null | Record | undefined; + partitionKey?: PartitionKey; operationType: typeof BulkOperationType.Read; id: string; } export interface DeleteOperationInput { - partitionKey?: string | number | null | Record | undefined; + partitionKey?: PartitionKey; operationType: typeof BulkOperationType.Delete; id: string; } export interface ReplaceOperationInput { - partitionKey?: string | number | null | Record | undefined; + partitionKey?: PartitionKey; ifMatch?: string; ifNoneMatch?: string; operationType: typeof BulkOperationType.Replace; @@ -107,7 +110,7 @@ export interface ReplaceOperationInput { } export interface PatchOperationInput { - partitionKey?: string | number | null | Record | undefined; + partitionKey?: PartitionKey; ifMatch?: string; ifNoneMatch?: string; operationType: typeof BulkOperationType.Patch; @@ -155,59 +158,63 @@ export function hasResource( (operation as OperationWithItem).resourceBody !== undefined ); } - -export function getPartitionKeyToHash(operation: Operation, partitionProperty: string): any { - const toHashKey = hasResource(operation) - ? deepFind(operation.resourceBody, partitionProperty) - : (operation.partitionKey && operation.partitionKey.replace(/[[\]"']/g, "")) || - operation.partitionKey; - // We check for empty object since replace will stringify the value - // The second check avoids cases where the partitionKey value is actually the string '{}' - if (toHashKey === "{}" && operation.partitionKey === "[{}]") { - return {}; - } - if (toHashKey === "null" && operation.partitionKey === "[null]") { - return null; +/** + * Maps + * @param operationInput + * @param definition + * @param options + * @returns + */ +export function prepareOperations( + operationInput: OperationInput, + definition: PartitionKeyDefinition, + options: RequestOptions = {} +): { + operation: Operation, + partitionKey: PrimitivePartitionKeyValue[] +} { + + populateIdsIfNeeded(operationInput, options); + + let partitionKey: PrimitivePartitionKeyValue[]; + if(operationInput.hasOwnProperty("partitionKey")) { + if(operationInput.partitionKey === undefined) { + partitionKey = definition.paths.map(() => NonePartitionKeyLiteral) + } else { + partitionKey = mapPartitionToInternal(operationInput.partitionKey) + } + } else { + switch (operationInput.operationType) { + case BulkOperationType.Create: + case BulkOperationType.Replace: + case BulkOperationType.Upsert: + partitionKey = stripUndefined(extractPartitionKey(operationInput.resourceBody, definition), ""); + break; + case BulkOperationType.Read: + case BulkOperationType.Delete: + case BulkOperationType.Patch: + partitionKey = definition.paths.map(() => NonePartitionKeyLiteral); + } } - if (toHashKey === "0" && operation.partitionKey === "[0]") { - return 0; + return { + operation: { ...operationInput, partitionKey: JSON.stringify(partitionKey) } as Operation, + partitionKey } - return toHashKey; } -export function decorateOperation( - operation: OperationInput, - definition: PartitionKeyDefinition, - options: RequestOptions = {} -): Operation { - if ( - operation.operationType === BulkOperationType.Create || - operation.operationType === BulkOperationType.Upsert - ) { - if ( - (operation.resourceBody.id === undefined || operation.resourceBody.id === "") && - !options.disableAutomaticIdGeneration - ) { - operation.resourceBody.id = uuid(); +/** + * For operations requiring Id genrate random uuids. + * @param operationInput + * @param options + */ +function populateIdsIfNeeded(operationInput: OperationInput, options: RequestOptions) { + if (operationInput.operationType === BulkOperationType.Create || + operationInput.operationType === BulkOperationType.Upsert) { + if ((operationInput.resourceBody.id === undefined || operationInput.resourceBody.id === "") && + !options.disableAutomaticIdGeneration) { + operationInput.resourceBody.id = uuid(); } } - if ("partitionKey" in operation) { - const extracted = extractPartitionKey(operation, { paths: ["/partitionKey"] }); - return { ...operation, partitionKey: JSON.stringify(extracted) } as Operation; - } else if ( - operation.operationType === BulkOperationType.Create || - operation.operationType === BulkOperationType.Replace || - operation.operationType === BulkOperationType.Upsert - ) { - const pk = extractPartitionKey(operation.resourceBody, definition); - return { ...operation, partitionKey: JSON.stringify(pk) } as Operation; - } else if ( - operation.operationType === BulkOperationType.Read || - operation.operationType === BulkOperationType.Delete - ) { - return { ...operation, partitionKey: "[{}]" }; - } - return operation as Operation; } export function decorateBatchOperation( @@ -227,19 +234,4 @@ export function decorateBatchOperation( } return operation as Operation; } -/** - * Util function for finding partition key values nested in objects at slash (/) separated paths - * @hidden - */ -export function deepFind(document: T, path: P): string | JSONObject { - const apath = path.split("/"); - let h: any = document; - for (const p of apath) { - if (p in h) h = h[p]; - else { - console.warn(`Partition key not found, using undefined: ${path} at ${p}`); - return "{}"; - } - } - return h; -} + diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/hash.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/hash.ts new file mode 100644 index 000000000000..c77b99497638 --- /dev/null +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/hash.ts @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { PartitionKeyDefinition, PartitionKeyDefinitionVersion, PartitionKeyKind, PrimitivePartitionKeyValue } from "../../documents"; +import { hashMultiHashPartitionKey } from "./multiHash"; +import { hashV1PartitionKey } from "./v1"; +import { hashV2PartitionKey } from "./v2"; + +export function hashPartitionKey(partitionKey: PrimitivePartitionKeyValue[], partitionDefinition: PartitionKeyDefinition): string { + const kind: PartitionKeyKind = partitionDefinition?.kind || PartitionKeyKind.Hash; // Default value. + const isV2 = partitionDefinition && partitionDefinition.version && partitionDefinition.version === PartitionKeyDefinitionVersion.V2 + switch (kind) { + case PartitionKeyKind.Hash: + return isV2 ? hashV2PartitionKey(partitionKey) : hashV1PartitionKey(partitionKey); + case PartitionKeyKind.MultiHash: + return hashMultiHashPartitionKey(partitionKey); + } +} \ No newline at end of file diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/multiHash.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/multiHash.ts new file mode 100644 index 000000000000..44933179154a --- /dev/null +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/multiHash.ts @@ -0,0 +1,8 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +import { PrimitivePartitionKeyValue } from "../../documents"; +import { hashV2PartitionKey } from "./v2"; + +export function hashMultiHashPartitionKey(partitionKey: PrimitivePartitionKeyValue[]): string { + return partitionKey.map(keys => hashV2PartitionKey([keys])).join(""); + } \ No newline at end of file diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts index 66794b8a9924..0d433b1d4692 100644 --- a/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/v1.ts @@ -5,20 +5,21 @@ import { doubleToByteArrayJSBI, writeNumberForBinaryEncodingJSBI } from "./encod import { writeStringForBinaryEncoding } from "./encoding/string"; import { BytePrefix } from "./encoding/prefix"; import MurmurHash from "./murmurHash"; +import { PrimitivePartitionKeyValue } from "../../documents"; const MAX_STRING_CHARS = 100; -type v1Key = string | number | boolean | null | Record | undefined; - -export function hashV1PartitionKey(partitionKey: v1Key): string { - const toHash = prefixKeyByType(partitionKey); +export function hashV1PartitionKey(partitionKey: PrimitivePartitionKeyValue[]): string { + const key = partitionKey[0]; + const toHash = prefixKeyByType(key); const hash = MurmurHash.x86.hash32(toHash); const encodedJSBI = writeNumberForBinaryEncodingJSBI(hash); - const encodedValue = encodeByType(partitionKey); - return Buffer.concat([encodedJSBI, encodedValue]).toString("hex").toUpperCase(); + const encodedValue = encodeByType(key); + const finalHash = Buffer.concat([encodedJSBI, encodedValue]).toString("hex").toUpperCase(); + return finalHash; } -function prefixKeyByType(key: v1Key): Buffer { +function prefixKeyByType(key: PrimitivePartitionKeyValue): Buffer { let bytes: Buffer; switch (typeof key) { case "string": { @@ -53,7 +54,7 @@ function prefixKeyByType(key: v1Key): Buffer { } } -function encodeByType(key: v1Key): Buffer { +function encodeByType(key: PrimitivePartitionKeyValue): Buffer { switch (typeof key) { case "string": { const truncated = key.substr(0, MAX_STRING_CHARS); diff --git a/sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts b/sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts index d2138195ad7f..806741f16834 100644 --- a/sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts +++ b/sdk/cosmosdb/cosmos/src/utils/hashing/v2.ts @@ -1,21 +1,21 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +import { PrimitivePartitionKeyValue } from "../../documents"; import { doubleToByteArrayJSBI } from "./encoding/number"; import { BytePrefix } from "./encoding/prefix"; import MurmurHash from "./murmurHash"; -type v2Key = string | number | boolean | null | Record | undefined; - -export function hashV2PartitionKey(partitionKey: v2Key): string { - const toHash = prefixKeyByType(partitionKey); +export function hashV2PartitionKey(partitionKey: PrimitivePartitionKeyValue[]): string { + let toHash: Buffer = Buffer.concat(partitionKey.map(prefixKeyByType)) const hash = MurmurHash.x64.hash128(toHash); const reverseBuff: Buffer = reverse(Buffer.from(hash, "hex")); reverseBuff[0] &= 0x3f; return reverseBuff.toString("hex").toUpperCase(); } -function prefixKeyByType(key: v2Key): Buffer { + +function prefixKeyByType(key: PrimitivePartitionKeyValue): Buffer { let bytes: Buffer; switch (typeof key) { case "string": { diff --git a/sdk/cosmosdb/cosmos/src/utils/typeChecks.ts b/sdk/cosmosdb/cosmos/src/utils/typeChecks.ts new file mode 100644 index 000000000000..ddd6ffd05a50 --- /dev/null +++ b/sdk/cosmosdb/cosmos/src/utils/typeChecks.ts @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +export type NonUndefinable = T extends undefined ? never : T; + +/** + * Utility function to avoid writing boilder plate code while checking for + * undefined values. It throws Error if the input value is undefined. + * @param value Value which is potentially undefined. + * @param msg Error Message to throw if value is undefined. + * @returns + */ +export function stripUndefined(value: T, msg?: string): NonUndefinable { + if(value !== undefined) { + return value as NonUndefinable; + } + throw new Error(msg || "Unexpected 'undefined' value encountered"); +} diff --git a/sdk/cosmosdb/cosmos/test/internal/unit/hashing/v1.spec.ts b/sdk/cosmosdb/cosmos/test/internal/unit/hashing/v1.spec.ts index 5a72780dfe53..95871f76b33d 100644 --- a/sdk/cosmosdb/cosmos/test/internal/unit/hashing/v1.spec.ts +++ b/sdk/cosmosdb/cosmos/test/internal/unit/hashing/v1.spec.ts @@ -8,56 +8,56 @@ describe("effectivePartitionKey", function () { describe("computes v1 key", function () { const toMatch = [ { - key: "partitionKey", + key: ["partitionKey"], output: "05C1E1B3D9CD2608716273756A756A706F4C667A00", }, { - key: "redmond", + key: ["redmond"], output: "05C1EFE313830C087366656E706F6500", }, { - key: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + key: ["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"], output: "05C1EB5921F706086262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626262626200", }, { - key: "", + key: [""], output: "05C1CF33970FF80800", }, { - key: "aa", + key: ["aa"], output: "05C1C7B7270FE008626200", }, { - key: null, + key: [null], output: "05C1ED45D7475601", }, { - key: true, + key: [true], output: "05C1D7C5A903D803", }, { - key: false, + key: [false], output: "05C1DB857D857C02", }, { - key: {}, + key: [{}], output: "05C1D529E345DC00", }, { - key: 5, + key: [5], output: "05C1D9C1C5517C05C014", }, { - key: 5.5, + key: [5.5], output: "05C1D7A771716C05C016", }, { - key: 12313.1221, + key: [12313.1221], output: "05C1ED154D592E05C0C90723F50FC925D8", }, { - key: 123456789, + key: [123456789], output: "05C1D9E1A5311C05C19DB7CD8B40", }, ]; diff --git a/sdk/cosmosdb/cosmos/test/internal/unit/hashing/v2.spec.ts b/sdk/cosmosdb/cosmos/test/internal/unit/hashing/v2.spec.ts index b1634cfc3506..51700151a5a0 100644 --- a/sdk/cosmosdb/cosmos/test/internal/unit/hashing/v2.spec.ts +++ b/sdk/cosmosdb/cosmos/test/internal/unit/hashing/v2.spec.ts @@ -8,51 +8,51 @@ describe("effectivePartitionKey", function () { describe("computes v2 key", function () { const toMatch = [ { - key: "redmond", + key: ["redmond"], output: "22E342F38A486A088463DFF7838A5963", }, { - key: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + key: ["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"], output: "0BA3E9CA8EE4C14538828D1612A4B652", }, { - key: "", + key: [""], output: "32E9366E637A71B4E710384B2F4970A0", }, { - key: "aa", + key: ["aa"], output: "05033626483AE80D00E44FBD35362B19", }, { - key: null, + key: [null], output: "378867E4430E67857ACE5C908374FE16", }, { - key: true, + key: [true], output: "0E711127C5B5A8E4726AC6DD306A3E59", }, { - key: false, + key: [false], output: "2FE1BE91E90A3439635E0E9E37361EF2", }, { - key: {}, + key: [{}], output: "11622DAA78F835834610ABE56EFF5CB5", }, { - key: 5, + key: [5], output: "19C08621B135968252FB34B4CF66F811", }, { - key: 5.5, + key: [5.5], output: "0E2EE47829D1AF775EEFB6540FD1D0ED", }, { - key: 12313.1221, + key: [12313.1221], output: "27E7ECA8F2EE3E53424DE8D5220631C6", }, { - key: 123456789, + key: [123456789], output: "1F56D2538088EBA82CCF988F36E16760", }, ]; diff --git a/sdk/cosmosdb/cosmos/test/internal/unit/utils/batch.spec.ts b/sdk/cosmosdb/cosmos/test/internal/unit/utils/batch.spec.ts deleted file mode 100644 index f9af90f73958..000000000000 --- a/sdk/cosmosdb/cosmos/test/internal/unit/utils/batch.spec.ts +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT license. - -import assert from "assert"; -import { deepFind } from "../../../../src/utils/batch"; - -describe("batch utils", function () { - it("deep finds nested partition key values in objects", function () { - const testTwiceNested = { - nested: { - nested2: { - key: "value", - }, - }, - }; - const testNested = { - nested: { - key: "value", - }, - }; - const testBase = { - key: "value", - }; - assert.equal(deepFind(testNested, "nested/key"), "value"); - assert.equal(deepFind(testBase, "key"), "value"); - assert.equal(deepFind(testTwiceNested, "nested/nested2/key"), "value"); - }); -}); diff --git a/sdk/cosmosdb/cosmos/test/public/common/TestHelpers.ts b/sdk/cosmosdb/cosmos/test/public/common/TestHelpers.ts index 388992e35b66..5850c4a08913 100644 --- a/sdk/cosmosdb/cosmos/test/public/common/TestHelpers.ts +++ b/sdk/cosmosdb/cosmos/test/public/common/TestHelpers.ts @@ -6,6 +6,9 @@ import { CosmosClient, Database, DatabaseDefinition, + extractPartitionKey, + PartitionKey, + PartitionKeyDefinition, PermissionDefinition, RequestOptions, Response, @@ -101,13 +104,11 @@ export async function bulkInsertItems( export async function bulkReadItems( container: Container, documents: any[], - partitionKeyProperty: string + partitionKeyDef: PartitionKeyDefinition ): Promise { return Promise.all( documents.map(async (document) => { - const partitionKey = Object.prototype.hasOwnProperty.call(document, partitionKeyProperty) - ? document[partitionKeyProperty] - : undefined; + const partitionKey = extractPartitionKey(document, partitionKeyDef) // TODO: should we block or do all requests in parallel? const { resource: doc } = await container.item(document.id, partitionKey).read(); @@ -119,13 +120,11 @@ export async function bulkReadItems( export async function bulkReplaceItems( container: Container, documents: any[], - partitionKeyProperty: string + partitionKeyDef: PartitionKeyDefinition ): Promise { return Promise.all( documents.map(async (document) => { - const partitionKey = Object.prototype.hasOwnProperty.call(document, partitionKeyProperty) - ? document[partitionKeyProperty] - : undefined; + const partitionKey = extractPartitionKey(document, partitionKeyDef) const { resource: doc } = await container.item(document.id, partitionKey).replace(document); const { _etag: _1, _ts: _2, ...expectedModifiedDocument } = document; // eslint-disable-line @typescript-eslint/no-unused-vars const { _etag: _4, _ts: _3, ...actualModifiedDocument } = doc; // eslint-disable-line @typescript-eslint/no-unused-vars @@ -138,13 +137,11 @@ export async function bulkReplaceItems( export async function bulkDeleteItems( container: Container, documents: any[], - partitionKeyProperty: string + partitionKeyDef: PartitionKeyDefinition ): Promise { await Promise.all( documents.map(async (document) => { - const partitionKey = Object.prototype.hasOwnProperty.call(document, partitionKeyProperty) - ? document[partitionKeyProperty] - : undefined; + const partitionKey = extractPartitionKey(document, partitionKeyDef) await container.item(document.id, partitionKey).delete(); }) @@ -154,25 +151,22 @@ export async function bulkDeleteItems( export async function bulkQueryItemsWithPartitionKey( container: Container, documents: any[], - partitionKeyPropertyName: string + query: string, + parameterGenerator: (doc: any) => {name: string, value: any}[], ): Promise { for (const document of documents) { - if (!Object.prototype.hasOwnProperty.call(document, partitionKeyPropertyName)) { + const parameters = parameterGenerator(document); + const shouldSkip = parameters.reduce((previous, current) => previous || current['value']=== undefined, false) + if(shouldSkip) { continue; } - const querySpec = { - query: "SELECT * FROM root r WHERE r." + partitionKeyPropertyName + "=@key", - parameters: [ - { - name: "@key", - value: document[partitionKeyPropertyName], - }, - ], + query: query, + parameters: parameters }; const { resources } = await container.items.query(querySpec).fetchAll(); - assert.equal(resources.length, 1, "Expected exactly 1 document"); + assert.equal(resources.length, 1, `Expected exactly 1 document, doc: ${JSON.stringify(document)}, query: '${query}', parameters: ${JSON.stringify(parameters)}`); assert.equal(JSON.stringify(resources[0]), JSON.stringify(document)); } } @@ -195,13 +189,14 @@ export async function replaceOrUpsertItem( container: Container, body: unknown, options: RequestOptions, - isUpsertTest: boolean + isUpsertTest: boolean, + partitionKey?: PartitionKey ): Promise> { if (isUpsertTest) { return container.items.upsert(body, options); } else { const bodyWithId = body as { id: string }; - return container.item(bodyWithId.id, undefined).replace(body, options); + return container.item(bodyWithId.id, partitionKey).replace(body, options); } }