From a5d0f7a5ceaa4eb0cf19c3ee1b457e6a994eb6d6 Mon Sep 17 00:00:00 2001 From: Radek Gruchalski Date: Tue, 30 Jul 2024 23:55:07 +0200 Subject: [PATCH 1/6] A naive implementation of the blockBlobClient stageBlockFromURL operation --- src/blob/handlers/BlockBlobHandler.ts | 310 +++++++++++++++++++++++++- tests/blob/apis/blockblob.test.ts | 77 +++++++ 2 files changed, 384 insertions(+), 3 deletions(-) diff --git a/src/blob/handlers/BlockBlobHandler.ts b/src/blob/handlers/BlockBlobHandler.ts index 025809987..1a9adee2b 100644 --- a/src/blob/handlers/BlockBlobHandler.ts +++ b/src/blob/handlers/BlockBlobHandler.ts @@ -1,3 +1,6 @@ +import { URLBuilder } from "@azure/ms-rest-js"; +import axios, { AxiosResponse } from "axios"; + import { convertRawHeadersToMetadata } from "../../common/utils/utils"; import { getMD5FromStream, @@ -11,10 +14,15 @@ import * as Models from "../generated/artifacts/models"; import Context from "../generated/Context"; import IBlockBlobHandler from "../generated/handlers/IBlockBlobHandler"; import { parseXML } from "../generated/utils/xml"; +import { extractStoragePartsFromPath } from "../middlewares/blobStorageContext.middleware"; import { BlobModel, BlockModel } from "../persistence/IBlobMetadataStore"; -import { BLOB_API_VERSION } from "../utils/constants"; +import { BLOB_API_VERSION, HeaderConstants } from "../utils/constants"; import BaseHandler from "./BaseHandler"; -import { getTagsFromString } from "../utils/utils"; +import { + getTagsFromString, + deserializeRangeHeader, + getBlobTagsCount, +} from "../utils/utils"; /** * BlobHandler handles Azure Storage BlockBlob related requests. @@ -270,7 +278,75 @@ export default class BlockBlobHandler options: Models.BlockBlobStageBlockFromURLOptionalParams, context: Context ): Promise { - throw new NotImplementedError(context.contextId); + + const blobCtx = new BlobStorageContext(context); + + // TODO: Check dest Lease status, and set to available if it's expired, see sample in BlobHandler.setMetadata() + const url = this.NewUriFromCopySource(sourceUrl, context); + const [ + sourceAccount, + sourceContainer, + sourceBlob + ] = extractStoragePartsFromPath(url.hostname, url.pathname, blobCtx.disableProductStyleUrl); + const snapshot = url.searchParams.get("snapshot") || ""; + + if ( + sourceAccount === undefined || + sourceContainer === undefined || + sourceBlob === undefined + ) { + throw StorageErrorFactory.getBlobNotFound(context.contextId!); + } + + const sig = url.searchParams.get("sig"); + if ((sourceAccount !== blobCtx.account) || (sig !== null)) { + await this.validateCopySource(sourceUrl, sourceAccount, context); + } + + const downloadBlobRes = await this.metadataStore.downloadBlob( + context, + sourceAccount, + sourceContainer, + sourceBlob, + snapshot, + options.leaseAccessConditions, + ); + + if (downloadBlobRes.properties.contentLength === undefined) { + throw StorageErrorFactory.getConditionNotMet(context.contextId!); + } + + const downloadBlockBlobRes = await this.downloadBlockBlobOrAppendBlob( + { snapshot: snapshot, leaseAccessConditions: options.leaseAccessConditions }, + context, + downloadBlobRes, + ); + + if (downloadBlockBlobRes.body === undefined) { + throw StorageErrorFactory.getConditionNotMet(context.contextId!); + } + + const stageBlockRes = await this.stageBlock(blockId, + downloadBlobRes.properties.contentLength, + downloadBlockBlobRes.body, + { leaseAccessConditions: options.leaseAccessConditions }, + context + ); + + const response: Models.BlockBlobStageBlockFromURLResponse = { + statusCode: stageBlockRes.statusCode, + contentMD5: stageBlockRes.contentMD5, + date: stageBlockRes.date, + encryptionKeySha256: stageBlockRes.encryptionKeySha256, + encryptionScope: stageBlockRes.encryptionScope, + errorCode: stageBlockRes.errorCode, + isServerEncrypted: stageBlockRes.isServerEncrypted, + requestId: stageBlockRes.requestId, + version: stageBlockRes.version, + xMsContentCrc64: stageBlockRes.xMsContentCrc64, + }; + + return response } public async commitBlockList( @@ -501,4 +577,232 @@ export default class BlockBlobHandler ); } } + + // from BlobHandler, surely there must be a better way + + private async validateCopySource(copySource: string, sourceAccount: string, context: Context): Promise { + // Currently the only cross-account copy support is from/to the same Azurite instance. In either case access + // is determined by performing a request to the copy source to see if the authentication is valid. + const blobCtx = new BlobStorageContext(context); + + const currentServer = blobCtx.request!.getHeader("Host") || ""; + const url = this.NewUriFromCopySource(copySource, context); + if (currentServer !== url.host) { + this.logger.error( + `BlobHandler:startCopyFromURL() Source account ${url} is not on the same Azurite instance as target account ${blobCtx.account}`, + context.contextId + ); + + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 404, + "The specified resource does not exist" + ); + } + + this.logger.debug( + `BlobHandler:startCopyFromURL() Validating access to the source account ${sourceAccount}`, + context.contextId + ); + + // In order to retrieve proper error details we make a metadata request to the copy source. If we instead issue + // a HEAD request then the error details are not returned and reporting authentication failures to the caller + // becomes a black box. + const metadataUrl = URLBuilder.parse(copySource); + metadataUrl.setQueryParameter("comp", "metadata"); + const validationResponse: AxiosResponse = await axios.get( + metadataUrl.toString(), + { + // Instructs axios to not throw an error for non-2xx responses + validateStatus: () => true + } + ); + if (validationResponse.status === 200) { + this.logger.debug( + `BlobHandler:startCopyFromURL() Successfully validated access to source account ${sourceAccount}`, + context.contextId + ); + } else { + this.logger.debug( + `BlobHandler:startCopyFromURL() Access denied to source account ${sourceAccount} StatusCode=${validationResponse.status}, AuthenticationErrorDetail=${validationResponse.data}`, + context.contextId + ); + + if (validationResponse.status === 404) { + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + validationResponse.status, + "The specified resource does not exist" + ); + } else { + // For non-successful responses attempt to unwrap the error message from the metadata call. + let message: string = + "Could not verify the copy source within the specified time."; + if ( + validationResponse.headers[HeaderConstants.CONTENT_TYPE] === + "application/xml" + ) { + const authenticationError = await parseXML(validationResponse.data); + if (authenticationError.Message !== undefined) { + message = authenticationError.Message.replace(/\n+/gm, ""); + } + } + + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + validationResponse.status, + message + ); + } + } + } + + private NewUriFromCopySource(copySource: string, context: Context): URL { + try { + return new URL(copySource) + } + catch + { + throw StorageErrorFactory.getInvalidHeaderValue( + context.contextId, + { + HeaderName: "x-ms-copy-source", + HeaderValue: copySource + }) + } + } + + /** + * Download block blob or append blob. + * + * @private + * @param {Models.BlobDownloadOptionalParams} options + * @param {Context} context + * @param {BlobModel} blob + * @returns {Promise} + * @memberof BlobHandler + */ + private async downloadBlockBlobOrAppendBlob( + options: Models.BlobDownloadOptionalParams, + context: Context, + blob: BlobModel + ): Promise { + if (blob.isCommitted === false) { + throw StorageErrorFactory.getBlobNotFound(context.contextId!); + } + + // Deserializer doesn't handle range header currently, manually parse range headers here + const rangesParts = deserializeRangeHeader( + context.request!.getHeader("range"), + context.request!.getHeader("x-ms-range") + ); + const rangeStart = rangesParts[0]; + let rangeEnd = rangesParts[1]; + + // Start Range is bigger than blob length + if (rangeStart > blob.properties.contentLength!) { + throw StorageErrorFactory.getInvalidPageRange(context.contextId!); + } + + // Will automatically shift request with longer data end than blob size to blob size + if (rangeEnd + 1 >= blob.properties.contentLength!) { + // report error is blob size is 0, and rangeEnd is specified but not 0 + if (blob.properties.contentLength == 0 && rangeEnd !== 0 && rangeEnd !== Infinity) { + throw StorageErrorFactory.getInvalidPageRange2(context.contextId!); + } + else { + rangeEnd = blob.properties.contentLength! - 1; + } + } + + const contentLength = rangeEnd - rangeStart + 1; + const partialRead = contentLength !== blob.properties.contentLength!; + + this.logger.info( + // tslint:disable-next-line:max-line-length + `BlobHandler:downloadBlockBlobOrAppendBlob() NormalizedDownloadRange=bytes=${rangeStart}-${rangeEnd} RequiredContentLength=${contentLength}`, + context.contextId + ); + + let bodyGetter: () => Promise; + const blocks = blob.committedBlocksInOrder; + if (blocks === undefined || blocks.length === 0) { + bodyGetter = async () => { + if (blob.persistency === undefined) { + return this.extentStore.readExtent(undefined, context.contextId); + } + return this.extentStore.readExtent( + { + id: blob.persistency.id, + offset: blob.persistency.offset + rangeStart, + count: Math.min(blob.persistency.count, contentLength) + }, + context.contextId + ); + }; + } else { + bodyGetter = async () => { + return this.extentStore.readExtents( + blocks.map((block) => block.persistency), + rangeStart, + rangeEnd + 1 - rangeStart, + context.contextId + ); + }; + } + + let contentRange: string | undefined; + if ( + context.request!.getHeader("range") || + context.request!.getHeader("x-ms-range") + ) { + contentRange = `bytes ${rangeStart}-${rangeEnd}/${blob.properties + .contentLength!}`; + } + + let body: NodeJS.ReadableStream | undefined = await bodyGetter(); + let contentMD5: Uint8Array | undefined; + if (!partialRead) { + contentMD5 = blob.properties.contentMD5; + } + if ( + contentLength <= 4 * 1024 * 1024 && + contentMD5 === undefined && + body !== undefined + ) { + contentMD5 = await getMD5FromStream(body); + body = await bodyGetter(); + } + + const response: Models.BlobDownloadResponse = { + statusCode: contentRange ? 206 : 200, + body, + metadata: blob.metadata, + eTag: blob.properties.etag, + requestId: context.contextId, + date: context.startTime!, + version: BLOB_API_VERSION, + ...blob.properties, + cacheControl: context.request!.getQuery("rscc") ?? blob.properties.cacheControl, + contentDisposition: context.request!.getQuery("rscd") ?? blob.properties.contentDisposition, + contentEncoding: context.request!.getQuery("rsce") ?? blob.properties.contentEncoding, + contentLanguage: context.request!.getQuery("rscl") ?? blob.properties.contentLanguage, + contentType: context.request!.getQuery("rsct") ?? blob.properties.contentType, + blobContentMD5: blob.properties.contentMD5, + acceptRanges: "bytes", + contentLength, + contentRange, + contentMD5: contentRange ? (context.request!.getHeader("x-ms-range-get-content-md5") ? contentMD5: undefined) : contentMD5, + tagCount: getBlobTagsCount(blob.blobTags), + isServerEncrypted: true, + clientRequestId: options.requestId, + creationTime: blob.properties.creationTime, + blobCommittedBlockCount: + blob.properties.blobType === Models.BlobType.AppendBlob + ? (blob.committedBlocksInOrder || []).length + : undefined, + }; + + return response; + } } diff --git a/tests/blob/apis/blockblob.test.ts b/tests/blob/apis/blockblob.test.ts index b483a2e7a..320db0c48 100644 --- a/tests/blob/apis/blockblob.test.ts +++ b/tests/blob/apis/blockblob.test.ts @@ -70,6 +70,83 @@ describe("BlockBlobAPIs", () => { await containerClient.delete(); }); + it("Blob should be staged from URL and committed @loki @sql", async () => { + const source1Blob = getUniqueName("blob"); + const source2Blob = getUniqueName("blob"); + const destBlob = getUniqueName("blob"); + + const source1BlobClient = containerClient.getBlockBlobClient(source1Blob); + const source2BlobClient = containerClient.getBlockBlobClient(source2Blob); + const destBlobClient = containerClient.getBlockBlobClient(destBlob); + + const metadata = { key: "value" }; + const blobHTTPHeaders = { + blobCacheControl: "blobCacheControl", + blobContentDisposition: "blobContentDisposition", + blobContentEncoding: "blobContentEncoding", + blobContentLanguage: "blobContentLanguage", + blobContentType: "blobContentType" + }; + + const payload1 = "Blob should be staged " + const payload2 = "from URL and committed." + + // Upload first object: + const result_upload1 = await source1BlobClient.upload(payload1, payload1.length, { + metadata, + blobHTTPHeaders + }); + assert.strictEqual( + result_upload1._response.request.headers.get("x-ms-client-request-id"), + result_upload1.clientRequestId + ); + + // Upload second object object: + const result_upload2 = await source2BlobClient.upload(payload2, payload2.length, { + metadata, + blobHTTPHeaders + }); + assert.strictEqual( + result_upload2._response.request.headers.get("x-ms-client-request-id"), + result_upload2.clientRequestId + ); + + // Stage blocks from URL + const result_stage1 = await destBlobClient.stageBlockFromURL( + base64encode("1"), + source1BlobClient.url + ); + assert.strictEqual( + result_stage1._response.request.headers.get("x-ms-client-request-id"), + result_stage1._response.request.requestId + ); + + const result_stage2 = await destBlobClient.stageBlockFromURL( + base64encode("2"), + source2BlobClient.url + ); + assert.strictEqual( + result_stage2._response.request.headers.get("x-ms-client-request-id"), + result_stage2._response.request.requestId + ); + + const result_commit = await destBlobClient.commitBlockList( + [base64encode("1"), base64encode("2")], + ); + assert.strictEqual( + result_commit._response.request.headers.get("x-ms-client-request-id"), + result_commit._response.request.requestId + ); + + const destContentLength = (await destBlobClient.getProperties()).contentLength + + assert.strictEqual(destContentLength, (payload1 + payload2).length); + + const buffer = await destBlobClient.downloadToBuffer(0, destContentLength); + assert.strictEqual(buffer.toString(), payload1 + payload2); + + }); + it("Block blob upload should refresh lease state @loki @sql", async () => { await blockBlobClient.upload('a', 1); From 558a5b6ac17e176584f881b7928b68b43f7461c3 Mon Sep 17 00:00:00 2001 From: Radek Gruchalski Date: Wed, 31 Jul 2024 08:28:57 +0200 Subject: [PATCH 2/6] Sorted imports --- src/blob/handlers/BlockBlobHandler.ts | 40 ++++++++++++--------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/src/blob/handlers/BlockBlobHandler.ts b/src/blob/handlers/BlockBlobHandler.ts index 1a9adee2b..a90e1ea68 100644 --- a/src/blob/handlers/BlockBlobHandler.ts +++ b/src/blob/handlers/BlockBlobHandler.ts @@ -1,28 +1,22 @@ -import { URLBuilder } from "@azure/ms-rest-js"; -import axios, { AxiosResponse } from "axios"; +import axios, { AxiosResponse } from 'axios'; + +import { URLBuilder } from '@azure/ms-rest-js'; -import { convertRawHeadersToMetadata } from "../../common/utils/utils"; -import { - getMD5FromStream, - getMD5FromString, - newEtag -} from "../../common/utils/utils"; -import BlobStorageContext from "../context/BlobStorageContext"; -import NotImplementedError from "../errors/NotImplementedError"; -import StorageErrorFactory from "../errors/StorageErrorFactory"; -import * as Models from "../generated/artifacts/models"; -import Context from "../generated/Context"; -import IBlockBlobHandler from "../generated/handlers/IBlockBlobHandler"; -import { parseXML } from "../generated/utils/xml"; -import { extractStoragePartsFromPath } from "../middlewares/blobStorageContext.middleware"; -import { BlobModel, BlockModel } from "../persistence/IBlobMetadataStore"; -import { BLOB_API_VERSION, HeaderConstants } from "../utils/constants"; -import BaseHandler from "./BaseHandler"; import { - getTagsFromString, - deserializeRangeHeader, - getBlobTagsCount, -} from "../utils/utils"; + convertRawHeadersToMetadata, getMD5FromStream, getMD5FromString, newEtag +} from '../../common/utils/utils'; +import BlobStorageContext from '../context/BlobStorageContext'; +import NotImplementedError from '../errors/NotImplementedError'; +import StorageErrorFactory from '../errors/StorageErrorFactory'; +import * as Models from '../generated/artifacts/models'; +import Context from '../generated/Context'; +import IBlockBlobHandler from '../generated/handlers/IBlockBlobHandler'; +import { parseXML } from '../generated/utils/xml'; +import { extractStoragePartsFromPath } from '../middlewares/blobStorageContext.middleware'; +import { BlobModel, BlockModel } from '../persistence/IBlobMetadataStore'; +import { BLOB_API_VERSION, HeaderConstants } from '../utils/constants'; +import { deserializeRangeHeader, getBlobTagsCount, getTagsFromString } from '../utils/utils'; +import BaseHandler from './BaseHandler'; /** * BlobHandler handles Azure Storage BlockBlob related requests. From 7b913e42475beec419a98dc39ad7a3fdf3959e28 Mon Sep 17 00:00:00 2001 From: Radek Gruchalski Date: Sun, 4 Aug 2024 15:25:19 +0200 Subject: [PATCH 3/6] Move code shared between BlobHandler and BlockBlobHandler to blob/utils/utils.ts --- src/blob/handlers/BlobHandler.ts | 287 +++----------------------- src/blob/handlers/BlockBlobHandler.ts | 246 +--------------------- src/blob/utils/utils.ts | 258 ++++++++++++++++++++++- src/common/utils/utils.ts | 8 +- 4 files changed, 290 insertions(+), 509 deletions(-) diff --git a/src/blob/handlers/BlobHandler.ts b/src/blob/handlers/BlobHandler.ts index 981f9a278..147848bd8 100644 --- a/src/blob/handlers/BlobHandler.ts +++ b/src/blob/handlers/BlobHandler.ts @@ -1,38 +1,23 @@ -import { URLBuilder } from "@azure/ms-rest-js"; -import axios, { AxiosResponse } from "axios"; -import { URL } from "url"; - -import IExtentStore from "../../common/persistence/IExtentStore"; -import { - convertRawHeadersToMetadata, - getMD5FromStream -} from "../../common/utils/utils"; -import BlobStorageContext from "../context/BlobStorageContext"; -import NotImplementedError from "../errors/NotImplementedError"; -import StorageErrorFactory from "../errors/StorageErrorFactory"; -import * as Models from "../generated/artifacts/models"; -import Context from "../generated/Context"; -import IBlobHandler from "../generated/handlers/IBlobHandler"; -import ILogger from "../generated/utils/ILogger"; -import { parseXML } from "../generated/utils/xml"; -import { extractStoragePartsFromPath } from "../middlewares/blobStorageContext.middleware"; -import IBlobMetadataStore, { - BlobModel -} from "../persistence/IBlobMetadataStore"; +import IExtentStore from '../../common/persistence/IExtentStore'; +import { convertRawHeadersToMetadata, getMD5FromStream } from '../../common/utils/utils'; +import BlobStorageContext from '../context/BlobStorageContext'; +import NotImplementedError from '../errors/NotImplementedError'; +import StorageErrorFactory from '../errors/StorageErrorFactory'; +import * as Models from '../generated/artifacts/models'; +import Context from '../generated/Context'; +import IBlobHandler from '../generated/handlers/IBlobHandler'; +import ILogger from '../generated/utils/ILogger'; +import { extractStoragePartsFromPath } from '../middlewares/blobStorageContext.middleware'; +import IBlobMetadataStore, { BlobModel } from '../persistence/IBlobMetadataStore'; import { - BLOB_API_VERSION, - EMULATOR_ACCOUNT_KIND, - EMULATOR_ACCOUNT_SKUNAME, - HeaderConstants -} from "../utils/constants"; + BLOB_API_VERSION, EMULATOR_ACCOUNT_KIND, EMULATOR_ACCOUNT_SKUNAME, HeaderConstants +} from '../utils/constants'; import { - deserializePageBlobRangeHeader, - deserializeRangeHeader, - getBlobTagsCount, - validateBlobTag -} from "../utils/utils"; -import BaseHandler from "./BaseHandler"; -import IPageBlobRangesManager from "./IPageBlobRangesManager"; + deserializePageBlobRangeHeader, downloadBlockBlobOrAppendBlob, getBlobTagsCount, + NewUriFromCopySource, validateBlobTag, validateCopySource +} from '../utils/utils'; +import BaseHandler from './BaseHandler'; +import IPageBlobRangesManager from './IPageBlobRangesManager'; /** * BlobHandler handles Azure Storage Blob related requests. @@ -81,11 +66,11 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { ); if (blob.properties.blobType === Models.BlobType.BlockBlob) { - return this.downloadBlockBlobOrAppendBlob(options, context, blob); + return downloadBlockBlobOrAppendBlob(this.logger, "BlobHandler", this.extentStore, options, context, blob); } else if (blob.properties.blobType === Models.BlobType.PageBlob) { return this.downloadPageBlob(options, context, blob); } else if (blob.properties.blobType === Models.BlobType.AppendBlob) { - return this.downloadBlockBlobOrAppendBlob(options, context, blob); + return downloadBlockBlobOrAppendBlob(this.logger, "BlobHandler", this.extentStore, options, context, blob); } else { throw StorageErrorFactory.getInvalidOperation(context.contextId!); } @@ -646,7 +631,7 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { const blob = blobCtx.blob!; // TODO: Check dest Lease status, and set to available if it's expired, see sample in BlobHandler.setMetadata() - const url = this.NewUriFromCopySource(copySource, context); + const url = NewUriFromCopySource(copySource, context); const [ sourceAccount, sourceContainer, @@ -664,7 +649,7 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { const sig = url.searchParams.get("sig"); if ((sourceAccount !== blobCtx.account) || (sig !== null)) { - await this.validateCopySource(copySource, sourceAccount, context); + await validateCopySource(this.logger, "BlobHandler", copySource, sourceAccount, context); } // Preserve metadata key case @@ -702,82 +687,6 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { return response; } - private async validateCopySource(copySource: string, sourceAccount: string, context: Context): Promise { - // Currently the only cross-account copy support is from/to the same Azurite instance. In either case access - // is determined by performing a request to the copy source to see if the authentication is valid. - const blobCtx = new BlobStorageContext(context); - - const currentServer = blobCtx.request!.getHeader("Host") || ""; - const url = this.NewUriFromCopySource(copySource, context); - if (currentServer !== url.host) { - this.logger.error( - `BlobHandler:startCopyFromURL() Source account ${url} is not on the same Azurite instance as target account ${blobCtx.account}`, - context.contextId - ); - - throw StorageErrorFactory.getCannotVerifyCopySource( - context.contextId!, - 404, - "The specified resource does not exist" - ); - } - - this.logger.debug( - `BlobHandler:startCopyFromURL() Validating access to the source account ${sourceAccount}`, - context.contextId - ); - - // In order to retrieve proper error details we make a metadata request to the copy source. If we instead issue - // a HEAD request then the error details are not returned and reporting authentication failures to the caller - // becomes a black box. - const metadataUrl = URLBuilder.parse(copySource); - metadataUrl.setQueryParameter("comp", "metadata"); - const validationResponse: AxiosResponse = await axios.get( - metadataUrl.toString(), - { - // Instructs axios to not throw an error for non-2xx responses - validateStatus: () => true - } - ); - if (validationResponse.status === 200) { - this.logger.debug( - `BlobHandler:startCopyFromURL() Successfully validated access to source account ${sourceAccount}`, - context.contextId - ); - } else { - this.logger.debug( - `BlobHandler:startCopyFromURL() Access denied to source account ${sourceAccount} StatusCode=${validationResponse.status}, AuthenticationErrorDetail=${validationResponse.data}`, - context.contextId - ); - - if (validationResponse.status === 404) { - throw StorageErrorFactory.getCannotVerifyCopySource( - context.contextId!, - validationResponse.status, - "The specified resource does not exist" - ); - } else { - // For non-successful responses attempt to unwrap the error message from the metadata call. - let message: string = - "Could not verify the copy source within the specified time."; - if ( - validationResponse.headers[HeaderConstants.CONTENT_TYPE] === - "application/xml" - ) { - const authenticationError = await parseXML(validationResponse.data); - if (authenticationError.Message !== undefined) { - message = authenticationError.Message.replace(/\n+/gm, ""); - } - } - - throw StorageErrorFactory.getCannotVerifyCopySource( - context.contextId!, - validationResponse.status, - message - ); - } - } - } /** * Abort copy from Url. @@ -845,7 +754,7 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { const blob = blobCtx.blob!; // TODO: Check dest Lease status, and set to available if it's expired, see sample in BlobHandler.setMetadata() - const url = this.NewUriFromCopySource(copySource, context); + const url = NewUriFromCopySource(copySource, context); const [ sourceAccount, sourceContainer, @@ -862,7 +771,7 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { } if (sourceAccount !== blobCtx.account) { - await this.validateCopySource(copySource, sourceAccount, context); + await validateCopySource(this.logger, "BlobHandler", copySource, sourceAccount, context); } // Specifying x-ms-copy-source-tag-option as COPY and x-ms-tags will result in error @@ -989,140 +898,6 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { return this.getAccountInfo(context); } - /** - * Download block blob or append blob. - * - * @private - * @param {Models.BlobDownloadOptionalParams} options - * @param {Context} context - * @param {BlobModel} blob - * @returns {Promise} - * @memberof BlobHandler - */ - private async downloadBlockBlobOrAppendBlob( - options: Models.BlobDownloadOptionalParams, - context: Context, - blob: BlobModel - ): Promise { - if (blob.isCommitted === false) { - throw StorageErrorFactory.getBlobNotFound(context.contextId!); - } - - // Deserializer doesn't handle range header currently, manually parse range headers here - const rangesParts = deserializeRangeHeader( - context.request!.getHeader("range"), - context.request!.getHeader("x-ms-range") - ); - const rangeStart = rangesParts[0]; - let rangeEnd = rangesParts[1]; - - // Start Range is bigger than blob length - if (rangeStart > blob.properties.contentLength!) { - throw StorageErrorFactory.getInvalidPageRange(context.contextId!); - } - - // Will automatically shift request with longer data end than blob size to blob size - if (rangeEnd + 1 >= blob.properties.contentLength!) { - // report error is blob size is 0, and rangeEnd is specified but not 0 - if (blob.properties.contentLength == 0 && rangeEnd !== 0 && rangeEnd !== Infinity) { - throw StorageErrorFactory.getInvalidPageRange2(context.contextId!); - } - else { - rangeEnd = blob.properties.contentLength! - 1; - } - } - - const contentLength = rangeEnd - rangeStart + 1; - const partialRead = contentLength !== blob.properties.contentLength!; - - this.logger.info( - // tslint:disable-next-line:max-line-length - `BlobHandler:downloadBlockBlobOrAppendBlob() NormalizedDownloadRange=bytes=${rangeStart}-${rangeEnd} RequiredContentLength=${contentLength}`, - context.contextId - ); - - let bodyGetter: () => Promise; - const blocks = blob.committedBlocksInOrder; - if (blocks === undefined || blocks.length === 0) { - bodyGetter = async () => { - if (blob.persistency === undefined) { - return this.extentStore.readExtent(undefined, context.contextId); - } - return this.extentStore.readExtent( - { - id: blob.persistency.id, - offset: blob.persistency.offset + rangeStart, - count: Math.min(blob.persistency.count, contentLength) - }, - context.contextId - ); - }; - } else { - bodyGetter = async () => { - return this.extentStore.readExtents( - blocks.map((block) => block.persistency), - rangeStart, - rangeEnd + 1 - rangeStart, - context.contextId - ); - }; - } - - let contentRange: string | undefined; - if ( - context.request!.getHeader("range") || - context.request!.getHeader("x-ms-range") - ) { - contentRange = `bytes ${rangeStart}-${rangeEnd}/${blob.properties - .contentLength!}`; - } - - let body: NodeJS.ReadableStream | undefined = await bodyGetter(); - let contentMD5: Uint8Array | undefined; - if (!partialRead) { - contentMD5 = blob.properties.contentMD5; - } - if ( - contentLength <= 4 * 1024 * 1024 && - contentMD5 === undefined && - body !== undefined - ) { - contentMD5 = await getMD5FromStream(body); - body = await bodyGetter(); - } - - const response: Models.BlobDownloadResponse = { - statusCode: contentRange ? 206 : 200, - body, - metadata: blob.metadata, - eTag: blob.properties.etag, - requestId: context.contextId, - date: context.startTime!, - version: BLOB_API_VERSION, - ...blob.properties, - cacheControl: context.request!.getQuery("rscc") ?? blob.properties.cacheControl, - contentDisposition: context.request!.getQuery("rscd") ?? blob.properties.contentDisposition, - contentEncoding: context.request!.getQuery("rsce") ?? blob.properties.contentEncoding, - contentLanguage: context.request!.getQuery("rscl") ?? blob.properties.contentLanguage, - contentType: context.request!.getQuery("rsct") ?? blob.properties.contentType, - blobContentMD5: blob.properties.contentMD5, - acceptRanges: "bytes", - contentLength, - contentRange, - contentMD5: contentRange ? (context.request!.getHeader("x-ms-range-get-content-md5") ? contentMD5: undefined) : contentMD5, - tagCount: getBlobTagsCount(blob.blobTags), - isServerEncrypted: true, - clientRequestId: options.requestId, - creationTime: blob.properties.creationTime, - blobCommittedBlockCount: - blob.properties.blobType === Models.BlobType.AppendBlob - ? (blob.committedBlocksInOrder || []).length - : undefined, - }; - - return response; - } - /** * Download page blob. * @@ -1331,18 +1106,4 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { return response; } - private NewUriFromCopySource(copySource: string, context: Context): URL { - try { - return new URL(copySource) - } - catch - { - throw StorageErrorFactory.getInvalidHeaderValue( - context.contextId, - { - HeaderName: "x-ms-copy-source", - HeaderValue: copySource - }) - } - } } diff --git a/src/blob/handlers/BlockBlobHandler.ts b/src/blob/handlers/BlockBlobHandler.ts index a90e1ea68..e480c7200 100644 --- a/src/blob/handlers/BlockBlobHandler.ts +++ b/src/blob/handlers/BlockBlobHandler.ts @@ -1,7 +1,3 @@ -import axios, { AxiosResponse } from 'axios'; - -import { URLBuilder } from '@azure/ms-rest-js'; - import { convertRawHeadersToMetadata, getMD5FromStream, getMD5FromString, newEtag } from '../../common/utils/utils'; @@ -14,8 +10,10 @@ import IBlockBlobHandler from '../generated/handlers/IBlockBlobHandler'; import { parseXML } from '../generated/utils/xml'; import { extractStoragePartsFromPath } from '../middlewares/blobStorageContext.middleware'; import { BlobModel, BlockModel } from '../persistence/IBlobMetadataStore'; -import { BLOB_API_VERSION, HeaderConstants } from '../utils/constants'; -import { deserializeRangeHeader, getBlobTagsCount, getTagsFromString } from '../utils/utils'; +import { BLOB_API_VERSION } from '../utils/constants'; +import { + downloadBlockBlobOrAppendBlob, getTagsFromString, NewUriFromCopySource, validateCopySource +} from '../utils/utils'; import BaseHandler from './BaseHandler'; /** @@ -276,7 +274,7 @@ export default class BlockBlobHandler const blobCtx = new BlobStorageContext(context); // TODO: Check dest Lease status, and set to available if it's expired, see sample in BlobHandler.setMetadata() - const url = this.NewUriFromCopySource(sourceUrl, context); + const url = NewUriFromCopySource(sourceUrl, context); const [ sourceAccount, sourceContainer, @@ -294,7 +292,7 @@ export default class BlockBlobHandler const sig = url.searchParams.get("sig"); if ((sourceAccount !== blobCtx.account) || (sig !== null)) { - await this.validateCopySource(sourceUrl, sourceAccount, context); + await validateCopySource(this.logger, "BlockBlobHandler", sourceUrl, sourceAccount, context); } const downloadBlobRes = await this.metadataStore.downloadBlob( @@ -310,7 +308,10 @@ export default class BlockBlobHandler throw StorageErrorFactory.getConditionNotMet(context.contextId!); } - const downloadBlockBlobRes = await this.downloadBlockBlobOrAppendBlob( + const downloadBlockBlobRes = await downloadBlockBlobOrAppendBlob( + this.logger, + "BlockBlobHandler", + this.extentStore, { snapshot: snapshot, leaseAccessConditions: options.leaseAccessConditions }, context, downloadBlobRes, @@ -572,231 +573,4 @@ export default class BlockBlobHandler } } - // from BlobHandler, surely there must be a better way - - private async validateCopySource(copySource: string, sourceAccount: string, context: Context): Promise { - // Currently the only cross-account copy support is from/to the same Azurite instance. In either case access - // is determined by performing a request to the copy source to see if the authentication is valid. - const blobCtx = new BlobStorageContext(context); - - const currentServer = blobCtx.request!.getHeader("Host") || ""; - const url = this.NewUriFromCopySource(copySource, context); - if (currentServer !== url.host) { - this.logger.error( - `BlobHandler:startCopyFromURL() Source account ${url} is not on the same Azurite instance as target account ${blobCtx.account}`, - context.contextId - ); - - throw StorageErrorFactory.getCannotVerifyCopySource( - context.contextId!, - 404, - "The specified resource does not exist" - ); - } - - this.logger.debug( - `BlobHandler:startCopyFromURL() Validating access to the source account ${sourceAccount}`, - context.contextId - ); - - // In order to retrieve proper error details we make a metadata request to the copy source. If we instead issue - // a HEAD request then the error details are not returned and reporting authentication failures to the caller - // becomes a black box. - const metadataUrl = URLBuilder.parse(copySource); - metadataUrl.setQueryParameter("comp", "metadata"); - const validationResponse: AxiosResponse = await axios.get( - metadataUrl.toString(), - { - // Instructs axios to not throw an error for non-2xx responses - validateStatus: () => true - } - ); - if (validationResponse.status === 200) { - this.logger.debug( - `BlobHandler:startCopyFromURL() Successfully validated access to source account ${sourceAccount}`, - context.contextId - ); - } else { - this.logger.debug( - `BlobHandler:startCopyFromURL() Access denied to source account ${sourceAccount} StatusCode=${validationResponse.status}, AuthenticationErrorDetail=${validationResponse.data}`, - context.contextId - ); - - if (validationResponse.status === 404) { - throw StorageErrorFactory.getCannotVerifyCopySource( - context.contextId!, - validationResponse.status, - "The specified resource does not exist" - ); - } else { - // For non-successful responses attempt to unwrap the error message from the metadata call. - let message: string = - "Could not verify the copy source within the specified time."; - if ( - validationResponse.headers[HeaderConstants.CONTENT_TYPE] === - "application/xml" - ) { - const authenticationError = await parseXML(validationResponse.data); - if (authenticationError.Message !== undefined) { - message = authenticationError.Message.replace(/\n+/gm, ""); - } - } - - throw StorageErrorFactory.getCannotVerifyCopySource( - context.contextId!, - validationResponse.status, - message - ); - } - } - } - - private NewUriFromCopySource(copySource: string, context: Context): URL { - try { - return new URL(copySource) - } - catch - { - throw StorageErrorFactory.getInvalidHeaderValue( - context.contextId, - { - HeaderName: "x-ms-copy-source", - HeaderValue: copySource - }) - } - } - - /** - * Download block blob or append blob. - * - * @private - * @param {Models.BlobDownloadOptionalParams} options - * @param {Context} context - * @param {BlobModel} blob - * @returns {Promise} - * @memberof BlobHandler - */ - private async downloadBlockBlobOrAppendBlob( - options: Models.BlobDownloadOptionalParams, - context: Context, - blob: BlobModel - ): Promise { - if (blob.isCommitted === false) { - throw StorageErrorFactory.getBlobNotFound(context.contextId!); - } - - // Deserializer doesn't handle range header currently, manually parse range headers here - const rangesParts = deserializeRangeHeader( - context.request!.getHeader("range"), - context.request!.getHeader("x-ms-range") - ); - const rangeStart = rangesParts[0]; - let rangeEnd = rangesParts[1]; - - // Start Range is bigger than blob length - if (rangeStart > blob.properties.contentLength!) { - throw StorageErrorFactory.getInvalidPageRange(context.contextId!); - } - - // Will automatically shift request with longer data end than blob size to blob size - if (rangeEnd + 1 >= blob.properties.contentLength!) { - // report error is blob size is 0, and rangeEnd is specified but not 0 - if (blob.properties.contentLength == 0 && rangeEnd !== 0 && rangeEnd !== Infinity) { - throw StorageErrorFactory.getInvalidPageRange2(context.contextId!); - } - else { - rangeEnd = blob.properties.contentLength! - 1; - } - } - - const contentLength = rangeEnd - rangeStart + 1; - const partialRead = contentLength !== blob.properties.contentLength!; - - this.logger.info( - // tslint:disable-next-line:max-line-length - `BlobHandler:downloadBlockBlobOrAppendBlob() NormalizedDownloadRange=bytes=${rangeStart}-${rangeEnd} RequiredContentLength=${contentLength}`, - context.contextId - ); - - let bodyGetter: () => Promise; - const blocks = blob.committedBlocksInOrder; - if (blocks === undefined || blocks.length === 0) { - bodyGetter = async () => { - if (blob.persistency === undefined) { - return this.extentStore.readExtent(undefined, context.contextId); - } - return this.extentStore.readExtent( - { - id: blob.persistency.id, - offset: blob.persistency.offset + rangeStart, - count: Math.min(blob.persistency.count, contentLength) - }, - context.contextId - ); - }; - } else { - bodyGetter = async () => { - return this.extentStore.readExtents( - blocks.map((block) => block.persistency), - rangeStart, - rangeEnd + 1 - rangeStart, - context.contextId - ); - }; - } - - let contentRange: string | undefined; - if ( - context.request!.getHeader("range") || - context.request!.getHeader("x-ms-range") - ) { - contentRange = `bytes ${rangeStart}-${rangeEnd}/${blob.properties - .contentLength!}`; - } - - let body: NodeJS.ReadableStream | undefined = await bodyGetter(); - let contentMD5: Uint8Array | undefined; - if (!partialRead) { - contentMD5 = blob.properties.contentMD5; - } - if ( - contentLength <= 4 * 1024 * 1024 && - contentMD5 === undefined && - body !== undefined - ) { - contentMD5 = await getMD5FromStream(body); - body = await bodyGetter(); - } - - const response: Models.BlobDownloadResponse = { - statusCode: contentRange ? 206 : 200, - body, - metadata: blob.metadata, - eTag: blob.properties.etag, - requestId: context.contextId, - date: context.startTime!, - version: BLOB_API_VERSION, - ...blob.properties, - cacheControl: context.request!.getQuery("rscc") ?? blob.properties.cacheControl, - contentDisposition: context.request!.getQuery("rscd") ?? blob.properties.contentDisposition, - contentEncoding: context.request!.getQuery("rsce") ?? blob.properties.contentEncoding, - contentLanguage: context.request!.getQuery("rscl") ?? blob.properties.contentLanguage, - contentType: context.request!.getQuery("rsct") ?? blob.properties.contentType, - blobContentMD5: blob.properties.contentMD5, - acceptRanges: "bytes", - contentLength, - contentRange, - contentMD5: contentRange ? (context.request!.getHeader("x-ms-range-get-content-md5") ? contentMD5: undefined) : contentMD5, - tagCount: getBlobTagsCount(blob.blobTags), - isServerEncrypted: true, - clientRequestId: options.requestId, - creationTime: blob.properties.creationTime, - blobCommittedBlockCount: - blob.properties.blobType === Models.BlobType.AppendBlob - ? (blob.committedBlocksInOrder || []).length - : undefined, - }; - - return response; - } } diff --git a/src/blob/utils/utils.ts b/src/blob/utils/utils.ts index 38df653c6..ca452391d 100644 --- a/src/blob/utils/utils.ts +++ b/src/blob/utils/utils.ts @@ -1,8 +1,20 @@ -import { createHmac } from "crypto"; -import { createWriteStream, PathLike } from "fs"; -import StorageErrorFactory from "../errors/StorageErrorFactory"; -import { USERDELEGATIONKEY_BASIC_KEY } from "./constants"; -import { BlobTag, BlobTags } from "@azure/storage-blob"; +import axios, { AxiosResponse } from 'axios'; +import { createHmac } from 'crypto'; +import { createWriteStream, PathLike } from 'fs'; + +import { URLBuilder } from '@azure/ms-rest-js'; +import { BlobTag, BlobTags } from '@azure/storage-blob'; + +import IExtentStore from '../../common/persistence/IExtentStore'; +import { getMD5FromStream } from '../../common/utils/utils'; +import BlobStorageContext from '../context/BlobStorageContext'; +import StorageErrorFactory from '../errors/StorageErrorFactory'; +import * as Models from '../generated/artifacts/models'; +import Context from '../generated/Context'; +import ILogger from '../generated/utils/ILogger'; +import { parseXML } from '../generated/utils/xml'; +import { BlobModel } from '../persistence/IBlobMetadataStore'; +import { BLOB_API_VERSION, HeaderConstants, USERDELEGATIONKEY_BASIC_KEY } from './constants'; export function checkApiVersion( inputApiVersion: string, @@ -242,4 +254,238 @@ function ContainsInvalidTagCharacter(s: string): boolean{ } } return false; -} \ No newline at end of file +} + +export async function validateCopySource( + logger: ILogger, + loggerPrefix: string, + copySource: string, + sourceAccount: string, + context: Context): Promise { + // Currently the only cross-account copy support is from/to the same Azurite instance. In either case access + // is determined by performing a request to the copy source to see if the authentication is valid. + const blobCtx = new BlobStorageContext(context); + + const currentServer = blobCtx.request!.getHeader("Host") || ""; + const url = NewUriFromCopySource(copySource, context); + if (currentServer !== url.host) { + logger.error( + `${loggerPrefix}:startCopyFromURL() Source account ${url} is not on the same Azurite instance as target account ${blobCtx.account}`, + context.contextId + ); + + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + 404, + "The specified resource does not exist" + ); + } + + logger.debug( + `${loggerPrefix}:startCopyFromURL() Validating access to the source account ${sourceAccount}`, + context.contextId + ); + + // In order to retrieve proper error details we make a metadata request to the copy source. If we instead issue + // a HEAD request then the error details are not returned and reporting authentication failures to the caller + // becomes a black box. + const metadataUrl = URLBuilder.parse(copySource); + metadataUrl.setQueryParameter("comp", "metadata"); + const validationResponse: AxiosResponse = await axios.get( + metadataUrl.toString(), + { + // Instructs axios to not throw an error for non-2xx responses + validateStatus: () => true + } + ); + if (validationResponse.status === 200) { + logger.debug( + `${loggerPrefix}:startCopyFromURL() Successfully validated access to source account ${sourceAccount}`, + context.contextId + ); + } else { + logger.debug( + `${loggerPrefix}:startCopyFromURL() Access denied to source account ${sourceAccount} StatusCode=${validationResponse.status}, AuthenticationErrorDetail=${validationResponse.data}`, + context.contextId + ); + + if (validationResponse.status === 404) { + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + validationResponse.status, + "The specified resource does not exist" + ); + } else { + // For non-successful responses attempt to unwrap the error message from the metadata call. + let message: string = + "Could not verify the copy source within the specified time."; + if ( + validationResponse.headers[HeaderConstants.CONTENT_TYPE] === + "application/xml" + ) { + const authenticationError = await parseXML(validationResponse.data); + if (authenticationError.Message !== undefined) { + message = authenticationError.Message.replace(/\n+/gm, ""); + } + } + + throw StorageErrorFactory.getCannotVerifyCopySource( + context.contextId!, + validationResponse.status, + message + ); + } + } +} + +export function NewUriFromCopySource(copySource: string, context: Context): URL { + try { + return new URL(copySource) + } + catch + { + throw StorageErrorFactory.getInvalidHeaderValue( + context.contextId, + { + HeaderName: "x-ms-copy-source", + HeaderValue: copySource + }) + } +} + +/** + * Download block blob or append blob. + * + * @param {ILogger} logger + * @param {string} loggerPrefix + * @param {IExtentStore} extentStore + * @param {Context} context + * @param {BlobModel} blob + * @returns {Promise} + */ +export async function downloadBlockBlobOrAppendBlob( + logger: ILogger, + loggerPrefix: string, + extentStore: IExtentStore, + options: Models.BlobDownloadOptionalParams, + context: Context, + blob: BlobModel +): Promise { + if (blob.isCommitted === false) { + throw StorageErrorFactory.getBlobNotFound(context.contextId!); + } + + // Deserializer doesn't handle range header currently, manually parse range headers here + const rangesParts = deserializeRangeHeader( + context.request!.getHeader("range"), + context.request!.getHeader("x-ms-range") + ); + const rangeStart = rangesParts[0]; + let rangeEnd = rangesParts[1]; + + // Start Range is bigger than blob length + if (rangeStart > blob.properties.contentLength!) { + throw StorageErrorFactory.getInvalidPageRange(context.contextId!); + } + + // Will automatically shift request with longer data end than blob size to blob size + if (rangeEnd + 1 >= blob.properties.contentLength!) { + // report error is blob size is 0, and rangeEnd is specified but not 0 + if (blob.properties.contentLength == 0 && rangeEnd !== 0 && rangeEnd !== Infinity) { + throw StorageErrorFactory.getInvalidPageRange2(context.contextId!); + } + else { + rangeEnd = blob.properties.contentLength! - 1; + } + } + + const contentLength = rangeEnd - rangeStart + 1; + const partialRead = contentLength !== blob.properties.contentLength!; + + logger.info( + // tslint:disable-next-line:max-line-length + `${loggerPrefix}:downloadBlockBlobOrAppendBlob() NormalizedDownloadRange=bytes=${rangeStart}-${rangeEnd} RequiredContentLength=${contentLength}`, + context.contextId + ); + + let bodyGetter: () => Promise; + const blocks = blob.committedBlocksInOrder; + if (blocks === undefined || blocks.length === 0) { + bodyGetter = async () => { + if (blob.persistency === undefined) { + return extentStore.readExtent(undefined, context.contextId); + } + return extentStore.readExtent( + { + id: blob.persistency.id, + offset: blob.persistency.offset + rangeStart, + count: Math.min(blob.persistency.count, contentLength) + }, + context.contextId + ); + }; + } else { + bodyGetter = async () => { + return extentStore.readExtents( + blocks.map((block) => block.persistency), + rangeStart, + rangeEnd + 1 - rangeStart, + context.contextId + ); + }; + } + + let contentRange: string | undefined; + if ( + context.request!.getHeader("range") || + context.request!.getHeader("x-ms-range") + ) { + contentRange = `bytes ${rangeStart}-${rangeEnd}/${blob.properties + .contentLength!}`; + } + + let body: NodeJS.ReadableStream | undefined = await bodyGetter(); + let contentMD5: Uint8Array | undefined; + if (!partialRead) { + contentMD5 = blob.properties.contentMD5; + } + if ( + contentLength <= 4 * 1024 * 1024 && + contentMD5 === undefined && + body !== undefined + ) { + contentMD5 = await getMD5FromStream(body); + body = await bodyGetter(); + } + + const response: Models.BlobDownloadResponse = { + statusCode: contentRange ? 206 : 200, + body, + metadata: blob.metadata, + eTag: blob.properties.etag, + requestId: context.contextId, + date: context.startTime!, + version: BLOB_API_VERSION, + ...blob.properties, + cacheControl: context.request!.getQuery("rscc") ?? blob.properties.cacheControl, + contentDisposition: context.request!.getQuery("rscd") ?? blob.properties.contentDisposition, + contentEncoding: context.request!.getQuery("rsce") ?? blob.properties.contentEncoding, + contentLanguage: context.request!.getQuery("rscl") ?? blob.properties.contentLanguage, + contentType: context.request!.getQuery("rsct") ?? blob.properties.contentType, + blobContentMD5: blob.properties.contentMD5, + acceptRanges: "bytes", + contentLength, + contentRange, + contentMD5: contentRange ? (context.request!.getHeader("x-ms-range-get-content-md5") ? contentMD5: undefined) : contentMD5, + tagCount: getBlobTagsCount(blob.blobTags), + isServerEncrypted: true, + clientRequestId: options.requestId, + creationTime: blob.properties.creationTime, + blobCommittedBlockCount: + blob.properties.blobType === Models.BlobType.AppendBlob + ? (blob.committedBlocksInOrder || []).length + : undefined, + }; + + return response; +} diff --git a/src/common/utils/utils.ts b/src/common/utils/utils.ts index 8df02200f..f155fec44 100644 --- a/src/common/utils/utils.ts +++ b/src/common/utils/utils.ts @@ -1,8 +1,8 @@ -import { createHash, createHmac } from "crypto"; -import rimraf = require("rimraf"); -import { parse } from "url"; -import { promisify } from "util"; +import { createHash, createHmac } from 'crypto'; +import { parse } from 'url'; +import { promisify } from 'util'; +import rimraf = require("rimraf"); // LokiFsStructuredAdapter // tslint:disable-next-line:no-var-requires export const lfsa = require("lokijs/src/loki-fs-structured-adapter.js"); From bbb8bb97e2383336ce6f084dfb9e25c11434a59b Mon Sep 17 00:00:00 2001 From: Radek Gruchalski Date: Sun, 4 Aug 2024 15:25:31 +0200 Subject: [PATCH 4/6] Update support matrix --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 7b5633bf2..09751790d 100644 --- a/README.md +++ b/README.md @@ -1010,6 +1010,7 @@ Detailed support matrix: - Abort Copy Blob (Only supports copy within same Azurite instance) - Copy Blob From URL (Only supports copy within same Azurite instance, only on Loki) - Access control based on conditional headers + - Stage Blob From URL (Only supports copy within same Azurite instance, only on Loki) - Following features or REST APIs are NOT supported or limited supported in this release (will support more features per customers feedback in future releases) - SharedKey Lite From 2e35689f2cce2ef48a99c05fbe35f56bc8f44311 Mon Sep 17 00:00:00 2001 From: Radek Gruchalski Date: Sun, 4 Aug 2024 15:33:44 +0200 Subject: [PATCH 5/6] Test only for loki, no sql --- tests/blob/apis/blockblob.test.ts | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/tests/blob/apis/blockblob.test.ts b/tests/blob/apis/blockblob.test.ts index 320db0c48..d6e2a6caf 100644 --- a/tests/blob/apis/blockblob.test.ts +++ b/tests/blob/apis/blockblob.test.ts @@ -1,23 +1,16 @@ import { - StorageSharedKeyCredential, - BlobServiceClient, - newPipeline, - BlobSASPermissions -} from "@azure/storage-blob"; -import assert = require("assert"); -import crypto = require("crypto"); + BlobSASPermissions, BlobServiceClient, newPipeline, StorageSharedKeyCredential +} from '@azure/storage-blob'; -import { configLogger } from "../../../src/common/Logger"; -import BlobTestServerFactory from "../../BlobTestServerFactory"; +import { configLogger } from '../../../src/common/Logger'; +import { getMD5FromString } from '../../../src/common/utils/utils'; +import BlobTestServerFactory from '../../BlobTestServerFactory'; import { - base64encode, - bodyToString, - EMULATOR_ACCOUNT_KEY, - EMULATOR_ACCOUNT_NAME, - getUniqueName, - sleep -} from "../../testutils"; -import { getMD5FromString } from "../../../src/common/utils/utils"; + base64encode, bodyToString, EMULATOR_ACCOUNT_KEY, EMULATOR_ACCOUNT_NAME, getUniqueName, sleep +} from '../../testutils'; + +import assert = require("assert"); +import crypto = require("crypto"); // Set true to enable debug log configLogger(false); @@ -70,7 +63,7 @@ describe("BlockBlobAPIs", () => { await containerClient.delete(); }); - it("Blob should be staged from URL and committed @loki @sql", async () => { + it("Blob should be staged from URL and committed @loki", async () => { const source1Blob = getUniqueName("blob"); const source2Blob = getUniqueName("blob"); const destBlob = getUniqueName("blob"); From 063ca6e836bf01b50ff0f52691ce0638ab7fcf0c Mon Sep 17 00:00:00 2001 From: Radek Gruchalski Date: Tue, 6 Aug 2024 10:12:15 +0200 Subject: [PATCH 6/6] Fix readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 09751790d..4b404a6e2 100644 --- a/README.md +++ b/README.md @@ -1010,7 +1010,7 @@ Detailed support matrix: - Abort Copy Blob (Only supports copy within same Azurite instance) - Copy Blob From URL (Only supports copy within same Azurite instance, only on Loki) - Access control based on conditional headers - - Stage Blob From URL (Only supports copy within same Azurite instance, only on Loki) + - Stage Block From URL (Only supports copy within same Azurite instance, only on Loki) - Following features or REST APIs are NOT supported or limited supported in this release (will support more features per customers feedback in future releases) - SharedKey Lite