diff --git a/src/workerd/api/BUILD.bazel b/src/workerd/api/BUILD.bazel index 9cba4624afc..3c3528d0865 100644 --- a/src/workerd/api/BUILD.bazel +++ b/src/workerd/api/BUILD.bazel @@ -321,6 +321,12 @@ wd_test( data = ["queue-test.js"], ) +wd_test( + src = "r2-test.wd-test", + args = ["--experimental"], + data = ["r2-test.js"], +) + wd_test( src = "rtti-test.wd-test", args = ["--experimental"], diff --git a/src/workerd/api/r2-api.capnp b/src/workerd/api/r2-api.capnp index af42c90e0d1..a42ba3efc85 100644 --- a/src/workerd/api/r2-api.capnp +++ b/src/workerd/api/r2-api.capnp @@ -60,6 +60,10 @@ struct R2Conditional { # timestamp. } +struct R2SSECOptions { + key @0 :Text; +} + struct R2Checksums { # The JSON name of these fields must comform to the representation of the ChecksumAlgorithm in # the R2 gateway worker. @@ -93,6 +97,7 @@ struct R2GetRequest { range @1 :R2Range; rangeHeader @3 :Text; onlyIf @2 :R2Conditional; + ssec @4 :R2SSECOptions; } struct R2PutRequest { @@ -106,6 +111,7 @@ struct R2PutRequest { sha384 @7 :Data $Json.hex; sha512 @8 :Data $Json.hex; storageClass @9 :Text; + ssec @10 :R2SSECOptions; } struct R2CreateMultipartUploadRequest { @@ -113,12 +119,14 @@ struct R2CreateMultipartUploadRequest { customFields @1 :List(Record); httpFields @2 :R2HttpFields; storageClass @3 :Text; + ssec @4 :R2SSECOptions; } struct R2UploadPartRequest { object @0 :Text; uploadId @1 :Text; partNumber @2 :UInt32; + ssec @3 :R2SSECOptions; } struct R2CompleteMultipartUploadRequest { @@ -187,6 +195,11 @@ struct R2ErrorResponse { message @2 :Text; } +struct R2SSECResponse { + algorithm @0 :Text; + keyMd5 @1 :Text; +} + struct R2HeadResponse { name @0 :Text; # The name of the object. @@ -220,6 +233,9 @@ struct R2HeadResponse { storageClass @9 :Text; # The storage class of the object. Standard or Infrequent Access. # Provided on object creation to specify which storage tier R2 should use for this object. + + ssec @10 :R2SSECResponse; + # The algorithm/key hash used for encryption(if the user used SSE-C) } using R2GetResponse = R2HeadResponse; @@ -230,6 +246,8 @@ struct R2CreateMultipartUploadResponse { uploadId @0 :Text; # The unique identifier of this object, required for subsequent operations on # this multipart upload. + ssec @1 :R2SSECResponse; + # The algorithm/key hash used for encryption(if the user used SSE-C) } struct R2UploadPartResponse { diff --git a/src/workerd/api/r2-bucket.c++ b/src/workerd/api/r2-bucket.c++ index d1977a6f966..2c6893f4a77 100644 --- a/src/workerd/api/r2-bucket.c++ +++ b/src/workerd/api/r2-bucket.c++ @@ -21,6 +21,7 @@ #include #include +#include namespace workerd::api::public_beta { static bool isWholeNumber(double x) { @@ -154,10 +155,17 @@ static jsg::Ref parseObjectMetadata(R2HeadResponse::Reader responseReader, } } + jsg::Optional ssecKeyMd5; + + if (responseReader.hasSsec()) { + auto ssecBuilder = responseReader.getSsec(); + ssecKeyMd5 = kj::str(ssecBuilder.getKeyMd5()); + } + return jsg::alloc(kj::str(responseReader.getName()), kj::str(responseReader.getVersion()), responseReader.getSize(), kj::str(responseReader.getEtag()), kj::mv(checksums), uploaded, kj::mv(httpMetadata), kj::mv(customMetadata), range, - kj::str(responseReader.getStorageClass()), kj::fwd(args)...); + kj::str(responseReader.getStorageClass()), kj::mv(ssecKeyMd5), kj::fwd(args)...); } template @@ -253,6 +261,25 @@ void initOnlyIf(jsg::Lock& js, Builder& builder, Options& o) { } } +kj::Maybe buildSsecKey( + kj::Maybe, kj::String>> maybeRawSsecKey) { + KJ_IF_SOME(rawSsecKey, maybeRawSsecKey) { + KJ_SWITCH_ONEOF(rawSsecKey) { + KJ_CASE_ONEOF(keyString, kj::String) { + JSG_REQUIRE(std::regex_match(keyString.begin(), keyString.end(), std::regex("^[0-9a-f]+$")), + Error, "SSE-C Key has invalid format"); + JSG_REQUIRE(keyString.size() == 64, Error, "SSE-C Key must be 32 bytes in length"); + return kj::str(keyString); + } + KJ_CASE_ONEOF(keyBuff, kj::Array) { + JSG_REQUIRE(keyBuff.size() == 32, Error, "SSE-C Key must be 32 bytes in length"); + return kj::encodeHex(keyBuff); + } + } + } + return kj::none; +} + template void initGetOptions(jsg::Lock& js, Builder& builder, Options& o) { initOnlyIf(js, builder, o); @@ -296,6 +323,11 @@ void initGetOptions(jsg::Lock& js, Builder& builder, Options& o) { } } } + kj::Maybe maybeSsecKey = buildSsecKey(kj::mv(o.ssecKey)); + KJ_IF_SOME(ssecKey, maybeSsecKey) { + auto ssecBuilder = builder.initSsec(); + ssecBuilder.setKey(ssecKey); + } } static bool isQuotedEtag(kj::StringPtr etag) { @@ -559,6 +591,11 @@ jsg::Promise>> R2Bucket::put(jsg::Lock& KJ_IF_SOME(s, o.storageClass) { putBuilder.setStorageClass(s); } + kj::Maybe maybeSsecKey = buildSsecKey(kj::mv(o.ssecKey)); + KJ_IF_SOME(ssecKey, maybeSsecKey) { + auto ssecBuilder = putBuilder.initSsec(); + ssecBuilder.setKey(ssecKey); + } } auto requestJson = json.encode(requestBuilder); @@ -651,6 +688,11 @@ jsg::Promise> R2Bucket::createMultipartUpload(jsg::L KJ_IF_SOME(s, o.storageClass) { createMultipartUploadBuilder.setStorageClass(s); } + kj::Maybe maybeSsecKey = buildSsecKey(kj::mv(o.ssecKey)); + KJ_IF_SOME(ssecKey, maybeSsecKey) { + auto ssecBuilder = createMultipartUploadBuilder.initSsec(); + ssecBuilder.setKey(ssecKey); + } } auto requestJson = json.encode(requestBuilder); diff --git a/src/workerd/api/r2-bucket.h b/src/workerd/api/r2-bucket.h index d1228f13c29..d3a2cecd6fc 100644 --- a/src/workerd/api/r2-bucket.h +++ b/src/workerd/api/r2-bucket.h @@ -78,8 +78,9 @@ class R2Bucket: public jsg::Object { struct GetOptions { jsg::Optional>> onlyIf; jsg::Optional>> range; + jsg::Optional, kj::String>> ssecKey; - JSG_STRUCT(onlyIf, range); + JSG_STRUCT(onlyIf, range, ssecKey); JSG_STRUCT_TS_OVERRIDE(R2GetOptions); }; @@ -189,9 +190,18 @@ class R2Bucket: public jsg::Object { jsg::Optional, jsg::NonCoercible>> sha384; jsg::Optional, jsg::NonCoercible>> sha512; jsg::Optional storageClass; - - JSG_STRUCT( - onlyIf, httpMetadata, customMetadata, md5, sha1, sha256, sha384, sha512, storageClass); + jsg::Optional, kj::String>> ssecKey; + + JSG_STRUCT(onlyIf, + httpMetadata, + customMetadata, + md5, + sha1, + sha256, + sha384, + sha512, + storageClass, + ssecKey); JSG_STRUCT_TS_OVERRIDE(R2PutOptions); }; @@ -199,8 +209,9 @@ class R2Bucket: public jsg::Object { jsg::Optional>> httpMetadata; jsg::Optional> customMetadata; jsg::Optional storageClass; + jsg::Optional, kj::String>> ssecKey; - JSG_STRUCT(httpMetadata, customMetadata, storageClass); + JSG_STRUCT(httpMetadata, customMetadata, storageClass, ssecKey); JSG_STRUCT_TS_OVERRIDE(R2MultipartOptions); }; @@ -215,7 +226,8 @@ class R2Bucket: public jsg::Object { jsg::Optional httpMetadata, jsg::Optional> customMetadata, jsg::Optional range, - kj::String storageClass) + kj::String storageClass, + jsg::Optional ssecKeyMd5) : name(kj::mv(name)), version(kj::mv(version)), size(size), @@ -225,7 +237,8 @@ class R2Bucket: public jsg::Object { httpMetadata(kj::mv(httpMetadata)), customMetadata(kj::mv(customMetadata)), range(kj::mv(range)), - storageClass(kj::mv(storageClass)) {} + storageClass(kj::mv(storageClass)), + ssecKeyMd5(kj::mv(ssecKeyMd5)) {} kj::String getName() const { return kj::str(name); @@ -251,6 +264,9 @@ class R2Bucket: public jsg::Object { kj::StringPtr getStorageClass() const { return storageClass; } + jsg::Optional getSSECKeyMd5() const { + return ssecKeyMd5; + } jsg::Optional getHttpMetadata() const { return httpMetadata.map([](const HttpMetadata& m) { return m.clone(); }); @@ -285,6 +301,7 @@ class R2Bucket: public jsg::Object { JSG_LAZY_READONLY_INSTANCE_PROPERTY(customMetadata, getCustomMetadata); JSG_LAZY_READONLY_INSTANCE_PROPERTY(range, getRange); JSG_LAZY_READONLY_INSTANCE_PROPERTY(storageClass, getStorageClass); + JSG_LAZY_READONLY_INSTANCE_PROPERTY(ssecKeyMd5, getSSECKeyMd5); JSG_METHOD(writeHttpMetadata); JSG_TS_OVERRIDE(R2Object); } @@ -296,6 +313,7 @@ class R2Bucket: public jsg::Object { tracker.trackField("checksums", checksums); tracker.trackField("httpMetadata", httpMetadata); tracker.trackField("customMetadata", customMetadata); + tracker.trackField("ssecKeyMd5", ssecKeyMd5); } protected: @@ -310,6 +328,7 @@ class R2Bucket: public jsg::Object { jsg::Optional range; kj::String storageClass; + jsg::Optional ssecKeyMd5; friend class R2Bucket; }; @@ -325,6 +344,7 @@ class R2Bucket: public jsg::Object { jsg::Optional> customMetadata, jsg::Optional range, kj::String storageClass, + jsg::Optional ssecKeyMd5, jsg::Ref body) : HeadResult(kj::mv(name), kj::mv(version), @@ -335,7 +355,8 @@ class R2Bucket: public jsg::Object { kj::mv(KJ_ASSERT_NONNULL(httpMetadata)), kj::mv(KJ_ASSERT_NONNULL(customMetadata)), range, - kj::mv(storageClass)), + kj::mv(storageClass), + kj::mv(ssecKeyMd5)), body(kj::mv(body)) {} jsg::Ref getBody() { diff --git a/src/workerd/api/r2-multipart.c++ b/src/workerd/api/r2-multipart.c++ index 4c563767fcb..c3f00ac791f 100644 --- a/src/workerd/api/r2-multipart.c++ +++ b/src/workerd/api/r2-multipart.c++ @@ -6,6 +6,7 @@ #include "r2-bucket.h" #include "r2-rpc.h" +#include "workerd/jsg/jsg.h" #include #include @@ -13,15 +14,16 @@ #include #include #include +#include -#include -#include +#include namespace workerd::api::public_beta { jsg::Promise R2MultipartUpload::uploadPart(jsg::Lock& js, int partNumber, R2PutValue value, + jsg::Optional options, const jsg::TypeHandler>& errorType) { return js.evalNow([&] { JSG_REQUIRE(partNumber >= 1 && partNumber <= 10000, TypeError, @@ -44,6 +46,24 @@ jsg::Promise R2MultipartUpload::uploadPart(jsg: uploadPartBuilder.setUploadId(uploadId); uploadPartBuilder.setPartNumber(partNumber); uploadPartBuilder.setObject(key); + KJ_IF_SOME(options, options) { + KJ_IF_SOME(ssecKey, options.ssecKey) { + auto ssecBuilder = uploadPartBuilder.initSsec(); + KJ_SWITCH_ONEOF(ssecKey) { + KJ_CASE_ONEOF(keyString, kj::String) { + JSG_REQUIRE( + std::regex_match(keyString.begin(), keyString.end(), std::regex("^[0-9a-f]+$")), + Error, "SSE-C Key has invalid format"); + JSG_REQUIRE(keyString.size() == 64, Error, "SSE-C Key must be 32 bytes in length"); + ssecBuilder.setKey(kj::str(keyString)); + } + KJ_CASE_ONEOF(keyBuff, kj::Array) { + JSG_REQUIRE(keyBuff.size() == 32, Error, "SSE-C Key must be 32 bytes in length"); + ssecBuilder.setKey(kj::encodeHex(keyBuff)); + } + } + } + } auto requestJson = json.encode(requestBuilder); auto bucket = this->bucket->adminBucket.map([](auto&& s) { return kj::str(s); }); diff --git a/src/workerd/api/r2-multipart.h b/src/workerd/api/r2-multipart.h index c32ee3158a0..693d39f7dd8 100644 --- a/src/workerd/api/r2-multipart.h +++ b/src/workerd/api/r2-multipart.h @@ -19,6 +19,12 @@ class R2MultipartUpload: public jsg::Object { JSG_STRUCT(partNumber, etag); JSG_STRUCT_TS_OVERRIDE(R2UploadedPart); }; + struct UploadPartOptions { + jsg::Optional, kj::String>> ssecKey; + + JSG_STRUCT(ssecKey); + JSG_STRUCT_TS_OVERRIDE(R2UploadPartOptions); + }; R2MultipartUpload(kj::String key, kj::String uploadId, jsg::Ref bucket) : key(kj::mv(key)), @@ -35,6 +41,7 @@ class R2MultipartUpload: public jsg::Object { jsg::Promise uploadPart(jsg::Lock& js, int partNumber, R2PutValue value, + jsg::Optional options, const jsg::TypeHandler>& errorType); jsg::Promise abort(jsg::Lock& js, const jsg::TypeHandler>& errorType); jsg::Promise> complete(jsg::Lock& js, diff --git a/src/workerd/api/r2-test.js b/src/workerd/api/r2-test.js index 70a7437dbc8..4c1e4ad29e8 100644 --- a/src/workerd/api/r2-test.js +++ b/src/workerd/api/r2-test.js @@ -4,95 +4,245 @@ import assert from 'node:assert'; +const bufferKey = new Uint8Array([ + 185, 255, 145, 154, 120, 76, 122, 72, 191, 42, 8, 64, 86, 189, 185, 75, 105, + 37, 155, 123, 165, 158, 4, 42, 222, 13, 135, 52, 87, 154, 181, 227, +]); +const hexKey = + 'b9ff919a784c7a48bf2a084056bdb94b69259b7ba59e042ade0d8734579ab5e3'; +const keyMd5 = 'WGR5pEm07DroP3hYRAh8Yw=='; + +const objResponse = { + name: 'objectKey', + version: 'objectVersion', + size: '123', + etag: 'objectEtag', + uploaded: '1724767257918', + storageClass: 'Standard', +}; + export default { // Handler for HTTP request binding makes to R2 async fetch(request, env, ctx) { // We only expect PUT/Get assert(['GET', 'PUT'].includes(request.method)); - // Each request should have a metadata size header indicating how much - // we should read to understand what type of request this is - const metadataSizeString = request.headers.get('cf-r2-metadata-size'); - assert.notStrictEqual(metadataSizeString, null); + switch (request.method) { + case 'PUT': { + // Each request should have a metadata size header indicating how much + // we should read to understand what type of request this is + const metadataSizeString = request.headers.get('cf-r2-metadata-size'); + assert.notStrictEqual(metadataSizeString, null); - const metadataSize = parseInt(metadataSizeString); - assert(!Number.isNaN(metadataSize)); + const metadataSize = parseInt(metadataSizeString); + assert(!Number.isNaN(metadataSize)); - const reader = request.body.getReader({ mode: 'byob' }); - const jsonArray = new Uint8Array(metadataSize); - const { value } = await reader.readAtLeast(metadataSize, jsonArray); - reader.releaseLock(); + const reader = request.body.getReader({ mode: 'byob' }); + const jsonArray = new Uint8Array(metadataSize); + const { value } = await reader.readAtLeast(metadataSize, jsonArray); + reader.releaseLock(); - const jsonRequest = JSON.parse(new TextDecoder().decode(value)); + const jsonRequest = JSON.parse(new TextDecoder().decode(value)); - // Currently not using the body in these test so I'm going to just discard - for await (const _ of request.body) { - } - - // Assert it's the correct version - assert((jsonRequest.version = 1)); + // Currently not using the body in these test so I'm going to just discard + for await (const _ of request.body) { + } - if (request.method === 'PUT') { - assert(jsonRequest.method === 'put'); + // Assert it's the correct version + assert((jsonRequest.version = 1)); + assert( + [ + 'put', + 'createMultipartUpload', + 'uploadPart', + 'completeMultipartUpload', + ].includes(jsonRequest.method) + ); - if (jsonRequest.object === 'onlyIfStrongEtag') { - assert.deepStrictEqual(jsonRequest.onlyIf, { - etagMatches: [ - { - value: 'strongEtag', - type: 'strong', - }, - ], - etagDoesNotMatch: [ - { - value: 'strongEtag', - type: 'strong', - }, - ], - }); + switch (jsonRequest.object) { + case 'onlyIfStrongEtag': { + assert.deepStrictEqual(jsonRequest.onlyIf, { + etagMatches: [ + { + value: 'strongEtag', + type: 'strong', + }, + ], + etagDoesNotMatch: [ + { + value: 'strongEtag', + type: 'strong', + }, + ], + }); + break; + } + case 'onlyIfWildcard': { + assert.deepStrictEqual(jsonRequest.onlyIf, { + etagMatches: [ + { + type: 'wildcard', + }, + ], + etagDoesNotMatch: [ + { + type: 'wildcard', + }, + ], + }); + break; + } + case 'ssec': { + assert.deepStrictEqual(jsonRequest.ssec, { + key: hexKey, + }); + return Response.json({ + ...objResponse, + ssec: { + algorithm: 'aes256', + keyMd5, + }, + }); + } + case 'ssec-mu': { + if (jsonRequest.method === 'createMultipartUpload') { + assert.deepStrictEqual(jsonRequest.ssec, { + key: hexKey, + }); + return Response.json({ + uploadId: 'definitelyARealId', + }); + } + if (jsonRequest.method === 'uploadPart') { + assert.deepStrictEqual(jsonRequest.ssec, { + key: hexKey, + }); + return Response.json({ + etag: 'definitelyAValidEtag', + ssec: { + algorithm: 'aes256', + keyMd5, + }, + }); + } + if (jsonRequest.method === 'completeMultipartUpload') { + return Response.json({ + ...objResponse, + ssec: { + algorithm: 'aes256', + keyMd5, + }, + }); + } + } + } + return Response.json(objResponse); } - - if (jsonRequest.object === 'onlyIfWildcard') { - assert.deepStrictEqual(jsonRequest.onlyIf, { - etagMatches: [ - { - type: 'wildcard', - }, - ], - etagDoesNotMatch: [ - { - type: 'wildcard', + case 'GET': { + const rawHeader = request.headers.get('cf-r2-request'); + const jsonRequest = JSON.parse(rawHeader); + assert((jsonRequest.version = 1)); + assert(['get', 'head'].includes(jsonRequest.method)); + if (jsonRequest.object === 'ssec') { + const encoder = new TextEncoder(); + const metadata = encoder.encode( + JSON.stringify({ + ...objResponse, + ssec: { + algorithm: 'aes256', + keyMd5, + }, + }) + ); + const body = + jsonRequest.method === 'get' + ? new ReadableStream({ + start(controller) { + controller.enqueue(metadata); + controller.enqueue(encoder.encode('Bonk')); + controller.close(); + }, + }) + : metadata; + return new Response(body, { + headers: { + 'cf-r2-metadata-size': metadata.length.toString(), + 'content-length': metadata.length.toString(), }, - ], - }); + }); + } } - - return Response.json({ - name: 'objectKey', - version: 'objectVersion', - size: '123', - etag: 'objectEtag', - uploaded: '1724767257918', - storageClass: 'Standard', - }); } - throw new Error('unexpected'); }, async test(ctrl, env, ctx) { - await env.BUCKET.put('basic', 'content'); - await env.BUCKET.put('onlyIfStrongEtag', 'content', { - onlyIf: { - etagMatches: 'strongEtag', - etagDoesNotMatch: 'strongEtag', - }, - }); - await env.BUCKET.put('onlyIfWildcard', 'content', { - onlyIf: { - etagMatches: '*', - etagDoesNotMatch: '*', - }, - }); + { + // Conditionals + await env.BUCKET.put('basic', 'content'); + await env.BUCKET.put('onlyIfStrongEtag', 'content', { + onlyIf: { + etagMatches: 'strongEtag', + etagDoesNotMatch: 'strongEtag', + }, + }); + await env.BUCKET.put('onlyIfWildcard', 'content', { + onlyIf: { + etagMatches: '*', + etagDoesNotMatch: '*', + }, + }); + } + + { + // SSEC + for (const ssecKey of [bufferKey, hexKey]) { + { + const { ssecKeyMd5 } = await env.BUCKET.put('ssec', 'content', { + ssecKey, + }); + assert.strictEqual(ssecKeyMd5, keyMd5); + } + { + const { ssecKeyMd5 } = await env.BUCKET.get('ssec', { + ssecKey, + }); + assert.strictEqual(ssecKeyMd5, keyMd5); + } + { + const { ssecKeyMd5 } = await env.BUCKET.head('ssec', { + ssecKey, + }); + assert.strictEqual(ssecKeyMd5, keyMd5); + } + { + const multi = await env.BUCKET.createMultipartUpload('ssec-mu', { + ssecKey, + }); + assert.equal(multi.uploadId, 'definitelyARealId'); + } + { + const multi = await env.BUCKET.createMultipartUpload('ssec-mu', { + ssecKey, + }); + const part = await multi.uploadPart(1, 'hey', { + ssecKey, + }); + assert.equal(part.etag, 'definitelyAValidEtag'); + } + { + const multi = await env.BUCKET.createMultipartUpload('ssec-mu', { + ssecKey, + }); + const { ssecKeyMd5 } = await multi.complete([ + { + partNumber: 1, + etag: 'definitelyAValidEtag', + }, + ]); + assert.strictEqual(ssecKeyMd5, keyMd5); + } + } + } }, }; diff --git a/src/workerd/api/r2.h b/src/workerd/api/r2.h index 37d95230838..eb104bbef50 100644 --- a/src/workerd/api/r2.h +++ b/src/workerd/api/r2.h @@ -16,6 +16,7 @@ namespace workerd::api::public_beta { api::public_beta::R2Bucket::PutOptions, api::public_beta::R2Bucket::MultipartOptions, \ api::public_beta::R2Bucket::Checksums, api::public_beta::R2Bucket::StringChecksums, \ api::public_beta::R2Bucket::HttpMetadata, api::public_beta::R2Bucket::ListOptions, \ - api::public_beta::R2Bucket::ListResult + api::public_beta::R2Bucket::ListResult, \ + api::public_beta::R2MultipartUpload::UploadPartOptions // The list of r2 types that are added to worker.c++'s JSG_DECLARE_ISOLATE_TYPE } // namespace workerd::api::public_beta