feat: restrict block size (#242)

Adds a restriction to CAR uploads to ensure we don't store a block that cannot be transferred over the IPFS network.

packages/api/src/car.js

@@ -5,7 +5,7 @@ import { Block } from 'multiformats/block'
 import * as raw from 'multiformats/codecs/raw'
 import * as cbor from '@ipld/dag-cbor'
 import * as pb from '@ipld/dag-pb'
-import { GATEWAY, LOCAL_ADD_THRESHOLD, DAG_SIZE_CALC_LIMIT } from './constants.js'
+import { GATEWAY, LOCAL_ADD_THRESHOLD, DAG_SIZE_CALC_LIMIT, MAX_BLOCK_SIZE } from './constants.js'
 import { JSONResponse } from './utils/json-response.js'
 import { toPinStatusEnum } from './utils/pin.js'
 
@@ -104,6 +104,9 @@ export async function carPost (request, env, ctx) {
   }
 
   const blob = await request.blob()
+  const bytes = new Uint8Array(await blob.arrayBuffer())
+  const reader = await CarReader.fromBytes(bytes)
+  const chunkSize = await getBlocksSize(reader)
 
   // Ensure car blob.type is set; it is used by the cluster client to set the foramt=car flag on the /add call.
   const content = blob.slice(0, blob.size, 'application/car')
@@ -146,7 +149,7 @@ export async function carPost (request, env, ctx) {
       cid,
       name,
       type: 'Car',
-      chunkSize: await getBlocksSize(blob),
+      chunkSize,
       pins
     }
   })
@@ -189,28 +192,26 @@ export async function sizeOf (response) {
 
 /**
  * Returns the sum of all block sizes in the received Car.
- * @param {Blob} car
+ * @param {CarReader} reader
  */
-async function getBlocksSize (car) {
-  const bytes = new Uint8Array(await car.arrayBuffer())
-  const reader = await CarReader.fromBytes(bytes)
-
+async function getBlocksSize (reader) {
   let size = 0
   for await (const block of reader.blocks()) {
-    size += block.bytes.byteLength
+    const blockSize = block.bytes.byteLength
+    if (blockSize > MAX_BLOCK_SIZE) {
+      throw new Error(`block too big: ${blockSize} > ${MAX_BLOCK_SIZE}`)
+    }
+    size += blockSize
   }
-
   return size
 }
 
 /**
  * Returns the DAG size of the CAR but only if the graph is complete.
- * @param {Blob} car
+ * @param {CarReader} reader
  */
-async function getDagSize (car) {
+async function getDagSize (reader) {
   const decoders = [pb, raw, cbor]
-  const bytes = new Uint8Array(await car.arrayBuffer())
-  const reader = await CarReader.fromBytes(bytes)
   const [rootCid] = await reader.getRoots()
 
   const getBlock = async cid => {

packages/api/src/constants.js

@@ -3,3 +3,5 @@ export const JWT_ISSUER = 'web3-storage'
 export const METRICS_CACHE_MAX_AGE = 10 * 60 // in seconds (10 minutes)
 export const LOCAL_ADD_THRESHOLD = 1024 * 1024 * 2.5
 export const DAG_SIZE_CALC_LIMIT = 1024 * 1024 * 9
+// Maximum permitted block size in bytes.
+export const MAX_BLOCK_SIZE = 1 << 20 // 1MiB

packages/api/test/car.spec.js

@@ -1,10 +1,14 @@
-/* global describe it fetch */
+/* eslint-env mocha, browser */
 import assert from 'assert'
+import { CID } from 'multiformats/cid'
+import { sha256 } from 'multiformats/hashes/sha2'
+import * as pb from '@ipld/dag-pb'
+import { CarWriter } from '@ipld/car'
 import { endpoint } from './scripts/constants.js'
 import * as JWT from '../src/utils/jwt.js'
 import { SALT } from './scripts/worker-globals.js'
 import { createCar } from './scripts/car.js'
-import { JWT_ISSUER } from '../src/constants.js'
+import { JWT_ISSUER, MAX_BLOCK_SIZE } from '../src/constants.js'
 
 function getTestJWT (sub = 'test', name = 'test') {
   return JWT.sign({ sub, iss: JWT_ISSUER, iat: Date.now(), name }, SALT)
@@ -39,4 +43,34 @@ describe('POST /car', () => {
     assert(cid, 'Server response payload has `cid` property')
     assert.strictEqual(cid, expectedCid, 'Server responded with expected CID')
   })
+
+  it('should throw for blocks bigger than the maximum permitted size', async () => {
+    const token = await getTestJWT()
+
+    const bytes = pb.encode({ Data: new Uint8Array(MAX_BLOCK_SIZE + 1).fill(1), Links: [] })
+    const hash = await sha256.digest(bytes)
+    const cid = CID.create(1, pb.code, hash)
+
+    const { writer, out } = CarWriter.create(cid)
+    writer.put({ cid, bytes })
+    writer.close()
+
+    const carBytes = []
+    for await (const chunk of out) {
+      carBytes.push(chunk)
+    }
+
+    const res = await fetch(new URL('car', endpoint), {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${token}`,
+        'Content-Type': 'application/car'
+      },
+      body: new Blob(carBytes)
+    })
+
+    assert.notEqual(res.ok, true)
+    const { message } = await res.json()
+    assert.ok(message.includes('block too big'))
+  })
 })

packages/api/test/mocks/db/post_graphql.js

@@ -34,7 +34,10 @@ module.exports = ({ body }) => {
           _id: 'test-auth-token',
           name: 'Test Auth Token',
           secret: 'test-auth-token-secret',
-          created: Date.now()
+          created: Date.now(),
+          uploads: {
+            data: []
+          }
         }]
       }
     })