From 1ca2491b37cbc5c68c6515cb7290065b64c5956c Mon Sep 17 00:00:00 2001 From: iteye Date: Wed, 7 Aug 2024 15:08:23 +0800 Subject: [PATCH 1/9] sync get blob hash --- package.json | 6 ++++-- rollup.config.mjs | 16 +++++++++++++++- src/flatdirectory.js | 36 ++++++++++++++++++++++++------------ src/utils/uploader.js | 4 ++-- src/utils/wasm.js | 10 ++++++++++ src/worker.js | 29 +++++++++++++++++++++++++++++ yarn.lock | 5 +++++ 7 files changed, 89 insertions(+), 17 deletions(-) create mode 100644 src/utils/wasm.js create mode 100644 src/worker.js diff --git a/package.json b/package.json index 92be97c..ea9b282 100644 --- a/package.json +++ b/package.json @@ -15,13 +15,15 @@ } }, "scripts": { - "build": "rollup -c" + "build": "rollup -c", + "prepublishOnly": "npm run build" }, "dependencies": { "async-mutex": "^0.5.0", "dotenv": "^16.4.5", "ethers": "^6.13.1", - "kzg-wasm": "^0.4.0" + "kzg-wasm": "^0.4.0", + "workerpool": "^9.1.3" }, "repository": { "type": "git", diff --git a/rollup.config.mjs b/rollup.config.mjs index 5c838eb..5c29b8b 100644 --- a/rollup.config.mjs +++ b/rollup.config.mjs @@ -17,7 +17,7 @@ export default [ } ], plugins: [commonjs(), resolve()], - external: ["ethers", "kzg-wasm"] + external: ["ethers", "kzg-wasm", "workerpool"] }, { input: 'src/node/file.js', @@ -29,5 +29,19 @@ export default [ plugins: [commonjs(), resolve()], external: ["ethers"] }, + { + input: 'src/worker.js', + output: { + file: 'dist/worker.js', + format: 'umd', + globals: { + "workerpool": 'workerpool', + "ethers": 'ethers', + "kzg-wasm": "kzgWasm" + }, + }, + plugins: [commonjs(), resolve()], + external: ["ethers", "kzg-wasm", "workerpool"] + }, ]; diff --git a/src/flatdirectory.js b/src/flatdirectory.js index 6c94b93..233091f 100644 --- a/src/flatdirectory.js +++ b/src/flatdirectory.js @@ -16,6 +16,9 @@ import { stringToHex } from "./utils"; +const workerpool = require('workerpool'); +const pool = workerpool.pool(__dirname + '/worker.js'); + const REMOVE_FAIL = -1; const REMOVE_NORMAL = 0; const REMOVE_SUCCESS = 1; @@ -132,7 +135,7 @@ export class FlatDirectory { const provider = new ethers.JsonRpcProvider(this.#ethStorageRpc); const contract = new ethers.Contract(this.#contractAddr, FlatDirectoryAbi, provider); try { - const blobCount = await contract.countChunks(hexName); + const blobCount = await this.#countChunks(contract, hexName); for (let i = 0; i < blobCount; i++) { const result = await contract.readChunk(hexName, i); const chunk = ethers.getBytes(result[0]); @@ -210,7 +213,7 @@ export class FlatDirectory { let gasLimit = 0; const [cost, oldChunkLength, maxFeePerBlobGas, gasFeeData] = await Promise.all([ fileContract.upfrontPayment(), - fileContract.countChunks(hexName), + this.#countChunks(fileContract, hexName), this.#blobUploader.getBlobGasPrice(), this.#blobUploader.getGasPrice(), ]); @@ -222,19 +225,17 @@ export class FlatDirectory { const blobArr = encodeBlobs(data); const chunkIdArr = []; const chunkSizeArr = []; - const blobHashArr = []; const blobHashRequestArr = []; for (let j = 0; j < blobArr.length; j++) { chunkIdArr.push(i + j); chunkSizeArr.push(DEFAULT_BLOB_DATA_SIZE); - blobHashArr.push(this.#blobUploader.getBlobHash(blobArr[j])); blobHashRequestArr.push(fileContract.getChunkHash(hexName, i + j)); } // check change if (chunkIdArr[0] < oldChunkLength) { - const isChange = await this.#checkChange(fileContract, blobHashArr, blobHashRequestArr); + const isChange = await this.#checkChange(blobArr, blobHashRequestArr); if (!isChange) { continue; } @@ -246,6 +247,7 @@ export class FlatDirectory { totalStorageCost += value; // gas cost if (gasLimit === 0) { + const blobHashArr = await this.#getBlobHashes(blobArr); gasLimit = await fileContract.writeChunks.estimateGas(hexName, chunkIdArr, chunkSizeArr, { value: value, blobVersionedHashes: blobHashArr @@ -317,7 +319,7 @@ export class FlatDirectory { } const [oldChunkLength, gasFeeData] = await Promise.all([ - fileContract.countChunks(hexName), + this.#countChunks(fileContract, hexName), this.#blobUploader.getGasPrice(), ]); @@ -398,7 +400,7 @@ export class FlatDirectory { // check old data const [cost, oldBlobLength] = await Promise.all([ fileContract.upfrontPayment(), - fileContract.countChunks(hexName), + this.#countChunks(fileContract, hexName) ]); const clearState = await this.#clearOldFile(hexName, blobLength, oldBlobLength); if (clearState === REMOVE_FAIL) { @@ -414,7 +416,6 @@ export class FlatDirectory { const blobArr = encodeBlobs(data); const chunkIdArr = []; const chunkSizeArr = []; - const blobHashArr = []; const blobHashRequestArr = []; for (let j = 0; j < blobArr.length; j++) { chunkIdArr.push(i + j); @@ -424,14 +425,13 @@ export class FlatDirectory { } else { chunkSizeArr.push(DEFAULT_BLOB_DATA_SIZE); } - blobHashArr.push(this.#blobUploader.getBlobHash(blobArr[j])); blobHashRequestArr.push(fileContract.getChunkHash(hexName, i + j)); } // check change if (clearState === REMOVE_NORMAL) { try { - const isChange = await this.#checkChange(fileContract, blobHashArr, blobHashRequestArr); + const isChange = await this.#checkChange(blobArr, blobHashRequestArr); if (!isChange) { callback.onProgress(chunkIdArr[chunkIdArr.length - 1], blobLength, false); continue; @@ -520,7 +520,7 @@ export class FlatDirectory { } // check old data - const oldChunkLength = await fileContract.countChunks(hexName); + const oldChunkLength = await this.#countChunks(fileContract, hexName); const clearState = await this.#clearOldFile(hexName, chunkLength, oldChunkLength); if (clearState === REMOVE_FAIL) { callback.onFail(new Error(`FlatDirectory: Failed to delete old data!`)); @@ -586,8 +586,9 @@ export class FlatDirectory { } } - async #checkChange(fileContract, blobHashArr, blobHashRequestArr) { + async #checkChange(blobArr, blobHashRequestArr) { let hasChange = false; + const blobHashArr = await this.#getBlobHashes(blobArr); const dataHashArr = await Promise.all(blobHashRequestArr); for (let i = 0; i < blobHashArr.length; i++) { if (blobHashArr[i] !== dataHashArr[i]) { @@ -641,4 +642,15 @@ export class FlatDirectory { const txReceipt = await txResponse.wait(); return txReceipt && txReceipt.status; } + + async #countChunks(fileContract, hexName) { + const count = await fileContract.countChunks(hexName); + // Bigint to number + return Number(count); + } + + async #getBlobHashes(blobArr) { + const promises = blobArr.map(blob => pool.exec('getBlobHash', [blob])); + return await Promise.all(promises); + } } diff --git a/src/utils/uploader.js b/src/utils/uploader.js index cb740af..b777503 100644 --- a/src/utils/uploader.js +++ b/src/utils/uploader.js @@ -1,6 +1,6 @@ import {ethers} from "ethers"; -import {loadKZG} from 'kzg-wasm'; import {Mutex} from 'async-mutex'; +import {initializeKzg} from './wasm'; function computeVersionedHash(commitment, blobCommitmentVersion) { const computedVersionedHash = new Uint8Array(32); @@ -51,7 +51,7 @@ export class BlobUploader { async init() { if (!this.#kzg) { - this.#kzg = await loadKZG(); + this.#kzg = await initializeKzg(); } return this.#kzg; } diff --git a/src/utils/wasm.js b/src/utils/wasm.js new file mode 100644 index 0000000..44f7426 --- /dev/null +++ b/src/utils/wasm.js @@ -0,0 +1,10 @@ +import {loadKZG} from 'kzg-wasm'; + +let kzgInstance = null; + +export async function initializeKzg() { + if (!kzgInstance) { + kzgInstance = await loadKZG(); + } + return kzgInstance; +} diff --git a/src/worker.js b/src/worker.js new file mode 100644 index 0000000..0eb905f --- /dev/null +++ b/src/worker.js @@ -0,0 +1,29 @@ +// worker.js +import workerpool from 'workerpool'; +import {ethers} from "ethers"; +import {initializeKzg} from './utils/wasm'; + +function computeVersionedHash(commitment, blobCommitmentVersion) { + const computedVersionedHash = new Uint8Array(32); + computedVersionedHash.set([blobCommitmentVersion], 0); + const hash = ethers.getBytes(ethers.sha256(commitment)); + computedVersionedHash.set(hash.subarray(1), 1); + return computedVersionedHash; +} + +function commitmentsToVersionedHashes(commitment) { + return computeVersionedHash(commitment, 0x01); +} + +async function getBlobHash(blob) { + const kzg = await initializeKzg(); + const commit = kzg.blobToKzgCommitment(blob); + const localHash = commitmentsToVersionedHashes(commit); + const hash = new Uint8Array(32); + hash.set(localHash.subarray(0, 32 - 8)); + return ethers.hexlify(hash); +} + +workerpool.worker({ + getBlobHash: getBlobHash +}); diff --git a/yarn.lock b/yarn.lock index 4af0a41..94093a1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -583,6 +583,11 @@ undici-types@~5.26.4: resolved "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== +workerpool@^9.1.3: + version "9.1.3" + resolved "https://registry.npmjs.org/workerpool/-/workerpool-9.1.3.tgz#34b81f50f777a0e549c6dfaa0926575735e3f4b4" + integrity sha512-LhUrk4tbxJRDQmRrrFWA9EnboXI79fe0ZNTy3u8m+dqPN1EkVSIsQYAB8OF/fkyhG8Rtup+c/bzj/+bzbG8fqg== + wrappy@1: version "1.0.2" resolved "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" From 986383bb24aaba30d487a9df81775b5e1847f685 Mon Sep 17 00:00:00 2001 From: iteye Date: Wed, 7 Aug 2024 15:39:25 +0800 Subject: [PATCH 2/9] change package --- rollup.config.mjs | 2 +- src/utils/uploader.js | 24 ++++-------------------- src/utils/util.js | 20 ++++++++++++++++++++ src/worker.js | 29 ----------------------------- src/{utils => worker}/wasm.js | 0 src/worker/worker.js | 13 +++++++++++++ 6 files changed, 38 insertions(+), 50 deletions(-) delete mode 100644 src/worker.js rename src/{utils => worker}/wasm.js (100%) create mode 100644 src/worker/worker.js diff --git a/rollup.config.mjs b/rollup.config.mjs index 5c29b8b..7552bc1 100644 --- a/rollup.config.mjs +++ b/rollup.config.mjs @@ -30,7 +30,7 @@ export default [ external: ["ethers"] }, { - input: 'src/worker.js', + input: 'src/worker/worker.js', output: { file: 'dist/worker.js', format: 'umd', diff --git a/src/utils/uploader.js b/src/utils/uploader.js index b777503..8571107 100644 --- a/src/utils/uploader.js +++ b/src/utils/uploader.js @@ -1,18 +1,7 @@ import {ethers} from "ethers"; +import {loadKZG} from 'kzg-wasm'; import {Mutex} from 'async-mutex'; -import {initializeKzg} from './wasm'; - -function computeVersionedHash(commitment, blobCommitmentVersion) { - const computedVersionedHash = new Uint8Array(32); - computedVersionedHash.set([blobCommitmentVersion], 0); - const hash = ethers.getBytes(ethers.sha256(commitment)); - computedVersionedHash.set(hash.subarray(1), 1); - return computedVersionedHash; -} - -function commitmentsToVersionedHashes(commitment) { - return computeVersionedHash(commitment, 0x01); -} +import {getHash, commitmentsToVersionedHashes} from "./util"; // blob gas price const MIN_BLOB_GASPRICE = 1n; @@ -51,7 +40,7 @@ export class BlobUploader { async init() { if (!this.#kzg) { - this.#kzg = await initializeKzg(); + this.#kzg = await loadKZG(); } return this.#kzg; } @@ -125,11 +114,6 @@ export class BlobUploader { } getBlobHash(blob) { - const kzg = this.#kzg; - const commit = kzg.blobToKzgCommitment(blob); - const localHash = commitmentsToVersionedHashes(commit); - const hash = new Uint8Array(32); - hash.set(localHash.subarray(0, 32 - 8)); - return ethers.hexlify(hash); + return getHash(this.#kzg, blob); } } diff --git a/src/utils/util.js b/src/utils/util.js index 27ac48c..9dd8f06 100644 --- a/src/utils/util.js +++ b/src/utils/util.js @@ -25,3 +25,23 @@ export function isFile(content) { content.isNodeJs; return (content instanceof File) || isNodeFile; } + +function computeVersionedHash(commitment, blobCommitmentVersion) { + const computedVersionedHash = new Uint8Array(32); + computedVersionedHash.set([blobCommitmentVersion], 0); + const hash = ethers.getBytes(ethers.sha256(commitment)); + computedVersionedHash.set(hash.subarray(1), 1); + return computedVersionedHash; +} + +export function commitmentsToVersionedHashes(commitment) { + return computeVersionedHash(commitment, 0x01); +} + +export function getHash(kzg, blob) { + const commit = kzg.blobToKzgCommitment(blob); + const localHash = commitmentsToVersionedHashes(commit); + const hash = new Uint8Array(32); + hash.set(localHash.subarray(0, 32 - 8)); + return ethers.hexlify(hash); +} diff --git a/src/worker.js b/src/worker.js deleted file mode 100644 index 0eb905f..0000000 --- a/src/worker.js +++ /dev/null @@ -1,29 +0,0 @@ -// worker.js -import workerpool from 'workerpool'; -import {ethers} from "ethers"; -import {initializeKzg} from './utils/wasm'; - -function computeVersionedHash(commitment, blobCommitmentVersion) { - const computedVersionedHash = new Uint8Array(32); - computedVersionedHash.set([blobCommitmentVersion], 0); - const hash = ethers.getBytes(ethers.sha256(commitment)); - computedVersionedHash.set(hash.subarray(1), 1); - return computedVersionedHash; -} - -function commitmentsToVersionedHashes(commitment) { - return computeVersionedHash(commitment, 0x01); -} - -async function getBlobHash(blob) { - const kzg = await initializeKzg(); - const commit = kzg.blobToKzgCommitment(blob); - const localHash = commitmentsToVersionedHashes(commit); - const hash = new Uint8Array(32); - hash.set(localHash.subarray(0, 32 - 8)); - return ethers.hexlify(hash); -} - -workerpool.worker({ - getBlobHash: getBlobHash -}); diff --git a/src/utils/wasm.js b/src/worker/wasm.js similarity index 100% rename from src/utils/wasm.js rename to src/worker/wasm.js diff --git a/src/worker/worker.js b/src/worker/worker.js new file mode 100644 index 0000000..9bd5edf --- /dev/null +++ b/src/worker/worker.js @@ -0,0 +1,13 @@ +// worker.js +import workerpool from 'workerpool'; +import {getHash} from "../utils"; +import {initializeKzg} from './wasm'; + +async function getBlobHash(blob) { + const kzg = await initializeKzg(); + return getHash(kzg, blob); +} + +workerpool.worker({ + getBlobHash: getBlobHash +}); From 7fb30c7f79a95ff75db8e867fa0a19e62a45dc0e Mon Sep 17 00:00:00 2001 From: iteye Date: Wed, 7 Aug 2024 15:46:19 +0800 Subject: [PATCH 3/9] change log --- src/flatdirectory.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/flatdirectory.js b/src/flatdirectory.js index 233091f..434bf40 100644 --- a/src/flatdirectory.js +++ b/src/flatdirectory.js @@ -617,7 +617,7 @@ export class FlatDirectory { } // send const txResponse = await this.#blobUploader.sendTxLock(tx, blobArr); - console.log(`FlatDirectory: The ${chunkIdArr} chunks hash is ${txResponse.hash}`, key); + console.log(`FlatDirectory: The ${chunkIdArr} chunks hash is ${txResponse.hash}`, "", key); const txReceipt = await txResponse.wait(); return txReceipt && txReceipt.status; } @@ -638,7 +638,7 @@ export class FlatDirectory { // send const txResponse = await this.#blobUploader.sendTxLock(tx); - console.log(`FlatDirectory: The ${chunkId} chunk hash is ${txResponse.hash}`, key); + console.log(`FlatDirectory: The ${chunkId} chunk hash is ${txResponse.hash}`, "", key); const txReceipt = await txResponse.wait(); return txReceipt && txReceipt.status; } From 109273c0ae50ca6a2de994d5a5485b32b2b64d40 Mon Sep 17 00:00:00 2001 From: iteye Date: Wed, 7 Aug 2024 18:59:27 +0800 Subject: [PATCH 4/9] fix esm error --- package.json | 3 +-- rollup.config.mjs | 21 ++------------------- src/flatdirectory.js | 5 +---- src/worker/wasm.js | 10 ---------- src/worker/worker.js | 13 ------------- yarn.lock | 5 ----- 6 files changed, 4 insertions(+), 53 deletions(-) delete mode 100644 src/worker/wasm.js delete mode 100644 src/worker/worker.js diff --git a/package.json b/package.json index ea9b282..b5a58fe 100644 --- a/package.json +++ b/package.json @@ -22,8 +22,7 @@ "async-mutex": "^0.5.0", "dotenv": "^16.4.5", "ethers": "^6.13.1", - "kzg-wasm": "^0.4.0", - "workerpool": "^9.1.3" + "kzg-wasm": "^0.4.0" }, "repository": { "type": "git", diff --git a/rollup.config.mjs b/rollup.config.mjs index 7552bc1..6e020fe 100644 --- a/rollup.config.mjs +++ b/rollup.config.mjs @@ -8,40 +8,23 @@ export default [ { file: 'dist/index.cjs.js', format: 'cjs', - sourcemap: true }, { file: 'dist/index.esm.js', format: 'esm', - sourcemap: true } ], plugins: [commonjs(), resolve()], - external: ["ethers", "kzg-wasm", "workerpool"] + external: ["ethers", "kzg-wasm"] }, { input: 'src/node/file.js', output: { file: 'dist/file.cjs.js', format: 'cjs', - sourcemap: true }, plugins: [commonjs(), resolve()], external: ["ethers"] - }, - { - input: 'src/worker/worker.js', - output: { - file: 'dist/worker.js', - format: 'umd', - globals: { - "workerpool": 'workerpool', - "ethers": 'ethers', - "kzg-wasm": "kzgWasm" - }, - }, - plugins: [commonjs(), resolve()], - external: ["ethers", "kzg-wasm", "workerpool"] - }, + } ]; diff --git a/src/flatdirectory.js b/src/flatdirectory.js index 434bf40..11e2d72 100644 --- a/src/flatdirectory.js +++ b/src/flatdirectory.js @@ -16,9 +16,6 @@ import { stringToHex } from "./utils"; -const workerpool = require('workerpool'); -const pool = workerpool.pool(__dirname + '/worker.js'); - const REMOVE_FAIL = -1; const REMOVE_NORMAL = 0; const REMOVE_SUCCESS = 1; @@ -650,7 +647,7 @@ export class FlatDirectory { } async #getBlobHashes(blobArr) { - const promises = blobArr.map(blob => pool.exec('getBlobHash', [blob])); + const promises = blobArr.map(blob => this.#blobUploader.getBlobHash(blob)); return await Promise.all(promises); } } diff --git a/src/worker/wasm.js b/src/worker/wasm.js deleted file mode 100644 index 44f7426..0000000 --- a/src/worker/wasm.js +++ /dev/null @@ -1,10 +0,0 @@ -import {loadKZG} from 'kzg-wasm'; - -let kzgInstance = null; - -export async function initializeKzg() { - if (!kzgInstance) { - kzgInstance = await loadKZG(); - } - return kzgInstance; -} diff --git a/src/worker/worker.js b/src/worker/worker.js deleted file mode 100644 index 9bd5edf..0000000 --- a/src/worker/worker.js +++ /dev/null @@ -1,13 +0,0 @@ -// worker.js -import workerpool from 'workerpool'; -import {getHash} from "../utils"; -import {initializeKzg} from './wasm'; - -async function getBlobHash(blob) { - const kzg = await initializeKzg(); - return getHash(kzg, blob); -} - -workerpool.worker({ - getBlobHash: getBlobHash -}); diff --git a/yarn.lock b/yarn.lock index 94093a1..4af0a41 100644 --- a/yarn.lock +++ b/yarn.lock @@ -583,11 +583,6 @@ undici-types@~5.26.4: resolved "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== -workerpool@^9.1.3: - version "9.1.3" - resolved "https://registry.npmjs.org/workerpool/-/workerpool-9.1.3.tgz#34b81f50f777a0e549c6dfaa0926575735e3f4b4" - integrity sha512-LhUrk4tbxJRDQmRrrFWA9EnboXI79fe0ZNTy3u8m+dqPN1EkVSIsQYAB8OF/fkyhG8Rtup+c/bzj/+bzbG8fqg== - wrappy@1: version "1.0.2" resolved "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" From d0ec797be3a304c6111fc3a0cc2752eb783fd6dd Mon Sep 17 00:00:00 2001 From: iteye Date: Thu, 8 Aug 2024 11:35:43 +0800 Subject: [PATCH 5/9] change lib --- package.json | 5 +++-- rollup.config.mjs | 17 +++++++++++++++-- src/flatdirectory.js | 8 +++++++- src/worker/worker.js | 21 +++++++++++++++++++++ yarn.lock | 5 +++++ 5 files changed, 51 insertions(+), 5 deletions(-) create mode 100644 src/worker/worker.js diff --git a/package.json b/package.json index b5a58fe..fa78ae4 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "eip-4844 blobs upload sdk", "main": "dist/index.cjs.js", "module": "dist/index.esm.js", - "types": "types/index.d.ts", + "types": "index.d.ts", "exports": { ".": { "require": "./dist/index.cjs.js", @@ -22,7 +22,8 @@ "async-mutex": "^0.5.0", "dotenv": "^16.4.5", "ethers": "^6.13.1", - "kzg-wasm": "^0.4.0" + "kzg-wasm": "^0.4.0", + "workerpool": "^9.1.3" }, "repository": { "type": "git", diff --git a/rollup.config.mjs b/rollup.config.mjs index 6e020fe..ebfd94a 100644 --- a/rollup.config.mjs +++ b/rollup.config.mjs @@ -8,23 +8,36 @@ export default [ { file: 'dist/index.cjs.js', format: 'cjs', + sourcemap: true }, { file: 'dist/index.esm.js', format: 'esm', + sourcemap: true } ], plugins: [commonjs(), resolve()], - external: ["ethers", "kzg-wasm"] + external: ["ethers", "kzg-wasm", "workerpool"] }, { input: 'src/node/file.js', output: { file: 'dist/file.cjs.js', format: 'cjs', + sourcemap: true, }, plugins: [commonjs(), resolve()], external: ["ethers"] - } + }, + { + input: 'src/worker/worker.js', + output: { + file: 'dist/worker.cjs.js', + format: 'cjs', + sourcemap: true, + }, + plugins: [commonjs(), resolve()], + external: ["ethers", "kzg-wasm", "workerpool"] + }, ]; diff --git a/src/flatdirectory.js b/src/flatdirectory.js index 11e2d72..f148efc 100644 --- a/src/flatdirectory.js +++ b/src/flatdirectory.js @@ -16,6 +16,9 @@ import { stringToHex } from "./utils"; +import workerpool from 'workerpool'; +const pool = workerpool.pool(__dirname + '/worker.cjs.js'); + const REMOVE_FAIL = -1; const REMOVE_NORMAL = 0; const REMOVE_SUCCESS = 1; @@ -647,7 +650,10 @@ export class FlatDirectory { } async #getBlobHashes(blobArr) { - const promises = blobArr.map(blob => this.#blobUploader.getBlobHash(blob)); + const isNode = typeof process !== 'undefined' && !!process.versions && !!process.versions.node; + const promises = isNode + ? blobArr.map(blob => pool.exec('getBlobHash', [blob])) + : blobArr.map(blob => this.#blobUploader.getBlobHash(blob)); return await Promise.all(promises); } } diff --git a/src/worker/worker.js b/src/worker/worker.js new file mode 100644 index 0000000..064ea7e --- /dev/null +++ b/src/worker/worker.js @@ -0,0 +1,21 @@ +import workerpool from 'workerpool'; +import {loadKZG} from 'kzg-wasm'; +import {getHash} from "../utils"; + +let kzgInstance = null; + +async function initializeKzg() { + if (!kzgInstance) { + kzgInstance = await loadKZG(); + } + return kzgInstance; +} + +async function getBlobHash(blob) { + const kzg = await initializeKzg(); + return getHash(kzg, blob); +} + +workerpool.worker({ + getBlobHash: getBlobHash, +}); diff --git a/yarn.lock b/yarn.lock index 4af0a41..94093a1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -583,6 +583,11 @@ undici-types@~5.26.4: resolved "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== +workerpool@^9.1.3: + version "9.1.3" + resolved "https://registry.npmjs.org/workerpool/-/workerpool-9.1.3.tgz#34b81f50f777a0e549c6dfaa0926575735e3f4b4" + integrity sha512-LhUrk4tbxJRDQmRrrFWA9EnboXI79fe0ZNTy3u8m+dqPN1EkVSIsQYAB8OF/fkyhG8Rtup+c/bzj/+bzbG8fqg== + wrappy@1: version "1.0.2" resolved "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" From 12f000bd40bcb4caad2fdda389fee517f00cea9b Mon Sep 17 00:00:00 2001 From: iteye Date: Thu, 8 Aug 2024 12:37:37 +0800 Subject: [PATCH 6/9] fix error --- src/flatdirectory.js | 42 ++++++++++++++++++++++++++---------------- src/utils/uploader.js | 15 ++++++++++----- src/utils/util.js | 3 +-- src/worker/worker.js | 6 +++--- 4 files changed, 40 insertions(+), 26 deletions(-) diff --git a/src/flatdirectory.js b/src/flatdirectory.js index f148efc..c62c325 100644 --- a/src/flatdirectory.js +++ b/src/flatdirectory.js @@ -11,7 +11,7 @@ import { } from './param'; import { BlobUploader, encodeBlobs, - getChainId, getFileChunk, + getChainId, getFileChunk, getHash, isBuffer, isFile, stringToHex } from "./utils"; @@ -233,9 +233,11 @@ export class FlatDirectory { blobHashRequestArr.push(fileContract.getChunkHash(hexName, i + j)); } + let blobHashArr; // check change if (chunkIdArr[0] < oldChunkLength) { - const isChange = await this.#checkChange(blobArr, blobHashRequestArr); + blobHashArr = await this.#getBlobHashes(blobArr); + const isChange = await this.#checkChange(blobHashArr, blobHashRequestArr); if (!isChange) { continue; } @@ -247,7 +249,7 @@ export class FlatDirectory { totalStorageCost += value; // gas cost if (gasLimit === 0) { - const blobHashArr = await this.#getBlobHashes(blobArr); + blobHashArr = blobHashArr ? blobHashArr : await this.#getBlobHashes(blobArr); gasLimit = await fileContract.writeChunks.estimateGas(hexName, chunkIdArr, chunkSizeArr, { value: value, blobVersionedHashes: blobHashArr @@ -427,11 +429,13 @@ export class FlatDirectory { } blobHashRequestArr.push(fileContract.getChunkHash(hexName, i + j)); } + const blobCommitmentArr = await this.#getBlobCommitments(blobArr); // check change if (clearState === REMOVE_NORMAL) { try { - const isChange = await this.#checkChange(blobArr, blobHashRequestArr); + const blobHashArr = this.#getHashes(blobCommitmentArr); + const isChange = await this.#checkChange(blobHashArr, blobHashRequestArr); if (!isChange) { callback.onProgress(chunkIdArr[chunkIdArr.length - 1], blobLength, false); continue; @@ -444,7 +448,7 @@ export class FlatDirectory { // upload try { - const status = await this.#uploadBlob(fileContract, key, hexName, blobArr, chunkIdArr, chunkSizeArr, cost, gasIncPct); + const status = await this.#uploadBlob(fileContract, key, hexName, blobArr, blobCommitmentArr, chunkIdArr, chunkSizeArr, cost, gasIncPct); if (!status) { callback.onFail(new Error("FlatDirectory: Sending transaction failed.")); break; @@ -586,20 +590,17 @@ export class FlatDirectory { } } - async #checkChange(blobArr, blobHashRequestArr) { - let hasChange = false; - const blobHashArr = await this.#getBlobHashes(blobArr); + async #checkChange(blobHashArr, blobHashRequestArr) { const dataHashArr = await Promise.all(blobHashRequestArr); for (let i = 0; i < blobHashArr.length; i++) { if (blobHashArr[i] !== dataHashArr[i]) { - hasChange = true; - break; + return true; } } - return hasChange; + return false; } - async #uploadBlob(fileContract, key, hexName, blobArr, chunkIdArr, chunkSizeArr, cost, gasIncPct) { + async #uploadBlob(fileContract, key, hexName, blobArr, blobCommitmentArr, chunkIdArr, chunkSizeArr, cost, gasIncPct) { // create tx const value = cost * BigInt(blobArr.length); const tx = await fileContract.writeChunks.populateTransaction(hexName, chunkIdArr, chunkSizeArr, { @@ -616,7 +617,7 @@ export class FlatDirectory { tx.maxFeePerBlobGas = blobGas * BigInt(100 + gasIncPct) / BigInt(100); } // send - const txResponse = await this.#blobUploader.sendTxLock(tx, blobArr); + const txResponse = await this.#blobUploader.sendTxLock(tx, blobArr, blobCommitmentArr); console.log(`FlatDirectory: The ${chunkIdArr} chunks hash is ${txResponse.hash}`, "", key); const txReceipt = await txResponse.wait(); return txReceipt && txReceipt.status; @@ -649,11 +650,20 @@ export class FlatDirectory { return Number(count); } - async #getBlobHashes(blobArr) { + async #getBlobCommitments(blobArr) { const isNode = typeof process !== 'undefined' && !!process.versions && !!process.versions.node; const promises = isNode - ? blobArr.map(blob => pool.exec('getBlobHash', [blob])) - : blobArr.map(blob => this.#blobUploader.getBlobHash(blob)); + ? blobArr.map(blob => pool.exec('getCommitment', [blob])) + : blobArr.map(blob => this.#blobUploader.getCommitment(blob)); return await Promise.all(promises); } + + #getHashes(blobCommitmentArr) { + return blobCommitmentArr.map(comment => getHash(comment)); + } + + async #getBlobHashes(blobArr) { + const commitments = await this.#getBlobCommitments(blobArr); + return this.#getHashes(commitments); + } } diff --git a/src/utils/uploader.js b/src/utils/uploader.js index 8571107..e0caac8 100644 --- a/src/utils/uploader.js +++ b/src/utils/uploader.js @@ -69,7 +69,7 @@ export class BlobUploader { return null; } - async sendTx(tx, blobs) { + async sendTx(tx, blobs = null, commitments = null) { if (!blobs) { return await this.#wallet.sendTransaction(tx); } @@ -84,7 +84,7 @@ export class BlobUploader { const versionedHashes = []; for (let i = 0; i < blobs.length; i++) { const blob = blobs[i]; - const commitment = kzg.blobToKzgCommitment(blob); + const commitment = (commitments && commitments.length > i) ? commitments[i] : kzg.blobToKzgCommitment(blob); const proof = kzg.computeBlobKzgProof(blob, commitment); ethersBlobs.push({ data: blob, @@ -104,16 +104,21 @@ export class BlobUploader { return await this.#wallet.sendTransaction(tx); } - async sendTxLock(tx, blobs) { + async sendTxLock(tx, blobs = null, commitments = null) { const release = await this.#mutex.acquire(); try { - return await this.sendTx(tx, blobs); + return await this.sendTx(tx, blobs, commitments); } finally { release(); } } + getCommitment(blob) { + return this.#kzg.blobToKzgCommitment(blob); + } + getBlobHash(blob) { - return getHash(this.#kzg, blob); + const commit = this.getCommitment(blob); + return getHash(commit); } } diff --git a/src/utils/util.js b/src/utils/util.js index 9dd8f06..d7337f2 100644 --- a/src/utils/util.js +++ b/src/utils/util.js @@ -38,8 +38,7 @@ export function commitmentsToVersionedHashes(commitment) { return computeVersionedHash(commitment, 0x01); } -export function getHash(kzg, blob) { - const commit = kzg.blobToKzgCommitment(blob); +export function getHash(commit) { const localHash = commitmentsToVersionedHashes(commit); const hash = new Uint8Array(32); hash.set(localHash.subarray(0, 32 - 8)); diff --git a/src/worker/worker.js b/src/worker/worker.js index 064ea7e..cb1aa35 100644 --- a/src/worker/worker.js +++ b/src/worker/worker.js @@ -11,11 +11,11 @@ async function initializeKzg() { return kzgInstance; } -async function getBlobHash(blob) { +async function getCommitment(blob) { const kzg = await initializeKzg(); - return getHash(kzg, blob); + return kzg.blobToKzgCommitment(blob); } workerpool.worker({ - getBlobHash: getBlobHash, + getCommitment: getCommitment, }); From f0c2403cb603adfa7f03e386e344c9aac804f83e Mon Sep 17 00:00:00 2001 From: iteye Date: Thu, 8 Aug 2024 12:38:52 +0800 Subject: [PATCH 7/9] remove code --- rollup.config.mjs | 2 +- src/worker/worker.js | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/rollup.config.mjs b/rollup.config.mjs index ebfd94a..e14dd3b 100644 --- a/rollup.config.mjs +++ b/rollup.config.mjs @@ -37,7 +37,7 @@ export default [ sourcemap: true, }, plugins: [commonjs(), resolve()], - external: ["ethers", "kzg-wasm", "workerpool"] + external: ["kzg-wasm", "workerpool"] }, ]; diff --git a/src/worker/worker.js b/src/worker/worker.js index cb1aa35..c3f3309 100644 --- a/src/worker/worker.js +++ b/src/worker/worker.js @@ -1,6 +1,5 @@ import workerpool from 'workerpool'; import {loadKZG} from 'kzg-wasm'; -import {getHash} from "../utils"; let kzgInstance = null; From 0801fbab8e91cc5249e9006a6ebecab16f22be77 Mon Sep 17 00:00:00 2001 From: iteye Date: Wed, 14 Aug 2024 10:10:35 +0800 Subject: [PATCH 8/9] Code optimization --- src/flatdirectory.js | 252 ++++++++++++++++++++----------------------- 1 file changed, 117 insertions(+), 135 deletions(-) diff --git a/src/flatdirectory.js b/src/flatdirectory.js index c62c325..a39531c 100644 --- a/src/flatdirectory.js +++ b/src/flatdirectory.js @@ -187,17 +187,14 @@ export class FlatDirectory { // private method async #estimateCostByBlob(request) { + const {key, content, gasIncPct = 0} = request; + if (!this.#isSupportBlob) { throw new Error(`FlatDirectory: The contract does not support blob upload!`); } - const {key, content, gasIncPct} = request; - let blobLength = 0; - if (isFile(content)) { - blobLength = Math.ceil(content.size / DEFAULT_BLOB_DATA_SIZE); - } else if (isBuffer(content)) { - blobLength = Math.ceil(content.length / DEFAULT_BLOB_DATA_SIZE); - } else { + const blobLength = this.#getBlobLength(content); + if (blobLength === -1) { throw new Error(`FlatDirectory: Invalid upload content!`); } @@ -220,18 +217,12 @@ export class FlatDirectory { // send for (let i = 0; i < blobLength; i += MAX_BLOB_COUNT) { - const data = isBuffer(content) ? Buffer.from(content).subarray(i * DEFAULT_BLOB_DATA_SIZE, (i + MAX_BLOB_COUNT) * DEFAULT_BLOB_DATA_SIZE) : - await getFileChunk(content, content.size, i * DEFAULT_BLOB_DATA_SIZE, (i + MAX_BLOB_COUNT) * DEFAULT_BLOB_DATA_SIZE); - const blobArr = encodeBlobs(data); - const chunkIdArr = []; - const chunkSizeArr = []; - const blobHashRequestArr = []; - for (let j = 0; j < blobArr.length; j++) { - chunkIdArr.push(i + j); - chunkSizeArr.push(DEFAULT_BLOB_DATA_SIZE); - - blobHashRequestArr.push(fileContract.getChunkHash(hexName, i + j)); - } + const { + blobArr, + chunkIdArr, + chunkSizeArr, + blobHashRequestArr + } = await this.#getBlobInfo(fileContract, content, hexName, blobLength, i); let blobHashArr; // check change @@ -255,17 +246,9 @@ export class FlatDirectory { blobVersionedHashes: blobHashArr }); } - if (gasIncPct) { - const gasPrice = (gasFeeData.maxFeePerGas + gasFeeData.maxPriorityFeePerGas) * BigInt(100 + gasIncPct) / BigInt(100); - const gasCost = gasPrice * gasLimit; - const blobGasPrice = maxFeePerBlobGas * BigInt(100 + gasIncPct) / BigInt(100); - const blobGasCost = blobGasPrice * BigInt(BLOB_SIZE); - totalGasCost += gasCost + blobGasCost; - } else { - const gasCost = (gasFeeData.maxFeePerGas + gasFeeData.maxPriorityFeePerGas) * gasLimit; - const blobGasCost = maxFeePerBlobGas * BigInt(BLOB_SIZE); - totalGasCost += gasCost + blobGasCost; - } + const gasCost = (gasFeeData.maxFeePerGas + gasFeeData.maxPriorityFeePerGas) * BigInt(100 + gasIncPct) / BigInt(100) * gasLimit; + const blobGasCost = maxFeePerBlobGas * BigInt(100 + gasIncPct) / BigInt(100) * BigInt(BLOB_SIZE); + totalGasCost += gasCost + blobGasCost; } return { @@ -275,41 +258,10 @@ export class FlatDirectory { } async #estimateCostByCallData(request) { - const {key, content, gasIncPct} = request; + const {key, content, gasIncPct = 0} = request; - let chunkDataSize = 0; - let chunkLength = 1; - if (isFile(content)) { - chunkDataSize = content.size; - if (GALILEO_CHAIN_ID === this.#chainId) { - if (content.size > 475 * 1024) { - // Data need to be sliced if file > 475K - chunkDataSize = 475 * 1024; - chunkLength = Math.ceil(content.size / (475 * 1024)); - } - } else { - if (content.size > 24 * 1024 - 326) { - // Data need to be sliced if file > 24K - chunkDataSize = 24 * 1024 - 326; - chunkLength = Math.ceil(content.size / (24 * 1024 - 326)); - } - } - } else if (isBuffer(content)) { - chunkDataSize = content.length; - if (GALILEO_CHAIN_ID === this.#chainId) { - if (content.length > 475 * 1024) { - // Data need to be sliced if file > 475K - chunkDataSize = 475 * 1024; - chunkLength = Math.ceil(content.length / (475 * 1024)); - } - } else { - if (content.length > 24 * 1024 - 326) { - // Data need to be sliced if file > 24K - chunkDataSize = 24 * 1024 - 326; - chunkLength = Math.ceil(content.length / (24 * 1024 - 326)); - } - } - } else { + const {chunkDataSize, chunkLength} = this.#getChunkLength(content); + if (chunkDataSize === -1) { throw new Error(`FlatDirectory: Invalid upload content!`); } @@ -354,12 +306,8 @@ export class FlatDirectory { }); } totalStorageCost += cost; - if (gasIncPct) { - totalGasCost += (gasFeeData.maxFeePerGas + gasFeeData.maxPriorityFeePerGas) - * BigInt(100 + gasIncPct) / BigInt(100) * gasLimit; - } else { - totalGasCost += (gasFeeData.maxFeePerGas + gasFeeData.maxPriorityFeePerGas) * gasLimit; - } + totalGasCost += (gasFeeData.maxFeePerGas + gasFeeData.maxPriorityFeePerGas) + * BigInt(100 + gasIncPct) / BigInt(100) * gasLimit; } return { @@ -380,6 +328,13 @@ export class FlatDirectory { return; } + const blobLength = this.#getBlobLength(content); + if (blobLength === -1) { + callback.onFail(new Error(`FlatDirectory: Invalid upload content!`)); + callback.onFinish(totalUploadChunks, totalUploadSize, totalStorageCost); + return; + } + const hexName = stringToHex(key); const fileContract = new ethers.Contract(this.#contractAddr, FlatDirectoryAbi, this.#wallet); const fileMod = await fileContract.getStorageMode(hexName); @@ -389,16 +344,7 @@ export class FlatDirectory { return; } - let blobLength; - if (isFile(content)) { - blobLength = Math.ceil(content.size / DEFAULT_BLOB_DATA_SIZE); - } else if (isBuffer(content)) { - blobLength = Math.ceil(content.length / DEFAULT_BLOB_DATA_SIZE); - } else { - callback.onFail(new Error(`FlatDirectory: Invalid upload content!`)); - callback.onFinish(totalUploadChunks, totalUploadSize, totalStorageCost); - return; - } + // check old data const [cost, oldBlobLength] = await Promise.all([ fileContract.upfrontPayment(), @@ -413,22 +359,12 @@ export class FlatDirectory { // send for (let i = 0; i < blobLength; i += MAX_BLOB_COUNT) { - const data = isBuffer(content) ? Buffer.from(content).subarray(i * DEFAULT_BLOB_DATA_SIZE, (i + MAX_BLOB_COUNT) * DEFAULT_BLOB_DATA_SIZE) : - await getFileChunk(content, content.size, i * DEFAULT_BLOB_DATA_SIZE, (i + MAX_BLOB_COUNT) * DEFAULT_BLOB_DATA_SIZE); - const blobArr = encodeBlobs(data); - const chunkIdArr = []; - const chunkSizeArr = []; - const blobHashRequestArr = []; - for (let j = 0; j < blobArr.length; j++) { - chunkIdArr.push(i + j); - if (i + j === blobLength - 1) { - const size = isBuffer(content) ? content.length : content.size; - chunkSizeArr.push(size - DEFAULT_BLOB_DATA_SIZE * (blobLength - 1)); - } else { - chunkSizeArr.push(DEFAULT_BLOB_DATA_SIZE); - } - blobHashRequestArr.push(fileContract.getChunkHash(hexName, i + j)); - } + const { + blobArr, + chunkIdArr, + chunkSizeArr, + blobHashRequestArr + } = await this.#getBlobInfo(fileContract, content, hexName, blobLength, i); const blobCommitmentArr = await this.#getBlobCommitments(blobArr); // check change @@ -476,6 +412,13 @@ export class FlatDirectory { let totalStorageCost = 0n; const {key, content, callback, gasIncPct} = request; + const {chunkDataSize, chunkLength} = this.#getChunkLength(content); + if (chunkDataSize === -1) { + callback.onFail(new Error(`FlatDirectory: Invalid upload content!`)); + callback.onFinish(totalUploadChunks, totalUploadSize, totalStorageCost); + return; + } + const hexName = stringToHex(key); const fileContract = new ethers.Contract(this.#contractAddr, FlatDirectoryAbi, this.#wallet); const fileMod = await fileContract.getStorageMode(hexName); @@ -485,44 +428,6 @@ export class FlatDirectory { return; } - let chunkDataSize; - let chunkLength = 1; - if (isFile(content)) { - chunkDataSize = content.size; - if (GALILEO_CHAIN_ID === this.#chainId) { - if (content.size > 475 * 1024) { - // Data need to be sliced if file > 475K - chunkDataSize = 475 * 1024; - chunkLength = Math.ceil(content.size / (475 * 1024)); - } - } else { - if (content.size > 24 * 1024 - 326) { - // Data need to be sliced if file > 24K - chunkDataSize = 24 * 1024 - 326; - chunkLength = Math.ceil(content.size / (24 * 1024 - 326)); - } - } - } else if (isBuffer(content)) { - chunkDataSize = content.length; - if (GALILEO_CHAIN_ID === this.#chainId) { - if (content.length > 475 * 1024) { - // Data need to be sliced if file > 475K - chunkDataSize = 475 * 1024; - chunkLength = Math.ceil(content.length / (475 * 1024)); - } - } else { - if (content.length > 24 * 1024 - 326) { - // Data need to be sliced if file > 24K - chunkDataSize = 24 * 1024 - 326; - chunkLength = Math.ceil(content.length / (24 * 1024 - 326)); - } - } - } else { - callback.onFail(new Error(`FlatDirectory: Invalid upload content!`)); - callback.onFinish(totalUploadChunks, totalUploadSize, totalStorageCost); - return; - } - // check old data const oldChunkLength = await this.#countChunks(fileContract, hexName); const clearState = await this.#clearOldFile(hexName, chunkLength, oldChunkLength); @@ -535,6 +440,7 @@ export class FlatDirectory { for (let i = 0; i < chunkLength; i++) { const chunk = isBuffer(content) ? Buffer.from(content).subarray(i * chunkDataSize, (i + 1) * chunkDataSize) : await getFileChunk(content, content.size, i * chunkDataSize, (i + 1) * chunkDataSize); + // check is change if (clearState === REMOVE_NORMAL) { const localHash = ethers.keccak256(chunk); @@ -550,7 +456,6 @@ export class FlatDirectory { } } - // upload // upload try { const status = await this.#uploadCallData(fileContract, key, hexName, i, chunk, gasIncPct); @@ -666,4 +571,81 @@ export class FlatDirectory { const commitments = await this.#getBlobCommitments(blobArr); return this.#getHashes(commitments); } + + + #getBlobLength(content) { + let blobLength = -1; + if (isFile(content)) { + blobLength = Math.ceil(content.size / DEFAULT_BLOB_DATA_SIZE); + } else if (isBuffer(content)) { + blobLength = Math.ceil(content.length / DEFAULT_BLOB_DATA_SIZE); + } + return blobLength; + } + + async #getBlobInfo(fileContract, content, hexName, blobLength, index) { + const data = isBuffer(content) + ? Buffer.from(content).subarray(index * DEFAULT_BLOB_DATA_SIZE, (index + MAX_BLOB_COUNT) * DEFAULT_BLOB_DATA_SIZE) : + await getFileChunk(content, content.size, index * DEFAULT_BLOB_DATA_SIZE, (index + MAX_BLOB_COUNT) * DEFAULT_BLOB_DATA_SIZE); + const blobArr = encodeBlobs(data); + const chunkIdArr = []; + const chunkSizeArr = []; + const blobHashRequestArr = []; + for (let j = 0; j < blobArr.length; j++) { + chunkIdArr.push(index + j); + if (index + j === blobLength - 1) { + const size = isBuffer(content) ? content.length : content.size; + chunkSizeArr.push(size - DEFAULT_BLOB_DATA_SIZE * (blobLength - 1)); + } else { + chunkSizeArr.push(DEFAULT_BLOB_DATA_SIZE); + } + blobHashRequestArr.push(fileContract.getChunkHash(hexName, index + j)); + } + return { + blobArr, + chunkIdArr, + chunkSizeArr, + blobHashRequestArr + } + } + + #getChunkLength(content) { + let chunkDataSize = -1; + let chunkLength = 1; + if (isFile(content)) { + chunkDataSize = content.size; + if (GALILEO_CHAIN_ID === this.#chainId) { + if (content.size > 475 * 1024) { + // Data need to be sliced if file > 475K + chunkDataSize = 475 * 1024; + chunkLength = Math.ceil(content.size / (475 * 1024)); + } + } else { + if (content.size > 24 * 1024 - 326) { + // Data need to be sliced if file > 24K + chunkDataSize = 24 * 1024 - 326; + chunkLength = Math.ceil(content.size / (24 * 1024 - 326)); + } + } + } else if (isBuffer(content)) { + chunkDataSize = content.length; + if (GALILEO_CHAIN_ID === this.#chainId) { + if (content.length > 475 * 1024) { + // Data need to be sliced if file > 475K + chunkDataSize = 475 * 1024; + chunkLength = Math.ceil(content.length / (475 * 1024)); + } + } else { + if (content.length > 24 * 1024 - 326) { + // Data need to be sliced if file > 24K + chunkDataSize = 24 * 1024 - 326; + chunkLength = Math.ceil(content.length / (24 * 1024 - 326)); + } + } + } + return { + chunkDataSize, + chunkLength + } + } } From 306a1e4d862dd8910493f7e0171759ad7bfd3020 Mon Sep 17 00:00:00 2001 From: iteye Date: Wed, 14 Aug 2024 10:40:57 +0800 Subject: [PATCH 9/9] Code optimization --- src/flatdirectory.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/flatdirectory.js b/src/flatdirectory.js index a39531c..698859f 100644 --- a/src/flatdirectory.js +++ b/src/flatdirectory.js @@ -13,7 +13,7 @@ import { BlobUploader, encodeBlobs, getChainId, getFileChunk, getHash, isBuffer, isFile, - stringToHex + stringToHex, isNodejs } from "./utils"; import workerpool from 'workerpool'; @@ -556,8 +556,7 @@ export class FlatDirectory { } async #getBlobCommitments(blobArr) { - const isNode = typeof process !== 'undefined' && !!process.versions && !!process.versions.node; - const promises = isNode + const promises = isNodejs() ? blobArr.map(blob => pool.exec('getCommitment', [blob])) : blobArr.map(blob => this.#blobUploader.getCommitment(blob)); return await Promise.all(promises);