From e4194db9ba550dfa03d8d894e4c7a438dfc21246 Mon Sep 17 00:00:00 2001 From: Yixuan Xu Date: Fri, 31 Mar 2023 10:47:35 +0800 Subject: [PATCH 1/5] chore(bindings/nodejs): add deno benchmark --- bindings/nodejs/.env.example | 4 + bindings/nodejs/benchmark/deno.ts | 77 +++++++++++++++++++ .../nodejs/benchmark/{index.js => node.js} | 0 bindings/nodejs/package.json | 4 +- bindings/nodejs/yarn.lock | 8 ++ 5 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 bindings/nodejs/.env.example create mode 100644 bindings/nodejs/benchmark/deno.ts rename bindings/nodejs/benchmark/{index.js => node.js} (100%) diff --git a/bindings/nodejs/.env.example b/bindings/nodejs/.env.example new file mode 100644 index 000000000000..6574af0cba3a --- /dev/null +++ b/bindings/nodejs/.env.example @@ -0,0 +1,4 @@ +export AWS_ACCESS_KEY_ID= +export AWS_SECRET_ACCESS_KEY= +export AWS_S3_REGION= +export AWS_S3_ENDPOINT= \ No newline at end of file diff --git a/bindings/nodejs/benchmark/deno.ts b/bindings/nodejs/benchmark/deno.ts new file mode 100644 index 000000000000..cc7f707599b4 --- /dev/null +++ b/bindings/nodejs/benchmark/deno.ts @@ -0,0 +1,77 @@ +import { Operator } from 'npm:opendal' +import "https://deno.land/std@0.181.0/dotenv/load.ts"; +import { S3Client } from "https://deno.land/x/s3_lite_client@0.5.0/mod.ts"; + +const endpoint = Deno.env.get("AWS_S3_ENDPOINT") || "http://localhost:9000"; +const url = new URL(endpoint); +const port = url.port ? parseInt(url.port) : 9000; +const hostname = url.hostname; +const bucket = Deno.env.get('AWS_BUCKET') || "benchmark" + +const s3 = new S3Client({ + endPoint: hostname, + useSSL: false, + port: port, + region: Deno.env.get('AWS_REGION') || "us-east-1", + bucket, + accessKey: Deno.env.get('AWS_ACCESS_KEY_ID'), + secretKey: Deno.env.get('AWS_SECRET_ACCESS_KEY'), +}); + +const opendal = new Operator('s3', { + root: '/', + bucket, + endpoint, +}) + +const files = [{ + name: '4kb', + file: new Uint8Array(4 * 1024), +}, +{ + name: '256kb', + file: new Uint8Array(256 * 1024), +}, +{ + name: '4mb', + file: new Uint8Array(4 * 1024 * 1024), +}, +{ + name: '16mb', + file: new Uint8Array(16 * 1024 * 1024), +} +] + +const textDecoder = new TextDecoder() +const filenams = await Promise.all(files.map(async data => { + const filename = `${crypto.randomUUID()}-${data.name}-read-bench` + await s3.putObject(filename, textDecoder.decode(data.file)) + return filename +})) +files.map((data, i) => { + Deno.bench(`opendal: read ${data.name}`, { + group: `read ${data.name}` + }, async () => { + await opendal.read(filenams[i]) + }) + Deno.bench(`s3: read ${data.name}`, { + group: `read ${data.name}` + }, async () => { + await s3.getObject(filenams[i]) + }) +}) + +files.map(data => { + Deno.bench(`s3: write ${data.name}`, { + group: `write ${data.name}` + }, async () => { + const filename = `${crypto.randomUUID()}-${data.name}}-s3` + await s3.putObject(filename, textDecoder.decode(data.file)) + }) + Deno.bench(`opendal: write ${data.name}`, { + group: `write ${data.name}` + }, async () => { + const filename = `${crypto.randomUUID()}-${data.name}}-opendal` + await opendal.write(filename, textDecoder.decode(data.file)) + }) +}) \ No newline at end of file diff --git a/bindings/nodejs/benchmark/index.js b/bindings/nodejs/benchmark/node.js similarity index 100% rename from bindings/nodejs/benchmark/index.js rename to bindings/nodejs/benchmark/node.js diff --git a/bindings/nodejs/package.json b/bindings/nodejs/package.json index ab1cdb7a5160..8afe648f1980 100644 --- a/bindings/nodejs/package.json +++ b/bindings/nodejs/package.json @@ -51,6 +51,7 @@ "@swc/core": "^1.3.38", "@types/node": "^18.14.5", "benny": "^3.7.1", + "dotenv": "^16.0.3", "prettier": "^2.8.4", "typedoc": "^0.23.28", "typescript": "^5.0.2" @@ -66,7 +67,8 @@ "format": "prettier --write .", "prepublishOnly": "napi prepublish -t npm", "test": "cucumber-js", - "bench": "node ./benchmark/index.js", + "bench": "node -r dotenv/config ./benchmark/node.js dotenv_config_path=./.env", + "bench:deno": "deno bench ./benchmark/deno.ts --reload=npm:opendal --allow-read --allow-ffi --allow-net --allow-env", "version": "napi version" }, "prettier": { diff --git a/bindings/nodejs/yarn.lock b/bindings/nodejs/yarn.lock index 2e18e3f489fc..67e976c992cc 100644 --- a/bindings/nodejs/yarn.lock +++ b/bindings/nodejs/yarn.lock @@ -1814,6 +1814,13 @@ __metadata: languageName: node linkType: hard +"dotenv@npm:^16.0.3": + version: 16.0.3 + resolution: "dotenv@npm:16.0.3" + checksum: afcf03f373d7a6d62c7e9afea6328e62851d627a4e73f2e12d0a8deae1cd375892004f3021883f8aec85932cd2834b091f568ced92b4774625b321db83b827f8 + languageName: node + linkType: hard + "emoji-regex@npm:^8.0.0": version: 8.0.0 resolution: "emoji-regex@npm:8.0.0" @@ -2238,6 +2245,7 @@ __metadata: "@swc/core": ^1.3.38 "@types/node": ^18.14.5 benny: ^3.7.1 + dotenv: ^16.0.3 prettier: ^2.8.4 typedoc: ^0.23.28 typescript: ^5.0.2 From c88fd6769992a2c706044ff5ae7fa66e7d60320d Mon Sep 17 00:00:00 2001 From: Yixuan Xu Date: Fri, 31 Mar 2023 11:25:55 +0800 Subject: [PATCH 2/5] add env --- bindings/nodejs/.env.example | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bindings/nodejs/.env.example b/bindings/nodejs/.env.example index 6574af0cba3a..f26ca40a8d2a 100644 --- a/bindings/nodejs/.env.example +++ b/bindings/nodejs/.env.example @@ -1,4 +1,5 @@ export AWS_ACCESS_KEY_ID= export AWS_SECRET_ACCESS_KEY= export AWS_S3_REGION= -export AWS_S3_ENDPOINT= \ No newline at end of file +export AWS_S3_ENDPOINT= +export AWS_BUCKET \ No newline at end of file From a62afaa9fe17fdc7a364acb240620a8162d040c1 Mon Sep 17 00:00:00 2001 From: Yixuan Xu Date: Fri, 31 Mar 2023 11:26:49 +0800 Subject: [PATCH 3/5] update .env.example --- bindings/nodejs/.env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/nodejs/.env.example b/bindings/nodejs/.env.example index f26ca40a8d2a..88f75b60ae0f 100644 --- a/bindings/nodejs/.env.example +++ b/bindings/nodejs/.env.example @@ -2,4 +2,4 @@ export AWS_ACCESS_KEY_ID= export AWS_SECRET_ACCESS_KEY= export AWS_S3_REGION= export AWS_S3_ENDPOINT= -export AWS_BUCKET \ No newline at end of file +export AWS_BUCKET= \ No newline at end of file From 9a447f63cec57aca83ac938f5d5b59569908dcf2 Mon Sep 17 00:00:00 2001 From: Yixuan Xu Date: Fri, 31 Mar 2023 11:29:36 +0800 Subject: [PATCH 4/5] add license headers --- bindings/nodejs/benchmark/deno.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/bindings/nodejs/benchmark/deno.ts b/bindings/nodejs/benchmark/deno.ts index cc7f707599b4..f87a48ec9139 100644 --- a/bindings/nodejs/benchmark/deno.ts +++ b/bindings/nodejs/benchmark/deno.ts @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + import { Operator } from 'npm:opendal' import "https://deno.land/std@0.181.0/dotenv/load.ts"; import { S3Client } from "https://deno.land/x/s3_lite_client@0.5.0/mod.ts"; From bb651cfba4ee297ff6eac61ed670820315e4f589 Mon Sep 17 00:00:00 2001 From: Yixuan Xu Date: Fri, 31 Mar 2023 11:36:40 +0800 Subject: [PATCH 5/5] run format --- bindings/nodejs/benchmark/deno.ts | 123 +++++++++++++++++------------- 1 file changed, 71 insertions(+), 52 deletions(-) diff --git a/bindings/nodejs/benchmark/deno.ts b/bindings/nodejs/benchmark/deno.ts index f87a48ec9139..f14f167a9bfe 100644 --- a/bindings/nodejs/benchmark/deno.ts +++ b/bindings/nodejs/benchmark/deno.ts @@ -18,79 +18,98 @@ */ import { Operator } from 'npm:opendal' -import "https://deno.land/std@0.181.0/dotenv/load.ts"; -import { S3Client } from "https://deno.land/x/s3_lite_client@0.5.0/mod.ts"; +import 'https://deno.land/std@0.181.0/dotenv/load.ts' +import { S3Client } from 'https://deno.land/x/s3_lite_client@0.5.0/mod.ts' -const endpoint = Deno.env.get("AWS_S3_ENDPOINT") || "http://localhost:9000"; -const url = new URL(endpoint); -const port = url.port ? parseInt(url.port) : 9000; -const hostname = url.hostname; -const bucket = Deno.env.get('AWS_BUCKET') || "benchmark" +const endpoint = Deno.env.get('AWS_S3_ENDPOINT') || 'http://localhost:9000' +const url = new URL(endpoint) +const port = url.port ? parseInt(url.port) : 9000 +const hostname = url.hostname +const bucket = Deno.env.get('AWS_BUCKET') || 'benchmark' const s3 = new S3Client({ - endPoint: hostname, - useSSL: false, - port: port, - region: Deno.env.get('AWS_REGION') || "us-east-1", - bucket, - accessKey: Deno.env.get('AWS_ACCESS_KEY_ID'), - secretKey: Deno.env.get('AWS_SECRET_ACCESS_KEY'), -}); + endPoint: hostname, + useSSL: false, + port: port, + region: Deno.env.get('AWS_REGION') || 'us-east-1', + bucket, + accessKey: Deno.env.get('AWS_ACCESS_KEY_ID'), + secretKey: Deno.env.get('AWS_SECRET_ACCESS_KEY'), +}) const opendal = new Operator('s3', { - root: '/', - bucket, - endpoint, + root: '/', + bucket, + endpoint, }) -const files = [{ +const files = [ + { name: '4kb', file: new Uint8Array(4 * 1024), -}, -{ + }, + { name: '256kb', file: new Uint8Array(256 * 1024), -}, -{ + }, + { name: '4mb', file: new Uint8Array(4 * 1024 * 1024), -}, -{ + }, + { name: '16mb', file: new Uint8Array(16 * 1024 * 1024), -} + }, ] const textDecoder = new TextDecoder() -const filenams = await Promise.all(files.map(async data => { +const filenams = await Promise.all( + files.map(async (data) => { const filename = `${crypto.randomUUID()}-${data.name}-read-bench` await s3.putObject(filename, textDecoder.decode(data.file)) return filename -})) + }), +) files.map((data, i) => { - Deno.bench(`opendal: read ${data.name}`, { - group: `read ${data.name}` - }, async () => { - await opendal.read(filenams[i]) - }) - Deno.bench(`s3: read ${data.name}`, { - group: `read ${data.name}` - }, async () => { - await s3.getObject(filenams[i]) - }) + Deno.bench( + `opendal: read ${data.name}`, + { + group: `read ${data.name}`, + }, + async () => { + await opendal.read(filenams[i]) + }, + ) + Deno.bench( + `s3: read ${data.name}`, + { + group: `read ${data.name}`, + }, + async () => { + await s3.getObject(filenams[i]) + }, + ) }) -files.map(data => { - Deno.bench(`s3: write ${data.name}`, { - group: `write ${data.name}` - }, async () => { - const filename = `${crypto.randomUUID()}-${data.name}}-s3` - await s3.putObject(filename, textDecoder.decode(data.file)) - }) - Deno.bench(`opendal: write ${data.name}`, { - group: `write ${data.name}` - }, async () => { - const filename = `${crypto.randomUUID()}-${data.name}}-opendal` - await opendal.write(filename, textDecoder.decode(data.file)) - }) -}) \ No newline at end of file +files.map((data) => { + Deno.bench( + `s3: write ${data.name}`, + { + group: `write ${data.name}`, + }, + async () => { + const filename = `${crypto.randomUUID()}-${data.name}}-s3` + await s3.putObject(filename, textDecoder.decode(data.file)) + }, + ) + Deno.bench( + `opendal: write ${data.name}`, + { + group: `write ${data.name}`, + }, + async () => { + const filename = `${crypto.randomUUID()}-${data.name}}-opendal` + await opendal.write(filename, textDecoder.decode(data.file)) + }, + ) +})