diff --git a/.gitignore b/.gitignore index ea5dc0f0..baf253ce 100644 --- a/.gitignore +++ b/.gitignore @@ -2,11 +2,6 @@ node_modules /target **/*.rs.bk pkg/ -pkg_node/ -pkg_web/ -pkg2/ -pkg2_node/ -pkg2_web/ wasm-pack.log .idea/ www/data diff --git a/README.md b/README.md index 061867b8..9e6643dd 100644 --- a/README.md +++ b/README.md @@ -34,26 +34,26 @@ Presumably no one wants to use both `parquet` and `parquet2` at once, so the def | Entry point | Rust crates used | Description | | ----------------------- | ----------------------- | ------------------------------------------------------- | | `parquet-wasm` | `parquet` and `arrow` | "Bundler" build, to be used in bundlers such as Webpack | -| `parquet-wasm/node` | `parquet` and `arrow` | Node build, to be used with `require` in NodeJS | -| `parquet-wasm/web` | `parquet` and `arrow` | ESM, to be used directly from the Web as an ES Module | +| `parquet-wasm/node/arrow1` | `parquet` and `arrow` | Node build, to be used with `require` in NodeJS | +| `parquet-wasm/esm/arrow1` | `parquet` and `arrow` | ESM, to be used directly from the Web as an ES Module | | | | | -| `parquet-wasm/bundler2` | `parquet2` and `arrow2` | "Bundler" build, to be used in bundlers such as Webpack | -| `parquet-wasm/node2` | `parquet2` and `arrow2` | Node build, to be used with `require` in NodeJS | -| `parquet-wasm/web2` | `parquet2` and `arrow2` | ESM, to be used directly from the Web as an ES Module | +| `parquet-wasm/bundler/arrow2` | `parquet2` and `arrow2` | "Bundler" build, to be used in bundlers such as Webpack | +| `parquet-wasm/node/arrow2` | `parquet2` and `arrow2` | Node build, to be used with `require` in NodeJS | +| `parquet-wasm/esm/arrow2` | `parquet2` and `arrow2` | ESM, to be used directly from the Web as an ES Module | -Note that when using the `/web` and `/web2` bundles, the default export must be awaited. See [here](https://rustwasm.github.io/docs/wasm-bindgen/examples/without-a-bundler.html) for an example. +Note that when using the `esm` bundles, the default export must be awaited. See [here](https://rustwasm.github.io/docs/wasm-bindgen/examples/without-a-bundler.html) for an example. ### `parquet` API This implementation uses the [`arrow`](https://crates.io/crates/arrow) and [`parquet`](https://crates.io/crates/parquet) Rust crates. -Refer to the [API documentation](https://kylebarron.dev/parquet-wasm/modules/bundler.html) for more details and examples. +Refer to the [API documentation](https://kylebarron.dev/parquet-wasm/modules/bundler_arrow1.html) for more details and examples. ### `parquet2` API This implementation uses the [`arrow2`](https://crates.io/crates/arrow2) and [`parquet2`](https://crates.io/crates/parquet2) Rust crates. -Refer to the [API documentation](https://kylebarron.dev/parquet-wasm/modules/bundler2.html) for more details and examples. +Refer to the [API documentation](https://kylebarron.dev/parquet-wasm/modules/bundler_arrow2.html) for more details and examples. ### Debug functions @@ -106,7 +106,7 @@ The Parquet specification permits several compression codecs. This library curre - [x] Snappy - [x] Gzip - [x] Brotli -- [ ] ZSTD. Will be supported using the next versions of the upstream packages `parquet` and `parquet2`. +- [x] ZSTD. Supported in `arrow1`, will be supported in `arrow2` when the next version of the upstream `parquet2` package is released. - [ ] LZ4. Work is progressing but no support yet. ## Custom builds diff --git a/package.json b/package.json index d4b48651..b63560e0 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,10 @@ { "scripts": { "build": "bash ./scripts/build.sh", + "build:test": "bash ./scripts/build-node-tests.sh", "docs:build": "typedoc", "docs:publish": "gh-pages -d docs_build", + "docs:serve": "cd docs_build && python -m http.server 8081", "docs:watch": "typedoc --watch", "test": "ts-node node_modules/tape/bin/tape ./tests/js/index.ts" }, diff --git a/scripts/build-node-tests.sh b/scripts/build-node-tests.sh index d919d6c2..1dc7e139 100644 --- a/scripts/build-node-tests.sh +++ b/scripts/build-node-tests.sh @@ -2,26 +2,27 @@ # Build script used in Node-only CI tests -rm -rf pkg pkg_node pkg_web +rm -rf tmp_build pkg +mkdir -p tmp_build ###################################### # ARROW 1 (arrow-rs) the default feature -# Build node version into pkg_node +# Build node version into tmp_build/node echo "Building arrow-rs node" wasm-pack build \ - --dev \ - --out-dir pkg \ - --out-name node \ + --release \ + --out-dir tmp_build/node \ + --out-name arrow1 \ --target nodejs ###################################### # ARROW 2 turn on the feature manually -# Build node version into pkg2_node +# Build node version into tmp_build/node2 echo "Building arrow2 node" wasm-pack build \ - --dev \ - --out-dir pkg2_node \ - --out-name node2 \ + --release \ + --out-dir tmp_build/node2 \ + --out-name arrow2 \ --target nodejs \ --no-default-features \ --features arrow2 \ @@ -30,9 +31,14 @@ wasm-pack build \ --features parquet2_supported_compressions # Copy files into pkg/ -cp pkg2_node/{node2.d.ts,node2.js,node2_bg.wasm,node2_bg.wasm.d.ts} pkg/ +mkdir -p pkg/node +cp tmp_build/{node,node2}/arrow* pkg/node +cp tmp_build/node/{package.json,LICENSE_APACHE,LICENSE_MIT,README.md} pkg/ -# Update files array using JQ -jq '.files += ["node2.d.ts", "node2.js", "node2_bg.wasm", "node2_bg.wasm.d.ts"]' pkg/package.json > pkg/package.json.tmp -# Overwrite existing file +# Update files array in package.json using JQ +# Set main field to node/arrow1.js +# Set types field to node/arrow1.d.ts +jq '.files = ["*"] | .main="node/arrow1.js" | .types="node/arrow1.d.ts"' pkg/package.json > pkg/package.json.tmp + +# Overwrite existing package.json file mv pkg/package.json.tmp pkg/package.json diff --git a/scripts/build.sh b/scripts/build.sh index dc4eedb8..58da3ef1 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -1,40 +1,42 @@ #! /usr/bin/env bash -rm -rf pkg pkg_node pkg_web pkg2_node pkg2_web pkg2 +rm -rf tmp_build pkg +mkdir -p tmp_build + ###################################### # ARROW 1 (arrow-rs) the default feature -# Build node version into pkg_node +# Build node version into tmp_build/node echo "Building arrow-rs node" wasm-pack build \ --release \ - --out-dir pkg_node \ - --out-name node \ + --out-dir tmp_build/node \ + --out-name arrow1 \ --target nodejs -# Build web version into pkg_web -echo "Building arrow-rs web" +# Build web version into tmp_build/esm +echo "Building arrow-rs esm" wasm-pack build \ --release \ - --out-dir pkg_web \ - --out-name web \ + --out-dir tmp_build/esm \ + --out-name arrow1 \ --target web -# Build standard bundler version into pkg +# Build bundler version into tmp_build/bundler echo "Building arrow-rs bundler" wasm-pack build \ --release \ - --out-dir pkg \ - --out-name bundler \ + --out-dir tmp_build/bundler \ + --out-name arrow1 \ --target bundler ###################################### # ARROW 2 turn on the feature manually -# Build node version into pkg2_node +# Build node version into tmp_build/node2 echo "Building arrow2 node" wasm-pack build \ --release \ - --out-dir pkg2_node \ - --out-name node2 \ + --out-dir tmp_build/node2 \ + --out-name arrow2 \ --target nodejs \ --no-default-features \ --features arrow2 \ @@ -42,12 +44,12 @@ wasm-pack build \ --features writer \ --features parquet2_supported_compressions -# Build web version into pkg2_web -echo "Building arrow2 web" +# Build web version into tmp_build/esm2 +echo "Building arrow2 esm" wasm-pack build \ --release \ - --out-dir pkg2_web \ - --out-name web2 \ + --out-dir tmp_build/esm2 \ + --out-name arrow2 \ --target web \ --no-default-features \ --features arrow2 \ @@ -55,12 +57,12 @@ wasm-pack build \ --features writer \ --features parquet2_supported_compressions -# Build standard bundler version into pkg2 +# Build bundler version into tmp_build/bundler2 echo "Building arrow2 bundler" wasm-pack build \ --release \ - --out-dir pkg2 \ - --out-name bundler2 \ + --out-dir tmp_build/bundler2 \ + --out-name arrow2 \ --target bundler \ --no-default-features \ --features arrow2 \ @@ -69,21 +71,21 @@ wasm-pack build \ --features parquet2_supported_compressions # Copy files into pkg/ -cp pkg_node/{node.d.ts,node.js,node_bg.wasm,node_bg.wasm.d.ts} pkg/ -cp pkg_web/{web.d.ts,web.js,web_bg.wasm,web_bg.wasm.d.ts} pkg/ +mkdir -p pkg/{node,esm,bundler} + +cp tmp_build/{bundler,bundler2}/arrow* pkg/bundler/ +cp tmp_build/{esm,esm2}/arrow* pkg/esm +cp tmp_build/{node,node2}/arrow* pkg/node + +cp tmp_build/bundler/{package.json,LICENSE_APACHE,LICENSE_MIT,README.md} pkg/ -cp pkg2_node/{node2.d.ts,node2.js,node2_bg.wasm,node2_bg.wasm.d.ts} pkg/ -cp pkg2_web/{web2.d.ts,web2.js,web2_bg.wasm,web2_bg.wasm.d.ts} pkg/ -cp pkg2/{bundler2.d.ts,bundler2.js,bundler2_bg.wasm,bundler2_bg.wasm.d.ts} pkg/ +# Create minimal package.json in esm/ folder with type: module +echo '{"type": "module"}' > pkg/esm/package.json -# Update files array using JQ -jq '.files += [ - "node.d.ts", "node.js", "node_bg.wasm", "node_bg.wasm.d.ts", - "web.d.ts", "web.js", "web_bg.wasm", "web_bg.wasm.d.ts", +# Update files array in package.json using JQ +# Set module field to bundler/arrow1.js +# Set types field to bundler/arrow1.d.ts +jq '.files = ["*"] | .module="bundler/arrow1.js" | .types="bundler/arrow1.d.ts"' pkg/package.json > pkg/package.json.tmp - "node2.d.ts", "node2.js", "node2_bg.wasm", "node2_bg.wasm.d.ts", - "web2.d.ts", "web2.js", "web2_bg.wasm", "web2_bg.wasm.d.ts", - "bundler2.d.ts", "bundler2.js", "bundler2_bg.wasm", "bundler2_bg.wasm.d.ts" - ]' pkg/package.json > pkg/package.json.tmp -# Overwrite existing file +# Overwrite existing package.json file mv pkg/package.json.tmp pkg/package.json diff --git a/tests/js/arrow1.ts b/tests/js/arrow1.ts index 1d2f51f5..c9549d8e 100644 --- a/tests/js/arrow1.ts +++ b/tests/js/arrow1.ts @@ -1,7 +1,7 @@ import * as test from "tape"; -import * as wasm from "../../pkg/node"; +import * as wasm from "../../pkg/node/arrow1"; import { readFileSync } from "fs"; -import { tableFromIPC, tableToIPC, Table } from "apache-arrow"; +import { tableFromIPC, tableToIPC } from "apache-arrow"; import { testArrowTablesEqual, readExpectedArrowData } from "./utils"; // Path from repo root diff --git a/tests/js/arrow2.ts b/tests/js/arrow2.ts index 0fb04b7b..d52ad70f 100644 --- a/tests/js/arrow2.ts +++ b/tests/js/arrow2.ts @@ -1,7 +1,7 @@ import * as test from "tape"; -import * as wasm from "../../pkg/node2"; +import * as wasm from "../../pkg/node/arrow2"; import { readFileSync } from "fs"; -import { tableFromIPC, tableToIPC, Table } from "apache-arrow"; +import { tableFromIPC, tableToIPC } from "apache-arrow"; import { testArrowTablesEqual, readExpectedArrowData } from "./utils"; // Path from repo root diff --git a/tsconfig.docs.json b/tsconfig.docs.json index b7738e2f..feefd9a3 100644 --- a/tsconfig.docs.json +++ b/tsconfig.docs.json @@ -1,3 +1,3 @@ { - "include": ["pkg/*.d.ts"] + "include": ["pkg/**/*.d.ts"] } diff --git a/typedoc.json b/typedoc.json index 7cc89366..b0351213 100644 --- a/typedoc.json +++ b/typedoc.json @@ -3,12 +3,12 @@ "cleanOutputDir": true, "darkHighlightTheme": "github-dark", "entryPoints": [ - "pkg/node.d.ts", - "pkg/node2.d.ts", - "pkg/bundler.d.ts", - "pkg/bundler2.d.ts", - "pkg/web.d.ts", - "pkg/web2.d.ts" + "pkg/node/arrow1.d.ts", + "pkg/node/arrow2.d.ts", + "pkg/bundler/arrow1.d.ts", + "pkg/bundler/arrow2.d.ts", + "pkg/esm/arrow1.d.ts", + "pkg/esm/arrow2.d.ts" ], "lightHighlightTheme": "github-light", "tsconfig": "tsconfig.docs.json",