Update bundling

.gitignore

@@ -2,11 +2,6 @@ node_modules
 /target
 **/*.rs.bk
 pkg/
-pkg_node/
-pkg_web/
-pkg2/
-pkg2_node/
-pkg2_web/
 wasm-pack.log
 .idea/
 www/data

README.md

@@ -34,26 +34,26 @@ Presumably no one wants to use both `parquet` and `parquet2` at once, so the def
 | Entry point             | Rust crates used        | Description                                             |
 | ----------------------- | ----------------------- | ------------------------------------------------------- |
 | `parquet-wasm`          | `parquet` and `arrow`   | "Bundler" build, to be used in bundlers such as Webpack |
-| `parquet-wasm/node`     | `parquet` and `arrow`   | Node build, to be used with `require` in NodeJS         |
-| `parquet-wasm/web`      | `parquet` and `arrow`   | ESM, to be used directly from the Web as an ES Module   |
+| `parquet-wasm/node/arrow1`     | `parquet` and `arrow`   | Node build, to be used with `require` in NodeJS         |
+| `parquet-wasm/esm/arrow1`      | `parquet` and `arrow`   | ESM, to be used directly from the Web as an ES Module   |
 |                         |                         |                                                         |
-| `parquet-wasm/bundler2` | `parquet2` and `arrow2` | "Bundler" build, to be used in bundlers such as Webpack |
-| `parquet-wasm/node2`    | `parquet2` and `arrow2` | Node build, to be used with `require` in NodeJS         |
-| `parquet-wasm/web2`     | `parquet2` and `arrow2` | ESM, to be used directly from the Web as an ES Module   |
+| `parquet-wasm/bundler/arrow2` | `parquet2` and `arrow2` | "Bundler" build, to be used in bundlers such as Webpack |
+| `parquet-wasm/node/arrow2`    | `parquet2` and `arrow2` | Node build, to be used with `require` in NodeJS         |
+| `parquet-wasm/esm/arrow2`     | `parquet2` and `arrow2` | ESM, to be used directly from the Web as an ES Module   |
 
-Note that when using the `/web` and `/web2` bundles, the default export must be awaited. See [here](https://rustwasm.github.io/docs/wasm-bindgen/examples/without-a-bundler.html) for an example.
+Note that when using the `esm` bundles, the default export must be awaited. See [here](https://rustwasm.github.io/docs/wasm-bindgen/examples/without-a-bundler.html) for an example.
 
 ### `parquet` API
 
 This implementation uses the [`arrow`](https://crates.io/crates/arrow) and [`parquet`](https://crates.io/crates/parquet) Rust crates.
 
-Refer to the [API documentation](https://kylebarron.dev/parquet-wasm/modules/bundler.html) for more details and examples.
+Refer to the [API documentation](https://kylebarron.dev/parquet-wasm/modules/bundler_arrow1.html) for more details and examples.
 
 ### `parquet2` API
 
 This implementation uses the [`arrow2`](https://crates.io/crates/arrow2) and [`parquet2`](https://crates.io/crates/parquet2) Rust crates.
 
-Refer to the [API documentation](https://kylebarron.dev/parquet-wasm/modules/bundler2.html) for more details and examples.
+Refer to the [API documentation](https://kylebarron.dev/parquet-wasm/modules/bundler_arrow2.html) for more details and examples.
 
 ### Debug functions
 
@@ -106,7 +106,7 @@ The Parquet specification permits several compression codecs. This library curre
 - [x] Snappy
 - [x] Gzip
 - [x] Brotli
-- [ ] ZSTD. Will be supported using the next versions of the upstream packages `parquet` and `parquet2`.
+- [x] ZSTD. Supported in `arrow1`, will be supported in `arrow2` when the next version of the upstream `parquet2` package is released.
 - [ ] LZ4. Work is progressing but no support yet.
 
 ## Custom builds

package.json

@@ -1,8 +1,10 @@
 {
   "scripts": {
     "build": "bash ./scripts/build.sh",
+    "build:test": "bash ./scripts/build-node-tests.sh",
     "docs:build": "typedoc",
     "docs:publish": "gh-pages -d docs_build",
+    "docs:serve": "cd docs_build && python -m http.server 8081",
     "docs:watch": "typedoc --watch",
     "test": "ts-node node_modules/tape/bin/tape ./tests/js/index.ts"
   },

scripts/build-node-tests.sh

@@ -2,26 +2,27 @@
 
 # Build script used in Node-only CI tests
 
-rm -rf pkg pkg_node pkg_web
+rm -rf tmp_build pkg
+mkdir -p tmp_build
 
 ######################################
 # ARROW 1 (arrow-rs) the default feature
-# Build node version into pkg_node
+# Build node version into tmp_build/node
 echo "Building arrow-rs node"
 wasm-pack build \
-  --dev \
-  --out-dir pkg \
-  --out-name node \
+  --release \
+  --out-dir tmp_build/node \
+  --out-name arrow1 \
   --target nodejs
 
 ######################################
 # ARROW 2 turn on the feature manually
-# Build node version into pkg2_node
+# Build node version into tmp_build/node2
 echo "Building arrow2 node"
 wasm-pack build \
-  --dev \
-  --out-dir pkg2_node \
-  --out-name node2 \
+  --release \
+  --out-dir tmp_build/node2 \
+  --out-name arrow2 \
   --target nodejs \
   --no-default-features \
   --features arrow2 \
@@ -30,9 +31,14 @@ wasm-pack build \
   --features parquet2_supported_compressions
 
 # Copy files into pkg/
-cp pkg2_node/{node2.d.ts,node2.js,node2_bg.wasm,node2_bg.wasm.d.ts} pkg/
+mkdir -p pkg/node
+cp tmp_build/{node,node2}/arrow* pkg/node
+cp tmp_build/node/{package.json,LICENSE_APACHE,LICENSE_MIT,README.md} pkg/
 
-# Update files array using JQ
-jq '.files += ["node2.d.ts", "node2.js", "node2_bg.wasm", "node2_bg.wasm.d.ts"]' pkg/package.json > pkg/package.json.tmp
-# Overwrite existing file
+# Update files array in package.json using JQ
+# Set main field to node/arrow1.js
+# Set types field to node/arrow1.d.ts
+jq '.files = ["*"] | .main="node/arrow1.js" | .types="node/arrow1.d.ts"' pkg/package.json > pkg/package.json.tmp
+
+# Overwrite existing package.json file
 mv pkg/package.json.tmp pkg/package.json

scripts/build.sh

@@ -1,66 +1,68 @@
 #! /usr/bin/env bash
-rm -rf pkg pkg_node pkg_web pkg2_node pkg2_web pkg2
+rm -rf tmp_build pkg
+mkdir -p tmp_build
+
 
 ######################################
 # ARROW 1 (arrow-rs) the default feature
-# Build node version into pkg_node
+# Build node version into tmp_build/node
 echo "Building arrow-rs node"
 wasm-pack build \
   --release \
-  --out-dir pkg_node \
-  --out-name node \
+  --out-dir tmp_build/node \
+  --out-name arrow1 \
   --target nodejs
 
-# Build web version into pkg_web
-echo "Building arrow-rs web"
+# Build web version into tmp_build/esm
+echo "Building arrow-rs esm"
 wasm-pack build \
   --release \
-  --out-dir pkg_web \
-  --out-name web \
+  --out-dir tmp_build/esm \
+  --out-name arrow1 \
   --target web
 
-# Build standard bundler version into pkg
+# Build bundler version into tmp_build/bundler
 echo "Building arrow-rs bundler"
 wasm-pack build \
   --release \
-  --out-dir pkg \
-  --out-name bundler \
+  --out-dir tmp_build/bundler \
+  --out-name arrow1 \
   --target bundler
 
 ######################################
 # ARROW 2 turn on the feature manually
-# Build node version into pkg2_node
+# Build node version into tmp_build/node2
 echo "Building arrow2 node"
 wasm-pack build \
   --release \
-  --out-dir pkg2_node \
-  --out-name node2 \
+  --out-dir tmp_build/node2 \
+  --out-name arrow2 \
   --target nodejs \
   --no-default-features \
   --features arrow2 \
   --features reader \
   --features writer \
   --features parquet2_supported_compressions
 
-# Build web version into pkg2_web
-echo "Building arrow2 web"
+# Build web version into tmp_build/esm2
+echo "Building arrow2 esm"
 wasm-pack build \
   --release \
-  --out-dir pkg2_web \
-  --out-name web2 \
+  --out-dir tmp_build/esm2 \
+  --out-name arrow2 \
   --target web \
   --no-default-features \
   --features arrow2 \
   --features reader \
   --features writer \
   --features parquet2_supported_compressions
 
-# Build standard bundler version into pkg2
+# Build bundler version into tmp_build/bundler2
 echo "Building arrow2 bundler"
 wasm-pack build \
   --release \
-  --out-dir pkg2 \
-  --out-name bundler2 \
+  --out-dir tmp_build/bundler2 \
+  --out-name arrow2 \
   --target bundler \
   --no-default-features \
   --features arrow2 \
@@ -69,21 +71,21 @@ wasm-pack build \
   --features parquet2_supported_compressions
 
 # Copy files into pkg/
-cp pkg_node/{node.d.ts,node.js,node_bg.wasm,node_bg.wasm.d.ts} pkg/
-cp pkg_web/{web.d.ts,web.js,web_bg.wasm,web_bg.wasm.d.ts} pkg/
+mkdir -p pkg/{node,esm,bundler}
+
+cp tmp_build/{bundler,bundler2}/arrow* pkg/bundler/
+cp tmp_build/{esm,esm2}/arrow* pkg/esm
+cp tmp_build/{node,node2}/arrow* pkg/node
+
+cp tmp_build/bundler/{package.json,LICENSE_APACHE,LICENSE_MIT,README.md} pkg/
 
-cp pkg2_node/{node2.d.ts,node2.js,node2_bg.wasm,node2_bg.wasm.d.ts} pkg/
-cp pkg2_web/{web2.d.ts,web2.js,web2_bg.wasm,web2_bg.wasm.d.ts} pkg/
-cp pkg2/{bundler2.d.ts,bundler2.js,bundler2_bg.wasm,bundler2_bg.wasm.d.ts} pkg/
+# Create minimal package.json in esm/ folder with type: module
+echo '{"type": "module"}' > pkg/esm/package.json
 
-# Update files array using JQ
-jq '.files += [
-  "node.d.ts", "node.js", "node_bg.wasm", "node_bg.wasm.d.ts",
-  "web.d.ts", "web.js", "web_bg.wasm", "web_bg.wasm.d.ts",
+# Update files array in package.json using JQ
+# Set module field to bundler/arrow1.js
+# Set types field to bundler/arrow1.d.ts
+jq '.files = ["*"] | .module="bundler/arrow1.js" | .types="bundler/arrow1.d.ts"' pkg/package.json > pkg/package.json.tmp
 
-  "node2.d.ts", "node2.js", "node2_bg.wasm", "node2_bg.wasm.d.ts",
-  "web2.d.ts", "web2.js", "web2_bg.wasm", "web2_bg.wasm.d.ts",
-  "bundler2.d.ts", "bundler2.js", "bundler2_bg.wasm", "bundler2_bg.wasm.d.ts"
-  ]' pkg/package.json > pkg/package.json.tmp
-# Overwrite existing file
+# Overwrite existing package.json file
 mv pkg/package.json.tmp pkg/package.json

tests/js/arrow1.ts

@@ -1,7 +1,7 @@
 import * as test from "tape";
-import * as wasm from "../../pkg/node";
+import * as wasm from "../../pkg/node/arrow1";
 import { readFileSync } from "fs";
-import { tableFromIPC, tableToIPC, Table } from "apache-arrow";
+import { tableFromIPC, tableToIPC } from "apache-arrow";
 import { testArrowTablesEqual, readExpectedArrowData } from "./utils";
 
 // Path from repo root

tests/js/arrow2.ts

@@ -1,7 +1,7 @@
 import * as test from "tape";
-import * as wasm from "../../pkg/node2";
+import * as wasm from "../../pkg/node/arrow2";
 import { readFileSync } from "fs";
-import { tableFromIPC, tableToIPC, Table } from "apache-arrow";
+import { tableFromIPC, tableToIPC } from "apache-arrow";
 import { testArrowTablesEqual, readExpectedArrowData } from "./utils";
 
 // Path from repo root

tsconfig.docs.json

@@ -1,3 +1,3 @@
 {
-  "include": ["pkg/*.d.ts"]
+  "include": ["pkg/**/*.d.ts"]
 }

typedoc.json

@@ -3,12 +3,12 @@
   "cleanOutputDir": true,
   "darkHighlightTheme": "github-dark",
   "entryPoints": [
-    "pkg/node.d.ts",
-    "pkg/node2.d.ts",
-    "pkg/bundler.d.ts",
-    "pkg/bundler2.d.ts",
-    "pkg/web.d.ts",
-    "pkg/web2.d.ts"
+    "pkg/node/arrow1.d.ts",
+    "pkg/node/arrow2.d.ts",
+    "pkg/bundler/arrow1.d.ts",
+    "pkg/bundler/arrow2.d.ts",
+    "pkg/esm/arrow1.d.ts",
+    "pkg/esm/arrow2.d.ts"
   ],
   "lightHighlightTheme": "github-light",
   "tsconfig": "tsconfig.docs.json",