Skip to content

Commit

Permalink
Support for compressed base64 wasm files
Browse files Browse the repository at this point in the history
Upgrade to use WebAssembly.instantiateStreaming
  • Loading branch information
mainnet-pat committed Sep 6, 2024
1 parent ade0151 commit b17bfc1
Show file tree
Hide file tree
Showing 12 changed files with 95 additions and 22 deletions.
3 changes: 2 additions & 1 deletion .eslintrc
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@
"node_modules",
"build",
"coverage",
"src/lib/schema/ajv/*.js"
"src/lib/schema/ajv/*.js",
"src/lib/bin/secp256k1/secp256k1.js"
],
"extends": ["bitauth"],
// "globals": { "BigInt": true, "console": true, "WebAssembly": true },
Expand Down
40 changes: 39 additions & 1 deletion src/lib/bin/hashes.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,47 @@
import { binsAreEqual } from '../format/hex.js';

export type HashFunction = {
final: (rawState: Uint8Array) => Uint8Array;
hash: (input: Uint8Array) => Uint8Array;
init: () => Uint8Array;
update: (rawState: Uint8Array, input: Uint8Array) => Uint8Array;
};

/* eslint-disable @typescript-eslint/require-await, functional/no-return-void, functional/no-expression-statements, @typescript-eslint/no-magic-numbers, @typescript-eslint/naming-convention */
/**
* Reads in a wasm binary as an ArrayBuffer, checks if it was compressed and
* decompresses it if necessary. Returns a Response object with the wasm binary compatible with WebAssembly.instantiateStreaming.
*/
export const streamWasmArrayBuffer = async (
wasmArrayBuffer: ArrayBuffer,
): Promise<Response> => {
// currently, we consume the data in a single chunk, but when ReadableStream.from() becomes widely available, we can switch to it
const wasmStream = new ReadableStream({
start(controller): void {
controller.enqueue(new Uint8Array(wasmArrayBuffer));
controller.close();
},
});

// if source is uncompressed, then return as is
if (
binsAreEqual(
new Uint8Array(wasmArrayBuffer, 0, 4),
new Uint8Array([0x00, 0x61, 0x73, 0x6d]),
)
) {
return new Response(wasmStream, {
headers: new Headers({ 'content-type': 'application/wasm' }),
});
}

// otherwise, decompress the source
return new Response(wasmStream.pipeThrough(new DecompressionStream('gzip')), {
headers: new Headers({ 'content-type': 'application/wasm' }),
});
};
/* eslint-enable @typescript-eslint/require-await, functional/no-return-void, functional/no-expression-statements, @typescript-eslint/no-magic-numbers, @typescript-eslint/naming-convention */

/* eslint-disable functional/no-conditional-statements, functional/no-let, functional/no-expression-statements, no-underscore-dangle, functional/no-try-statements, @typescript-eslint/no-magic-numbers, @typescript-eslint/max-params, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-non-null-assertion */
/**
* Note, most of this method is translated and boiled-down from the wasm-pack
Expand All @@ -19,8 +56,9 @@ export const instantiateRustWasm = async (
updateExportName: string,
finalExportName: string,
): Promise<HashFunction> => {
const webassemblyByteStream = await streamWasmArrayBuffer(webassemblyBytes);
const wasm = (
await WebAssembly.instantiate(webassemblyBytes, {
await WebAssembly.instantiateStreaming(webassemblyByteStream, {
[expectedImportModuleName]: {
/**
* This would only be called in cases where a `__wbindgen_malloc` failed.
Expand Down
2 changes: 1 addition & 1 deletion src/lib/bin/ripemd160/ripemd160.base64.ts

Large diffs are not rendered by default.

30 changes: 28 additions & 2 deletions src/lib/bin/secp256k1/secp256k1-wasm.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import {
getEmbeddedSecp256k1Binary,
instantiateSecp256k1Wasm,
instantiateSecp256k1WasmBytes,
streamWasmArrayBuffer,
} from '../../lib.js';

// test vectors from `zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo zoo wrong` (`xprv9s21ZrQH143K2PfMvkNViFc1fgumGqBew45JD8SxA59Jc5M66n3diqb92JjvaR61zT9P89Grys12kdtV4EFVo6tMwER7U2hcUmZ9VfMYPLC`), m/0 and m/1:
Expand Down Expand Up @@ -491,10 +492,35 @@ const testSecp256k1Wasm = (

const binary = getEmbeddedSecp256k1Binary();

test('[crypto] getEmbeddedSecp256k1Binary returns the proper binary', (t) => {
test('[crypto] getEmbeddedSecp256k1Binary returns the proper binary', async (t) => {
const path = join(new URL('.', import.meta.url).pathname, 'secp256k1.wasm');
const binaryFromDisk = readFileSync(path).buffer;
t.deepEqual(binary, binaryFromDisk);
const eventuallyDecompressedBinary = await (
await streamWasmArrayBuffer(binary)
).arrayBuffer();
t.deepEqual(eventuallyDecompressedBinary, binaryFromDisk);

// compress the wasm binary
const stream = new ReadableStream({
start(controller): void {
controller.enqueue(new Uint8Array(binaryFromDisk));
controller.close();
},
}).pipeThrough(new CompressionStream('gzip'));

const compressedBinary = await new Response(stream).arrayBuffer();

// decompress the compressed wasm binary
const decompressedBinary = await (
await streamWasmArrayBuffer(compressedBinary)
).arrayBuffer();
t.deepEqual(decompressedBinary, binaryFromDisk);

// check that the uncompressed binary produced by `streamWasmArrayBuffer` is the same as the binary from disk
const uncompressedBinary = await (
await streamWasmArrayBuffer(binaryFromDisk)
).arrayBuffer();
t.deepEqual(uncompressedBinary, binaryFromDisk);
});

test('[crypto] Secp256k1Wasm instantiated with embedded binary', async (t) => {
Expand Down
15 changes: 10 additions & 5 deletions src/lib/bin/secp256k1/secp256k1-wasm.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
/* eslint-disable no-underscore-dangle, @typescript-eslint/max-params, @typescript-eslint/naming-convention */
// cSpell:ignore memcpy, anyfunc
import { base64ToBin } from '../../format/format.js';
import { streamWasmArrayBuffer } from '../hashes.js';

import type { Secp256k1Wasm } from './secp256k1-wasm-types.js';
import { CompressionFlag, ContextFlag } from './secp256k1-wasm-types.js';
Expand Down Expand Up @@ -352,11 +353,15 @@ export const instantiateSecp256k1WasmBytes = async (
global: { Infinity, NaN },
};

return WebAssembly.instantiate(webassemblyBytes, info).then((result) => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment
getErrNoLocation = result.instance.exports['___errno_location'] as any;
return wrapSecp256k1Wasm(result.instance, heapU8, heapU32);
});
const webassemblyByteStream = await streamWasmArrayBuffer(webassemblyBytes);

return WebAssembly.instantiateStreaming(webassemblyByteStream, info).then(
(result) => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/no-unsafe-assignment
getErrNoLocation = result.instance.exports['___errno_location'] as any;
return wrapSecp256k1Wasm(result.instance, heapU8, heapU32);
},
);
};
/* eslint-enable functional/immutable-data, functional/no-expression-statements, @typescript-eslint/no-magic-numbers, functional/no-conditional-statements, no-bitwise, functional/no-throw-statements */

Expand Down
2 changes: 1 addition & 1 deletion src/lib/bin/secp256k1/secp256k1.base64.ts

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion src/lib/bin/sha1/sha1.base64.ts

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion src/lib/bin/sha256/sha256.base64.ts

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion src/lib/bin/sha512/sha512.base64.ts

Large diffs are not rendered by default.

9 changes: 6 additions & 3 deletions src/lib/crypto/hash.spec.helper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { join } from 'path';
import test from 'ava';

import type { HashFunction } from '../lib.js';
import { utf8ToBin } from '../lib.js';
import { streamWasmArrayBuffer, utf8ToBin } from '../lib.js';

import fc from 'fast-check';
import hashJs from 'hash.js';
Expand Down Expand Up @@ -43,7 +43,7 @@ export const testHashFunction = <T extends HashFunction>({
nodeJsAlgorithm: 'ripemd160' | 'sha1' | 'sha256' | 'sha512';
}) => {
const binary = getEmbeddedBinary();
test(`[crypto] ${hashFunctionName} getEmbeddedBinary returns the proper binary`, (t) => {
test(`[crypto] ${hashFunctionName} getEmbeddedBinary returns the proper binary`, async (t) => {
const path = join(
new URL('.', import.meta.url).pathname,
'..',
Expand All @@ -52,7 +52,10 @@ export const testHashFunction = <T extends HashFunction>({
`${hashFunctionName}.wasm`,
);
const binaryFromDisk = readFileSync(path).buffer;
t.deepEqual(binary, binaryFromDisk);
const eventuallyDecompressedBinary = await (
await streamWasmArrayBuffer(binary)
).arrayBuffer();
t.deepEqual(eventuallyDecompressedBinary, binaryFromDisk);
});

test(`[crypto] ${hashFunctionName} instantiated with embedded binary`, async (t) => {
Expand Down
8 changes: 4 additions & 4 deletions wasm/docker/hashes.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ RUN /binaryen/bin/wasm-opt -O3 pkg/ripemd160_bg.wasm -o pkg/ripemd160.wasm
RUN cp pkg/ripemd160.wasm out/ripemd160
RUN cp pkg/ripemd160.d.ts out/ripemd160
RUN cp pkg/ripemd160.js out/ripemd160
RUN OUTPUT_TS_FILE=out/ripemd160/ripemd160.base64.ts; printf "/**\n * @hidden\n */\n// prettier-ignore\nexport const ripemd160Base64Bytes =\n '" > $OUTPUT_TS_FILE && base64 -w 0 pkg/ripemd160.wasm >> $OUTPUT_TS_FILE && printf "';\n" >> $OUTPUT_TS_FILE
RUN OUTPUT_TS_FILE=out/ripemd160/ripemd160.base64.ts; printf "/**\n * @hidden\n */\n// prettier-ignore\nexport const ripemd160Base64Bytes =\n '" > $OUTPUT_TS_FILE && cat pkg/ripemd160.wasm | gzip | base64 -w 0 >> $OUTPUT_TS_FILE && printf "';\n" >> $OUTPUT_TS_FILE
RUN cp -r /libauth/wasm/hashes/ripemd160/out/ripemd160 /libauth/bin

# sha256
Expand All @@ -34,7 +34,7 @@ RUN /binaryen/bin/wasm-opt -O3 pkg/sha256_bg.wasm -o pkg/sha256.wasm
RUN cp pkg/sha256.wasm out/sha256
RUN cp pkg/sha256.d.ts out/sha256
RUN cp pkg/sha256.js out/sha256
RUN OUTPUT_TS_FILE=out/sha256/sha256.base64.ts; printf "/**\n * @hidden\n */\n// prettier-ignore\nexport const sha256Base64Bytes =\n '" > $OUTPUT_TS_FILE && base64 -w 0 pkg/sha256.wasm >> $OUTPUT_TS_FILE && printf "';\n" >> $OUTPUT_TS_FILE
RUN OUTPUT_TS_FILE=out/sha256/sha256.base64.ts; printf "/**\n * @hidden\n */\n// prettier-ignore\nexport const sha256Base64Bytes =\n '" > $OUTPUT_TS_FILE && cat pkg/sha256.wasm | gzip | base64 -w 0 >> $OUTPUT_TS_FILE && printf "';\n" >> $OUTPUT_TS_FILE
RUN cp -r /libauth/wasm/hashes/sha256/out/sha256 /libauth/bin

# sha512
Expand All @@ -46,7 +46,7 @@ RUN /binaryen/bin/wasm-opt -O3 pkg/sha512_bg.wasm -o pkg/sha512.wasm
RUN cp pkg/sha512.wasm out/sha512
RUN cp pkg/sha512.d.ts out/sha512
RUN cp pkg/sha512.js out/sha512
RUN OUTPUT_TS_FILE=out/sha512/sha512.base64.ts; printf "/**\n * @hidden\n */\n// prettier-ignore\nexport const sha512Base64Bytes =\n '" > $OUTPUT_TS_FILE && base64 -w 0 pkg/sha512.wasm >> $OUTPUT_TS_FILE && printf "';\n" >> $OUTPUT_TS_FILE
RUN OUTPUT_TS_FILE=out/sha512/sha512.base64.ts; printf "/**\n * @hidden\n */\n// prettier-ignore\nexport const sha512Base64Bytes =\n '" > $OUTPUT_TS_FILE && cat pkg/sha512.wasm | gzip | base64 -w 0 >> $OUTPUT_TS_FILE && printf "';\n" >> $OUTPUT_TS_FILE
RUN cp -r /libauth/wasm/hashes/sha512/out/sha512 /libauth/bin

# sha1
Expand All @@ -58,7 +58,7 @@ RUN /binaryen/bin/wasm-opt -O3 pkg/sha1_bg.wasm -o pkg/sha1.wasm
RUN cp pkg/sha1.wasm out/sha1
RUN cp pkg/sha1.d.ts out/sha1
RUN cp pkg/sha1.js out/sha1
RUN OUTPUT_TS_FILE=out/sha1/sha1.base64.ts; printf "/**\n * @hidden\n */\n// prettier-ignore\nexport const sha1Base64Bytes =\n '" > $OUTPUT_TS_FILE && base64 -w 0 pkg/sha1.wasm >> $OUTPUT_TS_FILE && printf "';\n" >> $OUTPUT_TS_FILE
RUN OUTPUT_TS_FILE=out/sha1/sha1.base64.ts; printf "/**\n * @hidden\n */\n// prettier-ignore\nexport const sha1Base64Bytes =\n '" > $OUTPUT_TS_FILE && cat pkg/sha1.wasm | gzip | base64 -w 0 >> $OUTPUT_TS_FILE && printf "';\n" >> $OUTPUT_TS_FILE
RUN cp -r /libauth/wasm/hashes/sha1/out/sha1 /libauth/bin

WORKDIR /libauth/wasm/hashes/
Expand Down
2 changes: 1 addition & 1 deletion wasm/docker/secp256k1.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ RUN emcc src/libsecp256k1_la-secp256k1.o \
]' \
-o out/secp256k1/secp256k1.js

RUN OUTPUT_TS_FILE=out/secp256k1/secp256k1.base64.ts; printf "/**\n * @hidden\n */\n// prettier-ignore\nexport const secp256k1Base64Bytes =\n '" > $OUTPUT_TS_FILE && base64 -w 0 out/secp256k1/secp256k1.wasm >> $OUTPUT_TS_FILE && printf "';\n" >> $OUTPUT_TS_FILE
RUN OUTPUT_TS_FILE=out/secp256k1/secp256k1.base64.ts; printf "/**\n * @hidden\n */\n// prettier-ignore\nexport const secp256k1Base64Bytes =\n '" > $OUTPUT_TS_FILE && cat out/secp256k1/secp256k1.wasm | gzip | base64 -w 0 >> $OUTPUT_TS_FILE && printf "';\n" >> $OUTPUT_TS_FILE

RUN cp -r /libauth/wasm/secp256k1/out /libauth/bin

Expand Down

0 comments on commit b17bfc1

Please sign in to comment.