diff --git a/packages/cli/lang/en.json b/packages/cli/lang/en.json index 99326c9585..fd1da137c2 100644 --- a/packages/cli/lang/en.json +++ b/packages/cli/lang/en.json @@ -129,7 +129,7 @@ "lib_compiler_noNodeModules": "could not locate {folder} in parent directories of web3api manifest", "lib_compiler_noInvoke": "WASM module is missing the _w3_invoke export. This should never happen...", "lib_compiler_dup_code_folder": "Duplicate code generation folder found `{directory}`. Please ensure each module file is located in a unique directory.", - "lib_compiler_missing_export": "{missingExport} is not exported from the WASM module `{moduleName}`", + "lib_compiler_invalid_module": "Invalid Wasm module found. `{moduleName}` at {modulePath} is invalid. Error: {error}", "lib_compiler_cannotBuildInterfaceModules": "Cannot build modules for an Interface Web3API", "lib_compiler_outputMetadataText": "Metadata written", "lib_compiler_outputMetadataError": "Failed to output metadata", diff --git a/packages/cli/lang/es.json b/packages/cli/lang/es.json index 051f088571..6687ff6092 100644 --- a/packages/cli/lang/es.json +++ b/packages/cli/lang/es.json @@ -129,7 +129,7 @@ "lib_compiler_noNodeModules": "could not locate {folder} in parent directories of web3api manifest", "lib_compiler_noInvoke": "WASM module is missing the _w3_invoke export. This should never happen...", "lib_compiler_dup_code_folder": "Duplicate code generation folder found `{directory}`. Please ensure each module file is located in a unique directory.", - "lib_compiler_missing_export": "{missingExport} is not exported from the WASM module `{moduleName}`", + "lib_compiler_invalid_module": "Invalid Wasm module found. `{moduleName}` at {modulePath} is invalid. Error: {error}", "lib_compiler_cannotBuildInterfaceModules": "Cannot build modules for an Interface Web3API", "lib_compiler_outputMetadataText": "Metadata written", "lib_compiler_outputMetadataError": "Failed to output metadata", diff --git a/packages/cli/src/lib/Compiler.ts b/packages/cli/src/lib/Compiler.ts index e2beba901b..73a7b52d27 100644 --- a/packages/cli/src/lib/Compiler.ts +++ b/packages/cli/src/lib/Compiler.ts @@ -267,12 +267,12 @@ export class Compiler { // Build the sources const dockerImageId = await this._buildSourcesInDocker(); - // Validate the WASM exports + // Validate the Wasm modules await Promise.all( Object.keys(modulesToBuild) .filter((module: InvokableModules) => modulesToBuild[module]) .map((module: InvokableModules) => - this._validateExports(module, outputDir) + this._validateWasmModule(module, outputDir) ) ); @@ -528,16 +528,13 @@ export class Compiler { } } - private async _validateExports( + private async _validateWasmModule( moduleName: InvokableModules, buildDir: string ): Promise { - const wasmSource = fs.readFileSync( - path.join(buildDir, `${moduleName}.wasm`) - ); + const modulePath = path.join(buildDir, `${moduleName}.wasm`); + const wasmSource = fs.readFileSync(modulePath); - const mod = await WebAssembly.compile(wasmSource); - const memory = new WebAssembly.Memory({ initial: 1 }); const w3Imports: Record void> = { __w3_subinvoke: () => {}, __w3_subinvoke_result_len: () => {}, @@ -553,40 +550,24 @@ export class Compiler { __w3_abort: () => {}, }; - const instance = await WebAssembly.instantiate(mod, { - env: { - memory, - }, - w3: w3Imports, - }); - - const requiredExports = [ - ...WasmWeb3Api.requiredExports, - ...AsyncWasmInstance.requiredExports, - ]; - const missingExports: string[] = []; - - for (const requiredExport of requiredExports) { - if (!instance.exports[requiredExport]) { - missingExports.push(requiredExport); - } - } - - if (missingExports.length) { + try { + const memory = AsyncWasmInstance.createMemory({ module: wasmSource }); + await AsyncWasmInstance.createInstance({ + module: wasmSource, + imports: { + env: { + memory, + }, + w3: w3Imports, + }, + requiredExports: WasmWeb3Api.requiredExports, + }); + } catch (error) { throw Error( - intlMsg.lib_compiler_missing_export({ - missingExport: missingExports - .map((missingExport, index) => { - if (missingExports.length === 1) { - return missingExport; - } else if (index === missingExports.length - 1) { - return "& " + missingExport; - } else { - return missingExport + ", "; - } - }) - .join(), + intlMsg.lib_compiler_invalid_module({ + modulePath, moduleName, + error, }) ); } diff --git a/packages/js/asyncify/src/AsyncWasmInstance.ts b/packages/js/asyncify/src/AsyncWasmInstance.ts index 7896ddedf8..598a1fddbd 100644 --- a/packages/js/asyncify/src/AsyncWasmInstance.ts +++ b/packages/js/asyncify/src/AsyncWasmInstance.ts @@ -1,23 +1,7 @@ /* eslint-disable @typescript-eslint/naming-convention */ /* eslint-disable @typescript-eslint/ban-types */ /* eslint-disable @typescript-eslint/no-empty-function */ - -type MaybeAsync = Promise | T; - -function isPromise( - test?: MaybeAsync -): test is Promise { - return !!test && typeof (test as Promise).then === "function"; -} - -function proxyGet>( - obj: T, - transform: (value: unknown) => unknown -): T { - return new Proxy(obj, { - get: (obj: T, name: string) => transform(obj[name]), - }); -} +import { indexOfArray, isPromise, proxyGet } from "./utils"; type WasmMemory = WebAssembly.Memory; type WasmExports = WebAssembly.Exports; @@ -60,6 +44,62 @@ export class AsyncWasmInstance { private constructor() {} + public static createMemory(config: { module: ArrayBuffer }): WasmMemory { + const bytecode = new Uint8Array(config.module); + + // extract the initial memory page size, as it will + // throw an error if the imported page size differs: + // https://chromium.googlesource.com/v8/v8/+/644556e6ed0e6e4fac2dfabb441439820ec59813/src/wasm/module-instantiate.cc#924 + const envMemoryImportSignature = Uint8Array.from([ + // string length + 0x03, + // env ; import module name + 0x65, + 0x6e, + 0x76, + // string length + 0x06, + // memory ; import field name + 0x6d, + 0x65, + 0x6d, + 0x6f, + 0x72, + 0x79, + // import kind + 0x02, + // limits ; https://github.com/sunfishcode/wasm-reference-manual/blob/master/WebAssembly.md#resizable-limits + // limits ; flags + // 0x??, + // limits ; initial + // 0x__, + ]); + + const sigIdx = indexOfArray(bytecode, envMemoryImportSignature); + + if (sigIdx < 0) { + throw Error( + `Unable to find Wasm memory import section. ` + + `Modules must import memory from the "env" module's ` + + `"memory" field like so:\n` + + `(import "env" "memory" (memory (;0;) #))` + ); + } + + // Extract the initial memory page-range size + const memoryInitalLimits = bytecode.at( + sigIdx + envMemoryImportSignature.length + 1 + ); + + if (memoryInitalLimits === undefined) { + throw Error( + "No initial memory number found, this should never happen..." + ); + } + + return new WebAssembly.Memory({ initial: memoryInitalLimits }); + } + public static async createInstance(config: { module: ArrayBuffer; imports: WasmImports; diff --git a/packages/js/asyncify/src/utils.ts b/packages/js/asyncify/src/utils.ts new file mode 100644 index 0000000000..c57129c479 --- /dev/null +++ b/packages/js/asyncify/src/utils.ts @@ -0,0 +1,51 @@ +export type MaybeAsync = Promise | T; + +export function isPromise( + test?: MaybeAsync +): test is Promise { + return !!test && typeof (test as Promise).then === "function"; +} + +export function proxyGet>( + obj: T, + transform: (value: unknown) => unknown +): T { + return new Proxy(obj, { + get: (obj: T, name: string) => transform(obj[name]), + }); +} + +export function indexOfArray(source: Uint8Array, search: Uint8Array): number { + let run = true; + let start = 0; + + while (run) { + const idx = source.indexOf(search[0], start); + + // not found + if (idx < start) { + run = false; + continue; + } + + // Make sure the rest of the subarray contains the search pattern + const subBuff = source.subarray(idx, idx + search.length); + + let retry = false; + let i = 1; + for (; i < search.length && !retry; ++i) { + if (subBuff.at(i) !== search.at(i)) { + retry = true; + } + } + + if (retry) { + start = idx + i; + continue; + } else { + return idx; + } + } + + return -1; +} diff --git a/packages/js/client/src/wasm/WasmWeb3Api.ts b/packages/js/client/src/wasm/WasmWeb3Api.ts index 7066a81550..0768e32c35 100644 --- a/packages/js/client/src/wasm/WasmWeb3Api.ts +++ b/packages/js/client/src/wasm/WasmWeb3Api.ts @@ -170,7 +170,7 @@ export class WasmWeb3Api extends Api { ); }; - const memory = new WebAssembly.Memory({ initial: 1 }); + const memory = AsyncWasmInstance.createMemory({ module: wasm }); const instance = await AsyncWasmInstance.createInstance({ module: wasm, imports: createImports({ diff --git a/packages/js/plugins/http/jest.config.js b/packages/js/plugins/http/jest.config.js index 38888109ca..f0f063c0ce 100644 --- a/packages/js/plugins/http/jest.config.js +++ b/packages/js/plugins/http/jest.config.js @@ -9,5 +9,14 @@ module.exports = { "transform": { "^.+\\.(ts|tsx)$": "ts-jest" }, + modulePathIgnorePatterns: [ + "/src/__tests__/e2e/integration/" + ], + testPathIgnorePatterns: [ + "/src/__tests__/e2e/integration/" + ], + transformIgnorePatterns: [ + "/src/__tests__/e2e/integration/" + ], testEnvironment: 'node' } diff --git a/packages/js/plugins/http/package.json b/packages/js/plugins/http/package.json index ce4e2cfe18..dc615896fd 100644 --- a/packages/js/plugins/http/package.json +++ b/packages/js/plugins/http/package.json @@ -17,6 +17,7 @@ "codegen": "node ../../../../dependencies/node_modules/@web3api/cli/bin/w3 plugin codegen", "lint": "eslint --color -c ../../../../.eslintrc.js src/", "test": "jest --passWithNoTests --runInBand --verbose", + "test:ci": "jest --passWithNoTests --runInBand --verbose", "test:watch": "jest --watch --passWithNoTests --verbose" }, "dependencies": { diff --git a/packages/js/plugins/http/src/__tests__/e2e/e2e.spec.ts b/packages/js/plugins/http/src/__tests__/e2e/e2e.spec.ts index 9f92d6f78b..4b7fafbf63 100644 --- a/packages/js/plugins/http/src/__tests__/e2e/e2e.spec.ts +++ b/packages/js/plugins/http/src/__tests__/e2e/e2e.spec.ts @@ -2,15 +2,6 @@ import { httpPlugin } from "../.."; import { Response } from "../../w3"; import { Web3ApiClient } from "@web3api/client-js" -import { ensPlugin } from "@web3api/ens-plugin-js"; -import { ipfsPlugin } from "@web3api/ipfs-plugin-js"; -import { ethereumPlugin } from "@web3api/ethereum-plugin-js"; -import { - initTestEnvironment, - stopTestEnvironment, - buildAndDeployApi -} from "@web3api/test-env-js"; -import axios from "axios"; import nock from "nock"; jest.setTimeout(360000) @@ -274,120 +265,4 @@ describe("e2e tests for HttpPlugin", () => { }); }); - - describe("integration", () => { - - let client: Web3ApiClient; - let uri: string; - let ensAddress: string; - - beforeAll(async () => { - const { ethereum, ipfs } = await initTestEnvironment(); - const { data } = await axios.get("http://localhost:4040/deploy-ens"); - - ensAddress = data.ensAddress - - client = new Web3ApiClient({ - plugins: [ - { - uri: "w3://ens/http.web3api.eth", - plugin: httpPlugin(), - }, - { - uri: "w3://ens/ethereum.web3api.eth", - plugin: ethereumPlugin({ - networks: { - testnet: { - provider: ethereum - } - }, - defaultNetwork: "testnet" - }), - }, - { - uri: "w3://ens/ipfs.web3api.eth", - plugin: ipfsPlugin({ - provider: ipfs, - fallbackProviders: ["https://ipfs.io"] - }) - }, - { - uri: "w3://ens/ens.web3api.eth", - plugin: ensPlugin({ - addresses: { - testnet: ensAddress - } - }) - } - ], - }); - - const api = await buildAndDeployApi( - `${__dirname}/integration`, - ipfs, - ensAddress - ); - - uri = `ens/testnet/${api.ensDomain}`; - }); - - afterAll(async () => { - await stopTestEnvironment(); - }); - - it("get", async () => { - nock("http://www.example.com", { reqheaders: { 'X-Request-Header': "req-foo" } }) - .defaultReplyHeaders(defaultReplyHeaders) - .get("/api") - .query({ query: "foo" }) - .reply(200, '{data: "test-response"}', { 'X-Response-Header': "resp-foo" }) - - const response = await client.query<{ get: Response }>({ - uri, - query: `query { - get( - url: "http://www.example.com/api" - request: { - responseType: TEXT - urlParams: [{key: "query", value: "foo"}] - headers: [{key: "X-Request-Header", value: "req-foo"}] - } - ) - }` - }); - - expect(response.data).toBeDefined() - expect(response.errors).toBeUndefined() - expect(response.data?.get.status).toBe(200) - }); - - it("post", async () => { - nock("http://www.example.com", { reqheaders: { 'X-Request-Header': "req-foo" } }) - .defaultReplyHeaders(defaultReplyHeaders) - .post("/api", "{data: 'test-request'}") - .query({ query: "foo" }) - .reply(200, '{data: "test-response"}', { 'X-Response-Header': "resp-foo" }) - - const response = await client.query<{ post: Response }>({ - uri, - query: `query { - post( - url: "http://www.example.com/api" - request: { - responseType: TEXT - body: "{data: 'test-request'}" - urlParams: [{key: "query", value: "foo"}] - headers: [{key: "X-Request-Header", value: "req-foo"}] - } - ) - }` - }); - - expect(response.data).toBeTruthy(); - expect(response.errors).toBeFalsy(); - - expect(response.data?.post.status).toBe(200); - expect(response.data?.post.body).toBeTruthy(); - }); - }); }); diff --git a/packages/js/plugins/http/src/__tests__/e2e/integration.spec.ts b/packages/js/plugins/http/src/__tests__/e2e/integration.spec.ts new file mode 100644 index 0000000000..9aac17503f --- /dev/null +++ b/packages/js/plugins/http/src/__tests__/e2e/integration.spec.ts @@ -0,0 +1,140 @@ +import { httpPlugin } from "../.."; +import { Response } from "../../w3"; + +import { Web3ApiClient } from "@web3api/client-js" +import { ensPlugin } from "@web3api/ens-plugin-js"; +import { ipfsPlugin } from "@web3api/ipfs-plugin-js"; +import { ethereumPlugin } from "@web3api/ethereum-plugin-js"; +import { + initTestEnvironment, + stopTestEnvironment, + buildAndDeployApi +} from "@web3api/test-env-js"; +import axios from "axios"; +import nock from "nock"; + +jest.setTimeout(360000) + +const defaultReplyHeaders = { + 'access-control-allow-origin': '*', + 'access-control-allow-credentials': 'true' +} + +describe("e2e tests for HttpPlugin", () => { + + describe("integration", () => { + + let client: Web3ApiClient; + let uri: string; + let ensAddress: string; + + beforeAll(async () => { + const { ethereum, ipfs } = await initTestEnvironment(); + const { data } = await axios.get("http://localhost:4040/deploy-ens"); + + ensAddress = data.ensAddress + + client = new Web3ApiClient({ + plugins: [ + { + uri: "w3://ens/http.web3api.eth", + plugin: httpPlugin(), + }, + { + uri: "w3://ens/ethereum.web3api.eth", + plugin: ethereumPlugin({ + networks: { + testnet: { + provider: ethereum + } + }, + defaultNetwork: "testnet" + }), + }, + { + uri: "w3://ens/ipfs.web3api.eth", + plugin: ipfsPlugin({ + provider: ipfs, + fallbackProviders: ["https://ipfs.io"] + }) + }, + { + uri: "w3://ens/ens.web3api.eth", + plugin: ensPlugin({ + addresses: { + testnet: ensAddress + } + }) + } + ], + }); + + const api = await buildAndDeployApi( + `${__dirname}/integration`, + ipfs, + ensAddress + ); + + uri = `ens/testnet/${api.ensDomain}`; + }); + + afterAll(async () => { + await stopTestEnvironment(); + }); + + it("get", async () => { + nock("http://www.example.com", { reqheaders: { 'X-Request-Header': "req-foo" } }) + .defaultReplyHeaders(defaultReplyHeaders) + .get("/api") + .query({ query: "foo" }) + .reply(200, '{data: "test-response"}', { 'X-Response-Header': "resp-foo" }) + + const response = await client.query<{ get: Response }>({ + uri, + query: `query { + get( + url: "http://www.example.com/api" + request: { + responseType: TEXT + urlParams: [{key: "query", value: "foo"}] + headers: [{key: "X-Request-Header", value: "req-foo"}] + } + ) + }` + }); + + expect(response.data).toBeDefined() + expect(response.errors).toBeUndefined() + expect(response.data?.get.status).toBe(200) + }); + + it("post", async () => { + nock("http://www.example.com", { reqheaders: { 'X-Request-Header': "req-foo" } }) + .defaultReplyHeaders(defaultReplyHeaders) + .post("/api", "{data: 'test-request'}") + .query({ query: "foo" }) + .reply(200, '{data: "test-response"}', { 'X-Response-Header': "resp-foo" }) + + const response = await client.query<{ post: Response }>({ + uri, + query: `query { + post( + url: "http://www.example.com/api" + request: { + responseType: TEXT + body: "{data: 'test-request'}" + urlParams: [{key: "query", value: "foo"}] + headers: [{key: "X-Request-Header", value: "req-foo"}] + } + ) + }` + }); + + expect(response.data).toBeTruthy(); + expect(response.errors).toBeFalsy(); + + expect(response.data?.post.status).toBe(200); + expect(response.data?.post.body).toBeTruthy(); + }); + }); +}); diff --git a/packages/js/plugins/http/src/__tests__/unit/util.test.ts b/packages/js/plugins/http/src/__tests__/unit/util.test.ts index f14bbf38e7..e0d7d6a504 100644 --- a/packages/js/plugins/http/src/__tests__/unit/util.test.ts +++ b/packages/js/plugins/http/src/__tests__/unit/util.test.ts @@ -24,7 +24,7 @@ describe("converting axios response", () => { const response = fromAxiosResponse({ status: 200, statusText: "Ok", - data: "body-content", + data: Buffer.from("body-content"), headers: { ["Accept"]: "application-json", ["X-Header"]: "test-value" }, config: { responseType: "arraybuffer" }, }); diff --git a/packages/js/plugins/http/tsconfig.json b/packages/js/plugins/http/tsconfig.json index 85d516ce30..4a5f4564af 100644 --- a/packages/js/plugins/http/tsconfig.json +++ b/packages/js/plugins/http/tsconfig.json @@ -6,5 +6,7 @@ "include": [ "./src/**/*.ts" ], - "exclude": [] + "exclude": [ + "./**/.w3/**/*.ts" + ] } diff --git a/packages/js/test-env/package.json b/packages/js/test-env/package.json index 0ace9c504c..db8ef1f706 100644 --- a/packages/js/test-env/package.json +++ b/packages/js/test-env/package.json @@ -13,10 +13,7 @@ ], "scripts": { "build": "rimraf ./build && tsc --project tsconfig.build.json", - "lint": "eslint --color -c ../../../.eslintrc.js src/", - "test": "jest --passWithNoTests --runInBand --verbose", - "test:ci": "jest --passWithNoTests --runInBand --verbose", - "test:watch": "jest --watch --passWithNoTests --verbose" + "lint": "eslint --color -c ../../../.eslintrc.js src/" }, "dependencies": { "axios": "0.21.1",