Move from duckdb_web_tokenize to duckdb_web_tokenize_buffer

carlopi · Oct 30, 2024 · e89d2d7 · e89d2d7
1 parent 1954bc8
commit e89d2d7
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 1 deletion.
diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
@@ -333,6 +333,7 @@ if(EMSCRIPTEN)
           _duckdb_web_query_run_buffer, \
           _duckdb_web_reset, \
           _duckdb_web_tokenize, \
+          _duckdb_web_tokenize_buffer, \
           _duckdb_web_udf_scalar_create \
       ]' \
       -s EXPORTED_RUNTIME_METHODS='[\"ccall\", \"stackSave\", \"stackAlloc\", \"stackRestore\"]' \

diff --git a/lib/src/webdb_api.cc b/lib/src/webdb_api.cc
@@ -151,6 +151,13 @@ void duckdb_web_tokenize(WASMResponse* packed, const char* query) {
     auto tokens = webdb.Tokenize(query);
     WASMResponseBuffer::Get().Store(*packed, arrow::Result(std::move(tokens)));
 }
+/// Tokenize a query
+void duckdb_web_tokenize_buffer(WASMResponse* packed, const uint8_t* buffer, size_t buffer_length) {
+    GET_WEBDB(*packed);
+    std::string_view query(reinterpret_cast<const char*>(buffer), buffer_length);
+    auto tokens = webdb.Tokenize(query);
+    WASMResponseBuffer::Get().Store(*packed, arrow::Result(std::move(tokens)));
+}
 /// Create scalar UDF queries
 void duckdb_web_udf_scalar_create(WASMResponse* packed, ConnectionHdl connHdl, const char* args) {
     auto c = reinterpret_cast<WebDB::Connection*>(connHdl);

diff --git a/packages/duckdb-wasm/src/bindings/bindings_base.ts b/packages/duckdb-wasm/src/bindings/bindings_base.ts
@@ -134,11 +134,17 @@ export abstract class DuckDBBindingsBase implements DuckDBBindings {
 
     /** Tokenize a script */
     public tokenize(text: string): ScriptTokens {
-        const [s, d, n] = callSRet(this.mod, 'duckdb_web_tokenize', ['string'], [text]);
+        const BUF = TEXT_ENCODER.encode(text);
+        const bufferPtr = this.mod._malloc(BUF.length );
+        const bufferOfs = this.mod.HEAPU8.subarray(bufferPtr, bufferPtr + BUF.length );
+        bufferOfs.set(BUF);
+        const [s, d, n] = callSRet(this.mod, 'duckdb_web_tokenize_buffer', ['number', 'number'], [bufferPtr, BUF.length]);
         if (s !== StatusCode.SUCCESS) {
+            this.mod._free(bufferPtr);
             throw new Error(readString(this.mod, d, n));
         }
         const res = readString(this.mod, d, n);
+        this.mod._free(bufferPtr);
         dropResponseBuffers(this.mod);
         return JSON.parse(res) as ScriptTokens;
     }