duckdb · Mytherin · Jul 18, 2023 · Jul 17, 2023 · Jul 17, 2023 · Jul 17, 2023
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -378,6 +378,10 @@ jobs:
                   submodules: 'recursive'
                   fetch-depth: 0
 
+            - name: Prepare repository
+              run: |
+                  [ -f duckdb.patch ] && cd submodules/duckdb && git apply ../../duckdb.patch || echo "No patching needed"
+
             - uses: mymindstorm/setup-emsdk@v12
               with:
                   version: 'latest'
@@ -416,6 +420,10 @@ jobs:
                   submodules: 'recursive'
                   fetch-depth: 0
 
+            - name: Prepare repository
+              run: |
+                  [ -f duckdb.patch ] && cd submodules/duckdb && git apply ../../duckdb.patch || echo "No patching needed"
+
             - uses: mymindstorm/setup-emsdk@v12
               with:
                   version: 'latest'
@@ -454,6 +462,10 @@ jobs:
                   submodules: 'recursive'
                   fetch-depth: 0
 
+            - name: Prepare repository
+              run: |
+                  [ -f duckdb.patch ] && cd submodules/duckdb && git apply ../../duckdb.patch || echo "No patching needed"
+
             - uses: mymindstorm/setup-emsdk@v12
               with:
                   version: 'latest'

diff --git a/Makefile b/Makefile
@@ -276,7 +276,7 @@ wasm_debug: wasm_setup
 	${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh debug eh
 	${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh debug coi
 
-wasm: wasm_dev
+wasm: wasm_relperf
 
 .PHONY: wasm_star
 wasm_star: wasm_relsize wasm_relperf wasm_dev wasm_debug
@@ -285,11 +285,13 @@ wasm_star: wasm_relsize wasm_relperf wasm_dev wasm_debug
 .PHONY: js_debug
 js_debug: build/bootstrap wasm yarn_install
 	yarn workspace @duckdb/duckdb-wasm build:debug
+	yarn workspace @duckdb/duckdb-wasm test
 
 # Build the duckdb library in release mode
 .PHONY: js_release
 js_release: yarn_install
 	yarn workspace @duckdb/duckdb-wasm build:release
+	yarn workspace @duckdb/duckdb-wasm test
 
 # Build the duckdb docs
 .PHONY: docs

diff --git a/duckdb.patch b/duckdb.patch
@@ -0,0 +1,20 @@
+diff --git a/src/function/table/system/test_all_types.cpp b/src/function/table/system/test_all_types.cpp
+index 1652c3eccd..c9f3db2459 100644
+--- a/src/function/table/system/test_all_types.cpp
++++ b/src/function/table/system/test_all_types.cpp
+@@ -204,13 +204,14 @@ vector<TestType> TestAllTypesFun::GetTestTypes(bool use_large_enum) {
+ 	auto max_map_value = Value::MAP(ListType::GetChildType(map_type), map_values);
+ 	result.emplace_back(map_type, "map", std::move(min_map_value), std::move(max_map_value));
+
++#ifndef DUCKDB_FROM_DUCKDB_WASM
+ 	// union
+ 	child_list_t<LogicalType> members = {{"name", LogicalType::VARCHAR}, {"age", LogicalType::SMALLINT}};
+ 	auto union_type = LogicalType::UNION(members);
+ 	const Value &min = Value::UNION(members, 0, Value("Frank"));
+ 	const Value &max = Value::UNION(members, 1, Value::SMALLINT(5));
+ 	result.emplace_back(union_type, "union", min, max);
+-
++#endif
+ 	return result;
+ }
+
diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt
@@ -69,6 +69,8 @@ endif ()
 # -Os shrinks the module by 600KB but slows down TPC-H by a factor of 2!
 # Dedicated module?
 
+set(DUCKDB_PLATFORM "wasm_mvp")
+
 if(EMSCRIPTEN)
   # Release build
   if(CMAKE_BUILD_TYPE STREQUAL "Release")
@@ -107,6 +109,7 @@ if(EMSCRIPTEN)
   endif()
 
   if(WITH_WASM_EXCEPTIONS)
+    set(DUCKDB_PLATFORM "wasm_eh")
     set(CMAKE_CXX_FLAGS
         "${CMAKE_CXX_FLAGS} -fwasm-exceptions -DWEBDB_FAST_EXCEPTIONS=1")
   else()
@@ -126,6 +129,7 @@ if(EMSCRIPTEN)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -sUSE_PTHREADS=1 -DWEBDB_THREADS=1")
     set(DUCKDB_CXX_FLAGS "${CMAKE_CXX_FLAGS} -sUSE_PTHREADS=1")
     set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -sUSE_PTHREADS=1 -sPTHREAD_POOL_SIZE=4")
+    set(DUCKDB_PLATFORM "wasm_threads")
   else()
     set(CMAKE_CXX_FLAGS
         "${CMAKE_CXX_FLAGS} -sUSE_PTHREADS=0 -DDUCKDB_NO_THREADS=1")
@@ -134,6 +138,7 @@ if(EMSCRIPTEN)
     set(WASM_LINK_FLAGS "${WASM_LINK_FLAGS} -sUSE_PTHREADS=0")
     set(THREAD_LIBS)
   endif()
+  set(DUCKDB_CXX_FLAGS "${DUCKDB_CXX_FLAGS} -DDUCKDB_CUSTOM_PLATFORM=${DUCKDB_PLATFORM}")
 else()
   set(CMAKE_CXX_FLAGS
       "${CMAKE_CXX_FLAGS} -DWEBDB_FAST_EXCEPTIONS=1 -DWEBDB_THREADS=1")

diff --git a/lib/cmake/duckdb.cmake b/lib/cmake/duckdb.cmake
@@ -12,7 +12,7 @@ if(DUCKDB_LOCATION)
   set(DUCKDB_CORE_DIR ${DUCKDB_LOCATION})
 endif()
 
-set(DUCKDB_CXX_FLAGS "${DUCKDB_CXX_FLAGS} -Wno-unqualified-std-cast-call -DDUCKDB_DEBUG_NO_SAFETY")
+set(DUCKDB_CXX_FLAGS "${DUCKDB_CXX_FLAGS} -Wno-unqualified-std-cast-call -DDUCKDB_DEBUG_NO_SAFETY -DDUCKDB_FROM_DUCKDB_WASM")
 message("DUCKDB_CXX_FLAGS=${DUCKDB_CXX_FLAGS}")
 
 set(DUCKDB_EXTENSIONS "fts;excel;json")

diff --git a/packages/duckdb-wasm-app/package.json b/packages/duckdb-wasm-app/package.json
@@ -42,7 +42,7 @@
     },
     "devDependencies": {
         "@types/fontfaceobserver": "^2.1.0",
-        "@types/node": "^20.4.1",
+        "@types/node": "^20.4.2",
         "@types/react-dom": "^18.0.11",
         "@types/react-router-dom": "^5.3.3",
         "@types/react-virtualized": "^9.21.21",

diff --git a/packages/duckdb-wasm/package.json b/packages/duckdb-wasm/package.json
@@ -23,7 +23,7 @@
         "csv"
     ],
     "dependencies": {
-        "apache-arrow": "^11.0.0"
+        "apache-arrow": "^12.0.0"
     },
     "devDependencies": {
         "@types/emscripten": "^1.39.6",

diff --git a/packages/duckdb-wasm/src/bindings/connection.ts b/packages/duckdb-wasm/src/bindings/connection.ts
@@ -44,6 +44,7 @@ export class DuckDBConnection {
                 try {
                     resolve(this._bindings.pollPendingQuery(this._conn));
                 } catch (e: any) {
+                    console.log(e);
                     reject(e);
                 }
             });

diff --git a/packages/duckdb-wasm/src/bindings/runtime_browser.ts b/packages/duckdb-wasm/src/bindings/runtime_browser.ts
@@ -52,6 +52,7 @@ export const BROWSER_RUNTIME: DuckDBRuntime & {
             BROWSER_RUNTIME._fileInfoCache.set(fileId, file);
             return file;
         } catch (e: any) {
+            console.log(e);
             return null;
         }
     },
@@ -80,6 +81,7 @@ export const BROWSER_RUNTIME: DuckDBRuntime & {
 
             return BROWSER_RUNTIME._globalFileInfo;
         } catch (e: any) {
+            console.log(e);
             return null;
         }
     },
@@ -315,6 +317,7 @@ export const BROWSER_RUNTIME: DuckDBRuntime & {
                 mod.ccall('duckdb_web_fs_glob_add_path', null, ['string'], [path]);
             }
         } catch (e: any) {
+            console.log(e);
             failWith(mod, e.toString());
             return 0;
         }
@@ -338,6 +341,7 @@ export const BROWSER_RUNTIME: DuckDBRuntime & {
                 return xhr.status == 206 || xhr.status == 200;
             }
         } catch (e: any) {
+            console.log(e);
             return false;
         }
         return false;
@@ -434,6 +438,7 @@ export const BROWSER_RUNTIME: DuckDBRuntime & {
                             );
                         }
                     } catch (e) {
+                        console.log(e);
                         throw new Error(`Range request for ${file.dataUrl} failed with error: ${e}"`);
                     }
                 }
@@ -458,6 +463,7 @@ export const BROWSER_RUNTIME: DuckDBRuntime & {
             }
             return 0;
         } catch (e: any) {
+            console.log(e);
             failWith(mod, e.toString());
             return 0;
         }

diff --git a/packages/duckdb-wasm/src/bindings/runtime_node.ts b/packages/duckdb-wasm/src/bindings/runtime_node.ts
@@ -50,6 +50,7 @@ export const NODE_RUNTIME: DuckDBRuntime & {
             NODE_RUNTIME._fileInfoCache.set(fileId, info);
             return info as DuckDBFileInfo;
         } catch (e: any) {
+            console.log(e);
             failWith(mod, e.toString());
             return null;
         }
@@ -95,6 +96,7 @@ export const NODE_RUNTIME: DuckDBRuntime & {
                     failWith(mod, 'Unsupported data protocol');
             }
         } catch (e: any) {
+            console.log(e);
             failWith(mod, e.toString());
         }
         return 0;
@@ -120,6 +122,7 @@ export const NODE_RUNTIME: DuckDBRuntime & {
                     break;
             }
         } catch (e: any) {
+            console.log(e);
             failWith(mod, e.toString());
         }
         return 0;
@@ -139,6 +142,7 @@ export const NODE_RUNTIME: DuckDBRuntime & {
                     failWith(mod, 'Unsupported data protocol');
             }
         } catch (e: any) {
+            console.log(e);
             failWith(mod, e.toString());
         }
         return 0;
@@ -162,6 +166,7 @@ export const NODE_RUNTIME: DuckDBRuntime & {
                     failWith(mod, 'Unsupported data protocol');
             }
         } catch (e: any) {
+            console.log(e);
             failWith(mod, e.toString());
         }
         return 0;
@@ -186,6 +191,7 @@ export const NODE_RUNTIME: DuckDBRuntime & {
                     failWith(mod, 'Unsupported data protocol');
             }
         } catch (e: any) {
+            console.log(e);
             failWith(mod, e.toString());
         }
         return 0;
@@ -209,6 +215,7 @@ export const NODE_RUNTIME: DuckDBRuntime & {
                     failWith(mod, 'Unsupported data protocol');
             }
         } catch (e: any) {
+            console.log(e);
             failWith(mod, e.toString());
         }
         return 0;
@@ -219,6 +226,7 @@ export const NODE_RUNTIME: DuckDBRuntime & {
             const path = decodeText(mod.HEAPU8.subarray(pathPtr, pathPtr + pathLen));
             return fs.existsSync(path);
         } catch (e: any) {
+            console.log(e);
             failWith(mod, e.toString());
             return false;
         }
@@ -228,6 +236,7 @@ export const NODE_RUNTIME: DuckDBRuntime & {
             const path = decodeText(mod.HEAPU8.subarray(pathPtr, pathPtr + pathLen));
             return fs.mkdirSync(path);
         } catch (e: any) {
+            console.log(e);
             failWith(mod, e.toString());
             return 0;
         }
@@ -237,6 +246,7 @@ export const NODE_RUNTIME: DuckDBRuntime & {
             const path = decodeText(mod.HEAPU8.subarray(pathPtr, pathPtr + pathLen));
             return fs.rmdirSync(path);
         } catch (e: any) {
+            console.log(e);
             failWith(mod, e.toString());
             return 0;
         }
@@ -253,6 +263,7 @@ export const NODE_RUNTIME: DuckDBRuntime & {
                 mod.ccall('duckdb_web_fs_glob_add_path', null, ['string'], [entry]);
             }
         } catch (e: any) {
+            console.log(e);
             failWith(mod, e.toString());
             return 0;
         }
@@ -278,6 +289,7 @@ export const NODE_RUNTIME: DuckDBRuntime & {
             const path = decodeText(mod.HEAPU8.subarray(pathPtr, pathPtr + pathLen));
             return fs.existsSync(path);
         } catch (e: any) {
+            console.log(e);
             failWith(mod, e.toString());
             return false;
         }
@@ -287,6 +299,7 @@ export const NODE_RUNTIME: DuckDBRuntime & {
             const path = decodeText(mod.HEAPU8.subarray(pathPtr, pathPtr + pathLen));
             return fs.rmSync(path);
         } catch (e: any) {
+            console.log(e);
             failWith(mod, e.toString());
             return 0;
         }

diff --git a/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts b/packages/duckdb-wasm/src/parallel/worker_dispatcher.ts
@@ -90,6 +90,7 @@ export abstract class AsyncDuckDBDispatcher implements Logger {
                     });
                     this.sendOK(request);
                 } catch (e: any) {
+                    console.log(e);
                     this._bindings = null;
                     this.failWith(request, e);
                 }
@@ -395,6 +396,7 @@ export abstract class AsyncDuckDBDispatcher implements Logger {
                 }
             }
         } catch (e: any) {
+            console.log(e);
             return this.failWith(request, e);
         }
     }

diff --git a/packages/duckdb-wasm/test/bindings.test.ts b/packages/duckdb-wasm/test/bindings.test.ts
@@ -41,6 +41,20 @@ export function testBindings(db: () => duckdb.DuckDBBindings, baseURL: string):
             });
         });
 
+        describe('Check platform', () => {
+            it('Platform check', async () => {
+                await db().reset();
+                conn = db().connect();
+                const version = conn.query<{ name: arrow.Utf8 }>(
+                    "PRAGMA platform;",
+                );
+                const rows = version.getChildAt(0)?.toArray();
+                expect(rows.length).toEqual(1);
+                expect(rows[0].toString().substr(0,5)).toEqual("wasm_");
+                await db().reset();
+            });
+        });
+
         //describe('Open', () => {
         // XXX apparently synchronous XHR on the main thread does not allow for arraybuffer response type?
         // it('Remote TPCH 0_01', async () => {

diff --git a/submodules/arrow b/submodules/arrow
diff --git a/submodules/duckdb b/submodules/duckdb
diff --git a/submodules/rapidjson b/submodules/rapidjson
diff --git a/yarn.lock b/yarn.lock
@@ -954,10 +954,10 @@
   resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a"
   integrity sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==
 
-"@types/node@*", "@types/node@>=10.0.0", "@types/node@^20.4.1":
-  version "20.4.1"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.4.1.tgz#a6033a8718653c50ac4962977e14d0f984d9527d"
-  integrity sha512-JIzsAvJeA/5iY6Y/OxZbv1lUcc8dNSE77lb2gnBH+/PJ3lFR1Ccvgwl5JWnHAkNHcRsT0TbpVOsiMKZ1F/yyJg==
+"@types/node@*", "@types/node@>=10.0.0", "@types/node@^20.4.2":
+  version "20.4.2"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.4.2.tgz#129cc9ae69f93824f92fac653eebfb4812ab4af9"
+  integrity sha512-Dd0BYtWgnWJKwO1jkmTrzofjK2QXXcai0dmtzvIBhcA+RsG5h8R3xlyta0kGOZRNfL9GuRtb1knmPEhQrePCEw==
 
 "@types/node@18.7.23":
   version "18.7.23"
+11 −8		CMakeLists.txt
+2 −3		bin/unittestschema/address.json
+69 −0		bin/unittestschema/idandref.json
+10 −10		contrib/natvis/rapidjson.natvis
+1 −1		doc/dom.zh-cn.md
+6 −4		doc/features.md
+8 −0		doc/schema.md
+4 −1		example/schemavalidator/schemavalidator.cpp
+2 −1		include/rapidjson/allocators.h
+26 −10		include/rapidjson/document.h
+62 −8		include/rapidjson/error/en.h
+77 −8		include/rapidjson/error/error.h
+14 −7		include/rapidjson/internal/biginteger.h
+5 −1		include/rapidjson/internal/diyfp.h
+7 −3		include/rapidjson/internal/dtoa.h
+14 −0		include/rapidjson/internal/strfunc.h
+9 −6		include/rapidjson/internal/strtod.h
+84 −23		include/rapidjson/pointer.h
+9 −9		include/rapidjson/rapidjson.h
+22 −20		include/rapidjson/reader.h
+818 −204		include/rapidjson/schema.h
+481 −0		include/rapidjson/uri.h
+1 −0		test/unittest/CMakeLists.txt
+1 −1		test/unittest/cursorstreamwrappertest.cpp
+1 −0		test/unittest/dtoatest.cpp
+1 −1		test/unittest/encodedstreamtest.cpp
+1 −1		test/unittest/ostreamwrappertest.cpp
+65 −5		test/unittest/pointertest.cpp
+142 −0		test/unittest/readertest.cpp
+1,005 −17		test/unittest/schematest.cpp
+7 −7		test/unittest/simdtest.cpp
+725 −0		test/unittest/uritest.cpp
+18 −18		test/unittest/valuetest.cpp
+9 −0		test/valgrind.supp