Skip to content

Commit

Permalink
Merge branch 'main' into next
Browse files Browse the repository at this point in the history
  • Loading branch information
carlopi committed Aug 23, 2024
2 parents 39d713b + 3946612 commit 3c6f7cd
Show file tree
Hide file tree
Showing 13 changed files with 137 additions and 37 deletions.
Original file line number Diff line number Diff line change
@@ -1,24 +1,22 @@
name: Create Internal issue when the "High Priority" label is applied
name: Create or Label Mirror Issue
on:
issues:
types:
- labeled

env:
GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
# an event triggering this workflow is either an issue or a pull request,
# hence only one of the numbers will be filled in the TITLE_PREFIX
TITLE_PREFIX: "[duckdb-wasm/#${{ github.event.issue.number }}]"
PUBLIC_ISSUE_TITLE: ${{ github.event.issue.title }}

jobs:
create_or_label_issue:
if: github.event.label.name == 'High Priority'
if: github.event.label.name == 'reproduced' || github.event.label.name == 'under review'
runs-on: ubuntu-latest
steps:
- name: Get mirror issue number
run: |
gh issue list --repo duckdblabs/duckdb-internal --search "${TITLE_PREFIX}" --json title,number --jq ".[] | select(.title | startswith(\"$TITLE_PREFIX\")).number" > mirror_issue_number.txt
gh issue list --repo duckdblabs/duckdb-internal --search "${TITLE_PREFIX}" --json title,number --state all --jq ".[] | select(.title | startswith(\"$TITLE_PREFIX\")).number" > mirror_issue_number.txt
echo "MIRROR_ISSUE_NUMBER=$(cat mirror_issue_number.txt)" >> $GITHUB_ENV
- name: Print whether mirror issue exists
Expand All @@ -31,6 +29,8 @@ jobs:
- name: Create or label issue
run: |
export LABEL="Wasm"
if [ "$MIRROR_ISSUE_NUMBER" == "" ]; then
gh issue create --repo duckdblabs/duckdb-internal --label "Wasm" --title "$TITLE_PREFIX - $PUBLIC_ISSUE_TITLE" --body "See https://github.com/duckdb/duckdb-wasm/issues/${{ github.event.issue.number }}"
gh issue create --repo duckdblabs/duckdb-internal --label "$LABEL" --title "$TITLE_PREFIX - $PUBLIC_ISSUE_TITLE" --body "See https://github.com/duckdb/duckdb-wasm/issues/${{ github.event.issue.number }}"
fi
48 changes: 48 additions & 0 deletions .github/workflows/InternalIssuesUpdateMirror.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
name: Update Mirror Issue
on:
issues:
types:
- closed
- reopened

env:
GH_TOKEN: ${{ secrets.DUCKDBLABS_BOT_TOKEN }}
TITLE_PREFIX: "[duckdb-wasm/#${{ github.event.issue.number }}]"

jobs:
update_mirror_issue:
runs-on: ubuntu-latest
steps:
- name: Get mirror issue number
run: |
gh issue list --repo duckdblabs/duckdb-internal --search "${TITLE_PREFIX}" --json title,number --state all --jq ".[] | select(.title | startswith(\"$TITLE_PREFIX\")).number" > mirror_issue_number.txt
echo "MIRROR_ISSUE_NUMBER=$(cat mirror_issue_number.txt)" >> $GITHUB_ENV
- name: Print whether mirror issue exists
run: |
if [ "$MIRROR_ISSUE_NUMBER" == "" ]; then
echo "Mirror issue with title prefix '$TITLE_PREFIX' does not exist yet"
else
echo "Mirror issue with title prefix '$TITLE_PREFIX' exists with number $MIRROR_ISSUE_NUMBER"
fi
- name: Add comment with status to mirror issue
run: |
if [ "$MIRROR_ISSUE_NUMBER" != "" ]; then
gh issue comment --repo duckdblabs/duckdb-internal $MIRROR_ISSUE_NUMBER --body "The issue has been ${{ github.event.action }} (https://github.com/duckdb/duckdb-wasm/issues/${{ github.event.issue.number }})."
fi
- name: Add closed label to mirror issue
if: github.event.action == 'closed'
run: |
if [ "$MIRROR_ISSUE_NUMBER" != "" ]; then
gh issue edit --repo duckdblabs/duckdb-internal $MIRROR_ISSUE_NUMBER --add-label "public closed" --remove-label "public reopened"
fi
- name: Reopen mirror issue and add reopened label
if: github.event.action == 'reopened'
run: |
if [ "$MIRROR_ISSUE_NUMBER" != "" ]; then
gh issue reopen --repo duckdblabs/duckdb-internal $MIRROR_ISSUE_NUMBER
gh issue edit --repo duckdblabs/duckdb-internal $MIRROR_ISSUE_NUMBER --add-label "public reopened" --remove-label "public closed"
fi
16 changes: 8 additions & 8 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ jobs:
- name: Prepare repository
run: |
(cd ./submodules/duckdb && git fetch --all --tags)
make patch_duckdb
make apply_patches
cd submodules/duckdb && git tag ${{ inputs.override_git_describe }}
- name: Prepare environment
Expand Down Expand Up @@ -296,7 +296,7 @@ jobs:
- name: Prepare repository
run: |
(cd ./submodules/duckdb && git fetch --all --tags)
make patch_duckdb
make apply_patches
cd submodules/duckdb && git tag ${{ inputs.override_git_describe }}
- name: Prepare environment
Expand Down Expand Up @@ -362,7 +362,7 @@ jobs:

- name: Prepare repository
run: |
make patch_duckdb
make apply_patches
cd submodules/duckdb && git tag ${{ inputs.override_git_describe }}
- uses: mymindstorm/setup-emsdk@v14
Expand Down Expand Up @@ -405,7 +405,7 @@ jobs:

- name: Prepare repository
run: |
make patch_duckdb
make apply_patches
cd submodules/duckdb && git tag ${{ inputs.override_git_describe }}
- uses: mymindstorm/setup-emsdk@v14
Expand Down Expand Up @@ -448,7 +448,7 @@ jobs:

- name: Prepare repository
run: |
make patch_duckdb
make apply_patches
cd submodules/duckdb && git tag ${{ inputs.override_git_describe }}
- uses: mymindstorm/setup-emsdk@v14
Expand Down Expand Up @@ -492,7 +492,7 @@ jobs:

- name: Prepare repository
run: |
make patch_duckdb
make apply_patches
cd submodules/duckdb && git tag ${{ inputs.override_git_describe }}
- uses: mymindstorm/setup-emsdk@v14
Expand Down Expand Up @@ -535,7 +535,7 @@ jobs:

- name: Prepare repository
run: |
make patch_duckdb
make apply_patches
cd submodules/duckdb && git tag ${{ inputs.override_git_describe }}
- uses: mymindstorm/setup-emsdk@v14
Expand Down Expand Up @@ -578,7 +578,7 @@ jobs:

- name: Prepare repository
run: |
make patch_duckdb
make apply_patches
cd submodules/duckdb && git tag ${{ inputs.override_git_describe }}
- uses: mymindstorm/setup-emsdk@v14
Expand Down
2 changes: 1 addition & 1 deletion .vscode/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"editor.formatOnSave": true,
"editor.trimAutoWhitespace": true,
"editor.codeActionsOnSave": {
"source.fixAll.eslint": true
"source.fixAll.eslint": "explicit"
},
"files.insertFinalNewline": true,
"rust-analyzer.procMacro.enable": true,
Expand Down
10 changes: 6 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -368,11 +368,11 @@ serve_loadable_base: wasmpack shell docs

.PHONY: serve_local
serve_local: build_loadable_unsigned serve_loadable_base
http-server packages/duckdb-wasm-app/build/release -o "#queries=v0,SET-custom_extension_repository%3D'http%3A%2F%2F127.0.0.1%3A8080%2Fextension_repository'~" -a 127.0.0.1 -p 8080
npx http-server packages/duckdb-wasm-app/build/release -o "#queries=v0,SET-custom_extension_repository%3D'http%3A%2F%2F127.0.0.1%3A8080%2Fextension_repository'~" -a 127.0.0.1 -p 8080

.PHONY: serve
serve: build_loadable serve_loadable_base
http-server packages/duckdb-wasm-app/build/release -o
npx http-server packages/duckdb-wasm-app/build/release -o

.PHONY: app_server
app_server:
Expand Down Expand Up @@ -434,8 +434,10 @@ build/docker_ci_image:
touch build/docker_ci_image

patch_duckdb:
find patches/duckdb/* -type f -name '*.patch' -print0 | xargs -0 cat | patch -p1 --forward -d submodules/duckdb
find patches/arrow/* -type f -name '*.patch' -print0 | xargs -0 cat | patch -p1 --forward -d submodules/arrow
(find patches/duckdb/* -type f -name '*.patch' -print0 | xargs -0 cat | patch -p1 --forward -d submodules/duckdb) || true
(find patches/arrow/* -type f -name '*.patch' -print0 | xargs -0 cat | patch -p1 --forward -d submodules/arrow) || true

apply_patches: patch_duckdb

submodules:
git submodule update --init --recursive
Expand Down
2 changes: 2 additions & 0 deletions lib/include/duckdb/web/config.h
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,8 @@ struct WebDBConfig {
/// Whether to allow unsigned extensions
bool allow_unsigned_extensions = false;

std::string custom_user_agent = "";

/// Read from a document
static WebDBConfig ReadFrom(std::string_view args_json);
};
Expand Down
6 changes: 5 additions & 1 deletion lib/src/config.cc
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,8 @@ WebDBConfig WebDBConfig::ReadFrom(std::string_view args_json) {
.s3_secret_access_key = "",
.s3_session_token = "",
},
.allow_unsigned_extensions = false};
.allow_unsigned_extensions = false,
.custom_user_agent = ""};
rapidjson::Document doc;
rapidjson::ParseResult ok = doc.Parse(args_json.begin(), args_json.size());
if (ok) {
Expand Down Expand Up @@ -102,6 +103,9 @@ WebDBConfig WebDBConfig::ReadFrom(std::string_view args_json) {
config.filesystem.reliable_head_requests = fs["reliableHeadRequests"].GetBool();
}
}
if (doc.HasMember("customUserAgent") && doc["customUserAgent"].IsString()) {
config.custom_user_agent = doc["customUserAgent"].GetString();
}
}
if (!config.query.cast_bigint_to_double.has_value()) {
config.query.cast_bigint_to_double = !duckdb_web_test_platform_feature(PlatformFeature::BIGINT64ARRAY);
Expand Down
4 changes: 3 additions & 1 deletion lib/src/webdb.cc
Original file line number Diff line number Diff line change
Expand Up @@ -319,7 +319,8 @@ arrow::Result<duckdb::unique_ptr<duckdb::QueryResult>> WebDB::Connection::Execut
if (v.IsLosslessDouble())
values.emplace_back(v.GetDouble());
else if (v.IsString())
values.emplace_back(v.GetString());
// Use GetStringLenght otherwise null bytes will be counted as terminators
values.emplace_back(string_t(v.GetString(), v.GetStringLength()));
else if (v.IsNull())
values.emplace_back(nullptr);
else if (v.IsBool())
Expand Down Expand Up @@ -825,6 +826,7 @@ arrow::Status WebDB::Open(std::string_view args_json) {
db_config.options.use_temporary_directory = false;
db_config.options.access_mode = access_mode;
db_config.options.duckdb_api = "wasm";
db_config.options.custom_user_agent = config_->custom_user_agent;
auto db = make_shared_ptr<duckdb::DuckDB>(config_->path, &db_config);
#ifndef WASM_LOADABLE_EXTENSIONS
duckdb_web_parquet_init(db.get());
Expand Down
2 changes: 1 addition & 1 deletion packages/duckdb-wasm-app/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
"react-dnd-html5-backend": "^16.0.1",
"react-dom": "^18.2.0",
"react-popper-tooltip": "^4.4.2",
"react-router-dom": "^6.22.3",
"react-router-dom": "^6.26.1",
"react-virtualized": "9.22.5",
"xterm": "^5.3.0",
"xterm-addon-fit": "^0.8.0",
Expand Down
4 changes: 4 additions & 0 deletions packages/duckdb-wasm/src/bindings/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -62,4 +62,8 @@ export interface DuckDBConfig {
* Whether to allow unsigned extensions
*/
allowUnsignedExtensions?: boolean;
/**
* Custom user agent string
*/
customUserAgent?: string;
}
36 changes: 36 additions & 0 deletions packages/duckdb-wasm/test/regression/github_1833.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
import * as duckdb from '../../src';

// https://github.com/duckdb/duckdb-wasm/issues/1833
export function test1833(db: () => duckdb.AsyncDuckDB): void {
let conn: duckdb.AsyncDuckDBConnection;
beforeEach(async () => {
await db().flushFiles();
conn = await db().connect();
});
afterEach(async () => {
await conn.close();
await db().flushFiles();
await db().dropFiles();
});
describe('GitHub issues', () => {
it('1833', async () => {
await conn.query(`
CREATE TABLE "Test" (value VARCHAR)
`);
const stmt = await conn.prepare(`
INSERT INTO "Test" (value)
VALUES (?)
`);
await stmt.query('🦆🦆🦆🦆🦆');
await stmt.query('goo␀se');
await stmt.query('goo\u0000se');
const result = await conn.query(`
SELECT * FROM "Test"
`);
expect(result.schema.fields.length).toBe(1);
expect(result.schema.fields[0].name).toBe('value');
expect(result.toArray().length).toEqual(3);
expect(result.toArray()[2].value.length).toEqual(6);
});
});
}
2 changes: 2 additions & 0 deletions packages/duckdb-wasm/test/regression/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { test448 } from './github_448.test';
import { test470 } from './github_470.test';
import { test477 } from './github_477.test';
import { test1467 } from './github_1467.test';
import { test1833 } from './github_1833.test';

export function testRegressionAsync(adb: () => duckdb.AsyncDuckDB): void {
test332(adb);
Expand All @@ -15,4 +16,5 @@ export function testRegressionAsync(adb: () => duckdb.AsyncDuckDB): void {
test470(adb);
test477(adb);
test1467(adb);
test1833(adb);
}
30 changes: 15 additions & 15 deletions yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -845,10 +845,10 @@
resolved "https://registry.yarnpkg.com/@react-dnd/shallowequal/-/shallowequal-4.0.1.tgz#60582702124b494c028215895cc938c56909feac"
integrity sha512-GL86egYUtKjzUlLHP0ePl9ghHnfUjvdK/g0W3CrZg4vpfyB7YLpv3bsmNJyD59j/c+I6EEDuH178LlUdsMaFfg==

"@remix-run/router@1.15.3":
version "1.15.3"
resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.15.3.tgz#d2509048d69dbb72d5389a14945339f1430b2d3c"
integrity sha512-Oy8rmScVrVxWZVOpEF57ovlnhpZ8CCPlnIIumVcV9nFdiSIrus99+Lw78ekXyGvVDlIsFJbSfmSovJUhCWYV3w==
"@remix-run/router@1.19.1":
version "1.19.1"
resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.19.1.tgz#984771bfd1de2715f42394c87fb716c1349e014f"
integrity sha512-S45oynt/WH19bHbIXjtli6QmwNYvaz+vtnubvNpNDvUOoA/OWh6j1OikIP3G+v5GHdxyC6EXoChG3HgYGEUfcg==

"@restart/hooks@^0.4.9":
version "0.4.9"
Expand Down Expand Up @@ -6569,20 +6569,20 @@ react-popper@^2.3.0:
react-fast-compare "^3.0.1"
warning "^4.0.2"

react-router-dom@^6.22.3:
version "6.22.3"
resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.22.3.tgz#9781415667fd1361a475146c5826d9f16752a691"
integrity sha512-7ZILI7HjcE+p31oQvwbokjk6OA/bnFxrhJ19n82Ex9Ph8fNAq+Hm/7KchpMGlTgWhUxRHMMCut+vEtNpWpowKw==
react-router-dom@^6.26.1:
version "6.26.1"
resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.26.1.tgz#a408892b41767a49dc94b3564b0e7d8e3959f623"
integrity sha512-veut7m41S1fLql4pLhxeSW3jlqs+4MtjRLj0xvuCEXsxusJCbs6I8yn9BxzzDX2XDgafrccY6hwjmd/bL54tFw==
dependencies:
"@remix-run/router" "1.15.3"
react-router "6.22.3"
"@remix-run/router" "1.19.1"
react-router "6.26.1"

react-router@6.22.3:
version "6.22.3"
resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.22.3.tgz#9d9142f35e08be08c736a2082db5f0c9540a885e"
integrity sha512-dr2eb3Mj5zK2YISHK++foM9w4eBnO23eKnZEDs7c880P6oKbrjz/Svg9+nxqtHQK+oMW4OtjZca0RqPglXxguQ==
react-router@6.26.1:
version "6.26.1"
resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.26.1.tgz#88c64837e05ffab6899a49df2a1484a22471e4ce"
integrity sha512-kIwJveZNwp7teQRI5QmwWo39A5bXRyqpH0COKKmPnyD2vBvDwgFXSqDUYtt1h+FEyfnE8eXr7oe0MxRzVwCcvQ==
dependencies:
"@remix-run/router" "1.15.3"
"@remix-run/router" "1.19.1"

react-transition-group@^4.4.5:
version "4.4.5"
Expand Down

0 comments on commit 3c6f7cd

Please sign in to comment.