Main

Move workflow to master #646

Workflow file for this run

	name: 'Main'
	on:
	push:
	branches:
	- '**'
	- '!main'
	- '!feature'
	pull_request:
	create:
	tags:
	- v*
	workflow_dispatch:

	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref \|\| '' }}-${{ github.base_ref \|\| '' }}-${{ github.ref != 'refs/heads/main' \|\| github.sha }}
	cancel-in-progress: true

	jobs:
	clang_format:
	name: Clang-Format
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: 'recursive'

	- name: Lint ./lib
	uses: duckdb/duckdb-wasm-ci-env@master
	with:
	script: \|-
	python3 ./scripts/run_clang_format.py \
	--exclude ./lib/build \
	--exclude ./lib/third_party \
	-r ./lib/

	eslint:
	name: ESLint
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: 'recursive'

	- name: Cache node_modules
	uses: actions/cache@v3
	with:
	path: \|
	./node_modules
	./packages/benchmarks/node_modules
	./packages/duckdb-wasm/node_modules
	./packages/duckdb-wasm-shell/node_modules
	key: ${{ runner.os }}-yarn-${{ hashFiles('./yarn.lock') }}
	restore-keys: \|
	${{ runner.os }}-yarn-

	- name: Prepare repository
	uses: duckdb/duckdb-wasm-ci-env@v0.11
	with:
	script: \|-
	yarn install --frozen-lockfile --prefer-offline

	- name: Lint @duckdb/duckdb-wasm
	uses: duckdb/duckdb-wasm-ci-env@v0.11
	with:
	script: \|-
	yarn workspace @duckdb/duckdb-wasm run lint

	- name: Lint @duckdb/duckdb-wasm-shell
	uses: duckdb/duckdb-wasm-ci-env@v0.11
	with:
	script: \|-
	yarn workspace @duckdb/duckdb-wasm-shell run lint

	- name: Lint @duckdb/benchmarks
	uses: duckdb/duckdb-wasm-ci-env@v0.11
	with:
	script: \|-
	yarn workspace @duckdb/benchmarks run lint

	tpchgen:
	name: TPCH Generator
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: 'recursive'

	- name: Cache generator
	uses: actions/cache@v3
	id: cache-generator
	with:
	path: ./submodules/tpch-dbgen/dbgen/dbgen
	key: ${{ runner.os }}-tpch-dbgen

	- name: Build generator
	if: steps.cache-generator.outputs.cache-hit != 'true'
	uses: duckdb/duckdb-wasm-ci-env@master
	with:
	script: \|-
	make -C ./submodules/tpch-dbgen/dbgen/ dbgen

	- name: Upload artifact
	uses: actions/upload-artifact@v3
	with:
	name: tpch-dbgen
	path: \|
	./submodules/tpch-dbgen/dbgen/dbgen
	retention-days: 1

	dataprep:
	name: Dataprep
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: 'recursive'

	- name: Cache rust build
	uses: actions/cache@v3
	with:
	path: \|
	./.cargo/bin/
	./.cargo/registry/index/
	./.cargo/registry/cache/
	./.cargo/git/db/
	./target
	key: ${{ runner.os }}-dataprep-${{ hashFiles('./Cargo.lock') }}-${{ hashFiles('./tools/dataprep/src/*.rs') }}
	restore-keys: \|
	${{ runner.os }}-dataprep-

	- name: Build generator
	uses: duckdb/duckdb-wasm-ci-env@master
	with:
	script: \|-
	cargo build --manifest-path=./Cargo.toml --release -p dataprep

	- name: Upload artifact
	uses: actions/upload-artifact@v3
	with:
	name: dataprep
	path: \|
	./target/release/dataprep
	retention-days: 1

	duckdb_shell:
	name: DuckDB Shell
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: 'recursive'

	- name: Git submodule status
	run: \|
	git submodule status ./submodules/duckdb > git_submodule_status.txt

	- name: Cache ccache
	uses: actions/cache@v3
	with:
	path: \|
	./.ccache
	key: ${{ runner.os }}-duckdb-${{ hashFiles('git_submodule_status.txt') }}
	restore-keys: \|
	${{ runner.os }}-duckdb-

	- name: Build DuckDB shell
	uses: duckdb/duckdb-wasm-ci-env@master
	with:
	script: \|-
	ccache -z
	./scripts/build_duckdb_shell.sh
	ccache -s

	- name: Upload artifact
	uses: actions/upload-artifact@v3
	with:
	name: duckdb-shell
	path: ./submodules/duckdb/build/Release/duckdb
	retention-days: 1

	native_debug:
	name: Native / Debug
	runs-on: ubuntu-latest
	needs:
	- dataprep
	- tpchgen
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: 'recursive'
	fetch-depth: 0

	- uses: actions/download-artifact@v3
	with:
	name: dataprep
	path: ./target/release/

	- uses: actions/download-artifact@v3
	with:
	name: tpch-dbgen
	path: ./submodules/tpch-dbgen/dbgen/

	- name: Git submodule status
	run: \|
	git submodule status > git_submodule_status.txt

	- name: Cache native build
	uses: actions/cache@v3
	with:
	path: \|
	./.cargo/bin/
	./.cargo/registry/index/
	./.cargo/registry/cache/
	./.cargo/git/db/
	./.ccache
	./.emscripten_cache
	key: ${{ runner.os }}-native2-debug-${{ hashFiles('git_submodule_status.txt') }}-${{ hashFiles('lib/src/') }}-${{ hashFiles('lib/include/') }}
	restore-keys: \|
	${{ runner.os }}-native2-debug-${{ hashFiles('git_submodule_status.txt') }}
	${{ runner.os }}-native2-debug-
	${{ runner.os }}-native2-

	- name: Prepare repository
	run: \|
	(cd ./submodules/duckdb && git fetch --all --tags)
	[ -f duckdb.patch ] && cd submodules/duckdb && git apply ../../duckdb.patch \|\| echo "No patching needed"

	- name: Prepare environment
	uses: duckdb/duckdb-wasm-ci-env@master
	with:
	script: \|-
	mkdir -p ./lib/build/debug ./reports
	./scripts/generate_tpch_tbl.sh 0.01
	./scripts/generate_tpch_arrow.sh 0.01
	./scripts/generate_uni.sh
	ccache --max-size 200M
	ccache -s

	- name: Build project
	uses: duckdb/duckdb-wasm-ci-env@master
	with:
	script: \|-
	cmake \
	-S./lib/ \
	-B./lib/build/debug \
	-DCODE_COVERAGE=1 \
	-DCMAKE_C_COMPILER_LAUNCHER=ccache \
	-DCMAKE_C_COMPILER=clang \
	-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
	-DCMAKE_CXX_COMPILER=clang++ \
	-DCMAKE_BUILD_TYPE=Debug
	ccache -z
	make -C./lib/build/debug -j`nproc` \|\| exit 1
	ccache -s

	- name: Test project
	uses: duckdb/duckdb-wasm-ci-env@master
	with:
	script: \|-
	export LLVM_PROFILE_FILE=./reports/lib.profraw
	./lib/build/debug/tester \
	--source_dir=./lib/ \
	--gtest_output=xml:./reports/tests_lib_debug.xml

	- name: Code Coverage
	uses: duckdb/duckdb-wasm-ci-env@master
	with:
	script: \|-
	llvm-profdata merge \
	-sparse ./reports/lib.profraw \
	-o ./reports/lib.profdata
	llvm-cov show ./lib/build/debug/tester \
	--instr-profile ./reports/lib.profdata \
	-ignore-filename-regex submodules \
	> ./reports/coverage_lib.txt

	native_release:
	name: Native / Release
	runs-on: ubuntu-latest
	needs:
	- dataprep
	- tpchgen
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: 'recursive'
	fetch-depth: 0

	- uses: actions/download-artifact@v3
	with:
	name: dataprep
	path: ./target/release/

	- uses: actions/download-artifact@v3
	with:
	name: tpch-dbgen
	path: ./submodules/tpch-dbgen/dbgen/

	- name: Git submodule status
	run: \|
	git submodule status > git_submodule_status.txt

	- name: Cache native build
	uses: actions/cache@v3
	with:
	path: \|
	./.cargo/bin/
	./.cargo/registry/index/
	./.cargo/registry/cache/
	./.cargo/git/db/
	./.ccache
	./.emscripten_cache
	key: ${{ runner.os }}-native2-release-${{ hashFiles('git_submodule_status.txt') }}-${{ hashFiles('lib/src/') }}-${{ hashFiles('lib/include/') }}
	restore-keys: \|
	${{ runner.os }}-native2-release-${{ hashFiles('git_submodule_status.txt') }}
	${{ runner.os }}-native2-release-
	${{ runner.os }}-native2-

	- name: Prepare repository
	run: \|
	(cd ./submodules/duckdb && git fetch --all --tags)
	[ -f duckdb.patch ] && cd submodules/duckdb && git apply ../../duckdb.patch \|\| echo "No patching needed"

	- name: Prepare environment
	uses: duckdb/duckdb-wasm-ci-env@master
	with:
	script: \|-
	mkdir -p ./lib/build/release ./reports
	./scripts/generate_tpch_tbl.sh 0.01
	./scripts/generate_tpch_arrow.sh 0.01
	./scripts/generate_uni.sh
	ccache --max-size 200M
	ccache -s

	- name: Build project
	uses: duckdb/duckdb-wasm-ci-env@master
	with:
	script: \|-
	cmake \
	-S./lib/ \
	-B./lib/build/release \
	-DCODE_COVERAGE=1 \
	-DCMAKE_C_COMPILER_LAUNCHER=ccache \
	-DCMAKE_C_COMPILER=clang \
	-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
	-DCMAKE_CXX_COMPILER=clang++ \
	-DCMAKE_BUILD_TYPE=Release
	ccache -z
	make -C./lib/build/release -j`nproc` \|\| exit 1
	ccache -s

	- name: Test project
	uses: duckdb/duckdb-wasm-ci-env@master
	with:
	script: \|-
	export LLVM_PROFILE_FILE=./reports/lib.profraw
	./lib/build/release/tester \
	--source_dir=./lib/ \
	--gtest_output=xml:./reports/tests_lib_release.xml

	- name: Code Coverage
	uses: duckdb/duckdb-wasm-ci-env@master
	with:
	script: \|-
	llvm-profdata merge \
	-sparse ./reports/lib.profraw \
	-o ./reports/lib.profdata
	llvm-cov show ./lib/build/release/tester \
	--instr-profile ./reports/lib.profdata \
	-ignore-filename-regex submodules \
	> ./reports/coverage_lib.txt

	wasm_mvp:
	name: Wasm / mvp
	runs-on: ubuntu-latest
	needs:
	- dataprep
	- tpchgen
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: 'recursive'
	fetch-depth: 0

	- name: Prepare repository
	run: \|
	[ -f duckdb.patch ] && cd submodules/duckdb && git apply ../../duckdb.patch \|\| echo "No patching needed"

	- uses: mymindstorm/setup-emsdk@v12
	with:
	version: 'latest'

	- name: Setup Ccache
	uses: hendrikmuhs/ccache-action@main
	with:
	key: ${{ github.job }}

	- name: Git submodule status
	run: \|
	git submodule status > git_submodule_status.txt

	- name: Build Wasm module
	run: \|
	./scripts/wasm_build_lib.sh relperf mvp

	- name: Upload artifact
	uses: actions/upload-artifact@v3
	with:
	name: wasm-mvp
	path: \|
	./packages/duckdb-wasm/src/bindings/duckdb-mvp.js
	./packages/duckdb-wasm/src/bindings/duckdb-mvp.wasm
	retention-days: 1

	wasm_eh:
	name: Wasm / eh
	runs-on: ubuntu-latest
	needs:
	- dataprep
	- tpchgen
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: 'recursive'
	fetch-depth: 0

	- name: Prepare repository
	run: \|
	[ -f duckdb.patch ] && cd submodules/duckdb && git apply ../../duckdb.patch \|\| echo "No patching needed"

	- uses: mymindstorm/setup-emsdk@v12
	with:
	version: 'latest'

	- name: Setup Ccache
	uses: hendrikmuhs/ccache-action@main
	with:
	key: ${{ github.job }}

	- name: Git submodule status
	run: \|
	git submodule status > git_submodule_status.txt

	- name: Build Wasm module
	run: \|
	./scripts/wasm_build_lib.sh relperf eh

	- name: Upload artifact
	uses: actions/upload-artifact@v3
	with:
	name: wasm-eh
	path: \|
	./packages/duckdb-wasm/src/bindings/duckdb-eh.js
	./packages/duckdb-wasm/src/bindings/duckdb-eh.wasm
	retention-days: 1

	wasm_coi:
	name: Wasm / coi
	runs-on: ubuntu-latest
	needs:
	- dataprep
	- tpchgen
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: 'recursive'
	fetch-depth: 0

	- name: Prepare repository
	run: \|
	[ -f duckdb.patch ] && cd submodules/duckdb && git apply ../../duckdb.patch \|\| echo "No patching needed"

	- uses: mymindstorm/setup-emsdk@v12
	with:
	version: 'latest'

	- name: Setup Ccache
	uses: hendrikmuhs/ccache-action@main
	with:
	key: ${{ github.job }}

	- name: Git submodule status
	run: \|
	git submodule status > git_submodule_status.txt

	- name: Build Wasm module
	run: \|
	./scripts/wasm_build_lib.sh relperf coi

	- name: Upload artifact
	uses: actions/upload-artifact@v3
	with:
	name: wasm-coi
	path: \|
	./packages/duckdb-wasm/src/bindings/duckdb-coi.js
	./packages/duckdb-wasm/src/bindings/duckdb-coi.pthread.js
	./packages/duckdb-wasm/src/bindings/duckdb-coi.wasm
	retention-days: 1

	wasm_mvp_loadable:
	name: Wasm / mvp (loadable version)
	runs-on: ubuntu-latest
	needs:
	- dataprep
	- tpchgen
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: 'recursive'
	fetch-depth: 0

	- name: Prepare repository
	run: \|
	[ -f duckdb.patch ] && cd submodules/duckdb && git apply ../../duckdb.patch \|\| echo "No patching needed"

	- uses: mymindstorm/setup-emsdk@v12
	with:
	version: 'latest'

	- name: Setup Ccache
	uses: hendrikmuhs/ccache-action@main
	with:
	key: ${{ github.job }}

	- name: Git submodule status
	run: \|
	git submodule status > git_submodule_status.txt

	- name: Build Wasm module
	run: \|
	cp .github/config/extension_config_wasm.cmake submodules/duckdb/extension/extension_config.cmake
	DUCKDB_WASM_LOADABLE_EXTENSIONS="signed" GEN=ninja ./scripts/wasm_build_lib.sh relsize mvp
	bash ./scripts/build_loadable.sh relsize mvp

	- name: Upload artifact
	uses: actions/upload-artifact@v3
	with:
	name: wasm-mvp-loadable
	path: \|
	./packages/duckdb-wasm/src/bindings/duckdb-mvp.js
	./packages/duckdb-wasm/src/bindings/duckdb-mvp.wasm
	retention-days: 1

	wasm_eh_loadable:
	name: Wasm / eh (loadable version)
	runs-on: ubuntu-latest
	needs:
	- dataprep
	- tpchgen
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: 'recursive'
	fetch-depth: 0

	- name: Prepare repository
	run: \|
	[ -f duckdb.patch ] && cd submodules/duckdb && git apply ../../duckdb.patch \|\| echo "No patching needed"

	- uses: mymindstorm/setup-emsdk@v12
	with:
	version: 'latest'

	- name: Setup Ccache
	uses: hendrikmuhs/ccache-action@main
	with:
	key: ${{ github.job }}

	- name: Git submodule status
	run: \|
	git submodule status > git_submodule_status.txt

	- name: Build Wasm module
	run: \|
	cp .github/config/extension_config_wasm.cmake submodules/duckdb/extension/extension_config.cmake
	DUCKDB_WASM_LOADABLE_EXTENSIONS="signed" GEN=ninja ./scripts/wasm_build_lib.sh relsize eh
	bash ./scripts/build_loadable.sh relsize eh

	- name: Upload artifact
	uses: actions/upload-artifact@v3
	with:
	name: wasm-eh-loadable
	path: \|
	./packages/duckdb-wasm/src/bindings/duckdb-eh.js
	./packages/duckdb-wasm/src/bindings/duckdb-eh.wasm
	retention-days: 1

	js_libs:
	name: Js / Libraries
	runs-on: ubuntu-latest
	needs:
	- wasm_mvp
	- wasm_eh
	- wasm_coi
	- duckdb_shell
	- clang_format
	- eslint
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: 'recursive'
	fetch-depth: 0

	- uses: actions/setup-node@v3
	with:
	node-version: '18.x'
	registry-url: 'https://registry.npmjs.org'

	- name: Git submodule status
	run: \|
	git submodule status > git_submodule_status.txt

	- name: Cache rust build
	uses: actions/cache@v3
	with:
	path: \|
	./.cargo/bin/
	./.cargo/registry/index/
	./.cargo/registry/cache/
	./.cargo/git/db/
	./target
	key: ${{ runner.os }}-shell-${{ hashFiles('./Cargo.lock') }}-${{ hashFiles('./packages/duckdb-wasm-shell/crate/src/**') }}
	restore-keys: \|
	${{ runner.os }}-shell-

	- name: Cache node_modules
	uses: actions/cache@v3
	with:
	path: \|
	./node_modules
	./packages/benchmarks/node_modules
	./packages/duckdb-wasm/node_modules
	./packages/duckdb-wasm-shell/node_modules
	key: ${{ runner.os }}-yarn-${{ hashFiles('./yarn.lock') }}
	restore-keys: \|
	${{ runner.os }}-yarn-

	- uses: actions/download-artifact@v3
	with:
	name: dataprep
	path: ./target/release/

	- uses: actions/download-artifact@v3
	with:
	name: tpch-dbgen
	path: ./submodules/tpch-dbgen/dbgen/

	- uses: actions/download-artifact@v3
	with:
	name: duckdb-shell
	path: ./submodules/duckdb/build/Release/

	- uses: actions/download-artifact@v3
	with:
	name: wasm-mvp
	path: ./packages/duckdb-wasm/src/bindings/

	- uses: actions/download-artifact@v3
	with:
	name: wasm-eh
	path: ./packages/duckdb-wasm/src/bindings/

	- uses: actions/download-artifact@v3
	with:
	name: wasm-coi
	path: ./packages/duckdb-wasm/src/bindings/

	- name: Prepare repository
	run: \|
	git fetch --tags --no-recurse-submodules -f
	(cd ./submodules/duckdb && git fetch --all --tags)

	- name: Prepare environment
	uses: duckdb/duckdb-wasm-ci-env@master
	with:
	script: \|-
	git config --global --add safe.directory '*'
	mkdir -p ./lib/build/wasm/release ./reports
	yarn install --frozen-lockfile
	./scripts/generate_tpch_tbl.sh 0.01
	./scripts/generate_tpch_arrow.sh 0.01
	./scripts/generate_tpch_duckdb.sh 0.01
	./scripts/generate_tpch_sqlite.sh 0.01
	./scripts/generate_uni.sh
	./scripts/npm_version.sh

	- name: Build @duckdb/duckdb-wasm
	shell: bash
	run: \|
	rm -rf ./packages/duckdb-wasm/dist/
	yarn workspace @duckdb/duckdb-wasm build:release
	yarn workspace @duckdb/duckdb-wasm docs

	- name: Build @duckdb/duckdb-wasm-shell
	shell: bash
	run: \|
	rm -rf ./packages/duckdb-wasm-shell/dist/
	yarn workspace @duckdb/duckdb-wasm-shell install:wasmpack
	yarn workspace @duckdb/duckdb-wasm-shell build:release

	- name: Build @duckdb/duckdb-wasm-app
	shell: bash
	run: \|
	rm -rf ./packages/duckdb-wasm-app/build/
	yarn workspace @duckdb/duckdb-wasm-app build:release

	- name: Test @duckdb/duckdb-wasm on Chrome
	shell: bash
	run: \|
	CHROME_BIN=`which google-chrome` yarn workspace @duckdb/duckdb-wasm test:chrome

	# - name: Test @duckdb/duckdb-wasm on Firefox
	# uses: duckdb/duckdb-wasm-ci-env@v0.11
	# with:
	# script: \|-
	# yarn workspace @duckdb/duckdb-wasm test:firefox

	- name: Test @duckdb/duckdb-wasm on Node.js
	shell: bash
	run: \|
	yarn workspace @duckdb/duckdb-wasm test:node

	- name: Coverage @duckdb/duckdb-wasm
	shell: bash
	run: \|
	CHROME_BIN=`which google-chrome` yarn workspace @duckdb/duckdb-wasm test:chrome:coverage

	- name: Build examples
	shell: bash
	run: \|
	yarn install
	yarn workspace @duckdb/duckdb-wasm-examples-bare-node test
	yarn workspace @duckdb/duckdb-wasm-examples-bare-browser build
	yarn workspace @duckdb/duckdb-wasm-examples-esbuild-node build
	yarn workspace @duckdb/duckdb-wasm-examples-esbuild-node test
	yarn workspace @duckdb/duckdb-wasm-examples-esbuild-browser build

	- name: Package
	shell: bash
	run: zip -r duckdb-wasm-packages.zip ./packages

	- uses: actions/upload-artifact@v3
	with:
	name: duckdb-wasm-packages
	path: duckdb-wasm-packages.zip

	- name: Publish to NPM
	env:
	NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }}
	if: env.NODE_AUTH_TOKEN != null && (startsWith(github.ref, 'refs/tags/v') \|\| github.event_name == 'workflow_dispatch')
	run: \|
	./scripts/npm_publish_lib.sh
	./scripts/npm_publish_shell.sh
	./scripts/npm_publish_react.sh

	- name: Build @duckdb/benchmarks
	shell: bash
	run: \|
	rm -r packages/duckdb-wasm/node_modules/*
	yarn workspace @duckdb/benchmarks build
	yarn workspace @duckdb/benchmarks bench:system:sort:int
	yarn workspace @duckdb/benchmarks bench:system:join:2
	yarn workspace @duckdb/benchmarks bench:system:join:3
	yarn workspace @duckdb/benchmarks bench:system:regex
	yarn workspace @duckdb/benchmarks bench:system:sum:int
	yarn workspace @duckdb/benchmarks bench:system:sum:csv

	- name: Upload reports
	uses: actions/upload-artifact@v3
	with:
	name: reports_micro
	path: \|
	./reports/benchmark_system_sort_int.json
	./reports/benchmark_system_join_2.json
	./reports/benchmark_system_join_3.json
	./reports/benchmark_system_regex.json
	./reports/benchmark_system_sum_csv.json
	./reports/benchmark_system_sum_int.json
	retention-days: 1

	- name: Preparation TPCH 0.01
	if: github.ref == 'refs/heads/main'
	uses: duckdb/duckdb-wasm-ci-env@master
	with:
	script: \|-
	git config --global --add safe.directory '*'
	mkdir -p ./lib/build/wasm/release ./reports
	yarn install --frozen-lockfile
	./scripts/generate_tpch_tbl.sh 0.01
	./scripts/generate_tpch_arrow.sh 0.01
	./scripts/generate_tpch_duckdb.sh 0.01
	./scripts/generate_tpch_sqlite.sh 0.01

	- name: Benchmarks TPCH 0.01
	if: github.ref == 'refs/heads/main'
	shell: bash
	run: \|
	yarn workspace @duckdb/benchmarks bench:system:tpch:duckdb 0.01
	yarn workspace @duckdb/benchmarks bench:system:tpch:sqljs 0.01
	yarn workspace @duckdb/benchmarks bench:system:tpch:arquero 0.01
	yarn workspace @duckdb/benchmarks bench:system:tpch:lovefield 0.01

	- name: Upload reports TPCH 0.01
	if: github.ref == 'refs/heads/main'
	uses: actions/upload-artifact@v3
	with:
	name: reports_tpch_0_01
	path: \|
	./reports/benchmark_system_tpch_001_duckdb.json
	./reports/benchmark_system_tpch_001_sqljs.json
	./reports/benchmark_system_tpch_001_arquero.json
	./reports/benchmark_system_tpch_001_lovefield.json
	retention-days: 1

	- name: Preparation TPCH 0.1
	if: github.ref == 'refs/heads/main'
	uses: duckdb/duckdb-wasm-ci-env@master
	with:
	script: \|-
	git config --global --add safe.directory '*'
	mkdir -p ./lib/build/wasm/release ./reports
	yarn install --frozen-lockfile
	./scripts/generate_tpch_tbl.sh 0.1
	./scripts/generate_tpch_arrow.sh 0.1
	./scripts/generate_tpch_duckdb.sh 0.1
	./scripts/generate_tpch_sqlite.sh 0.1

	- name: Benchmarks TPCH 0.1
	if: github.ref == 'refs/heads/main'
	shell: bash
	run: \|
	yarn workspace @duckdb/benchmarks bench:system:tpch:duckdb 0.1
	yarn workspace @duckdb/benchmarks bench:system:tpch:sqljs 0.1
	yarn workspace @duckdb/benchmarks bench:system:tpch:arquero 0.1
	yarn workspace @duckdb/benchmarks bench:system:tpch:lovefield 0.1

	- name: Upload reports TPCH 0.1
	if: github.ref == 'refs/heads/main'
	uses: actions/upload-artifact@v3
	with:
	name: reports_tpch_0_1
	path: \|
	./reports/benchmark_system_tpch_01_duckdb.json
	./reports/benchmark_system_tpch_01_sqljs.json
	./reports/benchmark_system_tpch_01_arquero.json
	./reports/benchmark_system_tpch_01_lovefield.json
	retention-days: 1

	- name: Preparation TPCH 0.25
	if: github.ref == 'refs/heads/main'
	uses: duckdb/duckdb-wasm-ci-env@master
	with:
	script: \|-
	git config --global --add safe.directory '*'
	mkdir -p ./lib/build/wasm/release ./reports
	yarn install --frozen-lockfile
	./scripts/generate_tpch_tbl.sh 0.25
	./scripts/generate_tpch_arrow.sh 0.25
	./scripts/generate_tpch_duckdb.sh 0.25
	./scripts/generate_tpch_sqlite.sh 0.25

	- name: Benchmarks TPCH 0.25
	if: github.ref == 'refs/heads/main'
	shell: bash
	run: \|
	yarn workspace @duckdb/benchmarks bench:system:tpch:duckdb 0.25
	yarn workspace @duckdb/benchmarks bench:system:tpch:sqljs 0.25
	yarn workspace @duckdb/benchmarks bench:system:tpch:arquero 0.25
	yarn workspace @duckdb/benchmarks bench:system:tpch:lovefield 0.25

	- name: Upload reports TPCH 0.25
	if: github.ref == 'refs/heads/main'
	uses: actions/upload-artifact@v3
	with:
	name: reports_tpch_0_25
	path: \|
	./reports/benchmark_system_tpch_025_duckdb.json
	./reports/benchmark_system_tpch_025_sqljs.json
	./reports/benchmark_system_tpch_025_arquero.json
	./reports/benchmark_system_tpch_025_lovefield.json
	retention-days: 1

	- name: Preparation TPCH 0.5
	if: github.ref == 'refs/heads/main'
	uses: duckdb/duckdb-wasm-ci-env@master
	with:
	script: \|-
	git config --global --add safe.directory '*'
	mkdir -p ./lib/build/wasm/release ./reports
	yarn install --frozen-lockfile
	./scripts/generate_tpch_tbl.sh 0.5
	./scripts/generate_tpch_arrow.sh 0.5
	./scripts/generate_tpch_duckdb.sh 0.5
	./scripts/generate_tpch_sqlite.sh 0.5

	- name: Benchmarks TPCH 0.5
	if: github.ref == 'refs/heads/main'
	shell: bash
	run: \|
	yarn workspace @duckdb/benchmarks bench:system:tpch:duckdb 0.5
	yarn workspace @duckdb/benchmarks bench:system:tpch:sqljs 0.5
	yarn workspace @duckdb/benchmarks bench:system:tpch:arquero 0.5
	yarn workspace @duckdb/benchmarks bench:system:tpch:lovefield 0.5

	- name: Upload reports TPCH 0.5
	if: github.ref == 'refs/heads/main'
	uses: actions/upload-artifact@v3
	with:
	name: reports_tpch_0_5
	path: \|
	./reports/benchmark_system_tpch_05_duckdb.json
	./reports/benchmark_system_tpch_05_sqljs.json
	./reports/benchmark_system_tpch_05_arquero.json
	./reports/benchmark_system_tpch_05_lovefield.json
	retention-days: 1

	js_libs_loadable:
	name: Js / Libraries (loadable version)
	runs-on: ubuntu-latest
	needs:
	- wasm_mvp_loadable
	- wasm_eh_loadable
	- wasm_coi
	- duckdb_shell
	- clang_format
	- eslint
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: 'recursive'
	fetch-depth: 0

	- uses: actions/setup-node@v3
	with:
	node-version: '18.x'
	registry-url: 'https://registry.npmjs.org'

	- name: Git submodule status
	run: \|
	git submodule status > git_submodule_status.txt

	- name: Cache rust build
	uses: actions/cache@v3
	with:
	path: \|
	./.cargo/bin/
	./.cargo/registry/index/
	./.cargo/registry/cache/
	./.cargo/git/db/
	./target
	key: ${{ runner.os }}-shell-${{ hashFiles('./Cargo.lock') }}-${{ hashFiles('./packages/duckdb-wasm-shell/crate/src/**') }}
	restore-keys: \|
	${{ runner.os }}-shell-

	- name: Cache node_modules
	uses: actions/cache@v3
	with:
	path: \|
	./node_modules
	./packages/benchmarks/node_modules
	./packages/duckdb-wasm/node_modules
	./packages/duckdb-wasm-shell/node_modules
	key: ${{ runner.os }}-yarn-${{ hashFiles('./yarn.lock') }}
	restore-keys: \|
	${{ runner.os }}-yarn-

	- uses: actions/download-artifact@v3
	with:
	name: dataprep
	path: ./target/release/

	- uses: actions/download-artifact@v3
	with:
	name: tpch-dbgen
	path: ./submodules/tpch-dbgen/dbgen/

	- uses: actions/download-artifact@v3
	with:
	name: duckdb-shell
	path: ./submodules/duckdb/build/Release/

	- uses: actions/download-artifact@v3
	with:
	name: wasm-mvp-loadable
	path: ./packages/duckdb-wasm/src/bindings/

	- uses: actions/download-artifact@v3
	with:
	name: wasm-eh-loadable
	path: ./packages/duckdb-wasm/src/bindings/

	- uses: actions/download-artifact@v3
	with:
	name: wasm-coi
	path: ./packages/duckdb-wasm/src/bindings/

	- name: Prepare repository
	run: \|
	git fetch --tags --no-recurse-submodules -f
	(cd ./submodules/duckdb && git fetch --all --tags)

	- name: Prepare environment
	uses: duckdb/duckdb-wasm-ci-env@master
	with:
	script: \|-
	git config --global --add safe.directory '*'
	mkdir -p ./lib/build/wasm/release ./reports
	yarn install --frozen-lockfile
	./scripts/generate_tpch_tbl.sh 0.01
	./scripts/generate_tpch_arrow.sh 0.01
	./scripts/generate_tpch_duckdb.sh 0.01
	./scripts/generate_tpch_sqlite.sh 0.01
	./scripts/generate_uni.sh
	./scripts/npm_version.sh

	- name: Build @duckdb/duckdb-wasm
	shell: bash
	run: \|
	rm -rf ./packages/duckdb-wasm/dist/
	yarn workspace @duckdb/duckdb-wasm build:release
	yarn workspace @duckdb/duckdb-wasm docs

	- name: Build @duckdb/duckdb-wasm-shell
	shell: bash
	run: \|
	rm -rf ./packages/duckdb-wasm-shell/dist/
	yarn workspace @duckdb/duckdb-wasm-shell install:wasmpack
	yarn workspace @duckdb/duckdb-wasm-shell build:release

	- name: Build @duckdb/duckdb-wasm-app
	shell: bash
	run: \|
	rm -rf ./packages/duckdb-wasm-app/build/
	yarn workspace @duckdb/duckdb-wasm-app build:release

	- name: Test @duckdb/duckdb-wasm on Chrome
	shell: bash
	run: \|
	CHROME_BIN=`which google-chrome` yarn workspace @duckdb/duckdb-wasm test:chrome

	# - name: Test @duckdb/duckdb-wasm on Firefox
	# uses: duckdb/duckdb-wasm-ci-env@master
	# with:
	# script: \|-
	# yarn workspace @duckdb/duckdb-wasm test:firefox

	#- name: Test @duckdb/duckdb-wasm on Node.js
	# shell: bash
	# run: \|
	# yarn workspace @duckdb/duckdb-wasm test:node \|\| treu

	#- name: Coverage @duckdb/duckdb-wasm
	# shell: bash
	# run: \|
	# CHROME_BIN=`which google-chrome` yarn workspace @duckdb/duckdb-wasm test:chrome:coverage

	- name: Deploy GitHub Page
	if: github.ref == 'refs/heads/main'
	run: \|
	git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY
	git fetch origin gh-pages
	./scripts/deploy_pages.sh

	- name: Package
	shell: bash
	run: zip -r duckdb-wasm-packages.zip ./packages

	- uses: actions/upload-artifact@v3
	with:
	name: duckdb-wasm-packages-loadable
	path: duckdb-wasm-packages.zip

	merge_reports:
	name: Merge benchmark reports
	runs-on: ubuntu-latest
	if: github.ref == 'refs/heads/main'
	needs:
	- js_libs
	steps:
	- uses: actions/checkout@v4
	with:
	submodules: 'recursive'

	- uses: actions/download-artifact@v3
	with:
	name: dataprep
	path: ./target/release/

	- uses: actions/download-artifact@v3
	with:
	name: reports_micro
	path: ./reports/

	- uses: actions/download-artifact@v3
	with:
	name: reports_tpch_0_01
	path: ./reports/

	- uses: actions/download-artifact@v3
	with:
	name: reports_tpch_0_1
	path: ./reports/

	- uses: actions/download-artifact@v3
	with:
	name: reports_tpch_0_25
	path: ./reports/

	- uses: actions/download-artifact@v3
	with:
	name: reports_tpch_0_5
	path: ./reports/

	- name: Merge reports
	shell: bash
	run: \|
	mkdir duckdb
	cd duckdb
	wget https://github.com/duckdb/duckdb/releases/download/v0.8.1/duckdb_cli-linux-amd64.zip
	unzip duckdb_cli-linux-amd64.zip
	./duckdb -c "COPY (FROM read_json_auto('../reports/*.json', union_by_name=1)) TO 'benchmarks.json'"
	./duckdb -c "COPY (FROM read_json_auto('../reports/*.json', union_by_name=1)) TO 'benchmarks.csv'"

	- name: Upload artifact
	uses: actions/upload-artifact@v3
	with:
	name: reports_various
	path: \|
	./reports/benchmarks.arrow
	./duckdb/benchmarks.json
	./duckdb/benchmarks.csv
	retention-days: 1

	- name: Merge benchmark reports
	uses: duckdb/duckdb-wasm-ci-env@master
	with:
	script: \|-
	chmod +x ./target/release/dataprep
	./target/release/dataprep merge-benchmarks -r ./reports/

	- name: Upload artifact
	uses: actions/upload-artifact@v3
	with:
	name: reports_all
	path: \|
	./reports/benchmarks.arrow
	retention-days: 1

	- name: Deploy benchmark reports
	if: github.ref == 'refs/heads/main'
	run: \|
	git config --global user.name 'github-actions[bot]'
	git config --global user.email '41898282+github-actions[bot]@users.noreply.github.com'
	git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/$GITHUB_REPOSITORY
	git fetch origin gh-pages
	./scripts/deploy_benchmarks.sh

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Move workflow to master #646

Workflow file

Move workflow to master #646

Jobs

Run details

Workflow file for this run