matt/feat/support-scalar-udfs

.github/workflows/rust.yaml

@@ -33,12 +33,13 @@ jobs:
           rust-version: stable${{ matrix.host }}
           targets: ${{ matrix.target }}
           components: 'rustfmt, clippy'
+
       # download libduckdb
       - uses: robinraju/release-downloader@v1.4
         name: Download duckdb
         with:
           repository: "duckdb/duckdb"
-          tag: "v0.10.1"
+          tag: "v1.1.1"
           fileName: ${{ matrix.duckdb }}
           out-file-path: .
 
@@ -49,15 +50,25 @@ jobs:
         with:
           file_path: ${{ github.workspace }}/${{ matrix.duckdb }}
           extract_dir: libduckdb
+
       - run: cargo fmt --all -- --check
         if: matrix.os == 'ubuntu-latest'
-      - run: cargo clippy --all-targets --workspace --all-features -- -D warnings -A clippy::redundant-closure
+
+      # TODO: remove
+      - name: Workaround for https://github.com/pola-rs/polars/issues/19063
+        run: |
+          cargo update indexmap@2.6.0 --precise 2.5.0
+
+      - name: run cargo clippy
         if: matrix.os == 'ubuntu-latest'
-        name: run cargo clippy
         env:
           DUCKDB_LIB_DIR: ${{ github.workspace }}/libduckdb
           DUCKDB_INCLUDE_DIR: ${{ github.workspace }}/libduckdb
           LD_LIBRARY_PATH: ${{ github.workspace }}/libduckdb
+        run: |
+          cargo clippy --all-targets --workspace --all-features -- -D warnings -A clippy::redundant-closure
+
+
       - name: Run cargo-tarpaulin
         if: matrix.os == 'ubuntu-latest'
         uses: actions-rs/tarpaulin@v0.1
@@ -70,6 +81,7 @@ jobs:
           DUCKDB_LIB_DIR: ${{ github.workspace }}/libduckdb
           DUCKDB_INCLUDE_DIR: ${{ github.workspace }}/libduckdb
           LD_LIBRARY_PATH: ${{ github.workspace }}/libduckdb
+
       - name: Upload to codecov.io
         if: matrix.os == 'ubuntu-latest'
         uses: codecov/codecov-action@v1
@@ -88,19 +100,28 @@ jobs:
         with:
           name: PATH
           value: $env:PATH;${{ github.workspace }}/libduckdb
+
       - name: Run cargo-test
         if: matrix.os == 'windows-latest'
         run: cargo test --features "modern-full vtab-full vtab-loadable"
         env:
           DUCKDB_LIB_DIR: ${{ github.workspace }}/libduckdb
           DUCKDB_INCLUDE_DIR: ${{ github.workspace }}/libduckdb
+
       - name: Build loadable extension
         run: cargo build --example hello-ext --features="vtab-loadable"
         env:
           DUCKDB_LIB_DIR: ${{ github.workspace }}/libduckdb
           DUCKDB_INCLUDE_DIR: ${{ github.workspace }}/libduckdb
           LD_LIBRARY_PATH: ${{ github.workspace }}/libduckdb
 
+      - name: Build loadable extension
+        run: cargo build --example hello-ext-capi --features="vtab-loadable loadable-extension"
+        env:
+          DUCKDB_LIB_DIR: ${{ github.workspace }}/libduckdb
+          DUCKDB_INCLUDE_DIR: ${{ github.workspace }}/libduckdb
+          LD_LIBRARY_PATH: ${{ github.workspace }}/libduckdb
+
   Windows:
     name: Windows build from source
     needs: test
@@ -117,6 +138,7 @@ jobs:
         with:
           rust-version: stable
           targets: x86_64-pc-windows-msvc
+
       - run: cargo install cargo-examples
 
   Sanitizer:
@@ -140,7 +162,9 @@ jobs:
           # leak sanitization, but we don't care about backtraces here, so long
           # as the other tests have them.
           RUST_BACKTRACE: "0"
-        run: cargo -Z build-std test --features "modern-full extensions-full" --target x86_64-unknown-linux-gnu
+        run: |
+          # TODO switch back to modern-full once polars is fixed
+          cargo -Z build-std test --features "chrono serde_json url r2d2 uuid extensions-full" --target x86_64-unknown-linux-gnu --package duckdb
       - name: publish crates --dry-run
         uses: katyo/publish-crates@v2
         with:

.gitignore

@@ -27,6 +27,6 @@ Cargo.lock
 
 *.db
 
-crates/libduckdb-sys/duckdb-sources/
+crates/libduckdb-sys/duckdb-sources/*
 crates/libduckdb-sys/duckdb/
 crates/libduckdb-sys/._duckdb

.gitmodules

@@ -1,4 +1,3 @@
 [submodule "crates/libduckdb-sys/duckdb-sources"]
 	path = crates/libduckdb-sys/duckdb-sources
 	url = https://github.com/duckdb/duckdb
-	update = none

.rustfmt.toml

@@ -1,2 +1,7 @@
 max_width = 120
-imports_granularity = "Crate"
+imports_granularity = "Crate"
+reorder_imports = true
+fn_call_width = 72
+# indent_style = "Block"
+# tab_spaces = 2
+# group_imports="StdExternalCrate"

Cargo.toml

@@ -7,21 +7,21 @@ members = [
 ]
 
 [workspace.package]
-version = "0.10.2"
+version = "1.1.1"
 authors = ["wangfenjin <wangfenj@gmail.com>"]
 edition = "2021"
-repository = "https://github.com/wangfenjin/duckdb-rs"
-homepage = "https://github.com/wangfenjin/duckdb-rs"
+repository = "https://github.com/duckdb/duckdb-rs"
+homepage = "https://github.com/duckdb/duckdb-rs"
 documentation = "http://docs.rs/duckdb/"
 readme = "README.md"
 keywords = ["duckdb", "database", "ffi"]
 license = "MIT"
 categories = ["database"]
 
 [workspace.dependencies]
-duckdb = { version = "0.10.2", path = "crates/duckdb" }
-libduckdb-sys = { version = "0.10.2", path = "crates/libduckdb-sys" }
-duckdb-loadable-macros = { version = "0.1.1", path = "crates/duckdb-loadable-macros" }
+duckdb = { version = "1.1.1", path = "crates/duckdb" }
+libduckdb-sys = { version = "1.1.1", path = "crates/libduckdb-sys" }
+duckdb-loadable-macros = { version = "0.1.3", path = "crates/duckdb-loadable-macros" }
 autocfg = "1.0"
 bindgen = { version = "0.69", default-features = false }
 byteorder = "1.3"
@@ -34,14 +34,16 @@ doc-comment = "0.3"
 fallible-iterator = "0.3"
 fallible-streaming-iterator = "0.1"
 flate2 = "1.0"
-hashlink = "0.8"
+hashlink = "0.9"
 lazy_static = "1.4"
 memchr = "2.3"
 num = { version = "0.4", default-features = false }
+num-integer = "0.1.46"
 pkg-config = "0.3.24"
 polars = "0.35.4"
 polars-core = "0.35.4"
 pretty_assertions = "1.4.0"
+prettyplease = "0.2.20"
 proc-macro2 = "1.0.56"
 quote = "1.0.21"
 r2d2 = "0.8.9"
@@ -60,4 +62,4 @@ unicase = "2.6.0"
 url = "2.1"
 uuid = "1.0"
 vcpkg = "0.2"
-arrow = { version = "52", default-features = false }
+arrow = { version = "53", default-features = false }

Makefile

@@ -11,4 +11,4 @@ all:
 	cargo clippy --all-targets --workspace --features buildtime_bindgen --features modern-full -- -D warnings -A clippy::redundant-closure
 
 test:
-	cargo test --features bundled --features modern-full -- --nocapture
+	cargo test --features bundled --features modern-full -- --nocapture

add_rustfmt_hook.sh

@@ -8,7 +8,7 @@ command -v rustfmt >/dev/null 2>&1 || { echo >&2 "Rustfmt is required but it's n
 # write a whole script to pre-commit hook
 # NOTE: it will overwrite pre-commit file!
 cat > .git/hooks/pre-commit <<'EOF'
-#!/bin/bash -e
+#!/bin/bash
 declare -a rust_files=()
 files=$(git diff-index --name-only --cached HEAD)
 echo 'Formatting source files'

crates/duckdb-loadable-macros/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "duckdb-loadable-macros"
-version = "0.1.1"
+version = "0.1.3"
 authors.workspace = true
 edition.workspace = true
 license.workspace = true
@@ -14,6 +14,7 @@ description = "Native bindings to the libduckdb library, C API; build loadable e
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+darling = "0.20.10"
 proc-macro2 = { workspace = true }
 quote = { workspace = true }
 syn = { workspace = true, features = ["extra-traits", "full", "fold", "parsing"] }

crates/duckdb-loadable-macros/src/lib.rs

@@ -6,6 +6,108 @@ use syn::{parse_macro_input, spanned::Spanned, Item};
 use proc_macro::TokenStream;
 use quote::quote_spanned;
 
+use darling::{ast::NestedMeta, Error, FromMeta};
+
+/// For parsing the arguments to the duckdb_entrypoint_c_api macro
+#[derive(Debug, FromMeta)]
+struct CEntryPointMacroArgs {
+    #[darling(default)]
+    /// The name to be given to this extension. This name is used in the entrypoint function called by duckdb
+    ext_name: String,
+    /// The minimum C API version this extension requires. It is recommended to set this to the lowest possible version
+    /// at which your extension still compiles
+    min_duckdb_version: Option<String>,
+}
+
+/// Wraps an entrypoint function to expose an unsafe extern "C" function of the same name.
+/// Warning: experimental!
+#[proc_macro_attribute]
+pub fn duckdb_entrypoint_c_api(attr: TokenStream, item: TokenStream) -> TokenStream {
+    let attr_args = match NestedMeta::parse_meta_list(attr.into()) {
+        Ok(v) => v,
+        Err(e) => {
+            return TokenStream::from(Error::from(e).write_errors());
+        }
+    };
+
+    let args = match CEntryPointMacroArgs::from_list(&attr_args) {
+        Ok(v) => v,
+        Err(e) => {
+            return TokenStream::from(e.write_errors());
+        }
+    };
+
+    // Set the minimum duckdb version (dev by default)
+    let minimum_duckdb_version = match args.min_duckdb_version {
+        Some(i) => i,
+        None => "dev".to_string(),
+    };
+
+    let ast = parse_macro_input!(item as syn::Item);
+
+    match ast {
+        Item::Fn(func) => {
+            let c_entrypoint = Ident::new(format!("{}_init_c_api", args.ext_name).as_str(), Span::call_site());
+            let prefixed_original_function = func.sig.ident.clone();
+            let c_entrypoint_internal = Ident::new(
+                format!("{}_init_c_api_internal", args.ext_name).as_str(),
+                Span::call_site(),
+            );
+
+            quote_spanned! {func.span()=>
+                /// # Safety
+                ///
+                /// Internal Entrypoint for error handling
+                pub unsafe fn #c_entrypoint_internal(info: ffi::duckdb_extension_info, access: *const ffi::duckdb_extension_access) -> Result<bool, Box<dyn std::error::Error>> {
+                    let have_api_struct = ffi::duckdb_rs_extension_api_init(info, access, #minimum_duckdb_version).unwrap();
+
+                    if !have_api_struct {
+                        // initialization failed to return an api struct, likely due to an API version mismatch, we can simply return here
+                        return Ok(false);
+                    }
+
+                    // TODO: handle error here?
+                    let db : ffi::duckdb_database = *(*access).get_database.unwrap()(info);
+                    let connection = Connection::open_from_raw(db.cast())?;
+
+                    #prefixed_original_function(connection)?;
+
+                    Ok(true)
+                }
+
+                /// # Safety
+                ///
+                /// Entrypoint that will be called by DuckDB
+                #[no_mangle]
+                pub unsafe extern "C" fn #c_entrypoint(info: ffi::duckdb_extension_info, access: *const ffi::duckdb_extension_access) -> bool {
+                    let init_result = #c_entrypoint_internal(info, access);
+
+                    if let Err(x) = init_result {
+                        let error_c_string = std::ffi::CString::new(x.to_string());
+
+                        match error_c_string {
+                            Ok(e) => {
+                                (*access).set_error.unwrap()(info, e.as_ptr());
+                            },
+                            Err(_e) => {
+                                let error_alloc_failure = c"An error occured but the extension failed to allocate memory for an error string";
+                                (*access).set_error.unwrap()(info, error_alloc_failure.as_ptr());
+                            }
+                        }
+                        return false;
+                    }
+
+                    init_result.unwrap()
+                }
+
+                #func
+            }
+            .into()
+        }
+        _ => panic!("Only function items are allowed on duckdb_entrypoint"),
+    }
+}
+
 /// Wraps an entrypoint function to expose an unsafe extern "C" function of the same name.
 #[proc_macro_attribute]
 pub fn duckdb_entrypoint(_attr: TokenStream, item: TokenStream) -> TokenStream {

crates/duckdb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "duckdb"
-version = "0.10.2"
+version = "1.1.1"
 authors.workspace = true
 edition.workspace = true
 repository.workspace = true
@@ -20,22 +20,23 @@ name = "duckdb"
 [features]
 default = []
 bundled = ["libduckdb-sys/bundled"]
-httpfs = ["libduckdb-sys/httpfs", "bundled"]
 json = ["libduckdb-sys/json", "bundled"]
 parquet = ["libduckdb-sys/parquet", "bundled"]
-openssl_vendored = ["libduckdb-sys/openssl_vendored", "bundled"]
-unstable_boringssl = ["libduckdb-sys/unstable_boringssl", "bundled"]
-openssl_bindgen = ["libduckdb-sys/openssl_bindgen", "bundled"]
 vtab = []
 vtab-loadable = ["vtab", "duckdb-loadable-macros"]
 vtab-excel = ["vtab", "calamine"]
 vtab-arrow = ["vtab", "num"]
 appender-arrow = ["vtab-arrow"]
 vtab-full = ["vtab-excel", "vtab-arrow", "appender-arrow"]
-extensions-full = ["httpfs", "json", "parquet", "vtab-full"]
+extensions-full = ["json", "parquet", "vtab-full"]
 buildtime_bindgen = ["libduckdb-sys/buildtime_bindgen"]
 modern-full = ["chrono", "serde_json", "url", "r2d2", "uuid", "polars"]
 polars = ["dep:polars"]
+# FIXME: These were added to make clippy happy: these features appear unused and should perhaps be removed
+column_decltype = []
+extra_check = []
+# Warning: experimental feature
+loadable-extension = ["libduckdb-sys/loadable-extension"]
 
 [dependencies]
 libduckdb-sys = { workspace = true }
@@ -60,7 +61,7 @@ calamine = { workspace = true, optional = true }
 num = { workspace = true, features = ["std"], optional = true }
 duckdb-loadable-macros = { workspace = true, optional = true }
 polars = { workspace = true, features = ["dtype-full"], optional = true }
-num-integer = {version = "0.1.46"}
+num-integer = { workspace = true }
 
 [dev-dependencies]
 doc-comment = { workspace = true }
@@ -94,3 +95,8 @@ all-features = false
 name = "hello-ext"
 crate-type = ["cdylib"]
 required-features = ["vtab-loadable"]
+
+[[example]]
+name = "hello-ext-capi"
+crate-type = ["cdylib"]
+required-features = ["vtab-loadable", "loadable-extension"]