diff --git a/Cargo.toml b/Cargo.toml index 154136d7..fc425fb0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,7 @@ duckdb-loadable-macros = { version = "0.1.2", path = "crates/duckdb-loadable-mac autocfg = "1.0" bindgen = { version = "0.69", default-features = false } byteorder = "1.3" -calamine = "0.22.0" +calamine = "0.22" cast = "0.3" cc = "1.0" chrono = "0.4.22" @@ -34,13 +34,15 @@ doc-comment = "0.3" fallible-iterator = "0.3" fallible-streaming-iterator = "0.1" flate2 = "1.0" -hashlink = "0.8" +hashlink = "0.9" lazy_static = "1.4" memchr = "2.3" num = { version = "0.4", default-features = false } +num-integer = "0.1.46" pkg-config = "0.3.24" -polars = "0.35.4" -polars-core = "0.35.4" +polars = "0.38" +polars-core = "0.38" +derive_more = "0.99" pretty_assertions = "1.4.0" proc-macro2 = "1.0.56" quote = "1.0.21" @@ -55,9 +57,14 @@ strum = "0.25" syn = "2.0.15" tar = "0.4.38" tempdir = "0.3.7" +thiserror = "1.0" tempfile = "3.1.0" unicase = "2.6.0" url = "2.1" uuid = "1.0" vcpkg = "0.2" arrow = { version = "52", default-features = false } +rusqlite = "0.31" +arrow_convert = "0.6" +itertools = "0.13" +criterion = { version = "0.5", features = [ "html_reports"] } \ No newline at end of file diff --git a/crates/duckdb/Cargo.toml b/crates/duckdb/Cargo.toml index 36b2ce90..fc9861b5 100644 --- a/crates/duckdb/Cargo.toml +++ b/crates/duckdb/Cargo.toml @@ -16,6 +16,7 @@ description = "Ergonomic wrapper for DuckDB" [lib] name = "duckdb" +bench = false [features] default = [] @@ -54,9 +55,13 @@ strum = { workspace = true, features = ["derive"] } r2d2 = { workspace = true, optional = true } calamine = { workspace = true, optional = true } num = { workspace = true, features = ["std"], optional = true } +derive_more = { workspace = true } duckdb-loadable-macros = { workspace = true, optional = true } polars = { workspace = true, features = ["dtype-full"], optional = true } -num-integer = {version = "0.1.46"} +num-integer = { workspace = true } +thiserror = { workspace = true } +arrow_convert = { workspace = true } +itertools = { workspace = true } [dev-dependencies] doc-comment = { workspace = true } @@ -69,13 +74,13 @@ rand = { workspace = true } tempdir = { workspace = true } polars-core = { workspace = true } pretty_assertions = { workspace = true } -# criterion = "0.3" +rusqlite = { workspace = true } +criterion = { workspace = true } # [[bench]] # name = "data_types" # harness = false - [package.metadata.docs.rs] features = [] all-features = false @@ -90,3 +95,12 @@ all-features = false name = "hello-ext" crate-type = ["cdylib"] required-features = ["vtab-loadable"] + +[[example]] +name = "appender" +crate-type = ["cdylib"] +required-features = ["appender-arrow"] + +[[bench]] +name = "issue-282" +harness = false diff --git a/crates/duckdb/benches/issue-282/data/.gitignore b/crates/duckdb/benches/issue-282/data/.gitignore new file mode 100644 index 00000000..6afd071c --- /dev/null +++ b/crates/duckdb/benches/issue-282/data/.gitignore @@ -0,0 +1 @@ +db.* \ No newline at end of file diff --git a/crates/duckdb/benches/issue-282/generate-database.sh b/crates/duckdb/benches/issue-282/generate-database.sh new file mode 100755 index 00000000..8eba1622 --- /dev/null +++ b/crates/duckdb/benches/issue-282/generate-database.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# SQLite +sqlite3 data/db.sqlite < rusqlite::Connection { + let db_path = concat!(env!("CARGO_MANIFEST_DIR"), "/benches/issue-282/data/db.sqlite"); + rusqlite::Connection::open(db_path).unwrap() +} + +pub fn duck_db() -> duckdb::Connection { + let db_path = concat!(env!("CARGO_MANIFEST_DIR"), "/benches/issue-282/data/db.duckdb"); + duckdb::Connection::open(db_path).unwrap() +} + +#[derive(Debug, ArrowField, ArrowSerialize, ArrowDeserialize)] +struct Income { + created_at: Option, + amount: Option, + category_id: i32, + wallet_id: Option, + meta: Option, +} + +impl Income { + fn select_duckdb_arrow( + conn: &duckdb::Connection, + start: u32, + end: u32, + ) -> Result, Box> { + let sql = format!( + "SELECT created_at, amount, category_id, wallet_id, meta \ + FROM 'income' \ + WHERE created_at >= {} AND created_at <= {}", + start, end + ); + let mut stmt = conn.prepare_cached(&sql)?; + let result = stmt.query_arrow_deserialized::([])?; + Ok(result) + } + + fn select_duckdb(conn: &duckdb::Connection, start: u32, end: u32) -> Result, Box> { + let mut arr = Vec::new(); + let sql = format!( + "SELECT created_at, amount, category_id, wallet_id, meta \ + FROM 'income' \ + WHERE created_at >= {} AND created_at <= {}", + start, end + ); + let mut stmt = conn.prepare_cached(&sql)?; + let result_iter = stmt.query_map([], |row| { + Ok(Self { + created_at: row.get(0)?, + amount: row.get(1)?, + category_id: row.get(2)?, + wallet_id: row.get(3)?, + meta: row.get(4)?, + }) + })?; + for result in result_iter { + arr.push(result?); + } + Ok(arr) + } + + fn select_sqlite( + conn: &rusqlite::Connection, + start: u32, + end: u32, + ) -> Result, Box> { + let mut arr = Vec::new(); + let sql = format!( + "SELECT created_at, amount, category_id, wallet_id, meta \ + FROM 'income' \ + WHERE created_at >= {} AND created_at <= {}", + start, end + ); + let mut stmt = conn.prepare(&sql)?; + let result_iter = stmt.query_map([], |row| { + Ok(Self { + created_at: row.get(0)?, + amount: row.get(1)?, + category_id: row.get(2)?, + wallet_id: row.get(3)?, + meta: row.get(4)?, + }) + })?; + for result in result_iter { + arr.push(result?); + } + Ok(arr) + } +} + +fn bench_sqlite(c: &mut Criterion) { + let sqlite_conn = sqlite_db(); + c.bench_function("sqlite_test", |b| { + b.iter(|| { + let out = Income::select_sqlite(&sqlite_conn, 1709292049, 1711375239).unwrap(); + out.len() + }) + }); +} + +fn bench_duckdb(c: &mut Criterion) { + let duckdb_conn = duck_db(); + c.bench_function("duckdb_test", |b| { + b.iter(|| { + let out = Income::select_duckdb(&duckdb_conn, 1709292049, 1711375239).unwrap(); + out.len() + }) + }); +} + +fn bench_duckdb_arrow(c: &mut Criterion) { + let duckdb_conn = duck_db(); + c.bench_function("duckdb_test_arrow", |b| { + b.iter(|| { + let out = Income::select_duckdb_arrow(&duckdb_conn, 1709292049, 1711375239).unwrap(); + out.len() + }) + }); +} + +// criterion_group!(benches, bench_duckdb_arrow); +criterion_group!(benches, bench_sqlite, bench_duckdb, bench_duckdb_arrow); +criterion_main!(benches); diff --git a/crates/duckdb/benches/issue-282/output.csv b/crates/duckdb/benches/issue-282/output.csv new file mode 100644 index 00000000..33996243 --- /dev/null +++ b/crates/duckdb/benches/issue-282/output.csv @@ -0,0 +1,1063 @@ +id,created_at,amount,category_id,wallet_id,meta +1,1665991212,800.0,19,2,"{""description"":""some"",""duration"":40}" +2,1666011332,3000.0,19,2,"{""description"":""some"",""duration"":40}" +3,1666092255,800.0,19,2,"{""description"":""some"",""duration"":40}" +4,1666092280,50.0,19,2,"{""description"":""some"",""duration"":40}" +5,1666092522,400.0,19,2,"{""description"":""some"",""duration"":40}" +6,1666092603,500.0,19,2,"{""description"":""some"",""duration"":40}" +7,1666099128,1800.0,19,2,"{""description"":""some"",""duration"":40}" +8,1666180623,1000.0,19,2,"{""description"":""some"",""duration"":40}" +9,1666259498,2500.0,19,2,"{""description"":""some"",""duration"":40}" +10,1666342098,1800.0,19,2,"{""description"":""some"",""duration"":40}" +11,1666343577,700.0,19,2,"{""description"":""some"",""duration"":40}" +12,1666436034,3000.0,19,2,"{""description"":""some"",""duration"":40}" +13,1666436046,1500.0,19,2,"{""description"":""some"",""duration"":40}" +14,1666618507,3000.0,19,2,"{""description"":""some"",""duration"":40}" +15,1666618523,800.0,19,2,"{""description"":""some"",""duration"":40}" +16,1666618538,200.0,19,2,"{""description"":""some"",""duration"":40}" +17,1666781099,800.0,19,2,"{""description"":""some"",""duration"":40}" +18,1666781118,100.0,19,2,"{""description"":""some"",""duration"":40}" +19,1666880169,3000.0,19,2,"{""description"":""some"",""duration"":40}" +20,1666880206,200.0,19,2,"{""description"":""some"",""duration"":40}" +21,1666880218,800.0,19,2,"{""description"":""some"",""duration"":40}" +22,1666880234,400.0,19,2,"{""description"":""some"",""duration"":40}" +23,1666880301,500.0,19,2,"{""description"":""some"",""duration"":40}" +24,1666967736,800.0,19,2,"{""description"":""some"",""duration"":40}" +25,1666967754,1800.0,19,2,"{""description"":""some"",""duration"":40}" +26,1666967769,3000.0,19,2,"{""description"":""some"",""duration"":40}" +27,1667039365,500.0,19,2,"{""description"":""some"",""duration"":40}" +28,1667039381,200.0,19,2,"{""description"":""some"",""duration"":40}" +29,1667042456,1800.0,19,2,"{""description"":""some"",""duration"":40}" +30,1667045415,800.0,19,2,"{""description"":""some"",""duration"":40}" +31,1667045434,400.0,19,2,"{""description"":""some"",""duration"":40}" +32,1667220004,500.0,19,2,"{""description"":""some"",""duration"":40}" +33,1667220014,200.0,19,2,"{""description"":""some"",""duration"":40}" +34,1667298931,800.0,19,2,"{""description"":""some"",""duration"":40}" +35,1667298941,800.0,19,2,"{""description"":""some"",""duration"":40}" +36,1667380456,800.0,19,2,"{""description"":""some"",""duration"":40}" +37,1667380467,400.0,19,2,"{""description"":""some"",""duration"":40}" +38,1667395164,3000.0,19,2,"{""description"":""some"",""duration"":40}" +39,1667473802,3000.0,19,2,"{""description"":""some"",""duration"":40}" +40,1667479293,200.0,19,2,"{""description"":""some"",""duration"":40}" +41,1667557832,500.0,19,2,"{""description"":""some"",""duration"":40}" +42,1667557842,500.0,19,2,"{""description"":""some"",""duration"":40}" +43,1667650725,3000.0,19,2,"{""description"":""some"",""duration"":40}" +44,1667930428,800.0,19,2,"{""description"":""some"",""duration"":40}" +45,1667987940,800.0,19,2,"{""description"":""some"",""duration"":40}" +46,1667987957,800.0,19,2,"{""description"":""some"",""duration"":40}" +47,1668002224,800.0,19,2,"{""description"":""some"",""duration"":40}" +48,1668002234,800.0,19,2,"{""description"":""some"",""duration"":40}" +49,1668002245,800.0,19,2,"{""description"":""some"",""duration"":40}" +50,1668064898,4000.0,19,2,"{""description"":""some"",""duration"":40}" +51,1668081047,1800.0,19,2,"{""description"":""some"",""duration"":40}" +52,1668090135,800.0,19,2,"{""description"":""some"",""duration"":40}" +53,1668090156,800.0,19,2,"{""description"":""some"",""duration"":40}" +54,1668090174,800.0,19,2,"{""description"":""some"",""duration"":40}" +55,1668090191,400.0,19,2,"{""description"":""some"",""duration"":40}" +56,1668176260,3000.0,19,2,"{""description"":""some"",""duration"":40}" +57,1668249763,1800.0,19,2,"{""description"":""some"",""duration"":40}" +58,1668249774,200.0,19,2,"{""description"":""some"",""duration"":40}" +59,1668422461,1800.0,19,2,"{""description"":""some"",""duration"":40}" +60,1668424057,800.0,19,2,"{""description"":""some"",""duration"":40}" +61,1668429753,300.0,19,2,"{""description"":""some"",""duration"":40}" +62,1668429903,800.0,19,2,"{""description"":""some"",""duration"":40}" +63,1668429920,400.0,19,2,"{""description"":""some"",""duration"":40}" +64,1668432205,500.0,19,2,"{""description"":""some"",""duration"":40}" +65,1668432218,200.0,19,2,"{""description"":""some"",""duration"":40}" +66,1668501917,800.0,19,2,"{""description"":""some"",""duration"":40}" +67,1668501955,200.0,19,2,"{""description"":""some"",""duration"":40}" +68,1668670998,800.0,19,2,"{""description"":""some"",""duration"":40}" +69,1668683440,1800.0,19,2,"{""description"":""some"",""duration"":40}" +70,1668772715,800.0,19,2,"{""description"":""some"",""duration"":40}" +71,1668889934,800.0,19,2,"{""description"":""some"",""duration"":40}" +72,1668889945,800.0,19,2,"{""description"":""some"",""duration"":40}" +73,1668889959,800.0,19,2,"{""description"":""some"",""duration"":40}" +74,1668889967,200.0,19,2,"{""description"":""some"",""duration"":40}" +75,1668889981,400.0,19,2,"{""description"":""some"",""duration"":40}" +76,1668889999,1800.0,19,2,"{""description"":""some"",""duration"":40}" +77,1669032406,3000.0,19,2,"{""description"":""some"",""duration"":40}" +78,1669032416,800.0,19,2,"{""description"":""some"",""duration"":40}" +79,1669039590,1800.0,19,2,"{""description"":""some"",""duration"":40}" +80,1669039602,500.0,19,2,"{""description"":""some"",""duration"":40}" +81,1669043306,800.0,19,2,"{""description"":""some"",""duration"":40}" +82,1669189532,800.0,19,2,"{""description"":""some"",""duration"":40}" +83,1669189541,800.0,19,2,"{""description"":""some"",""duration"":40}" +84,1669199163,800.0,19,2,"{""description"":""some"",""duration"":40}" +85,1669292359,500.0,19,2,"{""description"":""some"",""duration"":40}" +86,1669292368,200.0,19,2,"{""description"":""some"",""duration"":40}" +87,1669292378,800.0,19,2,"{""description"":""some"",""duration"":40}" +88,1669292398,1800.0,19,2,"{""description"":""some"",""duration"":40}" +89,1669371839,500.0,19,2,"{""description"":""some"",""duration"":40}" +90,1669371848,300.0,19,2,"{""description"":""some"",""duration"":40}" +91,1669371860,800.0,19,2,"{""description"":""some"",""duration"":40}" +92,1669410919,1800.0,19,2,"{""description"":""some"",""duration"":40}" +93,1669410967,400.0,19,2,"{""description"":""some"",""duration"":40}" +94,1669411110,500.0,19,2,"{""description"":""some"",""duration"":40}" +95,1669486955,2000.0,19,2,"{""description"":""some"",""duration"":40}" +96,1669637109,1800.0,19,2,"{""description"":""some"",""duration"":40}" +97,1669637124,200.0,19,2,"{""description"":""some"",""duration"":40}" +98,1669637135,150.0,19,2,"{""description"":""some"",""duration"":40}" +99,1669712508,3000.0,19,2,"{""description"":""some"",""duration"":40}" +100,1669720639,500.0,19,2,"{""description"":""some"",""duration"":40}" +101,1669747477,800.0,19,2,"{""description"":""some"",""duration"":40}" +102,1669813127,800.0,19,2,"{""description"":""some"",""duration"":40}" +103,1669813136,800.0,19,2,"{""description"":""some"",""duration"":40}" +104,1669813151,400.0,19,2,"{""description"":""some"",""duration"":40}" +105,1669813187,2000.0,19,2,"{""description"":""some"",""duration"":40}" +106,1669889997,800.0,19,2,"{""description"":""some"",""duration"":40}" +107,1669890009,5000.0,19,2,"{""description"":""some"",""duration"":40}" +108,1669907717,500.0,19,2,"{""description"":""some"",""duration"":40}" +109,1669907726,800.0,19,2,"{""description"":""some"",""duration"":40}" +110,1669976745,6500.0,19,2,"{""description"":""some"",""duration"":40}" +111,1669976949,1000.0,19,2,"{""description"":""some"",""duration"":40}" +112,1669996768,500.0,19,2,"{""description"":""some"",""duration"":40}" +113,1670067222,3000.0,19,2,"{""description"":""some"",""duration"":40}" +114,1670067231,800.0,19,2,"{""description"":""some"",""duration"":40}" +115,1670067239,800.0,19,2,"{""description"":""some"",""duration"":40}" +116,1670067251,500.0,19,2,"{""description"":""some"",""duration"":40}" +117,1670151315,4500.0,19,2,"{""description"":""some"",""duration"":40}" +118,1670220965,1500.0,19,2,"{""description"":""some"",""duration"":40}" +119,1670228907,700.0,19,2,"{""description"":""some"",""duration"":40}" +120,1670240285,500.0,19,2,"{""description"":""some"",""duration"":40}" +121,1670250803,1000.0,19,2,"{""description"":""some"",""duration"":40}" +122,1670250811,1000.0,19,2,"{""description"":""some"",""duration"":40}" +123,1670250822,1800.0,19,2,"{""description"":""some"",""duration"":40}" +124,1670251121,500.0,19,2,"{""description"":""some"",""duration"":40}" +125,1670259887,3500.0,19,2,"{""description"":""some"",""duration"":40}" +126,1670326583,1800.0,19,2,"{""description"":""some"",""duration"":40}" +127,1670403142,1000.0,19,2,"{""description"":""some"",""duration"":40}" +128,1670506048,1000.0,19,2,"{""description"":""some"",""duration"":40}" +129,1670586024,1000.0,19,2,"{""description"":""some"",""duration"":40}" +130,1670586034,1000.0,19,2,"{""description"":""some"",""duration"":40}" +131,1670586043,500.0,19,2,"{""description"":""some"",""duration"":40}" +132,1670864755,1000.0,19,2,"{""description"":""some"",""duration"":40}" +133,1670864825,400.0,19,2,"{""description"":""some"",""duration"":40}" +134,1670864836,500.0,19,2,"{""description"":""some"",""duration"":40}" +135,1670864848,1000.0,19,2,"{""description"":""some"",""duration"":40}" +136,1670864867,1000.0,19,2,"{""description"":""some"",""duration"":40}" +137,1671450861,1000.0,19,2,"{""description"":""some"",""duration"":40}" +138,1671450877,400.0,19,2,"{""description"":""some"",""duration"":40}" +139,1671450892,2000.0,19,2,"{""description"":""some"",""duration"":40}" +140,1671521885,1800.0,19,2,"{""description"":""some"",""duration"":40}" +141,1671525043,1000.0,19,2,"{""description"":""some"",""duration"":40}" +142,1671528490,1000.0,19,2,"{""description"":""some"",""duration"":40}" +143,1671528845,1000.0,19,2,"{""description"":""some"",""duration"":40}" +144,1671605508,11000.0,19,2,"{""description"":""some"",""duration"":40}" +145,1671702072,1000.0,19,2,"{""description"":""some"",""duration"":40}" +146,1671704433,1000.0,19,2,"{""description"":""some"",""duration"":40}" +147,1671710054,1000.0,19,2,"{""description"":""some"",""duration"":40}" +148,1671710064,500.0,19,2,"{""description"":""some"",""duration"":40}" +149,1671720933,1000.0,19,2,"{""description"":""some"",""duration"":40}" +150,1671798277,1000.0,19,2,"{""description"":""some"",""duration"":40}" +151,1671895489,1500.0,19,2,"{""description"":""some"",""duration"":40}" +152,1671895508,1000.0,19,2,"{""description"":""some"",""duration"":40}" +153,1671895524,3500.0,19,2,"{""description"":""some"",""duration"":40}" +154,1671895540,1000.0,19,2,"{""description"":""some"",""duration"":40}" +155,1671895551,3500.0,19,2,"{""description"":""some"",""duration"":40}" +156,1672043148,500.0,19,2,"{""description"":""some"",""duration"":40}" +157,1672043159,1000.0,19,2,"{""description"":""some"",""duration"":40}" +158,1672043170,700.0,19,2,"{""description"":""some"",""duration"":40}" +159,1672047482,700.0,19,2,"{""description"":""some"",""duration"":40}" +160,1672047501,300.0,19,2,"{""description"":""some"",""duration"":40}" +161,1672061644,1000.0,19,2,"{""description"":""some"",""duration"":40}" +162,1672061653,1800.0,19,2,"{""description"":""some"",""duration"":40}" +163,1672061669,800.0,19,2,"{""description"":""some"",""duration"":40}" +164,1672061683,200.0,19,2,"{""description"":""some"",""duration"":40}" +165,1672061696,1000.0,19,2,"{""description"":""some"",""duration"":40}" +166,1672125296,800.0,19,2,"{""description"":""some"",""duration"":40}" +167,1672125307,1000.0,19,2,"{""description"":""some"",""duration"":40}" +168,1672129676,1000.0,19,2,"{""description"":""some"",""duration"":40}" +169,1672129684,400.0,19,2,"{""description"":""some"",""duration"":40}" +170,1672135200,1000.0,19,2,"{""description"":""some"",""duration"":40}" +171,1672135217,800.0,19,2,"{""description"":""some"",""duration"":40}" +172,1672135227,200.0,19,2,"{""description"":""some"",""duration"":40}" +173,1672141191,2000.0,19,2,"{""description"":""some"",""duration"":40}" +174,1672145495,1000.0,19,2,"{""description"":""some"",""duration"":40}" +175,1672145512,400.0,19,2,"{""description"":""some"",""duration"":40}" +176,1672148170,1000.0,19,2,"{""description"":""some"",""duration"":40}" +177,1672148326,1000.0,19,2,"{""description"":""some"",""duration"":40}" +178,1672218345,1000.0,19,2,"{""description"":""some"",""duration"":40}" +179,1672218353,1000.0,19,2,"{""description"":""some"",""duration"":40}" +180,1672222138,1000.0,19,2,"{""description"":""some"",""duration"":40}" +181,1672227880,1000.0,19,2,"{""description"":""some"",""duration"":40}" +182,1672227938,400.0,19,2,"{""description"":""some"",""duration"":40}" +183,1672234010,700.0,19,2,"{""description"":""some"",""duration"":40}" +184,1672303709,1000.0,19,2,"{""description"":""some"",""duration"":40}" +185,1672308144,800.0,19,2,"{""description"":""some"",""duration"":40}" +186,1672313359,700.0,19,2,"{""description"":""some"",""duration"":40}" +187,1672313377,200.0,19,2,"{""description"":""some"",""duration"":40}" +188,1672318111,1000.0,19,2,"{""description"":""some"",""duration"":40}" +189,1672318132,300.0,19,2,"{""description"":""some"",""duration"":40}" +190,1672320161,1000.0,19,2,"{""description"":""some"",""duration"":40}" +191,1672320172,200.0,19,2,"{""description"":""some"",""duration"":40}" +192,1672326596,700.0,19,2,"{""description"":""some"",""duration"":40}" +193,1672326607,1000.0,19,2,"{""description"":""some"",""duration"":40}" +194,1672326618,200.0,19,2,"{""description"":""some"",""duration"":40}" +195,1672389579,1000.0,19,2,"{""description"":""some"",""duration"":40}" +196,1672393666,1000.0,19,2,"{""description"":""some"",""duration"":40}" +197,1672400309,2000.0,19,2,"{""description"":""some"",""duration"":40}" +198,1672400322,200.0,19,2,"{""description"":""some"",""duration"":40}" +199,1672400406,500.0,19,2,"{""description"":""some"",""duration"":40}" +200,1672405512,1000.0,19,2,"{""description"":""some"",""duration"":40}" +201,1672426125,4500.0,19,2,"{""description"":""some"",""duration"":40}" +202,1673258846,400.0,19,2,"{""description"":""some"",""duration"":40}" +203,1673275425,1000.0,19,2,"{""description"":""some"",""duration"":40}" +204,1673344593,1000.0,19,2,"{""description"":""some"",""duration"":40}" +205,1673344606,1000.0,19,2,"{""description"":""some"",""duration"":40}" +206,1673347846,1000.0,19,2,"{""description"":""some"",""duration"":40}" +207,1673354520,2000.0,19,2,"{""description"":""some"",""duration"":40}" +208,1673602888,20000.0,19,2,"{""description"":""some"",""duration"":40}" +209,1673609998,10.0,19,2,"{""description"":""some"",""duration"":40}" +210,1673610036,1000.0,19,2,"{""description"":""some"",""duration"":40}" +211,1673619701,3500.0,19,2,"{""description"":""some"",""duration"":40}" +212,1673619714,1000.0,19,2,"{""description"":""some"",""duration"":40}" +213,1674038890,1500.0,19,2,"{""description"":""some"",""duration"":40}" +214,1674038898,1000.0,19,2,"{""description"":""some"",""duration"":40}" +215,1674038906,1000.0,19,2,"{""description"":""some"",""duration"":40}" +216,1674124770,1800.0,19,2,"{""description"":""some"",""duration"":40}" +217,1674155991,1000.0,19,2,"{""description"":""some"",""duration"":40}" +218,1674156006,600.0,19,2,"{""description"":""some"",""duration"":40}" +219,1674214709,1000.0,19,2,"{""description"":""some"",""duration"":40}" +220,1674214719,1000.0,19,2,"{""description"":""some"",""duration"":40}" +221,1674214736,500.0,19,2,"{""description"":""some"",""duration"":40}" +222,1674214750,2000.0,19,2,"{""description"":""some"",""duration"":40}" +223,1674379144,1000.0,19,2,"{""description"":""some"",""duration"":40}" +224,1674379158,300.0,19,2,"{""description"":""some"",""duration"":40}" +225,1674471516,2000.0,19,2,"{""description"":""some"",""duration"":40}" +226,1674478214,1000.0,19,2,"{""description"":""some"",""duration"":40}" +227,1674478231,300.0,19,2,"{""description"":""some"",""duration"":40}" +228,1674540208,3000.0,19,2,"{""description"":""some"",""duration"":40}" +229,1674724411,1000.0,19,2,"{""description"":""some"",""duration"":40}" +230,1674997317,700.0,19,2,"{""description"":""some"",""duration"":40}" +231,1674997326,500.0,19,2,"{""description"":""some"",""duration"":40}" +232,1674997343,350.0,19,2,"{""description"":""some"",""duration"":40}" +233,1675022124,1000.0,19,2,"{""description"":""some"",""duration"":40}" +234,1675083832,25000.0,19,2,"{""description"":""some"",""duration"":40}" +235,1675083844,3500.0,19,2,"{""description"":""some"",""duration"":40}" +236,1675083857,2000.0,19,2,"{""description"":""some"",""duration"":40}" +237,1675174989,700.0,19,2,"{""description"":""some"",""duration"":40}" +238,1675175000,1000.0,19,2,"{""description"":""some"",""duration"":40}" +239,1675175017,1800.0,19,2,"{""description"":""some"",""duration"":40}" +240,1675191645,1000.0,19,2,"{""description"":""some"",""duration"":40}" +241,1675345422,1800.0,19,2,"{""description"":""some"",""duration"":40}" +242,1675345429,700.0,19,2,"{""description"":""some"",""duration"":40}" +243,1675509724,900.0,19,2,"{""description"":""some"",""duration"":40}" +244,1675509732,1000.0,19,2,"{""description"":""some"",""duration"":40}" +245,1675510973,10.0,19,2,"{""description"":""some"",""duration"":40}" +246,1675670571,1000.0,19,2,"{""description"":""some"",""duration"":40}" +247,1675670582,1000.0,19,2,"{""description"":""some"",""duration"":40}" +248,1675681184,1000.0,19,2,"{""description"":""some"",""duration"":40}" +249,1675682854,700.0,19,2,"{""description"":""some"",""duration"":40}" +250,1675837827,800.0,19,2,"{""description"":""some"",""duration"":40}" +251,1675843115,1000.0,19,2,"{""description"":""some"",""duration"":40}" +252,1675876971,3000.0,19,2,"{""description"":""some"",""duration"":40}" +253,1675932830,300.0,19,2,"{""description"":""some"",""duration"":40}" +254,1675932839,200.0,19,2,"{""description"":""some"",""duration"":40}" +255,1675944721,1000.0,19,2,"{""description"":""some"",""duration"":40}" +256,1676008329,2000.0,19,2,"{""description"":""some"",""duration"":40}" +257,1676022215,3500.0,19,2,"{""description"":""some"",""duration"":40}" +258,1676034759,3500.0,19,2,"{""description"":""some"",""duration"":40}" +259,1676040413,1700.0,19,2,"{""description"":""some"",""duration"":40}" +260,1676302586,1000.0,19,2,"{""description"":""some"",""duration"":40}" +261,1676304103,3500.0,19,2,"{""description"":""some"",""duration"":40}" +262,1676315047,20000.0,19,2,"{""description"":""some"",""duration"":40}" +263,1676357146,1000.0,19,2,"{""description"":""some"",""duration"":40}" +264,1676372490,1000.0,19,2,"{""description"":""some"",""duration"":40}" +265,1676372490,1000.0,19,2,"{""description"":""some"",""duration"":40}" +266,1676372512,200.0,19,2,"{""description"":""some"",""duration"":40}" +267,1676392182,2200.0,19,2,"{""description"":""some"",""duration"":40}" +268,1676394067,500.0,19,2,"{""description"":""some"",""duration"":40}" +269,1676394084,700.0,19,2,"{""description"":""some"",""duration"":40}" +270,1676394099,300.0,19,2,"{""description"":""some"",""duration"":40}" +271,1676481893,5000.0,19,2,"{""description"":""some"",""duration"":40}" +272,1676714787,1000.0,19,2,"{""description"":""some"",""duration"":40}" +273,1676714802,1000.0,19,2,"{""description"":""some"",""duration"":40}" +274,1676714814,3000.0,19,2,"{""description"":""some"",""duration"":40}" +275,1676714822,1000.0,19,2,"{""description"":""some"",""duration"":40}" +276,1676714831,300.0,19,2,"{""description"":""some"",""duration"":40}" +277,1676809111,500.0,19,2,"{""description"":""some"",""duration"":40}" +278,1676809188,1000.0,19,2,"{""description"":""some"",""duration"":40}" +279,1676809204,300.0,19,2,"{""description"":""some"",""duration"":40}" +280,1676809216,1000.0,19,2,"{""description"":""some"",""duration"":40}" +281,1676809225,300.0,19,2,"{""description"":""some"",""duration"":40}" +282,1676906359,1000.0,19,2,"{""description"":""some"",""duration"":40}" +283,1676906377,500.0,19,2,"{""description"":""some"",""duration"":40}" +284,1676906385,1000.0,19,2,"{""description"":""some"",""duration"":40}" +285,1676906393,300.0,19,2,"{""description"":""some"",""duration"":40}" +286,1676906402,400.0,19,2,"{""description"":""some"",""duration"":40}" +287,1676906415,1800.0,19,2,"{""description"":""some"",""duration"":40}" +288,1676906440,3500.0,19,2,"{""description"":""some"",""duration"":40}" +289,1677261147,1000.0,19,2,"{""description"":""some"",""duration"":40}" +290,1677261159,1800.0,19,2,"{""description"":""some"",""duration"":40}" +291,1677493836,1000.0,19,2,"{""description"":""some"",""duration"":40}" +292,1677524903,3500.0,19,2,"{""description"":""some"",""duration"":40}" +293,1677603379,1000.0,19,2,"{""description"":""some"",""duration"":40}" +294,1677603390,1000.0,19,2,"{""description"":""some"",""duration"":40}" +295,1677909684,1000.0,19,2,"{""description"":""some"",""duration"":40}" +296,1678015228,700.0,19,2,"{""description"":""some"",""duration"":40}" +297,1678119822,3500.0,19,2,"{""description"":""some"",""duration"":40}" +298,1678120298,2000.0,19,2,"{""description"":""some"",""duration"":40}" +299,1678198568,1000.0,19,2,"{""description"":""some"",""duration"":40}" +300,1678198581,500.0,19,2,"{""description"":""some"",""duration"":40}" +301,1678478679,1000.0,19,2,"{""description"":""some"",""duration"":40}" +302,1678527822,1000.0,19,2,"{""description"":""some"",""duration"":40}" +303,1678527834,500.0,19,2,"{""description"":""some"",""duration"":40}" +304,1678532181,2500.0,19,2,"{""description"":""some"",""duration"":40}" +305,1678541402,1000.0,19,2,"{""description"":""some"",""duration"":40}" +306,1678543730,1000.0,19,2,"{""description"":""some"",""duration"":40}" +307,1678717604,3000.0,19,2,"{""description"":""some"",""duration"":40}" +308,1678776984,1000.0,19,2,"{""description"":""some"",""duration"":40}" +309,1678777040,20000.0,19,2,"{""description"":""some"",""duration"":40}" +310,1678787613,700.0,19,2,"{""description"":""some"",""duration"":40}" +311,1678787625,500.0,19,2,"{""description"":""some"",""duration"":40}" +312,1678805199,1000.0,19,2,"{""description"":""some"",""duration"":40}" +313,1678805212,1800.0,19,2,"{""description"":""some"",""duration"":40}" +314,1678805611,700.0,19,2,"{""description"":""some"",""duration"":40}" +315,1678805621,500.0,19,2,"{""description"":""some"",""duration"":40}" +316,1678963626,1000.0,19,2,"{""description"":""some"",""duration"":40}" +317,1678963633,300.0,19,2,"{""description"":""some"",""duration"":40}" +318,1678966411,1000.0,19,2,"{""description"":""some"",""duration"":40}" +319,1679046241,4000.0,19,2,"{""description"":""some"",""duration"":40}" +320,1679056941,1000.0,19,2,"{""description"":""some"",""duration"":40}" +321,1679061998,1000.0,19,2,"{""description"":""some"",""duration"":40}" +322,1679068223,1000.0,19,2,"{""description"":""some"",""duration"":40}" +323,1679153697,14000.0,19,2,"{""description"":""some"",""duration"":40}" +324,1679224295,3000.0,19,2,"{""description"":""some"",""duration"":40}" +325,1679224306,2000.0,19,2,"{""description"":""some"",""duration"":40}" +326,1679224369,1000.0,19,2,"{""description"":""some"",""duration"":40}" +327,1679224422,300.0,19,2,"{""description"":""some"",""duration"":40}" +328,1679224486,700.0,19,2,"{""description"":""some"",""duration"":40}" +329,1679229904,1000.0,19,2,"{""description"":""some"",""duration"":40}" +330,1679229927,300.0,19,2,"{""description"":""some"",""duration"":40}" +331,1679332859,1000.0,19,2,"{""description"":""some"",""duration"":40}" +332,1679332870,3500.0,19,2,"{""description"":""some"",""duration"":40}" +333,1679332903,3500.0,19,2,"{""description"":""some"",""duration"":40}" +334,1679470210,3000.0,19,2,"{""description"":""some"",""duration"":40}" +335,1679470218,1000.0,19,2,"{""description"":""some"",""duration"":40}" +336,1679482021,1000.0,19,2,"{""description"":""some"",""duration"":40}" +337,1679563931,1000.0,19,2,"{""description"":""some"",""duration"":40}" +338,1679571532,1000.0,19,2,"{""description"":""some"",""duration"":40}" +339,1679571545,500.0,19,2,"{""description"":""some"",""duration"":40}" +340,1679571552,1000.0,19,2,"{""description"":""some"",""duration"":40}" +341,1679584622,4000.0,19,2,"{""description"":""some"",""duration"":40}" +342,1679684724,1000.0,19,2,"{""description"":""some"",""duration"":40}" +343,1679941789,3500.0,19,2,"{""description"":""some"",""duration"":40}" +344,1680003956,1000.0,19,2,"{""description"":""some"",""duration"":40}" +345,1680272450,1000.0,19,2,"{""description"":""some"",""duration"":40}" +346,1680272461,1000.0,19,2,"{""description"":""some"",""duration"":40}" +347,1680272470,1000.0,19,2,"{""description"":""some"",""duration"":40}" +348,1680510632,1000.0,19,2,"{""description"":""some"",""duration"":40}" +349,1680510641,300.0,19,2,"{""description"":""some"",""duration"":40}" +350,1680510652,300.0,19,2,"{""description"":""some"",""duration"":40}" +351,1680526841,1000.0,19,2,"{""description"":""some"",""duration"":40}" +352,1680526853,700.0,19,2,"{""description"":""some"",""duration"":40}" +353,1680526865,300.0,19,2,"{""description"":""some"",""duration"":40}" +354,1680526882,3000.0,19,2,"{""description"":""some"",""duration"":40}" +355,1680547813,3500.0,19,2,"{""description"":""some"",""duration"":40}" +356,1680599086,500.0,19,2,"{""description"":""some"",""duration"":40}" +357,1680617464,1000.0,19,2,"{""description"":""some"",""duration"":40}" +358,1680617474,500.0,19,2,"{""description"":""some"",""duration"":40}" +359,1680617495,1000.0,19,2,"{""description"":""some"",""duration"":40}" +360,1680617508,700.0,19,2,"{""description"":""some"",""duration"":40}" +361,1680617519,2500.0,19,2,"{""description"":""some"",""duration"":40}" +362,1680780471,500.0,19,2,"{""description"":""some"",""duration"":40}" +363,1680780485,700.0,19,2,"{""description"":""some"",""duration"":40}" +364,1680780497,200.0,19,2,"{""description"":""some"",""duration"":40}" +365,1680780505,1000.0,19,2,"{""description"":""some"",""duration"":40}" +366,1680801021,800.0,19,2,"{""description"":""some"",""duration"":40}" +367,1680853871,1000.0,19,2,"{""description"":""some"",""duration"":40}" +368,1680869380,15000.0,19,2,"{""description"":""some"",""duration"":40}" +369,1680879812,500.0,19,2,"{""description"":""some"",""duration"":40}" +370,1680879822,1000.0,19,2,"{""description"":""some"",""duration"":40}" +371,1680948931,1000.0,19,2,"{""description"":""some"",""duration"":40}" +372,1680956966,1000.0,19,2,"{""description"":""some"",""duration"":40}" +373,1680957007,3500.0,19,2,"{""description"":""some"",""duration"":40}" +374,1681222689,3500.0,19,2,"{""description"":""some"",""duration"":40}" +375,1681301475,1000.0,19,2,"{""description"":""some"",""duration"":40}" +376,1681301483,1000.0,19,2,"{""description"":""some"",""duration"":40}" +377,1681301495,500.0,19,2,"{""description"":""some"",""duration"":40}" +378,1681314721,1000.0,19,2,"{""description"":""some"",""duration"":40}" +379,1681375795,900.0,19,2,"{""description"":""some"",""duration"":40}" +380,1681381055,1000.0,19,2,"{""description"":""some"",""duration"":40}" +381,1681381063,200.0,19,2,"{""description"":""some"",""duration"":40}" +382,1681381081,700.0,19,2,"{""description"":""some"",""duration"":40}" +383,1681473375,950.0,19,2,"{""description"":""some"",""duration"":40}" +384,1681540834,20000.0,19,2,"{""description"":""some"",""duration"":40}" +385,1681556018,1000.0,19,2,"{""description"":""some"",""duration"":40}" +386,1681556025,300.0,19,2,"{""description"":""some"",""duration"":40}" +387,1681556033,1000.0,19,2,"{""description"":""some"",""duration"":40}" +388,1681559058,300.0,19,2,"{""description"":""some"",""duration"":40}" +389,1681731983,1200.0,19,2,"{""description"":""some"",""duration"":40}" +390,1681731997,3500.0,19,2,"{""description"":""some"",""duration"":40}" +391,1681819763,1000.0,19,2,"{""description"":""some"",""duration"":40}" +392,1681819813,300.0,19,2,"{""description"":""some"",""duration"":40}" +393,1681826833,1800.0,19,2,"{""description"":""some"",""duration"":40}" +394,1681826847,500.0,19,2,"{""description"":""some"",""duration"":40}" +395,1681983885,700.0,19,2,"{""description"":""some"",""duration"":40}" +396,1681983895,100.0,19,2,"{""description"":""some"",""duration"":40}" +397,1681995794,1000.0,19,2,"{""description"":""some"",""duration"":40}" +398,1682099095,500.0,19,2,"{""description"":""some"",""duration"":40}" +399,1682156794,1000.0,19,2,"{""description"":""some"",""duration"":40}" +400,1682572414,700.0,19,2,"{""description"":""some"",""duration"":40}" +401,1682572431,4000.0,19,2,"{""description"":""some"",""duration"":40}" +402,1682590229,1000.0,19,2,"{""description"":""some"",""duration"":40}" +403,1682590238,1000.0,19,2,"{""description"":""some"",""duration"":40}" +404,1682741765,500.0,19,2,"{""description"":""some"",""duration"":40}" +405,1682748977,3000.0,19,2,"{""description"":""some"",""duration"":40}" +406,1682748989,1000.0,19,2,"{""description"":""some"",""duration"":40}" +407,1682764243,1000.0,19,2,"{""description"":""some"",""duration"":40}" +408,1682764252,1000.0,19,2,"{""description"":""some"",""duration"":40}" +409,1682764265,300.0,19,2,"{""description"":""some"",""duration"":40}" +410,1682771152,1800.0,19,2,"{""description"":""some"",""duration"":40}" +411,1682771163,700.0,19,2,"{""description"":""some"",""duration"":40}" +412,1682773862,1000.0,19,2,"{""description"":""some"",""duration"":40}" +413,1683059696,10.0,19,2,"{""description"":""some"",""duration"":40}" +414,1683107172,1000.0,19,2,"{""description"":""some"",""duration"":40}" +415,1683113552,3000.0,19,2,"{""description"":""some"",""duration"":40}" +416,1683113563,1000.0,19,2,"{""description"":""some"",""duration"":40}" +417,1683124134,1000.0,19,2,"{""description"":""some"",""duration"":40}" +418,1683269852,800.0,19,2,"{""description"":""some"",""duration"":40}" +419,1683271255,700.0,19,2,"{""description"":""some"",""duration"":40}" +420,1683543424,1000.0,19,2,"{""description"":""some"",""duration"":40}" +421,1683554211,1000.0,19,2,"{""description"":""some"",""duration"":40}" +422,1683554224,300.0,19,2,"{""description"":""some"",""duration"":40}" +423,1683554236,1000.0,19,2,"{""description"":""some"",""duration"":40}" +424,1683554246,300.0,19,2,"{""description"":""some"",""duration"":40}" +425,1683557465,1000.0,19,2,"{""description"":""some"",""duration"":40}" +426,1683557477,350.0,19,2,"{""description"":""some"",""duration"":40}" +427,1683798328,1000.0,19,2,"{""description"":""some"",""duration"":40}" +428,1683798341,300.0,19,2,"{""description"":""some"",""duration"":40}" +429,1683798351,1000.0,19,2,"{""description"":""some"",""duration"":40}" +430,1683809307,1000.0,19,2,"{""description"":""some"",""duration"":40}" +431,1683809323,500.0,19,2,"{""description"":""some"",""duration"":40}" +432,1683809336,1000.0,19,2,"{""description"":""some"",""duration"":40}" +433,1684063474,700.0,19,2,"{""description"":""some"",""duration"":40}" +434,1684063508,300.0,19,2,"{""description"":""some"",""duration"":40}" +435,1684063533,2500.0,19,2,"{""description"":""some"",""duration"":40}" +436,1684068697,1000.0,19,2,"{""description"":""some"",""duration"":40}" +437,1684068708,300.0,19,2,"{""description"":""some"",""duration"":40}" +438,1684068723,2000.0,19,2,"{""description"":""some"",""duration"":40}" +439,1684068929,300.0,19,2,"{""description"":""some"",""duration"":40}" +440,1684142246,20000.0,19,2,"{""description"":""some"",""duration"":40}" +441,1684159623,10000.0,19,2,"{""description"":""some"",""duration"":40}" +442,1684159658,700.0,19,2,"{""description"":""some"",""duration"":40}" +443,1684234591,1000.0,19,2,"{""description"":""some"",""duration"":40}" +444,1684255447,1000.0,19,2,"{""description"":""some"",""duration"":40}" +445,1684255459,1000.0,19,2,"{""description"":""some"",""duration"":40}" +446,1684416481,1000.0,19,2,"{""description"":""some"",""duration"":40}" +447,1684423281,3500.0,19,2,"{""description"":""some"",""duration"":40}" +448,1684427403,1000.0,19,2,"{""description"":""some"",""duration"":40}" +449,1684507089,1000.0,19,2,"{""description"":""some"",""duration"":40}" +450,1684510171,2250.0,19,2,"{""description"":""some"",""duration"":40}" +451,1684515734,1000.0,19,2,"{""description"":""some"",""duration"":40}" +452,1684515742,1000.0,19,2,"{""description"":""some"",""duration"":40}" +453,1684515753,300.0,19,2,"{""description"":""some"",""duration"":40}" +454,1684581869,1000.0,19,2,"{""description"":""some"",""duration"":40}" +455,1684581878,300.0,19,2,"{""description"":""some"",""duration"":40}" +456,1684659380,1000.0,19,2,"{""description"":""some"",""duration"":40}" +457,1684659387,1000.0,19,2,"{""description"":""some"",""duration"":40}" +458,1684659400,300.0,19,2,"{""description"":""some"",""duration"":40}" +459,1684659410,1500.0,19,2,"{""description"":""some"",""duration"":40}" +460,1684659418,3000.0,19,2,"{""description"":""some"",""duration"":40}" +461,1684662528,500.0,19,2,"{""description"":""some"",""duration"":40}" +462,1684745403,1000.0,19,2,"{""description"":""some"",""duration"":40}" +463,1684844953,1000.0,19,2,"{""description"":""some"",""duration"":40}" +464,1684919223,1000.0,19,2,"{""description"":""some"",""duration"":40}" +465,1684919232,700.0,19,2,"{""description"":""some"",""duration"":40}" +466,1685251726,1000.0,19,2,"{""description"":""some"",""duration"":40}" +467,1685251726,1800.0,19,2,"{""description"":""some"",""duration"":40}" +468,1685251726,700.0,19,2,"{""description"":""some"",""duration"":40}" +469,1685495439,1000.0,19,2,"{""description"":""some"",""duration"":40}" +470,1685495439,700.0,19,2,"{""description"":""some"",""duration"":40}" +471,1685495439,1000.0,19,2,"{""description"":""some"",""duration"":40}" +472,1685495439,300.0,19,2,"{""description"":""some"",""duration"":40}" +473,1685625474,1000.0,19,2,"{""description"":""some"",""duration"":40}" +474,1685625474,200.0,19,2,"{""description"":""some"",""duration"":40}" +475,1685625474,1000.0,19,2,"{""description"":""some"",""duration"":40}" +476,1685625474,300.0,19,2,"{""description"":""some"",""duration"":40}" +477,1685625474,1000.0,19,2,"{""description"":""some"",""duration"":40}" +478,1685625474,1000.0,19,2,"{""description"":""some"",""duration"":40}" +479,1685625474,700.0,19,2,"{""description"":""some"",""duration"":40}" +480,1685625474,1000.0,19,2,"{""description"":""some"",""duration"":40}" +481,1685625474,1800.0,19,2,"{""description"":""some"",""duration"":40}" +482,1685625474,700.0,19,2,"{""description"":""some"",""duration"":40}" +483,1685625474,1000.0,19,2,"{""description"":""some"",""duration"":40}" +484,1685625474,3500.0,19,2,"{""description"":""some"",""duration"":40}" +485,1685625474,3000.0,19,2,"{""description"":""some"",""duration"":40}" +486,1685625474,3000.0,19,2,"{""description"":""some"",""duration"":40}" +487,1685993343,4000.0,19,2,"{""description"":""some"",""duration"":40}" +488,1686060913,1000.0,19,2,"{""description"":""some"",""duration"":40}" +489,1686060926,440.0,19,2,"{""description"":""some"",""duration"":40}" +490,1686060950,3500.0,19,2,"{""description"":""some"",""duration"":40}" +491,1686072609,1000.0,19,2,"{""description"":""some"",""duration"":40}" +492,1686072620,500.0,19,2,"{""description"":""some"",""duration"":40}" +493,1686142826,1800.0,19,2,"{""description"":""some"",""duration"":40}" +494,1686142837,200.0,19,2,"{""description"":""some"",""duration"":40}" +495,1686142846,1000.0,19,2,"{""description"":""some"",""duration"":40}" +496,1686142855,1000.0,19,2,"{""description"":""some"",""duration"":40}" +497,1686231940,1000.0,19,2,"{""description"":""some"",""duration"":40}" +498,1686231949,1000.0,19,2,"{""description"":""some"",""duration"":40}" +499,1686231956,1000.0,19,2,"{""description"":""some"",""duration"":40}" +500,1686231974,0.0,19,2,"{""description"":""some"",""duration"":40}" +501,1686296670,1000.0,19,2,"{""description"":""some"",""duration"":40}" +502,1686316196,1000.0,19,2,"{""description"":""some"",""duration"":40}" +503,1686316218,200.0,19,2,"{""description"":""some"",""duration"":40}" +504,1686316226,1000.0,19,2,"{""description"":""some"",""duration"":40}" +505,1686316234,500.0,19,2,"{""description"":""some"",""duration"":40}" +506,1686842773,15000.0,19,2,"{""description"":""some"",""duration"":40}" +507,1686849138,20000.0,19,2,"{""description"":""some"",""duration"":40}" +508,1686905957,1000.0,19,2,"{""description"":""some"",""duration"":40}" +509,1686905987,500.0,19,2,"{""description"":""some"",""duration"":40}" +510,1686993942,1000.0,19,2,"{""description"":""some"",""duration"":40}" +511,1687510179,1000.0,19,2,"{""description"":""some"",""duration"":40}" +512,1687510190,500.0,19,2,"{""description"":""some"",""duration"":40}" +513,1687510198,1000.0,19,2,"{""description"":""some"",""duration"":40}" +514,1687528818,700.0,19,2,"{""description"":""some"",""duration"":40}" +515,1687528830,1000.0,19,2,"{""description"":""some"",""duration"":40}" +516,1687528844,1000.0,19,2,"{""description"":""some"",""duration"":40}" +517,1687531541,700.0,19,2,"{""description"":""some"",""duration"":40}" +518,1687607686,1500.0,19,2,"{""description"":""some"",""duration"":40}" +519,1687607697,1000.0,19,2,"{""description"":""some"",""duration"":40}" +520,1687607707,300.0,19,2,"{""description"":""some"",""duration"":40}" +521,1687622885,1000.0,19,2,"{""description"":""some"",""duration"":40}" +522,1687622896,300.0,19,2,"{""description"":""some"",""duration"":40}" +523,1687776852,3500.0,19,2,"{""description"":""some"",""duration"":40}" +524,1687776866,800.0,19,2,"{""description"":""some"",""duration"":40}" +525,1687776883,1500.0,19,2,"{""description"":""some"",""duration"":40}" +526,1687786827,3000.0,19,2,"{""description"":""some"",""duration"":40}" +527,1687786836,3000.0,19,2,"{""description"":""some"",""duration"":40}" +528,1687901328,1000.0,19,2,"{""description"":""some"",""duration"":40}" +529,1687939090,1000.0,19,2,"{""description"":""some"",""duration"":40}" +530,1687961907,1000.0,19,2,"{""description"":""some"",""duration"":40}" +531,1688055227,3500.0,19,2,"{""description"":""some"",""duration"":40}" +532,1688203080,1800.0,19,2,"{""description"":""some"",""duration"":40}" +533,1688203088,500.0,19,2,"{""description"":""some"",""duration"":40}" +534,1688212129,1000.0,19,2,"{""description"":""some"",""duration"":40}" +535,1688212138,500.0,19,2,"{""description"":""some"",""duration"":40}" +536,1688212669,300.0,19,2,"{""description"":""some"",""duration"":40}" +537,1688377745,1000.0,19,2,"{""description"":""some"",""duration"":40}" +538,1688377763,1000.0,19,2,"{""description"":""some"",""duration"":40}" +539,1688730464,1000.0,19,2,"{""description"":""some"",""duration"":40}" +540,1688730473,1800.0,19,2,"{""description"":""some"",""duration"":40}" +541,1688805023,4500.0,19,2,"{""description"":""some"",""duration"":40}" +542,1688808683,1000.0,19,2,"{""description"":""some"",""duration"":40}" +543,1689180206,1000.0,19,2,"{""description"":""some"",""duration"":40}" +544,1689237819,3000.0,19,2,"{""description"":""some"",""duration"":40}" +545,1689254432,1000.0,19,2,"{""description"":""some"",""duration"":40}" +546,1689254441,1000.0,19,2,"{""description"":""some"",""duration"":40}" +547,1689254450,1000.0,19,2,"{""description"":""some"",""duration"":40}" +548,1689325295,1500.0,19,2,"{""description"":""some"",""duration"":40}" +549,1689325307,1500.0,19,2,"{""description"":""some"",""duration"":40}" +550,1689590069,1100.0,19,2,"{""description"":""some"",""duration"":40}" +551,1689590089,500.0,19,2,"{""description"":""some"",""duration"":40}" +552,1689590097,1000.0,19,2,"{""description"":""some"",""duration"":40}" +553,1689681790,1000.0,19,2,"{""description"":""some"",""duration"":40}" +554,1689913856,1000.0,19,2,"{""description"":""some"",""duration"":40}" +555,1689913865,1000.0,19,2,"{""description"":""some"",""duration"":40}" +556,1689913877,2000.0,19,2,"{""description"":""some"",""duration"":40}" +557,1689948414,1000.0,19,2,"{""description"":""some"",""duration"":40}" +558,1689948424,500.0,19,2,"{""description"":""some"",""duration"":40}" +559,1689948440,1000.0,19,2,"{""description"":""some"",""duration"":40}" +560,1689948452,1300.0,19,2,"{""description"":""some"",""duration"":40}" +561,1689948469,1300.0,19,2,"{""description"":""some"",""duration"":40}" +562,1690029025,1300.0,19,2,"{""description"":""some"",""duration"":40}" +563,1690029038,1300.0,19,2,"{""description"":""some"",""duration"":40}" +564,1690029045,1000.0,19,2,"{""description"":""some"",""duration"":40}" +565,1690120357,3500.0,19,2,"{""description"":""some"",""duration"":40}" +566,1690120374,4500.0,19,2,"{""description"":""some"",""duration"":40}" +567,1690211540,1300.0,19,2,"{""description"":""some"",""duration"":40}" +568,1690271007,1000.0,19,2,"{""description"":""some"",""duration"":40}" +569,1690471407,700.0,19,2,"{""description"":""some"",""duration"":40}" +570,1690546782,2000.0,19,2,"{""description"":""some"",""duration"":40}" +571,1690618437,1800.0,19,2,"{""description"":""some"",""duration"":40}" +572,1691056244,1000.0,19,2,"{""description"":""some"",""duration"":40}" +573,1691061626,1000.0,19,2,"{""description"":""some"",""duration"":40}" +574,1691061644,500.0,19,2,"{""description"":""some"",""duration"":40}" +575,1691061658,800.0,19,2,"{""description"":""some"",""duration"":40}" +576,1691067178,1000.0,19,2,"{""description"":""some"",""duration"":40}" +577,1691145984,800.0,19,2,"{""description"":""some"",""duration"":40}" +578,1691155256,1000.0,19,2,"{""description"":""some"",""duration"":40}" +579,1691155268,1000.0,19,2,"{""description"":""some"",""duration"":40}" +580,1691155286,300.0,19,2,"{""description"":""some"",""duration"":40}" +581,1691228808,2000.0,19,2,"{""description"":""some"",""duration"":40}" +582,1691325700,2500.0,19,2,"{""description"":""some"",""duration"":40}" +583,1691423922,500.0,19,2,"{""description"":""some"",""duration"":40}" +584,1691568834,1000.0,19,2,"{""description"":""some"",""duration"":40}" +585,1691568841,1000.0,19,2,"{""description"":""some"",""duration"":40}" +586,1691568853,2000.0,19,2,"{""description"":""some"",""duration"":40}" +587,1691659838,1000.0,19,2,"{""description"":""some"",""duration"":40}" +588,1691742970,6000.0,19,2,"{""description"":""some"",""duration"":40}" +589,1691750551,1800.0,19,2,"{""description"":""some"",""duration"":40}" +590,1691750567,700.0,19,2,"{""description"":""some"",""duration"":40}" +591,1691836061,1500.0,19,2,"{""description"":""some"",""duration"":40}" +592,1691836133,1000.0,19,2,"{""description"":""some"",""duration"":40}" +593,1691836142,200.0,19,2,"{""description"":""some"",""duration"":40}" +594,1691838372,1000.0,19,2,"{""description"":""some"",""duration"":40}" +595,1691838383,500.0,19,2,"{""description"":""some"",""duration"":40}" +596,1691849545,1000.0,19,2,"{""description"":""some"",""duration"":40}" +597,1691849552,1000.0,19,2,"{""description"":""some"",""duration"":40}" +598,1691849570,500.0,19,2,"{""description"":""some"",""duration"":40}" +599,1691851915,1000.0,19,2,"{""description"":""some"",""duration"":40}" +600,1691856926,1000.0,19,2,"{""description"":""some"",""duration"":40}" +601,1691913765,1000.0,19,2,"{""description"":""some"",""duration"":40}" +602,1692017490,700.0,19,2,"{""description"":""some"",""duration"":40}" +603,1692108560,1000.0,19,2,"{""description"":""some"",""duration"":40}" +604,1692108571,2000.0,19,2,"{""description"":""some"",""duration"":40}" +605,1692178948,1000.0,19,2,"{""description"":""some"",""duration"":40}" +606,1692178964,500.0,19,2,"{""description"":""some"",""duration"":40}" +607,1692184222,3000.0,19,2,"{""description"":""some"",""duration"":40}" +608,1692277211,700.0,19,2,"{""description"":""some"",""duration"":40}" +609,1692277236,200.0,19,2,"{""description"":""some"",""duration"":40}" +610,1692277257,900.0,19,2,"{""description"":""some"",""duration"":40}" +611,1692277279,2000.0,19,2,"{""description"":""some"",""duration"":40}" +612,1692277294,600.0,19,2,"{""description"":""some"",""duration"":40}" +613,1692277318,2000.0,19,2,"{""description"":""some"",""duration"":40}" +614,1692280467,1000.0,19,2,"{""description"":""some"",""duration"":40}" +615,1692280484,300.0,19,2,"{""description"":""some"",""duration"":40}" +616,1692289428,1000.0,19,2,"{""description"":""some"",""duration"":40}" +617,1692441584,1000.0,19,2,"{""description"":""some"",""duration"":40}" +618,1692447214,2000.0,19,2,"{""description"":""some"",""duration"":40}" +619,1692447222,1000.0,19,2,"{""description"":""some"",""duration"":40}" +620,1692447238,500.0,19,2,"{""description"":""some"",""duration"":40}" +621,1692524397,1000.0,19,2,"{""description"":""some"",""duration"":40}" +622,1692531489,700.0,19,2,"{""description"":""some"",""duration"":40}" +623,1692531649,500.0,19,2,"{""description"":""some"",""duration"":40}" +624,1692531657,900.0,19,2,"{""description"":""some"",""duration"":40}" +625,1692603771,1000.0,19,2,"{""description"":""some"",""duration"":40}" +626,1692618584,600.0,19,2,"{""description"":""some"",""duration"":40}" +627,1692618591,500.0,19,2,"{""description"":""some"",""duration"":40}" +628,1692618599,700.0,19,2,"{""description"":""some"",""duration"":40}" +629,1692806265,2000.0,19,2,"{""description"":""some"",""duration"":40}" +630,1692866389,1000.0,19,2,"{""description"":""some"",""duration"":40}" +631,1692866397,1000.0,19,2,"{""description"":""some"",""duration"":40}" +632,1692866408,1000.0,19,2,"{""description"":""some"",""duration"":40}" +633,1692870155,1000.0,19,2,"{""description"":""some"",""duration"":40}" +634,1692870166,500.0,19,2,"{""description"":""some"",""duration"":40}" +635,1692967112,1000.0,19,2,"{""description"":""some"",""duration"":40}" +636,1692967127,300.0,19,2,"{""description"":""some"",""duration"":40}" +637,1692971743,1000.0,19,2,"{""description"":""some"",""duration"":40}" +638,1692971762,500.0,19,2,"{""description"":""some"",""duration"":40}" +639,1692971780,500.0,19,2,"{""description"":""some"",""duration"":40}" +640,1692971791,1000.0,19,2,"{""description"":""some"",""duration"":40}" +641,1692971826,500.0,19,2,"{""description"":""some"",""duration"":40}" +642,1692971834,300.0,19,2,"{""description"":""some"",""duration"":40}" +643,1692974608,1000.0,19,2,"{""description"":""some"",""duration"":40}" +644,1693036293,1000.0,19,2,"{""description"":""some"",""duration"":40}" +645,1693036306,500.0,19,2,"{""description"":""some"",""duration"":40}" +646,1693040042,1000.0,19,2,"{""description"":""some"",""duration"":40}" +647,1693043292,1000.0,19,2,"{""description"":""some"",""duration"":40}" +648,1693043300,500.0,19,2,"{""description"":""some"",""duration"":40}" +649,1693226064,2000.0,19,2,"{""description"":""some"",""duration"":40}" +650,1693226076,700.0,19,2,"{""description"":""some"",""duration"":40}" +651,1693226086,300.0,19,2,"{""description"":""some"",""duration"":40}" +652,1693226117,2000.0,19,2,"{""description"":""some"",""duration"":40}" +653,1693303885,1000.0,19,2,"{""description"":""some"",""duration"":40}" +654,1693303893,1000.0,19,2,"{""description"":""some"",""duration"":40}" +655,1693385143,1000.0,19,2,"{""description"":""some"",""duration"":40}" +656,1693385155,300.0,19,2,"{""description"":""some"",""duration"":40}" +657,1693385176,500.0,19,2,"{""description"":""some"",""duration"":40}" +658,1693483053,600.0,19,2,"{""description"":""some"",""duration"":40}" +659,1693483065,700.0,19,2,"{""description"":""some"",""duration"":40}" +660,1693483081,500.0,19,2,"{""description"":""some"",""duration"":40}" +661,1693572936,1000.0,19,2,"{""description"":""some"",""duration"":40}" +662,1693578938,1000.0,19,2,"{""description"":""some"",""duration"":40}" +663,1693641451,1000.0,19,2,"{""description"":""some"",""duration"":40}" +664,1693641462,300.0,19,2,"{""description"":""some"",""duration"":40}" +665,1693657446,2000.0,19,2,"{""description"":""some"",""duration"":40}" +666,1693657467,2000.0,19,2,"{""description"":""some"",""duration"":40}" +667,1693850745,1000.0,19,2,"{""description"":""some"",""duration"":40}" +668,1693850758,1500.0,19,2,"{""description"":""some"",""duration"":40}" +669,1693985700,500.0,19,2,"{""description"":""some"",""duration"":40}" +670,1694018421,500.0,19,2,"{""description"":""some"",""duration"":40}" +671,1694087017,1000.0,19,2,"{""description"":""some"",""duration"":40}" +672,1694087028,500.0,19,2,"{""description"":""some"",""duration"":40}" +673,1694180430,1000.0,19,2,"{""description"":""some"",""duration"":40}" +674,1694260565,1000.0,19,2,"{""description"":""some"",""duration"":40}" +675,1694260574,500.0,19,2,"{""description"":""some"",""duration"":40}" +676,1694268371,1000.0,19,2,"{""description"":""some"",""duration"":40}" +677,1694425420,1000.0,19,2,"{""description"":""some"",""duration"":40}" +678,1694425434,2000.0,19,2,"{""description"":""some"",""duration"":40}" +679,1694437840,1000.0,19,2,"{""description"":""some"",""duration"":40}" +680,1694595492,1500.0,19,2,"{""description"":""some"",""duration"":40}" +681,1694607612,1000.0,19,2,"{""description"":""some"",""duration"":40}" +682,1694611951,1000.0,19,2,"{""description"":""some"",""duration"":40}" +683,1694689945,3500.0,19,2,"{""description"":""some"",""duration"":40}" +684,1695030060,1000.0,19,2,"{""description"":""some"",""duration"":40}" +685,1695044951,2000.0,19,2,"{""description"":""some"",""duration"":40}" +686,1695115942,1000.0,19,2,"{""description"":""some"",""duration"":40}" +687,1695126728,2000.0,19,2,"{""description"":""some"",""duration"":40}" +688,1695131546,1000.0,19,2,"{""description"":""some"",""duration"":40}" +689,1695131557,500.0,19,2,"{""description"":""some"",""duration"":40}" +690,1695131566,1000.0,19,2,"{""description"":""some"",""duration"":40}" +691,1695133639,1000.0,19,2,"{""description"":""some"",""duration"":40}" +692,1695204170,1000.0,19,2,"{""description"":""some"",""duration"":40}" +693,1695208128,1000.0,19,2,"{""description"":""some"",""duration"":40}" +694,1695208140,1000.0,19,2,"{""description"":""some"",""duration"":40}" +695,1695208152,300.0,19,2,"{""description"":""some"",""duration"":40}" +696,1695213265,1000.0,19,2,"{""description"":""some"",""duration"":40}" +697,1695213275,300.0,19,2,"{""description"":""some"",""duration"":40}" +698,1695226073,900.0,19,2,"{""description"":""some"",""duration"":40}" +699,1695369878,3000.0,19,2,"{""description"":""some"",""duration"":40}" +700,1695447410,1000.0,19,2,"{""description"":""some"",""duration"":40}" +701,1695447420,300.0,19,2,"{""description"":""some"",""duration"":40}" +702,1695462527,1000.0,19,2,"{""description"":""some"",""duration"":40}" +703,1695542006,4500.0,19,2,"{""description"":""some"",""duration"":40}" +704,1695724915,1000.0,19,2,"{""description"":""some"",""duration"":40}" +705,1695724929,300.0,19,2,"{""description"":""some"",""duration"":40}" +706,1695724960,1000.0,19,2,"{""description"":""some"",""duration"":40}" +707,1695724970,1000.0,19,2,"{""description"":""some"",""duration"":40}" +708,1695812407,1000.0,19,2,"{""description"":""some"",""duration"":40}" +709,1695977012,2000.0,19,2,"{""description"":""some"",""duration"":40}" +710,1695977084,1000.0,19,2,"{""description"":""some"",""duration"":40}" +711,1695977099,500.0,19,2,"{""description"":""some"",""duration"":40}" +712,1695978463,700.0,19,2,"{""description"":""some"",""duration"":40}" +713,1695989760,3000.0,19,2,"{""description"":""some"",""duration"":40}" +714,1695994538,2000.0,19,2,"{""description"":""some"",""duration"":40}" +715,1696064713,1000.0,19,2,"{""description"":""some"",""duration"":40}" +716,1696064725,700.0,19,2,"{""description"":""some"",""duration"":40}" +717,1696064734,500.0,19,2,"{""description"":""some"",""duration"":40}" +718,1696237522,2000.0,19,2,"{""description"":""some"",""duration"":40}" +719,1696237537,700.0,19,2,"{""description"":""some"",""duration"":40}" +720,1696409137,1000.0,19,2,"{""description"":""some"",""duration"":40}" +721,1696413405,1000.0,19,2,"{""description"":""some"",""duration"":40}" +722,1696413414,300.0,19,2,"{""description"":""some"",""duration"":40}" +723,1696419260,700.0,19,2,"{""description"":""some"",""duration"":40}" +724,1696419270,500.0,19,2,"{""description"":""some"",""duration"":40}" +725,1696498224,3500.0,19,2,"{""description"":""some"",""duration"":40}" +726,1696512272,600.0,19,2,"{""description"":""some"",""duration"":40}" +727,1696512306,600.0,19,2,"{""description"":""some"",""duration"":40}" +728,1696512314,1000.0,19,2,"{""description"":""some"",""duration"":40}" +729,1696584198,2000.0,19,2,"{""description"":""some"",""duration"":40}" +730,1696602696,2000.0,19,2,"{""description"":""some"",""duration"":40}" +731,1696763281,3000.0,19,2,"{""description"":""some"",""duration"":40}" +732,1696842215,1000.0,19,2,"{""description"":""some"",""duration"":40}" +733,1696842227,500.0,19,2,"{""description"":""some"",""duration"":40}" +734,1696863818,1000.0,19,2,"{""description"":""some"",""duration"":40}" +735,1696951368,2000.0,19,2,"{""description"":""some"",""duration"":40}" +736,1697016706,1000.0,19,2,"{""description"":""some"",""duration"":40}" +737,1697026949,2000.0,19,2,"{""description"":""some"",""duration"":40}" +738,1697193136,800.0,19,2,"{""description"":""some"",""duration"":40}" +739,1697193145,1000.0,19,2,"{""description"":""some"",""duration"":40}" +740,1697193156,500.0,19,2,"{""description"":""some"",""duration"":40}" +741,1697200701,2000.0,19,2,"{""description"":""some"",""duration"":40}" +742,1697200711,1000.0,19,2,"{""description"":""some"",""duration"":40}" +743,1697204569,1000.0,19,2,"{""description"":""some"",""duration"":40}" +744,1697446745,2000.0,19,2,"{""description"":""some"",""duration"":40}" +745,1697446756,300.0,19,2,"{""description"":""some"",""duration"":40}" +746,1697446765,1000.0,19,2,"{""description"":""some"",""duration"":40}" +747,1697618988,1000.0,19,2,"{""description"":""some"",""duration"":40}" +748,1697631857,1000.0,19,2,"{""description"":""some"",""duration"":40}" +749,1697708072,1000.0,19,2,"{""description"":""some"",""duration"":40}" +750,1697708180,1000.0,19,2,"{""description"":""some"",""duration"":40}" +751,1697708189,1300.0,19,2,"{""description"":""some"",""duration"":40}" +752,1697799272,20000.0,19,2,"{""description"":""some"",""duration"":40}" +753,1697799287,1000.0,19,2,"{""description"":""some"",""duration"":40}" +754,1698042003,2000.0,19,2,"{""description"":""some"",""duration"":40}" +755,1698042014,1000.0,19,2,"{""description"":""some"",""duration"":40}" +756,1698064325,1000.0,19,2,"{""description"":""some"",""duration"":40}" +757,1698069302,2000.0,19,2,"{""description"":""some"",""duration"":40}" +758,1698073704,1000.0,19,2,"{""description"":""some"",""duration"":40}" +759,1698073712,1000.0,19,2,"{""description"":""some"",""duration"":40}" +760,1698132322,3000.0,19,2,"{""description"":""some"",""duration"":40}" +761,1698132335,1000.0,19,2,"{""description"":""some"",""duration"":40}" +762,1698132352,500.0,19,2,"{""description"":""some"",""duration"":40}" +763,1698232445,1000.0,19,2,"{""description"":""some"",""duration"":40}" +764,1698232455,1000.0,19,2,"{""description"":""some"",""duration"":40}" +765,1698232465,500.0,19,2,"{""description"":""some"",""duration"":40}" +766,1698233758,700.0,19,2,"{""description"":""some"",""duration"":40}" +767,1698493234,1000.0,19,2,"{""description"":""some"",""duration"":40}" +768,1698493243,2000.0,19,2,"{""description"":""some"",""duration"":40}" +769,1698497541,1000.0,19,2,"{""description"":""some"",""duration"":40}" +770,1698501209,1000.0,19,2,"{""description"":""some"",""duration"":40}" +771,1698572939,5000.0,19,2,"{""description"":""some"",""duration"":40}" +772,1698831768,1000.0,19,2,"{""description"":""some"",""duration"":40}" +773,1698864277,2000.0,19,2,"{""description"":""some"",""duration"":40}" +774,1699270941,1000.0,19,2,"{""description"":""some"",""duration"":40}" +775,1699270965,2000.0,19,2,"{""description"":""some"",""duration"":40}" +776,1699270982,300.0,19,2,"{""description"":""some"",""duration"":40}" +777,1699270993,1000.0,19,2,"{""description"":""some"",""duration"":40}" +778,1699271062,700.0,19,2,"{""description"":""some"",""duration"":40}" +779,1699345725,1000.0,19,2,"{""description"":""some"",""duration"":40}" +780,1699364573,1000.0,19,2,"{""description"":""some"",""duration"":40}" +781,1699436162,1000.0,19,2,"{""description"":""some"",""duration"":40}" +782,1699444375,1000.0,19,2,"{""description"":""some"",""duration"":40}" +783,1699444389,500.0,19,2,"{""description"":""some"",""duration"":40}" +784,1699642609,2000.0,19,2,"{""description"":""some"",""duration"":40}" +785,1699642619,1000.0,19,2,"{""description"":""some"",""duration"":40}" +786,1700038094,1000.0,19,2,"{""description"":""some"",""duration"":40}" +787,1700038107,2000.0,19,2,"{""description"":""some"",""duration"":40}" +788,1700042242,1000.0,19,2,"{""description"":""some"",""duration"":40}" +789,1700042256,1000.0,19,2,"{""description"":""some"",""duration"":40}" +790,1700042267,1000.0,19,2,"{""description"":""some"",""duration"":40}" +791,1700056633,1000.0,19,2,"{""description"":""some"",""duration"":40}" +792,1700056654,300.0,19,2,"{""description"":""some"",""duration"":40}" +793,1700056664,1000.0,19,2,"{""description"":""some"",""duration"":40}" +794,1700205890,700.0,19,2,"{""description"":""some"",""duration"":40}" +795,1700229914,3500.0,19,2,"{""description"":""some"",""duration"":40}" +796,1700565540,20000.0,19,2,"{""description"":""some"",""duration"":40}" +797,1700565558,1000.0,19,2,"{""description"":""some"",""duration"":40}" +798,1700656546,5000.0,19,2,"{""description"":""some"",""duration"":40}" +799,1700736068,1500.0,19,2,"{""description"":""some"",""duration"":40}" +800,1700739329,1500.0,19,2,"{""description"":""some"",""duration"":40}" +801,1700739341,300.0,19,2,"{""description"":""some"",""duration"":40}" +802,1700902222,1000.0,19,2,"{""description"":""some"",""duration"":40}" +803,1700902246,1000.0,19,2,"{""description"":""some"",""duration"":40}" +804,1700902259,500.0,19,2,"{""description"":""some"",""duration"":40}" +805,1700902272,4000.0,19,2,"{""description"":""some"",""duration"":40}" +806,1700926026,1500.0,19,2,"{""description"":""some"",""duration"":40}" +807,1701097123,1000.0,19,2,"{""description"":""some"",""duration"":40}" +808,1701097139,2400.0,19,2,"{""description"":""some"",""duration"":40}" +809,1701097153,800.0,19,2,"{""description"":""some"",""duration"":40}" +810,1701245408,1000.0,19,2,"{""description"":""some"",""duration"":40}" +811,1701245421,2000.0,19,2,"{""description"":""some"",""duration"":40}" +812,1701251764,1000.0,19,2,"{""description"":""some"",""duration"":40}" +813,1701265932,7500.0,19,2,"{""description"":""some"",""duration"":40}" +814,1701418989,1000.0,19,2,"{""description"":""some"",""duration"":40}" +815,1701419000,300.0,19,2,"{""description"":""some"",""duration"":40}" +816,1701419014,1000.0,19,2,"{""description"":""some"",""duration"":40}" +817,1701419035,600.0,19,2,"{""description"":""some"",""duration"":40}" +818,1701419048,400.0,19,2,"{""description"":""some"",""duration"":40}" +819,1701419068,1000.0,19,2,"{""description"":""some"",""duration"":40}" +820,1701703329,2000.0,19,2,"{""description"":""some"",""duration"":40}" +821,1701703340,500.0,19,2,"{""description"":""some"",""duration"":40}" +822,1701770749,1500.0,19,2,"{""description"":""some"",""duration"":40}" +823,1701859469,1000.0,19,2,"{""description"":""some"",""duration"":40}" +824,1701947352,2600.0,19,2,"{""description"":""some"",""duration"":40}" +825,1701947366,1500.0,19,2,"{""description"":""some"",""duration"":40}" +826,1701950073,1500.0,19,2,"{""description"":""some"",""duration"":40}" +827,1701953358,1500.0,19,2,"{""description"":""some"",""duration"":40}" +828,1701953673,500.0,19,2,"{""description"":""some"",""duration"":40}" +829,1702024426,1000.0,19,2,"{""description"":""some"",""duration"":40}" +830,1702024439,2000.0,19,2,"{""description"":""some"",""duration"":40}" +831,1702026136,3500.0,19,2,"{""description"":""some"",""duration"":40}" +832,1702298711,1000.0,19,2,"{""description"":""some"",""duration"":40}" +833,1702298734,1000.0,19,2,"{""description"":""some"",""duration"":40}" +834,1702298739,300.0,19,2,"{""description"":""some"",""duration"":40}" +835,1702298744,400.0,19,2,"{""description"":""some"",""duration"":40}" +836,1702453750,6000.0,19,2,"{""description"":""some"",""duration"":40}" +837,1702466139,2000.0,19,2,"{""description"":""some"",""duration"":40}" +838,1702466153,2000.0,19,2,"{""description"":""some"",""duration"":40}" +839,1702636904,2000.0,19,2,"{""description"":""some"",""duration"":40}" +840,1702636912,1000.0,19,2,"{""description"":""some"",""duration"":40}" +841,1702636920,1000.0,19,2,"{""description"":""some"",""duration"":40}" +842,1702639560,1000.0,19,2,"{""description"":""some"",""duration"":40}" +843,1702888560,800.0,19,2,"{""description"":""some"",""duration"":40}" +844,1702888571,2000.0,19,2,"{""description"":""some"",""duration"":40}" +845,1702888578,1000.0,19,2,"{""description"":""some"",""duration"":40}" +846,1702982449,1500.0,19,2,"{""description"":""some"",""duration"":40}" +847,1702982460,500.0,19,2,"{""description"":""some"",""duration"":40}" +848,1702982476,1500.0,19,2,"{""description"":""some"",""duration"":40}" +849,1702982489,500.0,19,2,"{""description"":""some"",""duration"":40}" +850,1703071032,700.0,19,2,"{""description"":""some"",""duration"":40}" +851,1703071040,1000.0,19,2,"{""description"":""some"",""duration"":40}" +852,1703071050,1000.0,19,2,"{""description"":""some"",""duration"":40}" +853,1703071058,700.0,19,2,"{""description"":""some"",""duration"":40}" +854,1703071071,2450.0,19,2,"{""description"":""some"",""duration"":40}" +855,1703071087,1000.0,19,2,"{""description"":""some"",""duration"":40}" +856,1703071094,300.0,19,2,"{""description"":""some"",""duration"":40}" +857,1703071104,1000.0,19,2,"{""description"":""some"",""duration"":40}" +858,1703083455,1000.0,19,2,"{""description"":""some"",""duration"":40}" +859,1703083467,2500.0,19,2,"{""description"":""some"",""duration"":40}" +860,1703083475,1000.0,19,2,"{""description"":""some"",""duration"":40}" +861,1703083500,1000.0,19,2,"{""description"":""some"",""duration"":40}" +862,1703083511,500.0,19,2,"{""description"":""some"",""duration"":40}" +863,1703228194,2000.0,19,2,"{""description"":""some"",""duration"":40}" +864,1703228200,1000.0,19,2,"{""description"":""some"",""duration"":40}" +865,1703228220,200.0,19,2,"{""description"":""some"",""duration"":40}" +866,1703232931,1000.0,19,2,"{""description"":""some"",""duration"":40}" +867,1703232948,500.0,19,2,"{""description"":""some"",""duration"":40}" +868,1703241826,3500.0,19,2,"{""description"":""some"",""duration"":40}" +869,1703254522,1000.0,19,2,"{""description"":""some"",""duration"":40}" +870,1703254531,700.0,19,2,"{""description"":""some"",""duration"":40}" +871,1703254540,1200.0,19,2,"{""description"":""some"",""duration"":40}" +872,1703254551,2000.0,19,2,"{""description"":""some"",""duration"":40}" +873,1703254579,1000.0,19,2,"{""description"":""some"",""duration"":40}" +874,1703254590,500.0,19,2,"{""description"":""some"",""duration"":40}" +875,1703260577,900.0,19,2,"{""description"":""some"",""duration"":40}" +876,1703260587,1000.0,19,2,"{""description"":""some"",""duration"":40}" +877,1703260647,1000.0,19,2,"{""description"":""some"",""duration"":40}" +878,1703261966,200.0,19,2,"{""description"":""some"",""duration"":40}" +879,1703261984,2000.0,19,2,"{""description"":""some"",""duration"":40}" +880,1703504838,2000.0,19,2,"{""description"":""some"",""duration"":40}" +881,1703504841,1000.0,19,2,"{""description"":""some"",""duration"":40}" +882,1703504845,500.0,19,2,"{""description"":""some"",""duration"":40}" +883,1703507231,1000.0,19,2,"{""description"":""some"",""duration"":40}" +884,1703507239,1000.0,19,2,"{""description"":""some"",""duration"":40}" +885,1703511169,2000.0,19,2,"{""description"":""some"",""duration"":40}" +886,1703516553,1000.0,19,2,"{""description"":""some"",""duration"":40}" +887,1703516560,1000.0,19,2,"{""description"":""some"",""duration"":40}" +888,1703586431,2600.0,19,2,"{""description"":""some"",""duration"":40}" +889,1703586441,1500.0,19,2,"{""description"":""some"",""duration"":40}" +890,1703589397,1500.0,19,2,"{""description"":""some"",""duration"":40}" +891,1703589405,300.0,19,2,"{""description"":""some"",""duration"":40}" +892,1703669545,1000.0,19,2,"{""description"":""some"",""duration"":40}" +893,1703669555,500.0,19,2,"{""description"":""some"",""duration"":40}" +894,1703669564,1000.0,19,2,"{""description"":""some"",""duration"":40}" +895,1703669578,700.0,19,2,"{""description"":""some"",""duration"":40}" +896,1703669586,1000.0,19,2,"{""description"":""some"",""duration"":40}" +897,1703675009,2500.0,19,2,"{""description"":""some"",""duration"":40}" +898,1703680550,1000.0,19,2,"{""description"":""some"",""duration"":40}" +899,1703680561,500.0,19,2,"{""description"":""some"",""duration"":40}" +900,1703690736,1000.0,19,2,"{""description"":""some"",""duration"":40}" +901,1703690757,400.0,19,2,"{""description"":""some"",""duration"":40}" +902,1703690851,800.0,19,2,"{""description"":""some"",""duration"":40}" +903,1703690862,1000.0,19,2,"{""description"":""some"",""duration"":40}" +904,1703690880,500.0,19,2,"{""description"":""some"",""duration"":40}" +905,1703753876,2600.0,19,2,"{""description"":""some"",""duration"":40}" +906,1703753921,2400.0,19,2,"{""description"":""some"",""duration"":40}" +907,1703846567,2000.0,19,2,"{""description"":""some"",""duration"":40}" +908,1703846578,1000.0,19,2,"{""description"":""some"",""duration"":40}" +909,1703851354,2300.0,19,2,"{""description"":""some"",""duration"":40}" +910,1703851372,1000.0,19,2,"{""description"":""some"",""duration"":40}" +911,1703851386,1000.0,19,2,"{""description"":""some"",""duration"":40}" +912,1703851405,1000.0,19,2,"{""description"":""some"",""duration"":40}" +913,1703855103,1000.0,19,2,"{""description"":""some"",""duration"":40}" +914,1703855130,600.0,19,2,"{""description"":""some"",""duration"":40}" +915,1703858287,1000.0,19,2,"{""description"":""some"",""duration"":40}" +916,1703858297,300.0,19,2,"{""description"":""some"",""duration"":40}" +917,1704475114,3500.0,19,2,"{""description"":""some"",""duration"":40}" +918,1704946608,1000.0,19,2,"{""description"":""some"",""duration"":40}" +919,1704946615,1000.0,19,2,"{""description"":""some"",""duration"":40}" +920,1704946637,1000.0,19,2,"{""description"":""some"",""duration"":40}" +921,1705308258,2000.0,19,2,"{""description"":""some"",""duration"":40}" +922,1705308266,1000.0,19,2,"{""description"":""some"",""duration"":40}" +923,1705308277,500.0,19,2,"{""description"":""some"",""duration"":40}" +924,1705308284,1200.0,19,2,"{""description"":""some"",""duration"":40}" +925,1705340107,1200.0,19,2,"{""description"":""some"",""duration"":40}" +926,1705340119,300.0,19,2,"{""description"":""some"",""duration"":40}" +927,1705498948,3500.0,19,2,"{""description"":""some"",""duration"":40}" +928,1705498961,1000.0,19,2,"{""description"":""some"",""duration"":40}" +929,1705498970,300.0,19,2,"{""description"":""some"",""duration"":40}" +930,1705498977,700.0,19,2,"{""description"":""some"",""duration"":40}" +931,1705498986,1000.0,19,2,"{""description"":""some"",""duration"":40}" +932,1705499003,500.0,19,2,"{""description"":""some"",""duration"":40}" +933,1705521011,2400.0,19,2,"{""description"":""some"",""duration"":40}" +934,1705659907,1000.0,19,2,"{""description"":""some"",""duration"":40}" +935,1705672044,1000.0,19,2,"{""description"":""some"",""duration"":40}" +936,1705672058,1000.0,19,2,"{""description"":""some"",""duration"":40}" +937,1705685251,1000.0,19,2,"{""description"":""some"",""duration"":40}" +938,1706009980,2600.0,19,2,"{""description"":""some"",""duration"":40}" +939,1706009992,1500.0,19,2,"{""description"":""some"",""duration"":40}" +940,1706010004,1500.0,19,2,"{""description"":""some"",""duration"":40}" +941,1706010025,300.0,19,2,"{""description"":""some"",""duration"":40}" +942,1706091955,600.0,19,2,"{""description"":""some"",""duration"":40}" +943,1706091964,500.0,19,2,"{""description"":""some"",""duration"":40}" +944,1706091978,1000.0,19,2,"{""description"":""some"",""duration"":40}" +945,1706091991,500.0,19,2,"{""description"":""some"",""duration"":40}" +946,1706122818,700.0,19,2,"{""description"":""some"",""duration"":40}" +947,1706122826,700.0,19,2,"{""description"":""some"",""duration"":40}" +948,1706122840,1000.0,19,2,"{""description"":""some"",""duration"":40}" +949,1706122864,300.0,19,2,"{""description"":""some"",""duration"":40}" +950,1706122884,2600.0,19,2,"{""description"":""some"",""duration"":40}" +951,1706122904,2000.0,19,2,"{""description"":""some"",""duration"":40}" +952,1706122915,500.0,19,2,"{""description"":""some"",""duration"":40}" +953,1706122926,800.0,19,2,"{""description"":""some"",""duration"":40}" +954,1706288328,1000.0,19,2,"{""description"":""some"",""duration"":40}" +955,1706288335,500.0,19,2,"{""description"":""some"",""duration"":40}" +956,1706288342,1000.0,19,2,"{""description"":""some"",""duration"":40}" +957,1706288349,300.0,19,2,"{""description"":""some"",""duration"":40}" +958,1706288358,1500.0,19,2,"{""description"":""some"",""duration"":40}" +959,1706288367,1000.0,19,2,"{""description"":""some"",""duration"":40}" +960,1706288375,800.0,19,2,"{""description"":""some"",""duration"":40}" +961,1706288389,4000.0,19,2,"{""description"":""some"",""duration"":40}" +962,1706688568,2400.0,19,2,"{""description"":""some"",""duration"":40}" +963,1706688585,1000.0,19,2,"{""description"":""some"",""duration"":40}" +964,1706688593,4000.0,19,2,"{""description"":""some"",""duration"":40}" +965,1706693372,2000.0,19,2,"{""description"":""some"",""duration"":40}" +966,1706695183,1000.0,19,2,"{""description"":""some"",""duration"":40}" +967,1706696643,1300.0,19,2,"{""description"":""some"",""duration"":40}" +968,1706699548,1300.0,19,2,"{""description"":""some"",""duration"":40}" +969,1706701507,700.0,19,2,"{""description"":""some"",""duration"":40}" +970,1706705449,1500.0,19,2,"{""description"":""some"",""duration"":40}" +971,1706717659,1000.0,19,2,"{""description"":""some"",""duration"":40}" +972,1706870733,1300.0,19,2,"{""description"":""some"",""duration"":40}" +973,1706870744,1300.0,19,2,"{""description"":""some"",""duration"":40}" +974,1706870753,2400.0,19,2,"{""description"":""some"",""duration"":40}" +975,1706870762,1100.0,19,2,"{""description"":""some"",""duration"":40}" +976,1706878834,2000.0,19,2,"{""description"":""some"",""duration"":40}" +977,1708108933,1000.0,19,2,"{""description"":""some"",""duration"":40}" +978,1708108941,2000.0,19,2,"{""description"":""some"",""duration"":40}" +979,1708108945,1000.0,19,2,"{""description"":""some"",""duration"":40}" +980,1708108949,500.0,19,2,"{""description"":""some"",""duration"":40}" +981,1708108953,2400.0,19,2,"{""description"":""some"",""duration"":40}" +982,1708108956,1300.0,19,2,"{""description"":""some"",""duration"":40}" +983,1708108960,1000.0,19,2,"{""description"":""some"",""duration"":40}" +984,1708108963,1300.0,19,2,"{""description"":""some"",""duration"":40}" +985,1708108966,1000.0,19,2,"{""description"":""some"",""duration"":40}" +986,1708348230,1300.0,19,2,"{""description"":""some"",""duration"":40}" +987,1708348240,900.0,19,2,"{""description"":""some"",""duration"":40}" +988,1708348254,900.0,19,2,"{""description"":""some"",""duration"":40}" +989,1708348264,1300.0,19,2,"{""description"":""some"",""duration"":40}" +990,1708429295,1000.0,19,2,"{""description"":""some"",""duration"":40}" +991,1708429306,1300.0,19,2,"{""description"":""some"",""duration"":40}" +992,1708429318,500.0,19,2,"{""description"":""some"",""duration"":40}" +993,1708429327,1500.0,19,2,"{""description"":""some"",""duration"":40}" +994,1708429337,300.0,19,2,"{""description"":""some"",""duration"":40}" +995,1708438299,1500.0,19,2,"{""description"":""some"",""duration"":40}" +996,1708524076,1300.0,19,2,"{""description"":""some"",""duration"":40}" +997,1708524086,900.0,19,2,"{""description"":""some"",""duration"":40}" +998,1708524105,1300.0,19,2,"{""description"":""some"",""duration"":40}" +999,1708524114,300.0,19,2,"{""description"":""some"",""duration"":40}" +1000,1708524124,900.0,19,2,"{""description"":""some"",""duration"":40}" +1001,1708524133,300.0,19,2,"{""description"":""some"",""duration"":40}" +1002,1708527667,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1003,1708937921,1000.0,19,2,"{""description"":""some"",""duration"":40}" +1005,1708937946,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1006,1708938010,500.0,19,2,"{""description"":""some"",""duration"":40}" +1007,1708956931,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1008,1708956938,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1009,1709114657,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1010,1709114675,500.0,19,2,"{""description"":""some"",""duration"":40}" +1011,1709114706,900.0,19,2,"{""description"":""some"",""duration"":40}" +1012,1709114718,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1013,1709287407,1000.0,19,2,"{""description"":""some"",""duration"":40}" +1014,1709287419,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1015,1709287440,300.0,19,2,"{""description"":""some"",""duration"":40}" +1016,1709287454,2400.0,19,2,"{""description"":""some"",""duration"":40}" +1017,1709292049,1000.0,19,2,"{""description"":""some"",""duration"":40}" +1018,1709295355,1000.0,19,2,"{""description"":""some"",""duration"":40}" +1019,1709450798,5000.0,19,2,"{""description"":""some"",""duration"":40}" +1020,1709538152,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1021,1709542187,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1022,1709542200,100.0,19,2,"{""description"":""some"",""duration"":40}" +1023,1709550792,2000.0,19,2,"{""description"":""some"",""duration"":40}" +1024,1709732984,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1025,1709732996,2600.0,19,2,"{""description"":""some"",""duration"":40}" +1026,1709733004,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1027,1709733014,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1028,1709733025,500.0,19,2,"{""description"":""some"",""duration"":40}" +1029,1709822257,3000.0,19,2,"{""description"":""some"",""duration"":40}" +1030,1710347707,1000.0,19,2,"{""description"":""some"",""duration"":40}" +1031,1710347725,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1032,1710347733,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1033,1710347741,300.0,19,2,"{""description"":""some"",""duration"":40}" +1034,1710347748,900.0,19,2,"{""description"":""some"",""duration"":40}" +1035,1710347776,300.0,19,2,"{""description"":""some"",""duration"":40}" +1036,1710492247,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1037,1710499630,20000.0,19,2,"{""description"":""some"",""duration"":40}" +1038,1710754282,900.0,19,2,"{""description"":""some"",""duration"":40}" +1039,1710757850,600.0,19,2,"{""description"":""some"",""duration"":40}" +1040,1710757859,2000.0,19,2,"{""description"":""some"",""duration"":40}" +1041,1710775397,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1042,1710775427,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1043,1710930870,2000.0,19,2,"{""description"":""some"",""duration"":40}" +1044,1710930882,1000.0,19,2,"{""description"":""some"",""duration"":40}" +1045,1710938249,900.0,19,2,"{""description"":""some"",""duration"":40}" +1046,1710938315,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1047,1710938333,700.0,19,2,"{""description"":""some"",""duration"":40}" +1048,1710944451,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1049,1711010996,2000.0,19,2,"{""description"":""some"",""duration"":40}" +1050,1711011005,1000.0,19,2,"{""description"":""some"",""duration"":40}" +1051,1711014391,1500.0,19,2,"{""description"":""some"",""duration"":40}" +1052,1711014402,300.0,19,2,"{""description"":""some"",""duration"":40}" +1053,1711094574,2400.0,19,2,"{""description"":""some"",""duration"":40}" +1054,1711094585,1100.0,19,2,"{""description"":""some"",""duration"":40}" +1055,1711096245,900.0,19,2,"{""description"":""some"",""duration"":40}" +1056,1711096260,300.0,19,2,"{""description"":""some"",""duration"":40}" +1057,1711177704,7000.0,19,2,"{""description"":""some"",""duration"":40}" +1058,1711185522,2600.0,19,2,"{""description"":""some"",""duration"":40}" +1059,1711185534,600.0,19,2,"{""description"":""some"",""duration"":40}" +1060,1711375163,1300.0,19,2,"{""description"":""some"",""duration"":40}" +1061,1711375189,500.0,19,2,"{""description"":""some"",""duration"":40}" +1062,1711375224,2400.0,19,2,"{""description"":""some"",""duration"":40}" +1063,1711375239,1500.0,19,2,"{""description"":""some"",""duration"":40}" diff --git a/crates/duckdb/examples/appender.rs b/crates/duckdb/examples/appender.rs index c159829d..4350c4b3 100644 --- a/crates/duckdb/examples/appender.rs +++ b/crates/duckdb/examples/appender.rs @@ -1,9 +1,20 @@ extern crate duckdb; -use duckdb::{params, Connection, DropBehavior, Result}; +use std::time::Instant; + +use arrow_convert::{ArrowDeserialize, ArrowField, ArrowSerialize}; +use duckdb::{Connection, DropBehavior, Result}; + +#[derive(ArrowField, ArrowSerialize, ArrowDeserialize)] +struct User { + id: i32, + area: Option, + age: i8, + active: i8, +} fn main() -> Result<()> { - //let mut db = Connection::open("10m.db")?; + // let mut db = Connection::open("10m.db")?; let mut db = Connection::open_in_memory()?; let create_table_sql = " @@ -15,25 +26,20 @@ fn main() -> Result<()> { active TINYINT not null );"; db.execute_batch(create_table_sql)?; + let row_count = 10_000_000u32; + let data = firstn(row_count).collect_vec(); - let row_count = 10_000_000; { + let start = Instant::now(); let mut tx = db.transaction()?; tx.set_drop_behavior(DropBehavior::Commit); let mut app = tx.appender("test")?; // use generator - // for u in firstn(1_000_000) { - // app.append_row(params![u.id, u.area, u.age, u.active])?; - // } + app.append_rows_arrow(&data, true)?; - for i in 0..row_count { - app.append_row(params![ - i, - get_random_area_code(), - get_random_age(), - get_random_active(), - ])?; - } + let duration = start.elapsed(); + + println!("Time elapsed in transaction is: {:?}", duration); } let val = db.query_row("SELECT count(1) FROM test", [], |row| <(u32,)>::try_from(row))?; @@ -42,15 +48,7 @@ fn main() -> Result<()> { } #[allow(dead_code)] -struct User { - id: i32, - area: Option, - age: i8, - active: i8, -} - -#[allow(dead_code)] -fn firstn(n: i32) -> impl std::iter::Iterator { +fn firstn(n: u32) -> impl std::iter::Iterator { let mut id = 0; std::iter::from_fn(move || { if id >= n { @@ -58,7 +56,7 @@ fn firstn(n: i32) -> impl std::iter::Iterator { } id += 1; Some(User { - id, + id: id as i32, area: get_random_area_code(), age: get_random_age(), active: get_random_active(), @@ -66,6 +64,7 @@ fn firstn(n: i32) -> impl std::iter::Iterator { }) } +use itertools::Itertools; // Modified from https://github.com/avinassh/fast-sqlite3-inserts/blob/master/src/bin/common.rs use rand::{prelude::SliceRandom, Rng}; diff --git a/crates/duckdb/examples/hello-ext/main.rs b/crates/duckdb/examples/hello-ext/main.rs index 74ef02d0..78e8eee9 100644 --- a/crates/duckdb/examples/hello-ext/main.rs +++ b/crates/duckdb/examples/hello-ext/main.rs @@ -2,8 +2,9 @@ extern crate duckdb; extern crate duckdb_loadable_macros; extern crate libduckdb_sys; +use crate::duckdb::core::Inserter; use duckdb::{ - core::{DataChunk, Inserter, LogicalType, LogicalTypeId}, + core::{DataChunk, DataChunkHandle, FlatVector, LogicalTypeHandle, LogicalTypeId}, vtab::{BindInfo, Free, FunctionInfo, InitInfo, VTab}, Connection, Result, }; @@ -43,8 +44,8 @@ impl VTab for HelloVTab { type InitData = HelloInitData; type BindData = HelloBindData; - unsafe fn bind(bind: &BindInfo, data: *mut HelloBindData) -> Result<(), Box> { - bind.add_result_column("column0", LogicalType::new(LogicalTypeId::Varchar)); + fn bind(bind: &BindInfo, data: *mut HelloBindData) -> Result<(), Box> { + bind.add_result_column("column0", LogicalTypeHandle::from(LogicalTypeId::Varchar)); let param = bind.get_parameter(0).to_string(); unsafe { (*data).name = CString::new(param).unwrap().into_raw(); @@ -52,36 +53,38 @@ impl VTab for HelloVTab { Ok(()) } - unsafe fn init(_: &InitInfo, data: *mut HelloInitData) -> Result<(), Box> { + fn init(_: &InitInfo, data: *mut HelloInitData) -> Result<(), Box> { unsafe { (*data).done = false; } Ok(()) } - unsafe fn func(func: &FunctionInfo, output: &mut DataChunk) -> Result<(), Box> { + fn func(func: &FunctionInfo, output: DataChunkHandle) -> Result<(), Box> { let init_info = func.get_init_data::(); let bind_info = func.get_bind_data::(); + let mut data_chunk = DataChunk::from_existing_handle(output); unsafe { if (*init_info).done { - output.set_len(0); + data_chunk.set_len(0); } else { (*init_info).done = true; - let vector = output.flat_vector(0); + data_chunk.reserve(1); + let vector = data_chunk.columns[0].try_as_flat_mut().unwrap(); let name = CString::from_raw((*bind_info).name); let result = CString::new(format!("Hello {}", name.to_str()?))?; // Can't consume the CString (*bind_info).name = CString::into_raw(name); vector.insert(0, result); - output.set_len(1); + data_chunk.set_len(1); } } Ok(()) } - fn parameters() -> Option> { - Some(vec![LogicalType::new(LogicalTypeId::Varchar)]) + fn parameters() -> Option> { + Some(vec![LogicalTypeHandle::from(LogicalTypeId::Varchar)]) } } diff --git a/crates/duckdb/src/appender/arrow.rs b/crates/duckdb/src/appender/arrow.rs deleted file mode 100644 index 187af6f4..00000000 --- a/crates/duckdb/src/appender/arrow.rs +++ /dev/null @@ -1,83 +0,0 @@ -use super::{ffi, Appender, Result}; -use crate::{ - core::{DataChunk, LogicalType}, - error::result_from_duckdb_appender, - vtab::{record_batch_to_duckdb_data_chunk, to_duckdb_logical_type}, - Error, -}; -use arrow::record_batch::RecordBatch; -use ffi::duckdb_append_data_chunk; - -impl Appender<'_> { - /// Append one record_batch - /// - /// ## Example - /// - /// ```rust,no_run - /// # use duckdb::{Connection, Result, params}; - /// use arrow::record_batch::RecordBatch; - /// fn insert_record_batch(conn: &Connection,record_batch:RecordBatch) -> Result<()> { - /// let mut app = conn.appender("foo")?; - /// app.append_record_batch(record_batch)?; - /// Ok(()) - /// } - /// ``` - /// - /// # Failure - /// - /// Will return `Err` if append column count not the same with the table schema - #[inline] - pub fn append_record_batch(&mut self, record_batch: RecordBatch) -> Result<()> { - let schema = record_batch.schema(); - let mut logical_type: Vec = vec![]; - for field in schema.fields() { - let logical_t = to_duckdb_logical_type(field.data_type()) - .map_err(|_op| Error::ArrowTypeToDuckdbType(field.to_string(), field.data_type().clone()))?; - logical_type.push(logical_t); - } - - let mut data_chunk = DataChunk::new(&logical_type); - record_batch_to_duckdb_data_chunk(&record_batch, &mut data_chunk).map_err(|_op| Error::AppendError)?; - - let rc = unsafe { duckdb_append_data_chunk(self.app, data_chunk.get_ptr()) }; - result_from_duckdb_appender(rc, &mut self.app) - } -} - -#[cfg(test)] -mod test { - use crate::{Connection, Result}; - use arrow::{ - array::{Int8Array, StringArray}, - datatypes::{DataType, Field, Schema}, - record_batch::RecordBatch, - }; - use std::sync::Arc; - - #[test] - fn test_append_record_batch() -> Result<()> { - let db = Connection::open_in_memory()?; - db.execute_batch("CREATE TABLE foo(id TINYINT not null,area TINYINT not null,name Varchar)")?; - { - let id_array = Int8Array::from(vec![1, 2, 3, 4, 5]); - let area_array = Int8Array::from(vec![11, 22, 33, 44, 55]); - let name_array = StringArray::from(vec![Some("11"), None, None, Some("44"), None]); - let schema = Schema::new(vec![ - Field::new("id", DataType::Int8, true), - Field::new("area", DataType::Int8, true), - Field::new("area", DataType::Utf8, true), - ]); - let record_batch = RecordBatch::try_new( - Arc::new(schema), - vec![Arc::new(id_array), Arc::new(area_array), Arc::new(name_array)], - ) - .unwrap(); - let mut app = db.appender("foo")?; - app.append_record_batch(record_batch)?; - } - let mut stmt = db.prepare("SELECT id, area,name FROM foo")?; - let rbs: Vec = stmt.query_arrow([])?.collect(); - assert_eq!(rbs.iter().map(|op| op.num_rows()).sum::(), 5); - Ok(()) - } -} diff --git a/crates/duckdb/src/appender/mod.rs b/crates/duckdb/src/appender/mod.rs index 488db82f..3b1ea0fb 100644 --- a/crates/duckdb/src/appender/mod.rs +++ b/crates/duckdb/src/appender/mod.rs @@ -1,22 +1,135 @@ -use super::{ffi, AppenderParams, Connection, Result, ValueRef}; -use std::{ffi::c_void, fmt, os::raw::c_char}; +use std::{ + ffi::{c_char, c_void}, + fmt, +}; + +use ::arrow::array::RecordBatch; +use arrow_convert::{ + field::ArrowField, + serialize::{ArrowSerialize, FlattenRecordBatch, TryIntoArrow}, +}; +use ffi::{duckdb_append_data_chunk, duckdb_appender_flush}; +use itertools::Itertools; +use super::{ffi, Connection, Result}; +#[cfg(feature = "appender-arrow")] +use crate::vtab::arrow::record_batch_to_duckdb_data_chunk; use crate::{ + core::{ColumnInfo, DataChunk, DataChunkHandle, LogicalType, LogicalTypeHandle}, error::result_from_duckdb_appender, - types::{ToSql, ToSqlOutput}, - Error, + types::{ToSqlOutput, ValueRef}, + AppenderParams, Error, ToSql, }; /// Appender for fast import data pub struct Appender<'conn> { conn: &'conn Connection, app: ffi::duckdb_appender, -} -#[cfg(feature = "appender-arrow")] -mod arrow; + /// column layout stored as tree, for fast access and to create data chunks + columns: Vec, + + /// the type of the columns this table stores + column_types: Vec, + + /// chunks that have not been flushed + chunks: Vec, +} impl Appender<'_> { + #[inline] + pub(super) fn new(conn: &Connection, app: ffi::duckdb_appender) -> Appender<'_> { + let column_count = unsafe { ffi::duckdb_appender_column_count(app) }; + + // initialize column_types + let column_types = (0..column_count) + .map(|i| { + let handle = unsafe { LogicalTypeHandle::new(ffi::duckdb_appender_column_type(app, i)) }; + LogicalType::from(handle) + }) + .collect::>(); + + // initialize columns + let columns: Vec = (0..column_count) + .map(|i| ColumnInfo::new(&column_types[i as usize])) + .collect(); + + let chunks = vec![]; + + Appender { + conn, + app, + columns, + column_types, + chunks, + } + } + + #[cfg(feature = "appender-arrow")] + fn append_record_batch_inner(&mut self, record_batch: Vec) -> Result<()> { + for batch in record_batch { + let mut data_chunk = DataChunk::new(batch.num_rows(), &self.column_types, &self.columns); + record_batch_to_duckdb_data_chunk(&batch, &mut data_chunk).map_err(|_op| Error::AppendError)?; + self.chunks.push(data_chunk); + } + Ok(()) + } + + /// Append one record_batch + /// + /// ## Example + /// + /// ```rust,no_run + /// # use duckdb::{Connection, Result, params}; + /// use arrow::record_batch::RecordBatch; + /// fn insert_record_batch(conn: &Connection,record_batch:RecordBatch) -> Result<()> { + /// let mut app = conn.appender("foo")?; + /// app.append_record_batch(record_batch)?; + /// Ok(()) + /// } + /// ``` + /// + /// # Failure + /// + /// Will return `Err` if append column count not the same with the table schema + #[cfg(feature = "appender-arrow")] + pub fn append_record_batch(&mut self, record_batch: RecordBatch) -> Result<()> { + let batches = split_batch_for_data_chunk_capacity(record_batch, DataChunkHandle::max_capacity()); + self.append_record_batch_inner(batches) + } + + /// Append rows to the appender. + /// + /// # Arguments + /// + /// * `records` - An iterator over the records to be appended. + /// * `expand_struct` - A boolean indicating whether to expand the struct. + #[cfg(feature = "appender-arrow")] + pub fn append_rows_arrow<'a, T>( + &mut self, + records: impl IntoIterator, + expand_struct: bool, + ) -> Result<()> + where + T: ArrowSerialize + ArrowField + 'static, + { + let batches = records + .into_iter() + .chunks(DataChunkHandle::max_capacity()) + .into_iter() + .map(|chunk| { + let batch: RecordBatch = chunk.try_into_arrow()?; + if expand_struct { + batch.flatten() + } else { + Ok(batch) + } + }) + .collect::, _>>()?; + + self.append_record_batch_inner(batches) + } + /// Append multiple rows from Iterator /// /// ## Example @@ -143,25 +256,24 @@ impl Appender<'_> { Ok(()) } - #[inline] - pub(super) fn new(conn: &Connection, app: ffi::duckdb_appender) -> Appender<'_> { - Appender { conn, app } - } - /// Flush data into DB #[inline] pub fn flush(&mut self) -> Result<()> { - unsafe { - let res = ffi::duckdb_appender_flush(self.app); - result_from_duckdb_appender(res, &mut self.app) + // append data chunks + for chunk in self.chunks.drain(..) { + let rc = unsafe { duckdb_append_data_chunk(self.app, chunk.get_handle().ptr) }; + result_from_duckdb_appender(rc, &mut self.app)?; } + + let rc = unsafe { duckdb_appender_flush(self.app) }; + result_from_duckdb_appender(rc, &mut self.app) } } impl Drop for Appender<'_> { fn drop(&mut self) { if !self.app.is_null() { - let _ = self.flush(); // can't safely handle failures here + self.flush().expect("Failed to flush appender"); unsafe { ffi::duckdb_appender_close(self.app); ffi::duckdb_appender_destroy(&mut self.app); @@ -176,10 +288,63 @@ impl fmt::Debug for Appender<'_> { } } +/// Splits a given `RecordBatch` into multiple smaller batches based on the maximum number of rows per batch. +fn split_batch_for_data_chunk_capacity(batch: RecordBatch, data_chunk_max_rows: usize) -> Vec { + let rows_per_batch = data_chunk_max_rows.max(1); + let n_batches = (batch.num_rows() / rows_per_batch).max(1); + let mut out = Vec::with_capacity(n_batches + 1); + + let mut offset = 0; + while offset < batch.num_rows() { + let length = (rows_per_batch).min(batch.num_rows() - offset); + out.push(batch.slice(offset, length)); + + offset += length; + } + + out +} + #[cfg(test)] mod test { + use std::sync::Arc; + + use arrow::{ + array::{Int8Array, StringArray}, + datatypes::{DataType, Field, Schema}, + record_batch::RecordBatch, + }; + use crate::{Connection, Result}; + #[cfg(feature = "appender-arrow")] + #[test] + fn test_append_record_batch() -> Result<()> { + let db = Connection::open_in_memory()?; + db.execute_batch("CREATE TABLE foo(id TINYINT not null,area TINYINT not null,name Varchar)")?; + { + let id_array = Int8Array::from(vec![1, 2, 3, 4, 5]); + let area_array = Int8Array::from(vec![11, 22, 33, 44, 55]); + let name_array = StringArray::from(vec![Some("11"), None, None, Some("44"), None]); + let schema = Schema::new(vec![ + Field::new("id", DataType::Int8, true), + Field::new("area", DataType::Int8, true), + Field::new("area", DataType::Utf8, true), + ]); + let record_batch = RecordBatch::try_new( + Arc::new(schema), + vec![Arc::new(id_array), Arc::new(area_array), Arc::new(name_array)], + ) + .unwrap(); + let mut app = db.appender("foo")?; + app.append_record_batch(record_batch)?; + } + let mut stmt = db.prepare("SELECT id, area,name FROM foo")?; + let rbs: Vec = stmt.query_arrow([])?.collect(); + assert_eq!(rbs.iter().map(|op| op.num_rows()).sum::(), 5); + Ok(()) + } + #[test] fn test_append_one_row() -> Result<()> { let db = Connection::open_in_memory()?; @@ -266,9 +431,10 @@ mod test { #[test] #[cfg(feature = "chrono")] fn test_append_datetime() -> Result<()> { - use crate::params; use chrono::{NaiveDate, NaiveDateTime}; + use crate::params; + let db = Connection::open_in_memory()?; db.execute_batch("CREATE TABLE foo(x DATE, y TIMESTAMP)")?; diff --git a/crates/duckdb/src/core/column_info.rs b/crates/duckdb/src/core/column_info.rs new file mode 100644 index 00000000..5462f0b2 --- /dev/null +++ b/crates/duckdb/src/core/column_info.rs @@ -0,0 +1,133 @@ +use arrow::datatypes::DataType; +use derive_more::Unwrap; + +use super::LogicalType; + +#[derive(Debug)] +pub struct ListColumnInfo { + pub child: Box, +} + +// note: Also works for fixed size length +#[derive(Debug)] +pub struct ArrayColumnInfo { + pub array_size: usize, + pub child: Box, +} + +#[derive(Debug)] +pub struct StructColumnInfo { + pub children: Vec, +} + +#[derive(Debug)] +pub struct UnionColumnInfo { + pub members: Vec, +} + +/// Represents the duckdb schema for error handling and caching purposes. +#[derive(Unwrap, Debug)] +pub enum ColumnInfo { + Flat, + List(ListColumnInfo), + Array(ArrayColumnInfo), + Struct(StructColumnInfo), + Union(UnionColumnInfo), +} + +impl ColumnInfo { + pub fn new(logical_type: &LogicalType) -> Self { + match logical_type { + LogicalType::General { .. } | LogicalType::Decimal(_) => ColumnInfo::Flat, + LogicalType::List(ty) => { + let child_type = ty.child_type(); + let child_info = Self::new(&child_type); + + ColumnInfo::List(ListColumnInfo { + child: Box::new(child_info), + }) + } + LogicalType::Struct(ty) => { + let child_count = ty.child_count(); + + let child_column_infos = (0..child_count) + .map(|i| { + let child_type = ty.child_type(i); + Self::new(&child_type) + }) + .collect::>(); + + ColumnInfo::Struct(StructColumnInfo { + children: child_column_infos, + }) + } + LogicalType::Map(_) => unimplemented!(), + LogicalType::Union(ty) => { + let child_count = ty.member_count(); + + let member_column_infos = (0..child_count) + .map(|i| { + let child_type = ty.member_type(i); + Self::new(&child_type) + }) + .collect::>(); + + ColumnInfo::Union(UnionColumnInfo { + members: member_column_infos, + }) + } + LogicalType::Array(ty) => { + let array_size = ty.size(); + let child_type = ty.child_type(); + let child_info = Self::new(&child_type); + + ColumnInfo::Array(ArrayColumnInfo { + array_size, + child: Box::new(child_info), + }) + } + } + } + + pub fn new_from_data_type(data_type: &DataType) -> Self { + match data_type { + dt if dt.is_primitive() => ColumnInfo::Flat, + DataType::Binary | DataType::Utf8 | DataType::LargeUtf8 => ColumnInfo::Flat, + DataType::List(ty) => { + let child_info = Self::new_from_data_type(ty.data_type()); + ColumnInfo::List(ListColumnInfo { + child: Box::new(child_info), + }) + } + DataType::FixedSizeList(field, size) => { + let child_info = Self::new_from_data_type(field.data_type()); + ColumnInfo::Array(ArrayColumnInfo { + array_size: *size as usize, + child: Box::new(child_info), + }) + } + DataType::Struct(fields) => { + let children = fields + .iter() + .map(|field| { + let child_info = Self::new_from_data_type(field.data_type()); + child_info + }) + .collect::>(); + ColumnInfo::Struct(StructColumnInfo { children }) + } + DataType::Union(fields, _) => { + let members = fields + .iter() + .map(|(_, field)| { + let child_info = Self::new_from_data_type(field.data_type()); + child_info + }) + .collect::>(); + ColumnInfo::Union(UnionColumnInfo { members }) + } + DataType::Map(_, _) => todo!(), + dt => unimplemented!("data type: {} is not implemented yet.", dt), + } + } +} diff --git a/crates/duckdb/src/core/data_chunk.rs b/crates/duckdb/src/core/data_chunk.rs index 3bc6d874..d3640859 100644 --- a/crates/duckdb/src/core/data_chunk.rs +++ b/crates/duckdb/src/core/data_chunk.rs @@ -1,48 +1,69 @@ -use super::{ - logical_type::LogicalType, - vector::{ArrayVector, FlatVector, ListVector, StructVector}, -}; -use crate::ffi::{ - duckdb_create_data_chunk, duckdb_data_chunk, duckdb_data_chunk_get_column_count, duckdb_data_chunk_get_size, - duckdb_data_chunk_get_vector, duckdb_data_chunk_set_size, duckdb_destroy_data_chunk, +use std::sync::Arc; + +use libduckdb_sys::{duckdb_vector_get_column_type, duckdb_vector_size}; + +use super::{column_info::ColumnInfo, LogicalTypeHandle}; +use crate::{ + core::{LogicalType, Vector, VectorHandle}, + ffi::{ + duckdb_create_data_chunk, duckdb_data_chunk, duckdb_data_chunk_get_column_count, duckdb_data_chunk_get_size, + duckdb_data_chunk_get_vector, duckdb_data_chunk_set_size, duckdb_destroy_data_chunk, + }, }; /// DataChunk in DuckDB. -pub struct DataChunk { +pub struct DataChunkHandle { /// Pointer to the DataChunk in duckdb C API. - ptr: duckdb_data_chunk, + pub(crate) ptr: duckdb_data_chunk, - /// Whether this [DataChunk] own the [DataChunk::ptr]. + /// Whether this [DataChunkHandle] own the [DataChunk::ptr]. owned: bool, } -impl DataChunk { - /// Create a new [DataChunk] with the given [LogicalType]s. +impl Drop for DataChunkHandle { + fn drop(&mut self) { + if self.owned && !self.ptr.is_null() { + unsafe { duckdb_destroy_data_chunk(&mut self.ptr) } + } + self.ptr = std::ptr::null_mut(); + } +} + +impl DataChunkHandle { + pub(crate) unsafe fn new_unowned(ptr: duckdb_data_chunk) -> Self { + Self { ptr, owned: false } + } + + /// Create a new [DataChunkHandle] with the given [LogicalTypeHandle]s. pub fn new(logical_types: &[LogicalType]) -> Self { let num_columns = logical_types.len(); - let mut c_types = logical_types.iter().map(|t| t.ptr).collect::>(); + let mut c_types = logical_types.iter().map(|t| unsafe { t.ptr() }).collect::>(); let ptr = unsafe { duckdb_create_data_chunk(c_types.as_mut_ptr(), num_columns as u64) }; - DataChunk { ptr, owned: true } + Self { ptr, owned: true } } /// Get the vector at the specific column index: `idx`. - pub fn flat_vector(&self, idx: usize) -> FlatVector { - FlatVector::from(unsafe { duckdb_data_chunk_get_vector(self.ptr, idx as u64) }) + pub fn get_vector(&self, idx: usize) -> VectorHandle { + unsafe { VectorHandle::new_unchecked(duckdb_data_chunk_get_vector(self.ptr, idx as u64)) } } - /// Get a list vector from the column index. - pub fn list_vector(&self, idx: usize) -> ListVector { - ListVector::from(unsafe { duckdb_data_chunk_get_vector(self.ptr, idx as u64) }) + /// Get all the [LogicalTypes] in this [DataChunkHandle]. + pub fn get_logical_types(&self) -> Vec { + let num_columns = self.num_columns(); + (0..num_columns) + .map(|idx| unsafe { + let vector = duckdb_data_chunk_get_vector(self.ptr, idx as u64); + let handle = LogicalTypeHandle::new(duckdb_vector_get_column_type(vector)); + LogicalType::from(handle) + }) + .collect() } - /// Get a array vector from the column index. - pub fn array_vector(&self, idx: usize) -> ArrayVector { - ArrayVector::from(unsafe { duckdb_data_chunk_get_vector(self.ptr, idx as u64) }) - } - - /// Get struct vector at the column index: `idx`. - pub fn struct_vector(&self, idx: usize) -> StructVector { - StructVector::from(unsafe { duckdb_data_chunk_get_vector(self.ptr, idx as u64) }) + /// Get the maximum capacity of the data chunk. + /// Data should be capped at this size. + /// If the data exceeds this size, it should be split into multiple data chunks. + pub fn max_capacity() -> usize { + unsafe { duckdb_vector_size() as usize } } /// Set the size of the data chunk @@ -50,49 +71,88 @@ impl DataChunk { unsafe { duckdb_data_chunk_set_size(self.ptr, new_len as u64) }; } - /// Get the length / the number of rows in this [DataChunk]. + pub fn reserve(&self, size: usize) { + if size > self.len() { + self.set_len(size.next_power_of_two()); + } + } + + pub fn with_capacity(self, size: usize) -> Self { + self.reserve(size); + self + } + + /// Get the length / the number of rows in this [DataChunkHandle]. pub fn len(&self) -> usize { unsafe { duckdb_data_chunk_get_size(self.ptr) as usize } } - /// Check whether this [DataChunk] is empty. + pub fn num_columns(&self) -> usize { + unsafe { duckdb_data_chunk_get_column_count(self.ptr) as usize } + } + + /// Check whether this [DataChunkHandle] is empty. pub fn is_empty(&self) -> bool { self.len() == 0 } +} - /// Get the number of columns in this [DataChunk]. - pub fn num_columns(&self) -> usize { - unsafe { duckdb_data_chunk_get_column_count(self.ptr) as usize } +pub struct DataChunk { + handle: Arc, + pub columns: Vec, +} + +impl DataChunk { + // Create a new [DataChunk] construction from logical types and columns infos + pub fn new(capacity: usize, logical_types: &[LogicalType], columns: &[ColumnInfo]) -> Self { + let handle = Arc::new(DataChunkHandle::new(logical_types).with_capacity(capacity)); + let columns = Vector::new_from_column_infos(columns, handle.clone()); + Self { handle, columns } + } + + /// Create a new [DataChunk] from an existing [DataChunkHandle]. + pub fn from_existing_handle(handle: DataChunkHandle) -> Self { + let handle = Arc::new(handle); + let logical_types = handle.get_logical_types(); + let infos: Vec = logical_types.iter().map(ColumnInfo::new).collect(); + let columns = Vector::new_from_column_infos(&infos, handle.clone()); + Self { handle, columns } + } + + /// Create a new [DataChunk] from an arrow::RecordBatch. + pub fn from_arrow_schema(schema: &arrow::datatypes::SchemaRef, handle: DataChunkHandle) -> Self { + let handle = Arc::new(handle); + let columns = Vector::from_arrow_schema(schema, handle.clone()); + Self { handle, columns } } /// Get the ptr of duckdb_data_chunk in this [DataChunk]. - pub fn get_ptr(&self) -> duckdb_data_chunk { - self.ptr + pub fn get_handle(&self) -> Arc { + self.handle.clone() } -} -impl From for DataChunk { - fn from(ptr: duckdb_data_chunk) -> Self { - Self { ptr, owned: false } + /// Get the number of columns in this [DataChunk]. + pub fn num_columns(&self) -> usize { + self.columns.len() } -} -impl Drop for DataChunk { - fn drop(&mut self) { - if self.owned && !self.ptr.is_null() { - unsafe { duckdb_destroy_data_chunk(&mut self.ptr) } - self.ptr = std::ptr::null_mut(); - } + pub fn reserve(&mut self, capacity: usize) { + self.handle.reserve(capacity); + } + + pub fn set_len(&mut self, size: usize) { + self.handle.set_len(size); } } #[cfg(test)] mod test { use super::{super::logical_type::LogicalTypeId, *}; + use crate::core::{FlatVector, LogicalTypeHandle}; #[test] fn test_data_chunk_construction() { - let dc = DataChunk::new(&[LogicalType::new(LogicalTypeId::Integer)]); + let dc = DataChunkHandle::new(&[LogicalTypeHandle::from(LogicalTypeId::Integer).into()]); assert_eq!(dc.num_columns(), 1); @@ -101,33 +161,11 @@ mod test { #[test] fn test_vector() { - let datachunk = DataChunk::new(&[LogicalType::new(LogicalTypeId::Bigint)]); - let mut vector = datachunk.flat_vector(0); + let datachunk = DataChunkHandle::new(&[LogicalTypeHandle::from(LogicalTypeId::Bigint).into()]); + datachunk.reserve(1); + let mut vector = FlatVector::new(datachunk.get_vector(0), Arc::new(move || datachunk.len())); let data = vector.as_mut_slice::(); data[0] = 42; } - - #[test] - fn test_logi() { - let key = LogicalType::new(LogicalTypeId::Varchar); - - let value = LogicalType::new(LogicalTypeId::UTinyint); - - let map = LogicalType::map(&key, &value); - - assert_eq!(map.id(), LogicalTypeId::Map); - - // let union_ = LogicalType::new_union_type(HashMap::from([ - // ("number", LogicalType::new(LogicalTypeId::Bigint)), - // ("string", LogicalType::new(LogicalTypeId::Varchar)), - // ])); - // assert_eq!(union_.type_id(), LogicalTypeId::Union); - - // let struct_ = LogicalType::new_struct_type(HashMap::from([ - // ("number", LogicalType::new(LogicalTypeId::Bigint)), - // ("string", LogicalType::new(LogicalTypeId::Varchar)), - // ])); - // assert_eq!(struct_.type_id(), LogicalTypeId::Struct); - } } diff --git a/crates/duckdb/src/core/logical_type.rs b/crates/duckdb/src/core/logical_type.rs index 9ea83681..95a2e258 100644 --- a/crates/duckdb/src/core/logical_type.rs +++ b/crates/duckdb/src/core/logical_type.rs @@ -3,12 +3,15 @@ use std::{ fmt::Debug, }; +use derive_more::{Constructor, From}; +use strum::EnumTryAs; + use crate::ffi::*; /// Logical Type Id /// #[repr(u32)] -#[derive(Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum LogicalTypeId { /// Boolean Boolean = DUCKDB_TYPE_DUCKDB_TYPE_BOOLEAN, @@ -66,6 +69,8 @@ pub enum LogicalTypeId { Uuid = DUCKDB_TYPE_DUCKDB_TYPE_UUID, /// Union Union = DUCKDB_TYPE_DUCKDB_TYPE_UNION, + /// Array + Array = DUCKDB_TYPE_DUCKDB_TYPE_ARRAY, /// Timestamp TZ TimestampTZ = DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_TZ, } @@ -98,43 +103,43 @@ impl From for LogicalTypeId { DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_NS => Self::TimestampNs, DUCKDB_TYPE_DUCKDB_TYPE_ENUM => Self::Enum, DUCKDB_TYPE_DUCKDB_TYPE_LIST => Self::List, + DUCKDB_TYPE_DUCKDB_TYPE_ARRAY => Self::Array, DUCKDB_TYPE_DUCKDB_TYPE_STRUCT => Self::Struct, DUCKDB_TYPE_DUCKDB_TYPE_MAP => Self::Map, DUCKDB_TYPE_DUCKDB_TYPE_UUID => Self::Uuid, DUCKDB_TYPE_DUCKDB_TYPE_UNION => Self::Union, DUCKDB_TYPE_DUCKDB_TYPE_TIMESTAMP_TZ => Self::TimestampTZ, - _ => panic!(), + _ => unimplemented!("{value} not implemented"), } } } -/// DuckDB Logical Type. -/// -pub struct LogicalType { +#[derive(Debug)] +pub struct LogicalTypeHandle { pub(crate) ptr: duckdb_logical_type, } -impl Debug for LogicalType { - /// Debug implementation for LogicalType - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { - let id = self.id(); - match id { - LogicalTypeId::Struct => { - write!(f, "struct<")?; - for i in 0..self.num_children() { - if i > 0 { - write!(f, ", ")?; - } - write!(f, "{}: {:?}", self.child_name(i), self.child(i))?; - } - write!(f, ">") - } - _ => write!(f, "{:?}", self.id()), - } - } -} - -impl Drop for LogicalType { +// impl Debug for LogicalTypeHandle { +// /// Debug implementation for LogicalType +// fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { +// let id = self.id(); +// match id { +// LogicalTypeId::Struct => { +// write!(f, "struct<")?; +// for i in 0..self.num_children() { +// if i > 0 { +// write!(f, ", ")?; +// } +// write!(f, "{}: {:?}", self.child_name(i), self.struct_or_union_child(i))?; +// } +// write!(f, ">") +// } +// _ => write!(f, "{:?}", self.id()), +// } +// } +// } + +impl Drop for LogicalTypeHandle { /// Drop implementation for LogicalType fn drop(&mut self) { if !self.ptr.is_null() { @@ -147,25 +152,26 @@ impl Drop for LogicalType { } } -impl From for LogicalType { - /// Wrap a DuckDB logical type from C API - fn from(ptr: duckdb_logical_type) -> Self { - Self { ptr } - } -} - -impl LogicalType { - /// Create a new [LogicalType] from [LogicalTypeId] - pub fn new(id: LogicalTypeId) -> Self { +impl From for LogicalTypeHandle { + /// Create a new [LogicalTypeHandle] from [LogicalTypeId] + fn from(id: LogicalTypeId) -> Self { unsafe { Self { ptr: duckdb_create_logical_type(id as u32), } } } +} + +// constructors +impl LogicalTypeHandle { + /// Create a DuckDB logical type from C API + pub(crate) unsafe fn new(ptr: duckdb_logical_type) -> Self { + Self { ptr } + } /// Creates a map type from its child type. - pub fn map(key: &LogicalType, value: &LogicalType) -> Self { + pub fn map(key: &LogicalTypeHandle, value: &LogicalTypeHandle) -> Self { unsafe { Self { ptr: duckdb_create_map_type(key.ptr, value.ptr), @@ -174,7 +180,7 @@ impl LogicalType { } /// Creates a list type from its child type. - pub fn list(child_type: &LogicalType) -> Self { + pub fn list(child_type: &LogicalTypeHandle) -> Self { unsafe { Self { ptr: duckdb_create_list_type(child_type.ptr), @@ -182,11 +188,11 @@ impl LogicalType { } } - /// Creates an array type from its child type. - pub fn array(child_type: &LogicalType, array_size: u64) -> Self { + /// Creates a array type from its child type. + pub fn array(child_type: &LogicalTypeHandle, size: i32) -> Self { unsafe { Self { - ptr: duckdb_create_array_type(child_type.ptr, array_size), + ptr: duckdb_create_array_type(child_type.ptr, size as idx_t), } } } @@ -200,20 +206,8 @@ impl LogicalType { } } - /// Retrieves the decimal width - /// Returns 0 if the LogicalType is not a decimal - pub fn decimal_width(&self) -> u8 { - unsafe { duckdb_decimal_width(self.ptr) } - } - - /// Retrieves the decimal scale - /// Returns 0 if the LogicalType is not a decimal - pub fn decimal_scale(&self) -> u8 { - unsafe { duckdb_decimal_scale(self.ptr) } - } - /// Make a `LogicalType` for `struct` - pub fn struct_type(fields: &[(&str, LogicalType)]) -> Self { + pub fn struct_type(fields: &[(&str, LogicalTypeHandle)]) -> Self { let keys: Vec = fields.iter().map(|f| CString::new(f.0).unwrap()).collect(); let values: Vec = fields.iter().map(|it| it.1.ptr).collect(); let name_ptrs = keys.iter().map(|it| it.as_ptr()).collect::>(); @@ -230,7 +224,7 @@ impl LogicalType { } /// Make a `LogicalType` for `union` - pub fn union_type(fields: &[(&str, LogicalType)]) -> Self { + pub fn union_type(fields: &[(&str, LogicalTypeHandle)]) -> Self { let keys: Vec = fields.iter().map(|f| CString::new(f.0).unwrap()).collect(); let values: Vec = fields.iter().map(|it| it.1.ptr).collect(); let name_ptrs = keys.iter().map(|it| it.as_ptr()).collect::>(); @@ -251,91 +245,278 @@ impl LogicalType { let duckdb_type_id = unsafe { duckdb_get_type_id(self.ptr) }; duckdb_type_id.into() } +} + +#[derive(From, Constructor, Debug)] +pub struct DecimalLogicalType(LogicalTypeHandle); + +impl DecimalLogicalType { + /// Retrieves the decimal width + /// Returns 0 if the LogicalType is not a decimal + pub fn decimal_width(&self) -> u8 { + unsafe { duckdb_decimal_width(self.0.ptr) } + } - /// Logical type children num - pub fn num_children(&self) -> usize { - match self.id() { - LogicalTypeId::Struct => unsafe { duckdb_struct_type_child_count(self.ptr) as usize }, - LogicalTypeId::Union => unsafe { duckdb_union_type_member_count(self.ptr) as usize }, - LogicalTypeId::List => 1, - _ => 0, + /// Retrieves the decimal scale + /// Returns 0 if the LogicalType is not a decimal + pub fn decimal_scale(&self) -> u8 { + unsafe { duckdb_decimal_scale(self.0.ptr) } + } +} + +#[derive(From, Constructor, Debug)] +pub struct ListLogicalType(LogicalTypeHandle); + +impl ListLogicalType { + pub fn child_type(&self) -> LogicalType { + let handle = unsafe { LogicalTypeHandle::new(duckdb_list_type_child_type(self.0.ptr)) }; + + LogicalType::from(handle) + } +} + +#[derive(From, Constructor, Debug)] +pub struct UnionLogicalType(LogicalTypeHandle); + +impl UnionLogicalType { + pub fn member_count(&self) -> usize { + unsafe { duckdb_union_type_member_count(self.0.ptr) as usize } + } + + /// Logical type child by idx + pub fn member_type(&self, idx: usize) -> LogicalType { + let handle = unsafe { LogicalTypeHandle::new(duckdb_union_type_member_type(self.0.ptr, idx as u64)) }; + LogicalType::from(handle) + } + + /// Logical type child name by idx + pub fn member_name(&self, idx: usize) -> String { + unsafe { + let child_name_ptr = duckdb_union_type_member_name(self.0.ptr, idx as u64); + let c_str = CString::from_raw(child_name_ptr); + let name = c_str.to_str().unwrap(); + name.to_string() } } +} + +#[derive(From, Constructor, Debug)] +pub struct StructLogicalType(LogicalTypeHandle); + +impl StructLogicalType { + /// Logical type child by idx + pub fn child_type(&self, idx: usize) -> LogicalType { + let handle = unsafe { LogicalTypeHandle::new(duckdb_struct_type_child_type(self.0.ptr, idx as u64)) }; + LogicalType::from(handle) + } + + pub fn child_count(&self) -> usize { + unsafe { duckdb_struct_type_child_count(self.0.ptr) as usize } + } /// Logical type child name by idx /// /// Panics if the logical type is not a struct or union pub fn child_name(&self, idx: usize) -> String { unsafe { - let child_name_ptr = match self.id() { - LogicalTypeId::Struct => duckdb_struct_type_child_name(self.ptr, idx as u64), - LogicalTypeId::Union => duckdb_union_type_member_name(self.ptr, idx as u64), - _ => panic!("not a struct or union"), - }; + let child_name_ptr = duckdb_struct_type_child_name(self.0.ptr, idx as u64); let c_str = CString::from_raw(child_name_ptr); let name = c_str.to_str().unwrap(); name.to_string() } } +} - /// Logical type child by idx - pub fn child(&self, idx: usize) -> Self { - let c_logical_type = unsafe { - match self.id() { - LogicalTypeId::Struct => duckdb_struct_type_child_type(self.ptr, idx as u64), - LogicalTypeId::Union => duckdb_union_type_member_type(self.ptr, idx as u64), - _ => panic!("not a struct or union"), - } - }; - Self::from(c_logical_type) +#[derive(From, Constructor, Debug)] +pub struct MapLogicalType(LogicalTypeHandle); + +impl MapLogicalType { + pub fn key_type(&self) -> LogicalType { + let handle = unsafe { LogicalTypeHandle::new(duckdb_map_type_key_type(self.0.ptr)) }; + handle.into() + } + + pub fn value_type(&self) -> LogicalType { + let handle = unsafe { LogicalTypeHandle::new(duckdb_map_type_value_type(self.0.ptr)) }; + handle.into() + } +} + +#[derive(From, Constructor, Debug)] +pub struct ArrayLogicalType(LogicalTypeHandle); + +impl ArrayLogicalType { + pub fn size(&self) -> usize { + unsafe { duckdb_array_type_array_size(self.0.ptr) as usize } + } + + pub fn child_type(&self) -> LogicalType { + let handle = unsafe { LogicalTypeHandle::new(duckdb_array_type_child_type(self.0.ptr)) }; + LogicalType::from(handle) + } +} + +/// DuckDB Logical Type. +/// Modelled from https://duckdb.org/docs/sql/data_types/overview +#[derive(EnumTryAs, Debug)] +pub enum LogicalType { + // TODO: Possibly break this down further + General { + id: LogicalTypeId, + handle: LogicalTypeHandle, + }, + List(ListLogicalType), + Decimal(DecimalLogicalType), + Union(UnionLogicalType), + Struct(StructLogicalType), + Array(ArrayLogicalType), + Map(MapLogicalType), +} + +impl From for LogicalType { + fn from(handle: LogicalTypeHandle) -> Self { + let id = handle.id(); + match id { + LogicalTypeId::Boolean + | LogicalTypeId::Tinyint + | LogicalTypeId::Smallint + | LogicalTypeId::Integer + | LogicalTypeId::Bigint + | LogicalTypeId::UTinyint + | LogicalTypeId::USmallint + | LogicalTypeId::UInteger + | LogicalTypeId::UBigint + | LogicalTypeId::Float + | LogicalTypeId::Double + | LogicalTypeId::Timestamp + | LogicalTypeId::TimestampTZ + | LogicalTypeId::Date + | LogicalTypeId::Time + | LogicalTypeId::Interval + | LogicalTypeId::Hugeint + | LogicalTypeId::Varchar + | LogicalTypeId::Blob + | LogicalTypeId::TimestampS + | LogicalTypeId::TimestampMs + | LogicalTypeId::TimestampNs + | LogicalTypeId::Uuid + | LogicalTypeId::Enum => LogicalType::General { id, handle }, + LogicalTypeId::Decimal => Self::Decimal(handle.into()), + LogicalTypeId::List => Self::List(handle.into()), + LogicalTypeId::Struct => Self::Struct(handle.into()), + LogicalTypeId::Map => Self::Map(handle.into()), + LogicalTypeId::Union => Self::Union(handle.into()), + LogicalTypeId::Array => Self::Array(handle.into()), + } + } +} + +impl LogicalType { + /// Creates a map type from its child type. + pub fn new_map(key: &LogicalTypeHandle, value: &LogicalTypeHandle) -> Self { + Self::Map(LogicalTypeHandle::map(key, value).into()) + } + + /// Creates a list type from its child type. + pub fn new_list(child_type: &LogicalTypeHandle) -> Self { + Self::List(LogicalTypeHandle::list(child_type).into()) + } + + /// Creates a array type from its child type. + pub fn new_array(child_type: &LogicalTypeHandle, size: i32) -> Self { + Self::Array(LogicalTypeHandle::array(child_type, size).into()) + } + + /// Creates a decimal type from its `width` and `scale`. + pub fn new_decimal(width: u8, scale: u8) -> Self { + Self::Decimal(LogicalTypeHandle::decimal(width, scale).into()) + } + + /// Make a `LogicalType` for `struct` + pub fn new_struct_type(fields: &[(&str, LogicalTypeHandle)]) -> Self { + Self::Struct(LogicalTypeHandle::struct_type(fields).into()) + } + + /// Make a `LogicalType` for `union` + pub fn new_union_type(fields: &[(&str, LogicalTypeHandle)]) -> Self { + Self::Union(LogicalTypeHandle::union_type(fields).into()) + } + + pub fn logical_id(&self) -> LogicalTypeId { + match self { + LogicalType::General { id, .. } => *id, + LogicalType::Decimal(_) => LogicalTypeId::Decimal, + LogicalType::List(_) => LogicalTypeId::List, + LogicalType::Struct(_) => LogicalTypeId::Struct, + LogicalType::Map(_) => LogicalTypeId::Map, + LogicalType::Union(_) => LogicalTypeId::Union, + LogicalType::Array(_) => LogicalTypeId::Array, + } + } + + /// This method returns the number of children in the struct or union. + /// This is unsafe because dropping the LogicalTypeHandle will invalidate the pointer. + pub unsafe fn ptr(&self) -> duckdb_logical_type { + match self { + LogicalType::General { handle, .. } => handle.ptr, + LogicalType::Decimal(ty) => ty.0.ptr, + LogicalType::List(ty) => ty.0.ptr, + LogicalType::Struct(ty) => ty.0.ptr, + LogicalType::Map(ty) => ty.0.ptr, + LogicalType::Union(ty) => ty.0.ptr, + LogicalType::Array(ty) => ty.0.ptr, + } } } #[cfg(test)] mod test { - use crate::core::{LogicalType, LogicalTypeId}; + use super::{LogicalType, LogicalTypeHandle, LogicalTypeId}; #[test] fn test_struct() { - let fields = &[("hello", LogicalType::new(crate::core::LogicalTypeId::Boolean))]; - let typ = LogicalType::struct_type(fields); + let fields = &[("hello", LogicalTypeHandle::from(LogicalTypeId::Boolean))]; + let typ = LogicalType::new_struct_type(fields).try_as_struct().unwrap(); - assert_eq!(typ.num_children(), 1); + assert_eq!(typ.child_count(), 1); assert_eq!(typ.child_name(0), "hello"); - assert_eq!(typ.child(0).id(), crate::core::LogicalTypeId::Boolean); + assert_eq!(typ.child_type(0).logical_id(), LogicalTypeId::Boolean); } #[test] fn test_decimal() { - let typ = LogicalType::decimal(10, 2); + let typ = LogicalType::new_decimal(10, 2).try_as_decimal().unwrap(); - assert_eq!(typ.id(), crate::core::LogicalTypeId::Decimal); assert_eq!(typ.decimal_width(), 10); assert_eq!(typ.decimal_scale(), 2); } - #[test] - fn test_decimal_methods() { - let typ = LogicalType::new(crate::core::LogicalTypeId::Varchar); - - assert_eq!(typ.decimal_width(), 0); - assert_eq!(typ.decimal_scale(), 0); - } - #[test] fn test_union_type() { let fields = &[ - ("hello", LogicalType::new(LogicalTypeId::Boolean)), - ("world", LogicalType::new(LogicalTypeId::Integer)), + ("hello", LogicalTypeHandle::from(LogicalTypeId::Boolean)), + ("world", LogicalTypeHandle::from(LogicalTypeId::Integer)), ]; - let typ = LogicalType::union_type(fields); + let typ = LogicalType::new_union_type(fields).try_as_union().unwrap(); - assert_eq!(typ.num_children(), 2); + assert_eq!(typ.member_count(), 2); - assert_eq!(typ.child_name(0), "hello"); - assert_eq!(typ.child(0).id(), LogicalTypeId::Boolean); + assert_eq!(typ.member_name(0), "hello"); + assert_eq!(typ.member_type(0).logical_id(), LogicalTypeId::Boolean); - assert_eq!(typ.child_name(1), "world"); - assert_eq!(typ.child(1).id(), LogicalTypeId::Integer); + assert_eq!(typ.member_name(1), "world"); + assert_eq!(typ.member_type(1).logical_id(), LogicalTypeId::Integer); + } + + #[test] + fn test_map_type() { + let key = LogicalTypeHandle::from(LogicalTypeId::Varchar); + let value = LogicalTypeHandle::from(LogicalTypeId::UTinyint); + let map = LogicalTypeHandle::map(&key, &value); + + assert_eq!(map.id(), LogicalTypeId::Map); + let typ = LogicalType::new_map(&key, &value).try_as_map().unwrap(); + assert_eq!(typ.key_type().logical_id(), LogicalTypeId::Varchar); + assert_eq!(typ.value_type().logical_id(), LogicalTypeId::UTinyint); } } diff --git a/crates/duckdb/src/core/mod.rs b/crates/duckdb/src/core/mod.rs index 1726a35f..de721ecb 100644 --- a/crates/duckdb/src/core/mod.rs +++ b/crates/duckdb/src/core/mod.rs @@ -1,7 +1,9 @@ +mod column_info; mod data_chunk; mod logical_type; mod vector; -pub use data_chunk::DataChunk; -pub use logical_type::{LogicalType, LogicalTypeId}; +pub(crate) use column_info::*; +pub use data_chunk::*; +pub use logical_type::*; pub use vector::*; diff --git a/crates/duckdb/src/core/vector.rs b/crates/duckdb/src/core/vector.rs index 54f68607..f46071c5 100644 --- a/crates/duckdb/src/core/vector.rs +++ b/crates/duckdb/src/core/vector.rs @@ -1,63 +1,133 @@ -use std::{any::Any, ffi::CString, slice}; +use std::{ + ffi::{c_char, CString}, + fmt::Debug, + marker, slice, + sync::Arc, +}; -use libduckdb_sys::{duckdb_array_type_array_size, duckdb_array_vector_get_child}; +use arrow::datatypes::SchemaRef; +use derive_more::{Constructor, From}; +use libduckdb_sys::{ + duckdb_array_vector_get_child, duckdb_union_type_member_name, duckdb_vector_assign_string_element_len, idx_t, + DuckDbString, +}; -use super::LogicalType; +use super::{column_info::ColumnInfo, DataChunkHandle, LogicalTypeHandle}; use crate::ffi::{ duckdb_list_entry, duckdb_list_vector_get_child, duckdb_list_vector_get_size, duckdb_list_vector_reserve, duckdb_list_vector_set_size, duckdb_struct_type_child_count, duckdb_struct_type_child_name, duckdb_struct_vector_get_child, duckdb_validity_set_row_invalid, duckdb_vector, - duckdb_vector_assign_string_element, duckdb_vector_assign_string_element_len, - duckdb_vector_ensure_validity_writable, duckdb_vector_get_column_type, duckdb_vector_get_data, - duckdb_vector_get_validity, duckdb_vector_size, + duckdb_vector_assign_string_element, duckdb_vector_ensure_validity_writable, duckdb_vector_get_column_type, + duckdb_vector_get_data, duckdb_vector_get_validity, }; -/// Vector trait. -pub trait Vector { - /// Returns a reference to the underlying Any type that this trait object - fn as_any(&self) -> &dyn Any; - /// Returns a mutable reference to the underlying Any type that this trait object - fn as_mut_any(&mut self) -> &mut dyn Any; -} +/// There is no way to get the length of the FlatVector directly, +/// As the C Api is not exposed making us do this ugly hack. +type GetVectorCapacity = dyn Fn() -> usize; -/// A flat vector -pub struct FlatVector { +/// A handle to the vector created by data chunk +#[derive(Debug, Constructor, Clone)] +pub struct VectorHandle { ptr: duckdb_vector, - capacity: usize, + _marker: marker::PhantomData, } -impl From for FlatVector { - fn from(ptr: duckdb_vector) -> Self { +impl VectorHandle { + pub unsafe fn new_unchecked(ptr: duckdb_vector) -> Self { Self { ptr, - capacity: unsafe { duckdb_vector_size() as usize }, + _marker: marker::PhantomData, + } + } + + pub const fn cast(&self) -> VectorHandle { + VectorHandle { + ptr: self.ptr, + _marker: marker::PhantomData, + } + } + + pub const fn as_flat(&self) -> VectorHandle { + self.cast() + } + + pub const fn as_list(&self) -> VectorHandle { + self.cast() + } + + pub const fn as_array(&self) -> VectorHandle { + self.cast() + } + + pub const fn as_struct(&self) -> VectorHandle { + self.cast() + } + + pub const fn as_union(&self) -> VectorHandle { + self.cast() + } +} + +impl VectorHandle { + pub fn get_child(&self) -> VectorHandle { + unsafe { VectorHandle::new_unchecked(duckdb_list_vector_get_child(self.ptr)) } + } + + pub fn get_size(&self) -> usize { + unsafe { duckdb_list_vector_get_size(self.ptr) as usize } + } + + pub fn reserve_child_capacity(&mut self, capacity: usize) { + unsafe { + duckdb_list_vector_reserve(self.ptr, capacity as u64); + } + } + + /// Set the length of the list child vector. + pub fn set_child_len(&mut self, new_len: usize) { + unsafe { + duckdb_list_vector_set_size(self.ptr, new_len as u64); } } } -impl Vector for FlatVector { - fn as_any(&self) -> &dyn Any { - self +impl VectorHandle { + pub fn get_child(&self) -> VectorHandle { + unsafe { VectorHandle::new_unchecked(duckdb_array_vector_get_child(self.ptr)) } } +} - fn as_mut_any(&mut self) -> &mut dyn Any { - self +impl VectorHandle { + pub fn get_child<'b>(&self, idx: usize) -> VectorHandle { + unsafe { VectorHandle::new_unchecked(duckdb_struct_vector_get_child(self.ptr, idx as idx_t)) } } } -impl FlatVector { - fn with_capacity(ptr: duckdb_vector, capacity: usize) -> Self { - Self { ptr, capacity } +impl VectorHandle { + pub fn get_child<'b>(&self, idx: usize) -> VectorHandle { + unsafe { VectorHandle::new_unchecked(duckdb_struct_vector_get_child(self.ptr, idx as idx_t)) } } +} + +/// A flat vector +#[derive(Constructor)] +pub struct FlatVector { + handle: VectorHandle, + /// get the size of elements for this vector. + /// there is no good way to get the length just using the flat vector handle, + /// hence we need to retrive it from higher above, either a list vector or directly a data chunk + get_capacity: Arc, +} +impl FlatVector { /// Returns the capacity of the vector pub fn capacity(&self) -> usize { - self.capacity + (self.get_capacity)() } /// Returns an unsafe mutable pointer to the vector’s pub fn as_mut_ptr(&self) -> *mut T { - unsafe { duckdb_vector_get_data(self.ptr).cast() } + unsafe { duckdb_vector_get_data(self.handle.ptr.cast()).cast() } } /// Returns a slice of the vector @@ -71,22 +141,28 @@ impl FlatVector { } /// Returns the logical type of the vector - pub fn logical_type(&self) -> LogicalType { - LogicalType::from(unsafe { duckdb_vector_get_column_type(self.ptr) }) + pub fn logical_type(&self) -> LogicalTypeHandle { + unsafe { LogicalTypeHandle::new(duckdb_vector_get_column_type(self.handle.ptr)) } } /// Set row as null pub fn set_null(&mut self, row: usize) { unsafe { - duckdb_vector_ensure_validity_writable(self.ptr); - let idx = duckdb_vector_get_validity(self.ptr); + duckdb_vector_ensure_validity_writable(self.handle.ptr); + let idx = duckdb_vector_get_validity(self.handle.ptr); duckdb_validity_set_row_invalid(idx, row as u64); } } /// Copy data to the vector. pub fn copy(&mut self, data: &[T]) { - assert!(data.len() <= self.capacity()); + let capacity = self.capacity(); + assert!( + data.len() <= capacity, + "Provided data of length: {} exceeded capacity of {}", + data.len(), + capacity + ); self.as_mut_slice::()[0..data.len()].copy_from_slice(data); } } @@ -100,16 +176,21 @@ pub trait Inserter { impl Inserter for FlatVector { fn insert(&self, index: usize, value: CString) { unsafe { - duckdb_vector_assign_string_element(self.ptr, index as u64, value.as_ptr()); + duckdb_vector_assign_string_element(self.handle.ptr, index as u64, value.as_ptr()); } } } impl Inserter<&str> for FlatVector { fn insert(&self, index: usize, value: &str) { - let cstr = CString::new(value.as_bytes()).unwrap(); + let bytes = value.as_bytes(); unsafe { - duckdb_vector_assign_string_element(self.ptr, index as u64, cstr.as_ptr()); + duckdb_vector_assign_string_element_len( + self.handle.ptr, + index as u64, + bytes.as_ptr() as *const c_char, + bytes.len() as u64, + ); } } } @@ -120,7 +201,7 @@ impl Inserter<&[u8]> for FlatVector { unsafe { // This function also works for binary data. https://duckdb.org/docs/api/c/api#duckdb_vector_assign_string_element_len duckdb_vector_assign_string_element_len( - self.ptr, + self.handle.ptr, index as u64, value.as_ptr() as *const ::std::os::raw::c_char, value_size as u64, @@ -132,152 +213,250 @@ impl Inserter<&[u8]> for FlatVector { /// A list vector. pub struct ListVector { /// ListVector does not own the vector pointer. - entries: FlatVector, -} - -impl From for ListVector { - fn from(ptr: duckdb_vector) -> Self { - Self { - entries: FlatVector::from(ptr), - } - } + pub entry_list: FlatVector, + pub child: Box, } impl ListVector { - /// Returns the number of entries in the list vector. - pub fn len(&self) -> usize { - unsafe { duckdb_list_vector_get_size(self.entries.ptr) as usize } - } - /// Returns true if the list vector is empty. pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Returns the child vector. - // TODO: not ideal interface. Where should we keep capacity. - pub fn child(&self, capacity: usize) -> FlatVector { - self.reserve(capacity); - FlatVector::with_capacity(unsafe { duckdb_list_vector_get_child(self.entries.ptr) }, capacity) - } - - /// Set primitive data to the child node. - pub fn set_child(&self, data: &[T]) { - self.child(data.len()).copy(data); - self.set_len(data.len()); + self.child_len() == 0 } /// Set offset and length to the entry. pub fn set_entry(&mut self, idx: usize, offset: usize, length: usize) { - self.entries.as_mut_slice::()[idx].offset = offset as u64; - self.entries.as_mut_slice::()[idx].length = length as u64; + self.entry_list.as_mut_slice::()[idx] = duckdb_list_entry { + offset: offset as u64, + length: length as u64, + } + } + + /// Returns the number of entries in the list child vector. + pub fn child_len(&self) -> usize { + self.entry_list.handle.as_list().get_size() } /// Reserve the capacity for its child node. - fn reserve(&self, capacity: usize) { - unsafe { - duckdb_list_vector_reserve(self.entries.ptr, capacity as u64); - } + pub fn reserve_child_capacity(&mut self, capacity: usize) { + self.entry_list.handle.as_list().reserve_child_capacity(capacity) } - /// Set the length of the list vector. - pub fn set_len(&self, new_len: usize) { - unsafe { - duckdb_list_vector_set_size(self.entries.ptr, new_len as u64); - } + /// Set the length of the list child vector. + pub fn set_child_len(&mut self, new_len: usize) { + self.entry_list.handle.as_list().set_child_len(new_len) } } -/// A array vector. (fixed-size list) pub struct ArrayVector { - /// ArrayVector does not own the vector pointer. - ptr: duckdb_vector, + pub child: Box, + array_size: usize, // number of columns } -impl From for ArrayVector { - fn from(ptr: duckdb_vector) -> Self { - Self { ptr } +impl ArrayVector { + pub fn array_size(&self) -> usize { + self.array_size } } -impl ArrayVector { - /// Get the logical type of this ArrayVector. - pub fn logical_type(&self) -> LogicalType { - LogicalType::from(unsafe { duckdb_vector_get_column_type(self.ptr) }) - } +/// A struct vector. +pub struct StructVector { + handle: VectorHandle, + /// ListVector does not own the vector pointer. + pub children: Vec, +} - /// Returns the size of the array type. - pub fn get_array_size(&self) -> u64 { - let ty = self.logical_type(); - unsafe { duckdb_array_type_array_size(ty.ptr) as u64 } +impl StructVector { + /// Get the logical type of this struct vector. + pub fn logical_type(&self) -> LogicalTypeHandle { + unsafe { LogicalTypeHandle::new(duckdb_vector_get_column_type(self.handle.ptr)) } } - /// Returns the child vector. - /// capacity should be a multiple of the array size. - // TODO: not ideal interface. Where should we keep count. - pub fn child(&self, capacity: usize) -> FlatVector { - FlatVector::with_capacity(unsafe { duckdb_array_vector_get_child(self.ptr) }, capacity) + /// Get the name of the child by idx. + pub fn child_name(&self, idx: usize) -> DuckDbString { + let logical_type = self.logical_type(); + unsafe { DuckDbString::from_ptr(duckdb_struct_type_child_name(logical_type.ptr, idx as u64)) } } - /// Set primitive data to the child node. - pub fn set_child(&self, data: &[T]) { - self.child(data.len()).copy(data); + /// Get the number of children. + pub fn num_children(&self) -> usize { + let logical_type = self.logical_type(); + unsafe { duckdb_struct_type_child_count(logical_type.ptr) as usize } } } -/// A struct vector. -pub struct StructVector { - /// ListVector does not own the vector pointer. - ptr: duckdb_vector, +/// A union vector +pub struct UnionVector { + /// UnionVector does not own the vector pointer + handle: VectorHandle, + pub tags: FlatVector, + pub members: Vec, } -impl From for StructVector { - fn from(ptr: duckdb_vector) -> Self { - Self { ptr } +impl UnionVector { + /// Get the logical type of this struct vector. + pub fn logical_type(&self) -> LogicalTypeHandle { + unsafe { LogicalTypeHandle::new(duckdb_vector_get_column_type(self.handle.ptr)) } + } + + /// Retrieves the child type of the given union member at the specified index + /// Get the name of the union member. + pub fn member_name(&self, idx: usize) -> DuckDbString { + let logical_type = self.logical_type(); + unsafe { DuckDbString::from_ptr(duckdb_union_type_member_name(logical_type.ptr, idx as u64)) } } } -impl StructVector { - /// Returns the child by idx in the list vector. - pub fn child(&self, idx: usize) -> FlatVector { - FlatVector::from(unsafe { duckdb_struct_vector_get_child(self.ptr, idx as u64) }) +#[derive(From)] +pub enum Vector { + Flat(FlatVector), + List(ListVector), + Array(ArrayVector), + Struct(StructVector), + Union(UnionVector), +} + +impl Vector { + pub fn try_as_flat_mut(&mut self) -> Result<&mut FlatVector, Box> { + match self { + Vector::Flat(x) => Ok(x), + _ => Err("Vector is not a FlatVector".into()), + } } - /// Take the child as [StructVector]. - pub fn struct_vector_child(&self, idx: usize) -> StructVector { - Self::from(unsafe { duckdb_struct_vector_get_child(self.ptr, idx as u64) }) + pub fn try_as_list_mut(&mut self) -> Result<&mut ListVector, Box> { + match self { + Vector::List(x) => Ok(x), + _ => Err("Vector is not a ListVector".into()), + } } - /// Take the child as [ListVector]. - pub fn list_vector_child(&self, idx: usize) -> ListVector { - ListVector::from(unsafe { duckdb_struct_vector_get_child(self.ptr, idx as u64) }) + pub fn try_as_array_mut(&mut self) -> Result<&mut ArrayVector, Box> { + match self { + Vector::Array(x) => Ok(x), + _ => Err("Vector is not a ArrayVector".into()), + } } - /// Take the child as [ArrayVector]. - pub fn array_vector_child(&self, idx: usize) -> ArrayVector { - ArrayVector::from(unsafe { duckdb_struct_vector_get_child(self.ptr, idx as u64) }) + pub fn try_as_struct_mut(&mut self) -> Result<&mut StructVector, Box> { + match self { + Vector::Struct(x) => Ok(x), + _ => Err("Vector is not a StructVector".into()), + } } - /// Get the logical type of this struct vector. - pub fn logical_type(&self) -> LogicalType { - LogicalType::from(unsafe { duckdb_vector_get_column_type(self.ptr) }) + #[inline] + pub fn try_as_union_mut(&mut self) -> Result<&mut UnionVector, Box> { + match self { + Vector::Union(x) => Ok(x), + _ => Err("Vector is not a UnionVector".into()), + } } - /// Get the name of the child by idx. - pub fn child_name(&self, idx: usize) -> String { - let logical_type = self.logical_type(); - unsafe { - let child_name_ptr = duckdb_struct_type_child_name(logical_type.ptr, idx as u64); - let c_str = CString::from_raw(child_name_ptr); - let name = c_str.to_str().unwrap(); - // duckdb_free(child_name_ptr.cast()); - name.to_string() + fn new(column_info: &ColumnInfo, base_handle: VectorHandle, get_capacity: Arc) -> Self { + match column_info { + ColumnInfo::Flat => { + let vector = FlatVector::new(base_handle.cast(), get_capacity); + Vector::Flat(vector) + } + ColumnInfo::List(list) => { + let list_handle = base_handle.clone().as_list(); + let child_handle = list_handle.get_child(); + let entry_list = FlatVector::new(base_handle.as_flat(), get_capacity); + + let size_fn = Arc::new(move || list_handle.get_size()); + + let child = Box::new(Self::new(list.child.as_ref(), child_handle, size_fn)); + + ListVector { entry_list, child }.into() + } + ColumnInfo::Array(info) => { + let array_handle = base_handle.as_array(); + let child_handle = array_handle.get_child(); + let array_size = info.array_size; + let get_array_size = Arc::new(move || array_size * get_capacity()); + let child = Box::new(Self::new(info.child.as_ref(), child_handle, get_array_size)); + + let array_size = info.array_size; + + ArrayVector { child, array_size }.into() + } + ColumnInfo::Struct(info) => { + let struct_handle = base_handle.as_struct(); + let children = info + .children + .iter() + .enumerate() + .map(|(idx, child)| { + let child_handle = struct_handle.get_child(idx); + Self::new(child, child_handle, get_capacity.clone()) + }) + .collect(); + + StructVector { + handle: struct_handle, + children, + } + .into() + } + ColumnInfo::Union(info) => { + let union_handle = base_handle.as_union(); + // first child = tag, idx is 0 + let tag_vector = union_handle.get_child(0).as_flat(); + let tags = FlatVector::new(tag_vector, get_capacity.clone()); + + // members idx begin from 1 + let members = info + .members + .iter() + .enumerate() + .map(|(idx, child)| { + let child_handle = union_handle.get_child(idx + 1); + Self::new(child, child_handle.into(), get_capacity.clone()) + }) + .collect(); + + UnionVector { + handle: union_handle, + tags, + members, + } + .into() + } } } - /// Get the number of children. - pub fn num_children(&self) -> usize { - let logical_type = self.logical_type(); - unsafe { duckdb_struct_type_child_count(logical_type.ptr) as usize } + /// Create [Vector] lists from column infos and the data chunk. + pub fn new_from_column_infos(columns: &[ColumnInfo], data_chunk_handle: Arc) -> Vec { + let get_capacity = { + let data_chunk_handle = data_chunk_handle.clone(); + Arc::new(move || data_chunk_handle.len().max(DataChunkHandle::max_capacity())) + }; + (0..columns.len()) + .map(move |i| { + let column_info = &columns[i]; + let base_handle = data_chunk_handle.get_vector(i); + Self::new(column_info, base_handle, get_capacity.clone()) + }) + .collect() + } + + /// Create [Vector] lists from Arrow schema. + pub fn from_arrow_schema(schema: &SchemaRef, data_chunk_handle: Arc) -> Vec { + let get_capacity = { + let data_chunk_handle = data_chunk_handle.clone(); + Arc::new(move || data_chunk_handle.len().max(DataChunkHandle::max_capacity())) + }; + + schema + .fields + .iter() + .enumerate() + .map(|(i, field)| { + let data_type = field.data_type(); + let column_info = ColumnInfo::new_from_data_type(data_type); + let base_handle = data_chunk_handle.get_vector(i); + Self::new(&column_info, base_handle, get_capacity.clone()) + }) + .collect() } } diff --git a/crates/duckdb/src/error.rs b/crates/duckdb/src/error.rs index b56e2c35..432a0fd1 100644 --- a/crates/duckdb/src/error.rs +++ b/crates/duckdb/src/error.rs @@ -1,11 +1,12 @@ -use arrow::datatypes::DataType; +use std::{error, ffi::CStr, fmt, path::PathBuf, str}; + +use arrow::{datatypes::DataType, error::ArrowError}; use super::Result; use crate::{ ffi, types::{FromSqlError, Type}, }; -use std::{error, ffi::CStr, fmt, path::PathBuf, str}; /// Enum listing possible errors from duckdb. #[derive(Debug)] @@ -63,6 +64,9 @@ pub enum Error { /// Error when datatype to duckdb type ArrowTypeToDuckdbType(String, DataType), + /// Error when datatype to duckdb type + ArrowError(ArrowError), + /// Error when a query that was expected to insert one row did not insert /// any or insert many. StatementChangedRows(usize), @@ -122,6 +126,13 @@ impl From<::std::ffi::NulError> for Error { } } +impl From for Error { + #[cold] + fn from(err: ArrowError) -> Error { + Error::ArrowError(err) + } +} + const UNKNOWN_COLUMN: usize = usize::MAX; /// The conversion isn't precise, but it's convenient to have it @@ -186,6 +197,7 @@ impl fmt::Display for Error { Error::InvalidQuery => write!(f, "Query is not read-only"), Error::MultipleStatement => write!(f, "Multiple statements provided"), Error::AppendError => write!(f, "Append error"), + Error::ArrowError(_) => write!(f, "Arrow Error"), } } } @@ -210,6 +222,7 @@ impl error::Error for Error { | Error::InvalidQuery | Error::AppendError | Error::ArrowTypeToDuckdbType(..) + | Error::ArrowError(_) | Error::MultipleStatement => None, Error::FromSqlConversionFailure(_, _, ref err) | Error::ToSqlConversionFailure(ref err) => Some(&**err), } @@ -225,12 +238,12 @@ fn error_from_duckdb_code(code: ffi::duckdb_state, message: Option) -> R #[cold] #[inline] -pub fn result_from_duckdb_appender(code: ffi::duckdb_state, appender: *mut ffi::duckdb_appender) -> Result<()> { +pub fn result_from_duckdb_appender(code: ffi::duckdb_state, appender: &mut ffi::duckdb_appender) -> Result<()> { if code == ffi::DuckDBSuccess { return Ok(()); } unsafe { - let message = if (*appender).is_null() { + let message = if appender.is_null() { Some("appender is null".to_string()) } else { let c_err = ffi::duckdb_appender_error(*appender); diff --git a/crates/duckdb/src/extension.rs b/crates/duckdb/src/extension.rs index 47e3f3fa..1a6a9690 100644 --- a/crates/duckdb/src/extension.rs +++ b/crates/duckdb/src/extension.rs @@ -31,4 +31,15 @@ mod test { ); Ok(()) } + + // https://duckdb.org/docs/extensions/httpfs + #[test] + fn test_extension_httpfs() -> Result<()> { + let db = Connection::open_in_memory()?; + assert_eq!( + 300f32, + db.query_row::(r#"SELECT SUM(value) FROM read_parquet('https://github.com/duckdb/duckdb-rs/raw/main/crates/duckdb/examples/int32_decimal.parquet');"#, [], |r| r.get(0))? + ); + Ok(()) + } } diff --git a/crates/duckdb/src/statement.rs b/crates/duckdb/src/statement.rs index 1b0ceca8..6a00dbda 100644 --- a/crates/duckdb/src/statement.rs +++ b/crates/duckdb/src/statement.rs @@ -1,6 +1,10 @@ -use std::{convert, ffi::c_void, fmt, mem, os::raw::c_char, ptr, str}; +use std::{convert, ffi::c_void, fmt, mem, os::raw::c_char, ptr, str, sync::Arc}; -use arrow::{array::StructArray, datatypes::SchemaRef}; +use arrow::{ + array::StructArray, + datatypes::{DataType, SchemaRef}, +}; +use arrow_convert::deserialize::{ArrowArray, ArrowDeserialize}; use super::{ffi, AndThenRows, Connection, Error, MappedRows, Params, RawStatement, Result, Row, Rows, ValueRef}; #[cfg(feature = "polars")] @@ -109,6 +113,44 @@ impl Statement<'_> { Ok(Arrow::new(self)) } + /// Execute the prepared statement, returning a deserialized vector of type `T` from the resulting arrow RecordBatch. + /// + /// # Example + /// + /// ```rust,no_run + /// # use duckdb::{Result, Connection}; + /// # use arrow::record_batch::RecordBatch; + /// + /// fn query_and_deserialize(conn: &Connection) -> Result> { + /// let result: Vec = conn + /// .prepare("SELECT num FROM test")? + /// .query_arrow_deserialized([])?; + /// Ok(result) + /// } + /// ``` + pub fn query_arrow_deserialized(&mut self, params: impl Params) -> Result> + where + T: ArrowDeserialize + 'static, + { + let arrow_iter = self.query_arrow(params)?; + let mut result = Vec::new(); + + for rb in arrow_iter { + let fields = rb.schema_ref().fields(); + + let struct_or_union_array = if fields.len() == 1 && matches!(fields[0].data_type(), DataType::Union(_, _)) { + rb.column(0).clone() + } else { + Arc::new(StructArray::from(rb)) + }; + let chunk = T::ArrayType::iter_from_array_ref(&struct_or_union_array) + .map(::arrow_deserialize_internal); + result.extend(chunk); + } + + Ok(result) + } + /// Execute the prepared statement, returning a handle to the resulting /// vector of polars DataFrame. /// diff --git a/crates/duckdb/src/vtab/arrow.rs b/crates/duckdb/src/vtab/arrow.rs index 8adeec19..cb4b5c54 100644 --- a/crates/duckdb/src/vtab/arrow.rs +++ b/crates/duckdb/src/vtab/arrow.rs @@ -1,21 +1,23 @@ -use super::{BindInfo, DataChunk, Free, FunctionInfo, InitInfo, LogicalType, LogicalTypeId, VTab}; use std::ptr::null_mut; -use crate::core::{ArrayVector, FlatVector, Inserter, ListVector, StructVector, Vector}; -use arrow::array::{ - as_boolean_array, as_generic_binary_array, as_large_list_array, as_list_array, as_primitive_array, as_string_array, - as_struct_array, Array, ArrayData, AsArray, BinaryArray, BooleanArray, Decimal128Array, FixedSizeListArray, - GenericListArray, GenericStringArray, LargeStringArray, OffsetSizeTrait, PrimitiveArray, StructArray, -}; - use arrow::{ + array::{ + as_boolean_array, as_generic_binary_array, as_large_list_array, as_list_array, as_primitive_array, + as_string_array, as_struct_array, as_union_array, Array, ArrayData, ArrayRef, AsArray, BinaryArray, + BooleanArray, Decimal128Array, FixedSizeListArray, GenericStringArray, LargeStringArray, OffsetSizeTrait, + PrimitiveArray, StructArray, + }, datatypes::*, ffi::{from_ffi, FFI_ArrowArray, FFI_ArrowSchema}, record_batch::RecordBatch, }; - use num::{cast::AsPrimitive, ToPrimitive}; +use crate::{ + core::{DataChunk, DataChunkHandle, FlatVector, Inserter, LogicalTypeHandle, LogicalTypeId, Vector}, + vtab::{BindInfo, Free, FunctionInfo, InitInfo, VTab}, +}; + /// A pointer to the Arrow record batch for the table function. #[repr(C)] pub struct ArrowBindData { @@ -71,8 +73,10 @@ impl VTab for ArrowVTab { type BindData = ArrowBindData; type InitData = ArrowInitData; - unsafe fn bind(bind: &BindInfo, data: *mut ArrowBindData) -> Result<(), Box> { - (*data).rb = null_mut(); + fn bind(bind: &BindInfo, data: *mut ArrowBindData) -> Result<(), Box> { + unsafe { + (*data).rb = null_mut(); + } let param_count = bind.get_parameter_count(); if param_count != 2 { return Err(format!("Bad param count: {param_count}, expected 2").into()); @@ -92,14 +96,14 @@ impl VTab for ArrowVTab { Ok(()) } - unsafe fn init(_: &InitInfo, data: *mut ArrowInitData) -> Result<(), Box> { + fn init(_: &InitInfo, data: *mut ArrowInitData) -> Result<(), Box> { unsafe { (*data).done = false; } Ok(()) } - unsafe fn func(func: &FunctionInfo, output: &mut DataChunk) -> Result<(), Box> { + fn func(func: &FunctionInfo, output: DataChunkHandle) -> Result<(), Box> { let init_info = func.get_init_data::(); let bind_info = func.get_bind_data::(); unsafe { @@ -108,7 +112,8 @@ impl VTab for ArrowVTab { } else { let rb = Box::from_raw((*bind_info).rb); (*bind_info).rb = null_mut(); // erase ref in case of failure in record_batch_to_duckdb_data_chunk - record_batch_to_duckdb_data_chunk(&rb, output)?; + let mut chunk = DataChunk::from_arrow_schema(rb.schema_ref(), output.with_capacity(rb.num_rows())); + record_batch_to_duckdb_data_chunk(&rb, &mut chunk)?; (*bind_info).rb = Box::into_raw(rb); (*init_info).done = true; } @@ -116,10 +121,10 @@ impl VTab for ArrowVTab { Ok(()) } - fn parameters() -> Option> { + fn parameters() -> Option> { Some(vec![ - LogicalType::new(LogicalTypeId::UBigint), // file path - LogicalType::new(LogicalTypeId::UBigint), // sheet name + LogicalTypeHandle::from(LogicalTypeId::UBigint), // file path + LogicalTypeHandle::from(LogicalTypeId::UBigint), // sheet name ]) } } @@ -173,7 +178,7 @@ pub fn to_duckdb_type_id(data_type: &DataType) -> Result Result> { +pub fn to_duckdb_logical_type(data_type: &DataType) -> Result> { match data_type { DataType::Dictionary(_, value_type) => to_duckdb_logical_type(value_type), DataType::Struct(fields) => { @@ -181,23 +186,30 @@ pub fn to_duckdb_logical_type(data_type: &DataType) -> Result { + let mut shape = vec![]; + for (_, field) in fields.iter() { + shape.push((field.name().as_str(), to_duckdb_logical_type(field.data_type())?)); + } + Ok(LogicalTypeHandle::union_type(shape.as_slice())) } DataType::List(child) | DataType::LargeList(child) => { - Ok(LogicalType::list(&to_duckdb_logical_type(child.data_type())?)) + Ok(LogicalTypeHandle::list(&to_duckdb_logical_type(child.data_type())?)) } - DataType::FixedSizeList(child, array_size) => Ok(LogicalType::array( + DataType::FixedSizeList(child, array_size) => Ok(LogicalTypeHandle::array( &to_duckdb_logical_type(child.data_type())?, - *array_size as u64, + *array_size, )), DataType::Decimal128(width, scale) if *scale > 0 => { // DuckDB does not support negative decimal scales - Ok(LogicalType::decimal(*width, (*scale).try_into().unwrap())) + Ok(LogicalTypeHandle::decimal(*width, (*scale).try_into().unwrap())) } DataType::Boolean | DataType::Utf8 | DataType::LargeUtf8 | DataType::Binary | DataType::LargeBinary => { - Ok(LogicalType::new(to_duckdb_type_id(data_type)?)) + Ok(LogicalTypeHandle::from(to_duckdb_type_id(data_type)?)) } - dtype if dtype.is_primitive() => Ok(LogicalType::new(to_duckdb_type_id(data_type)?)), + dtype if dtype.is_primitive() => Ok(LogicalTypeHandle::from(to_duckdb_type_id(data_type)?)), _ => Err(format!( "Unsupported data type: {data_type}, please file an issue https://github.com/wangfenjin/duckdb-rs" ) @@ -205,12 +217,120 @@ pub fn to_duckdb_logical_type(data_type: &DataType) -> Result Result<(), Box> { + match input.data_type() { + dt if dt.is_primitive() || matches!(dt, DataType::Boolean) => { + primitive_array_to_vector(input, output.try_as_flat_mut()?) + } + DataType::Utf8 => { + let array = as_string_array(input.as_ref()); + let out: &mut FlatVector = output.try_as_flat_mut()?; + assert!(array.len() <= out.capacity()); + + string_array_to_vector(array, out); + Ok(()) + } + DataType::LargeUtf8 => { + let out: &mut FlatVector = output.try_as_flat_mut()?; + string_array_to_vector( + input + .as_ref() + .as_any() + .downcast_ref::() + .ok_or_else(|| Box::::from("Unable to downcast to LargeStringArray"))?, + out, + ); + Ok(()) + } + DataType::Binary => { + binary_array_to_vector(as_generic_binary_array(input.as_ref()), output.try_as_flat_mut()?); + Ok(()) + } + DataType::List(_) => { + let array = as_list_array(input.as_ref()); + let out = output.try_as_list_mut()?; + let value_array = array.values(); + for i in 0..array.len() { + let offset = array.value_offsets()[i]; + let length = array.value_length(i); + out.set_entry(i, offset.as_(), length.as_()); + } + + out.reserve_child_capacity(value_array.len()); + out.set_child_len(value_array.len()); + fill_duckdb_vector_from_arrow_vector(value_array, &mut out.child)?; + Ok(()) + } + DataType::LargeList(_) => { + let array = as_large_list_array(input.as_ref()); + let out = &mut output.try_as_list_mut()?; + let value_array = array.values(); + + for i in 0..array.len() { + let offset = array.value_offsets()[i]; + let length = array.value_length(i); + out.set_entry(i, offset.as_(), length.as_()); + } + + out.reserve_child_capacity(value_array.len()); + out.set_child_len(value_array.len()); + fill_duckdb_vector_from_arrow_vector(value_array, &mut out.child)?; + Ok(()) + } + DataType::FixedSizeList(_, _) => { + let array = as_fixed_size_list_array(input.as_ref()); + let out = &mut output.try_as_array_mut()?; + let value_array = array.values(); + fill_duckdb_vector_from_arrow_vector(value_array, &mut out.child)?; + Ok(()) + } + DataType::Struct(_) => { + let struct_array = as_struct_array(input.as_ref()); + let struct_vector = output.try_as_struct_mut()?; + for i in 0..struct_array.num_columns() { + fill_duckdb_vector_from_arrow_vector(struct_array.column(i), &mut struct_vector.children[i])?; + } + Ok(()) + } + DataType::Union(fields, mode) => { + assert_eq!( + mode, + &UnionMode::Sparse, + "duckdb only supports Sparse array for union types" + ); + + let union_array = as_union_array(input.as_ref()); + let union_vector = output.try_as_union_mut().unwrap(); + + // set the tag array + union_vector.tags.copy(union_array.type_ids()); + + // copy the members + for (i, (type_id, _)) in fields.iter().enumerate() { + let column = union_array.child(type_id); + fill_duckdb_vector_from_arrow_vector(column, &mut union_vector.members[i])?; + } + Ok(()) + } + df => { + unimplemented!( + "column {} is not supported yet, please file an issue https://github.com/wangfenjin/duckdb-rs", + df + ); + } + } +} + /// Converts a `RecordBatch` to a `DataChunk` in the DuckDB format. /// /// # Arguments /// /// * `batch` - A reference to the `RecordBatch` to be converted to a `DataChunk`. /// * `chunk` - A mutable reference to the `DataChunk` to store the converted data. +/// ``` pub fn record_batch_to_duckdb_data_chunk( batch: &RecordBatch, chunk: &mut DataChunk, @@ -219,47 +339,7 @@ pub fn record_batch_to_duckdb_data_chunk( assert_eq!(batch.num_columns(), chunk.num_columns()); for i in 0..batch.num_columns() { let col = batch.column(i); - match col.data_type() { - dt if dt.is_primitive() || matches!(dt, DataType::Boolean) => { - primitive_array_to_vector(col, &mut chunk.flat_vector(i))?; - } - DataType::Utf8 => { - string_array_to_vector(as_string_array(col.as_ref()), &mut chunk.flat_vector(i)); - } - DataType::LargeUtf8 => { - string_array_to_vector( - col.as_ref() - .as_any() - .downcast_ref::() - .ok_or_else(|| Box::::from("Unable to downcast to LargeStringArray"))?, - &mut chunk.flat_vector(i), - ); - } - DataType::Binary => { - binary_array_to_vector(as_generic_binary_array(col.as_ref()), &mut chunk.flat_vector(i)); - } - DataType::List(_) => { - list_array_to_vector(as_list_array(col.as_ref()), &mut chunk.list_vector(i))?; - } - DataType::LargeList(_) => { - list_array_to_vector(as_large_list_array(col.as_ref()), &mut chunk.list_vector(i))?; - } - DataType::FixedSizeList(_, _) => { - fixed_size_list_array_to_vector(as_fixed_size_list_array(col.as_ref()), &mut chunk.array_vector(i))?; - } - DataType::Struct(_) => { - let struct_array = as_struct_array(col.as_ref()); - let mut struct_vector = chunk.struct_vector(i); - struct_array_to_vector(struct_array, &mut struct_vector)?; - } - _ => { - return Err(format!( - "column {} is not supported yet, please file an issue https://github.com/wangfenjin/duckdb-rs", - batch.schema().field(i) - ) - .into()); - } - } + fill_duckdb_vector_from_arrow_vector(col, &mut chunk.columns[i])?; } chunk.set_len(batch.num_rows()); Ok(()) @@ -280,11 +360,11 @@ fn primitive_array_to_flat_vector(array: &PrimitiveArray< fn primitive_array_to_flat_vector_cast( data_type: DataType, array: &dyn Array, - out_vector: &mut dyn Vector, + out_vector: &mut FlatVector, ) { let array = arrow::compute::kernels::cast::cast(array, &data_type).unwrap(); - let out_vector: &mut FlatVector = out_vector.as_mut_any().downcast_mut().unwrap(); out_vector.copy::(array.as_primitive::().values()); + if let Some(nulls) = array.nulls() { for (i, null) in nulls.iter().enumerate() { if !null { @@ -294,77 +374,43 @@ fn primitive_array_to_flat_vector_cast( } } -fn primitive_array_to_vector(array: &dyn Array, out: &mut dyn Vector) -> Result<(), Box> { +fn primitive_array_to_vector(array: &dyn Array, out: &mut FlatVector) -> Result<(), Box> { match array.data_type() { DataType::Boolean => { - boolean_array_to_vector(as_boolean_array(array), out.as_mut_any().downcast_mut().unwrap()); + boolean_array_to_vector(as_boolean_array(array), out); } DataType::UInt8 => { - primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ); + primitive_array_to_flat_vector::(as_primitive_array(array), out); } DataType::UInt16 => { - primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ); + primitive_array_to_flat_vector::(as_primitive_array(array), out); } DataType::UInt32 => { - primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ); + primitive_array_to_flat_vector::(as_primitive_array(array), out); } DataType::UInt64 => { - primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ); + primitive_array_to_flat_vector::(as_primitive_array(array), out); } DataType::Int8 => { - primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ); + primitive_array_to_flat_vector::(as_primitive_array(array), out); } DataType::Int16 => { - primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ); + primitive_array_to_flat_vector::(as_primitive_array(array), out); } DataType::Int32 => { - primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ); + primitive_array_to_flat_vector::(as_primitive_array(array), out); } DataType::Int64 => { - primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ); + primitive_array_to_flat_vector::(as_primitive_array(array), out); } DataType::Float32 => { - primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ); + primitive_array_to_flat_vector::(as_primitive_array(array), out); } DataType::Float64 => { - primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ); + primitive_array_to_flat_vector::(as_primitive_array(array), out); } DataType::Decimal128(width, _) => { - decimal_array_to_vector( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - *width, - ); + decimal_array_to_vector(as_primitive_array(array), out, *width); } // DuckDB Only supports timetamp_tz in microsecond precision @@ -374,28 +420,19 @@ fn primitive_array_to_vector(array: &dyn Array, out: &mut dyn Vector) -> Result< out, ), DataType::Timestamp(unit, None) => match unit { - TimeUnit::Second => primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ), - TimeUnit::Millisecond => primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ), - TimeUnit::Microsecond => primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ), - TimeUnit::Nanosecond => primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ), + TimeUnit::Second => primitive_array_to_flat_vector::(as_primitive_array(array), out), + TimeUnit::Millisecond => { + primitive_array_to_flat_vector::(as_primitive_array(array), out) + } + TimeUnit::Microsecond => { + primitive_array_to_flat_vector::(as_primitive_array(array), out) + } + TimeUnit::Nanosecond => { + primitive_array_to_flat_vector::(as_primitive_array(array), out) + } }, DataType::Date32 => { - primitive_array_to_flat_vector::( - as_primitive_array(array), - out.as_mut_any().downcast_mut().unwrap(), - ); + primitive_array_to_flat_vector::(as_primitive_array(array), out); } DataType::Date64 => primitive_array_to_flat_vector_cast::(Date32Type::DATA_TYPE, array, out), DataType::Time32(_) => { @@ -478,106 +515,12 @@ fn binary_array_to_vector(array: &BinaryArray, out: &mut FlatVector) { } } -fn list_array_to_vector>( - array: &GenericListArray, - out: &mut ListVector, -) -> Result<(), Box> { - let value_array = array.values(); - let mut child = out.child(value_array.len()); - match value_array.data_type() { - dt if dt.is_primitive() => { - primitive_array_to_vector(value_array.as_ref(), &mut child)?; - } - DataType::Utf8 => { - string_array_to_vector(as_string_array(value_array.as_ref()), &mut child); - } - DataType::Binary => { - binary_array_to_vector(as_generic_binary_array(value_array.as_ref()), &mut child); - } - _ => { - return Err("Nested list is not supported yet.".into()); - } - } - - for i in 0..array.len() { - let offset = array.value_offsets()[i]; - let length = array.value_length(i); - out.set_entry(i, offset.as_(), length.as_()); - } - Ok(()) -} - -fn fixed_size_list_array_to_vector( - array: &FixedSizeListArray, - out: &mut ArrayVector, -) -> Result<(), Box> { - let value_array = array.values(); - let mut child = out.child(value_array.len()); - match value_array.data_type() { - dt if dt.is_primitive() => { - primitive_array_to_vector(value_array.as_ref(), &mut child)?; - } - DataType::Utf8 => { - string_array_to_vector(as_string_array(value_array.as_ref()), &mut child); - } - DataType::Binary => { - binary_array_to_vector(as_generic_binary_array(value_array.as_ref()), &mut child); - } - _ => { - return Err("Nested array is not supported yet.".into()); - } - } - - Ok(()) -} - /// Force downcast of an [`Array`], such as an [`ArrayRef`], to /// [`FixedSizeListArray`], panic'ing on failure. fn as_fixed_size_list_array(arr: &dyn Array) -> &FixedSizeListArray { arr.as_any().downcast_ref::().unwrap() } -fn struct_array_to_vector(array: &StructArray, out: &mut StructVector) -> Result<(), Box> { - for i in 0..array.num_columns() { - let column = array.column(i); - match column.data_type() { - dt if dt.is_primitive() || matches!(dt, DataType::Boolean) => { - primitive_array_to_vector(column, &mut out.child(i))?; - } - DataType::Utf8 => { - string_array_to_vector(as_string_array(column.as_ref()), &mut out.child(i)); - } - DataType::Binary => { - binary_array_to_vector(as_generic_binary_array(column.as_ref()), &mut out.child(i)); - } - DataType::List(_) => { - list_array_to_vector(as_list_array(column.as_ref()), &mut out.list_vector_child(i))?; - } - DataType::LargeList(_) => { - list_array_to_vector(as_large_list_array(column.as_ref()), &mut out.list_vector_child(i))?; - } - DataType::FixedSizeList(_, _) => { - fixed_size_list_array_to_vector( - as_fixed_size_list_array(column.as_ref()), - &mut out.array_vector_child(i), - )?; - } - DataType::Struct(_) => { - let struct_array = as_struct_array(column.as_ref()); - let mut struct_vector = out.struct_vector_child(i); - struct_array_to_vector(struct_array, &mut struct_vector)?; - } - _ => { - unimplemented!( - "Unsupported data type: {}, please file an issue https://github.com/wangfenjin/duckdb-rs", - column.data_type() - ); - } - } - } - Ok(()) -} - /// Pass RecordBatch to duckdb. /// /// # Safety @@ -613,20 +556,23 @@ pub fn arrow_ffi_to_query_params(array: FFI_ArrowArray, schema: FFI_ArrowSchema) #[cfg(test)] mod test { - use super::{arrow_recordbatch_to_query_params, ArrowVTab}; - use crate::{Connection, Result}; + use std::{error::Error, sync::Arc}; + use arrow::{ array::{ Array, ArrayRef, AsArray, BinaryArray, Date32Array, Date64Array, Decimal128Array, Decimal256Array, FixedSizeListArray, GenericByteArray, GenericListArray, Int32Array, LargeStringArray, ListArray, - OffsetSizeTrait, PrimitiveArray, StringArray, StructArray, Time32SecondArray, Time64MicrosecondArray, + OffsetSizeTrait, PrimitiveArray, StringArray, Time32SecondArray, Time64MicrosecondArray, TimestampMicrosecondArray, TimestampMillisecondArray, TimestampNanosecondArray, TimestampSecondArray, }, buffer::{OffsetBuffer, ScalarBuffer}, - datatypes::{i256, ArrowPrimitiveType, ByteArrayType, DataType, Field, Fields, Schema}, + datatypes::{i256, ArrowPrimitiveType, ByteArrayType, DataType, Field, Schema}, record_batch::RecordBatch, }; - use std::{error::Error, sync::Arc}; + use arrow_convert::{ArrowDeserialize, ArrowField, ArrowSerialize}; + + use super::{arrow_recordbatch_to_query_params, ArrowVTab}; + use crate::{Connection, Result}; #[test] fn test_vtab_arrow() -> Result<(), Box> { @@ -673,35 +619,31 @@ mod test { fn test_append_struct() -> Result<(), Box> { let db = Connection::open_in_memory()?; db.execute_batch("CREATE TABLE t1 (s STRUCT(v VARCHAR, i INTEGER))")?; + + #[derive(ArrowField, ArrowSerialize)] + struct T1Row { + v: String, + i: i32, + } + { - let struct_array = StructArray::from(vec![ - ( - Arc::new(Field::new("v", DataType::Utf8, true)), - Arc::new(StringArray::from(vec![Some("foo"), Some("bar")])) as ArrayRef, - ), - ( - Arc::new(Field::new("i", DataType::Int32, true)), - Arc::new(Int32Array::from(vec![Some(1), Some(2)])) as ArrayRef, - ), - ]); - - let schema = Schema::new(vec![Field::new( - "s", - DataType::Struct(Fields::from(vec![ - Field::new("v", DataType::Utf8, true), - Field::new("i", DataType::Int32, true), - ])), - true, - )]); - - let record_batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(struct_array)])?; + let rows = vec![ + T1Row { + v: "foo".to_string(), + i: 1, + }, + T1Row { + v: "bar".to_string(), + i: 2, + }, + ]; let mut app = db.appender("t1")?; - app.append_record_batch(record_batch)?; + app.append_rows_arrow(&rows, false)?; } + let mut stmt = db.prepare("SELECT s FROM t1")?; let rbs: Vec = stmt.query_arrow([])?.collect(); assert_eq!(rbs.iter().map(|op| op.num_rows()).sum::(), 2); - Ok(()) } @@ -894,6 +836,90 @@ mod test { Ok(()) } + #[test] + fn test_append_union() -> Result<(), Box> { + let db = Connection::open_in_memory()?; + + db.execute_batch("CREATE TABLE tbl1 (u UNION(num INT, str VARCHAR));")?; + + #[derive(PartialEq, Eq, Debug, ArrowField, ArrowSerialize, ArrowDeserialize)] + #[arrow_field(type = "sparse")] + enum TestUnion { + Num(i32), + Str(String), + } + + { + let rows = vec![TestUnion::Num(1), TestUnion::Str("foo".into()), TestUnion::Num(34)]; + + let mut app = db.appender("tbl1")?; + app.append_rows_arrow(&rows, false)?; + } + let mut stmt = db.prepare("SELECT u FROM tbl1")?; + let records: Vec = stmt.query_arrow_deserialized([])?; + assert_eq!( + records, + vec![TestUnion::Num(1), TestUnion::Str("foo".into()), TestUnion::Num(34)] + ); + Ok(()) + } + + #[test] + fn test_nested_struct_in_fixed_list() -> Result<(), Box> { + let db = Connection::open_in_memory()?; + + db.execute_batch( + r#" + CREATE TYPE POINT AS STRUCT(x INT, y INT); + CREATE TABLE tbl1 (line POINT[2]); + INSERT INTO tbl1 VALUES (ARRAY[{x: 1, y: 2}, {x: 3, y: 4}]); + "#, + )?; + + let mut stmt = db.prepare("SELECT line FROM tbl1")?; + let rbs: Vec = stmt.query_arrow([])?.collect(); + assert_eq!(rbs.iter().map(|op| op.num_rows()).sum::(), 1); + Ok(()) + } + + #[test] + fn test_list_column() -> Result<(), Box> { + let db = Connection::open_in_memory()?; + + db.execute_batch("CREATE TABLE tbl1 (scalar Int[])")?; + + { + let mut appender = db.appender("tbl1")?; + appender + .append_rows_arrow(&vec![vec![4, 5], vec![6, 7], vec![8, 9, 10]], false) + .unwrap(); + } + + let mut stmt = db.prepare("SELECT * FROM tbl1")?; + let rbs: Vec = stmt.query_arrow([])?.collect(); + assert_eq!(rbs.iter().map(|op| op.num_rows()).sum::(), 3); + Ok(()) + } + + #[test] + fn test_array_column() -> Result<(), Box> { + let db = Connection::open_in_memory()?; + + db.execute_batch("CREATE TABLE tbl1 (scalar CHAR[2])")?; + + { + let mut appender = db.appender("tbl1")?; + appender + .append_rows_arrow(&vec![[4, 5], [6, 7], [8, 9]], false) + .unwrap(); + } + + let mut stmt = db.prepare("SELECT * FROM tbl1")?; + let rbs: Vec = stmt.query_arrow([])?.collect(); + assert_eq!(rbs.iter().map(|op| op.num_rows()).sum::(), 3); + Ok(()) + } + #[test] fn test_primitive_roundtrip_contains_nulls() -> Result<(), Box> { let mut builder = arrow::array::PrimitiveBuilder::::new(); diff --git a/crates/duckdb/src/vtab/excel.rs b/crates/duckdb/src/vtab/excel.rs index 883aa7a9..3a514ed3 100644 --- a/crates/duckdb/src/vtab/excel.rs +++ b/crates/duckdb/src/vtab/excel.rs @@ -1,5 +1,5 @@ -use super::{BindInfo, DataChunk, Free, FunctionInfo, InitInfo, LogicalType, LogicalTypeId, VTab}; -use crate::core::Inserter; +use super::{BindInfo, Free, FunctionInfo, InitInfo, LogicalTypeHandle, VTab}; +use crate::core::{DataChunk, DataChunkHandle, Inserter, LogicalTypeId}; use calamine::{open_workbook_auto, DataType, Range, Reader}; #[repr(C)] @@ -33,7 +33,7 @@ impl VTab for ExcelVTab { type BindData = ExcelBindData; type InitData = ExcelInitData; - unsafe fn bind(bind: &BindInfo, data: *mut ExcelBindData) -> Result<(), Box> { + fn bind(bind: &BindInfo, data: *mut ExcelBindData) -> Result<(), Box> { let param_count = bind.get_parameter_count(); assert!(param_count == 2); let path = bind.get_parameter(0).to_string(); @@ -74,7 +74,7 @@ impl VTab for ExcelVTab { header[idx] .get_string() .unwrap_or_else(|| panic!("idx {} header empty?", idx)), - LogicalType::new(LogicalTypeId::Varchar), + LogicalTypeHandle::from(LogicalTypeId::Varchar), ); } DataType::Float(_) => { @@ -82,7 +82,7 @@ impl VTab for ExcelVTab { header[idx] .get_string() .unwrap_or_else(|| panic!("idx {} header empty?", idx)), - LogicalType::new(LogicalTypeId::Double), + LogicalTypeHandle::from(LogicalTypeId::Double), ); } DataType::Int(_) => { @@ -90,7 +90,7 @@ impl VTab for ExcelVTab { header[idx] .get_string() .unwrap_or_else(|| panic!("idx {} header empty?", idx)), - LogicalType::new(LogicalTypeId::Bigint), + LogicalTypeHandle::from(LogicalTypeId::Bigint), ); } DataType::Bool(_) => { @@ -98,7 +98,7 @@ impl VTab for ExcelVTab { header[idx] .get_string() .unwrap_or_else(|| panic!("idx {} header empty?", idx)), - LogicalType::new(LogicalTypeId::Boolean), + LogicalTypeHandle::from(LogicalTypeId::Boolean), ); } DataType::DateTime(_) => { @@ -106,7 +106,7 @@ impl VTab for ExcelVTab { header[idx] .get_string() .unwrap_or_else(|| panic!("idx {} header empty?", idx)), - LogicalType::new(LogicalTypeId::Date), + LogicalTypeHandle::from(LogicalTypeId::Date), ); } _ => { @@ -125,27 +125,30 @@ impl VTab for ExcelVTab { Ok(()) } - unsafe fn init(_: &InitInfo, data: *mut ExcelInitData) -> Result<(), Box> { + fn init(_: &InitInfo, data: *mut ExcelInitData) -> Result<(), Box> { unsafe { (*data).start = 1; } Ok(()) } - unsafe fn func(func: &FunctionInfo, output: &mut DataChunk) -> Result<(), Box> { + fn func(func: &FunctionInfo, output: DataChunkHandle) -> Result<(), Box> { let init_info = func.get_init_data::(); let bind_info = func.get_bind_data::(); + let mut data_chunk = DataChunk::from_existing_handle(output); + unsafe { if (*init_info).start >= (*bind_info).height { - output.set_len(0); + data_chunk.set_len(0); } else { let range = (*bind_info).range.as_ref().expect("range is null"); let height = std::cmp::min( - output.flat_vector(0).capacity(), + DataChunkHandle::max_capacity(), (*bind_info).height - (*init_info).start, ); for i in 0..(*bind_info).width { - let mut vector = output.flat_vector(i); + data_chunk.reserve(height); + let vector = data_chunk.columns[i].try_as_flat_mut().unwrap(); for j in 0..height { let cell = range.get(((*init_info).start + j, i)); if cell.is_none() { @@ -174,16 +177,16 @@ impl VTab for ExcelVTab { } } (*init_info).start += height; - output.set_len(height); + data_chunk.set_len(height); } } Ok(()) } - fn parameters() -> Option> { + fn parameters() -> Option> { Some(vec![ - LogicalType::new(LogicalTypeId::Varchar), // file path - LogicalType::new(LogicalTypeId::Varchar), // sheet name + LogicalTypeHandle::from(LogicalTypeId::Varchar), // file path + LogicalTypeHandle::from(LogicalTypeId::Varchar), // sheet name ]) } } @@ -203,7 +206,8 @@ mod test { .prepare("select count(*) from excel('./examples/Movies_Social_metadata.xlsx', 'Data')")? .query_row::([], |row| row.get(0))?; assert_eq!(3039, val); - let mut stmt = db.prepare("select genres, sum(movie_facebook_likes) from excel('./examples/Movies_Social_metadata.xlsx', 'Data') group by genres order by genres limit 4")?; + let mut stmt = + db.prepare("select genres, sum(movie_facebook_likes) from excel('./examples/Movies_Social_metadata.xlsx', 'Data') group by genres order by genres limit 4")?; // +-------------+---------------------------+ // | genres | sum(movie_facebook_likes) | // +-------------+---------------------------+ diff --git a/crates/duckdb/src/vtab/function.rs b/crates/duckdb/src/vtab/function.rs index a1ce75aa..9d14b510 100644 --- a/crates/duckdb/src/vtab/function.rs +++ b/crates/duckdb/src/vtab/function.rs @@ -9,7 +9,7 @@ use super::{ duckdb_table_function_set_init, duckdb_table_function_set_local_init, duckdb_table_function_set_name, duckdb_table_function_supports_projection_pushdown, idx_t, }, - LogicalType, Value, + LogicalTypeHandle, Value, }; use std::{ ffi::{c_void, CString}, @@ -28,7 +28,7 @@ impl BindInfo { /// # Arguments /// * `name`: The name of the column /// * `type`: The logical type of the column - pub fn add_result_column(&self, column_name: &str, column_type: LogicalType) { + pub fn add_result_column(&self, column_name: &str, column_type: LogicalTypeHandle) { let c_str = CString::new(column_name).unwrap(); unsafe { duckdb_bind_add_result_column(self.ptr, c_str.as_ptr() as *const c_char, column_type.ptr); @@ -226,7 +226,7 @@ impl TableFunction { /// /// # Arguments /// * `logical_type`: The type of the parameter to add. - pub fn add_parameter(&self, logical_type: &LogicalType) -> &Self { + pub fn add_parameter(&self, logical_type: &LogicalTypeHandle) -> &Self { unsafe { duckdb_table_function_add_parameter(self.ptr, logical_type.ptr); } @@ -238,7 +238,7 @@ impl TableFunction { /// # Arguments /// * `name`: The name of the parameter to add. /// * `logical_type`: The type of the parameter to add. - pub fn add_named_parameter(&self, name: &str, logical_type: &LogicalType) -> &Self { + pub fn add_named_parameter(&self, name: &str, logical_type: &LogicalTypeHandle) -> &Self { unsafe { let string = CString::new(name).unwrap(); duckdb_table_function_add_named_parameter(self.ptr, string.as_ptr(), logical_type.ptr); diff --git a/crates/duckdb/src/vtab/mod.rs b/crates/duckdb/src/vtab/mod.rs index d1e843c2..dd4c69cb 100644 --- a/crates/duckdb/src/vtab/mod.rs +++ b/crates/duckdb/src/vtab/mod.rs @@ -1,7 +1,12 @@ -use crate::{error::Error, inner_connection::InnerConnection, Connection, Result}; +use std::ffi::c_void; use super::{ffi, ffi::duckdb_free}; -use std::ffi::c_void; +use crate::{ + core::{DataChunkHandle, LogicalTypeHandle}, + error::Error, + inner_connection::InnerConnection, + Connection, Result, +}; mod function; mod value; @@ -17,15 +22,12 @@ pub use self::arrow::{ #[cfg(feature = "vtab-excel")] mod excel; +use std::mem::size_of; + +use ffi::{duckdb_bind_info, duckdb_data_chunk, duckdb_function_info, duckdb_init_info, duckdb_malloc}; pub use function::{BindInfo, FunctionInfo, InitInfo, TableFunction}; pub use value::Value; -use crate::core::{DataChunk, LogicalType, LogicalTypeId}; -use ffi::{duckdb_bind_info, duckdb_data_chunk, duckdb_function_info, duckdb_init_info}; - -use ffi::duckdb_malloc; -use std::mem::size_of; - /// duckdb_malloc a struct of type T /// used for the bind_info and init_info /// # Safety @@ -73,7 +75,7 @@ pub trait VTab: Sized { /// `bind` does not take ownership of `data`. /// - Concurrent access to `data` (if applicable) must be properly synchronized. /// - The `bind` object must be valid and correctly initialized. - unsafe fn bind(bind: &BindInfo, data: *mut Self::BindData) -> Result<(), Box>; + fn bind(bind: &BindInfo, data: *mut Self::BindData) -> Result<(), Box>; /// Initialize the table function /// /// # Safety @@ -85,7 +87,7 @@ pub trait VTab: Sized { /// - There is no data race when accessing `data`, meaning if `data` is accessed from multiple threads, /// proper synchronization is required. /// - The lifetime of `data` extends beyond the scope of this call to avoid use-after-free errors. - unsafe fn init(init: &InitInfo, data: *mut Self::InitData) -> Result<(), Box>; + fn init(init: &InitInfo, data: *mut Self::InitData) -> Result<(), Box>; /// The actual function /// /// # Safety @@ -100,7 +102,7 @@ pub trait VTab: Sized { /// - The `init_info` and `bind_info` data pointed to remains valid and is not freed until after this function completes. /// - No other threads are concurrently mutating the data pointed to by `init_info` and `bind_info` without proper synchronization. /// - The `output` parameter is correctly initialized and can safely be written to. - unsafe fn func(func: &FunctionInfo, output: &mut DataChunk) -> Result<(), Box>; + fn func(func: &FunctionInfo, output: DataChunkHandle) -> Result<(), Box>; /// Does the table function support pushdown /// default is false fn supports_pushdown() -> bool { @@ -108,12 +110,12 @@ pub trait VTab: Sized { } /// The parameters of the table function /// default is None - fn parameters() -> Option> { + fn parameters() -> Option> { None } /// The named parameters of the table function /// default is None - fn named_parameters() -> Option> { + fn named_parameters() -> Option> { None } } @@ -123,8 +125,8 @@ where T: VTab, { let info = FunctionInfo::from(info); - let mut output = DataChunk::from(output); - let result = T::func(&info, &mut output); + let data_chunk_handle = DataChunkHandle::new_unowned(output); + let result = T::func(&info, data_chunk_handle); if result.is_err() { info.set_error(&result.err().unwrap().to_string()); } @@ -192,13 +194,14 @@ impl InnerConnection { #[cfg(test)] mod test { - use super::*; use crate::core::Inserter; use std::{ error::Error, ffi::{c_char, CString}, }; + use super::*; + #[repr(C)] struct HelloBindData { name: *mut c_char, @@ -228,8 +231,8 @@ mod test { type InitData = HelloInitData; type BindData = HelloBindData; - unsafe fn bind(bind: &BindInfo, data: *mut HelloBindData) -> Result<(), Box> { - bind.add_result_column("column0", LogicalType::new(LogicalTypeId::Varchar)); + fn bind(bind: &BindInfo, data: *mut HelloBindData) -> Result<(), Box> { + bind.add_result_column("column0", LogicalTypeHandle::from(LogicalTypeId::Varchar)); let param = bind.get_parameter(0).to_string(); unsafe { (*data).name = CString::new(param).unwrap().into_raw(); @@ -237,14 +240,14 @@ mod test { Ok(()) } - unsafe fn init(_: &InitInfo, data: *mut HelloInitData) -> Result<(), Box> { + fn init(_: &InitInfo, data: *mut HelloInitData) -> Result<(), Box> { unsafe { (*data).done = false; } Ok(()) } - unsafe fn func(func: &FunctionInfo, output: &mut DataChunk) -> Result<(), Box> { + fn func(func: &FunctionInfo, output: DataChunkHandle) -> Result<(), Box> { let init_info = func.get_init_data::(); let bind_info = func.get_bind_data::(); @@ -252,31 +255,34 @@ mod test { if (*init_info).done { output.set_len(0); } else { + let mut data_chunk = DataChunk::from_existing_handle(output); (*init_info).done = true; - let vector = output.flat_vector(0); + data_chunk.reserve(1); + let vector = data_chunk.columns[0].try_as_flat_mut().unwrap(); let name = CString::from_raw((*bind_info).name); let result = CString::new(format!("Hello {}", name.to_str()?))?; // Can't consume the CString (*bind_info).name = CString::into_raw(name); vector.insert(0, result); - output.set_len(1); + data_chunk.set_len(1); } } Ok(()) } - fn parameters() -> Option> { - Some(vec![LogicalType::new(LogicalTypeId::Varchar)]) + fn parameters() -> Option> { + Some(vec![LogicalTypeHandle::from(LogicalTypeId::Varchar)]) } } struct HelloWithNamedVTab {} + impl VTab for HelloWithNamedVTab { type InitData = HelloInitData; type BindData = HelloBindData; - unsafe fn bind(bind: &BindInfo, data: *mut HelloBindData) -> Result<(), Box> { - bind.add_result_column("column0", LogicalType::new(LogicalTypeId::Varchar)); + fn bind(bind: &BindInfo, data: *mut HelloBindData) -> Result<(), Box> { + bind.add_result_column("column0", LogicalTypeHandle::from(LogicalTypeId::Varchar)); let param = bind.get_named_parameter("name").unwrap().to_string(); assert!(bind.get_named_parameter("unknown_name").is_none()); unsafe { @@ -285,16 +291,19 @@ mod test { Ok(()) } - unsafe fn init(init_info: &InitInfo, data: *mut HelloInitData) -> Result<(), Box> { + fn init(init_info: &InitInfo, data: *mut HelloInitData) -> Result<(), Box> { HelloVTab::init(init_info, data) } - unsafe fn func(func: &FunctionInfo, output: &mut DataChunk) -> Result<(), Box> { + fn func(func: &FunctionInfo, output: DataChunkHandle) -> Result<(), Box> { HelloVTab::func(func, output) } - fn named_parameters() -> Option> { - Some(vec![("name".to_string(), LogicalType::new(LogicalTypeId::Varchar))]) + fn named_parameters() -> Option> { + Some(vec![( + "name".to_string(), + LogicalTypeHandle::from(LogicalTypeId::Varchar), + )]) } } @@ -325,6 +334,8 @@ mod test { #[cfg(feature = "vtab-loadable")] use duckdb_loadable_macros::duckdb_entrypoint; + use crate::core::{DataChunk, LogicalTypeId}; + // this function is never called, but is still type checked // Exposes a extern C function named "libhello_ext_init" in the compiled dynamic library, // the "entrypoint" that duckdb will use to load the extension. diff --git a/crates/libduckdb-sys/src/lib.rs b/crates/libduckdb-sys/src/lib.rs index ae57cadf..5f683bd7 100644 --- a/crates/libduckdb-sys/src/lib.rs +++ b/crates/libduckdb-sys/src/lib.rs @@ -11,6 +11,9 @@ mod bindings { #[allow(clippy::all)] pub use bindings::*; +mod string; +pub use string::*; + pub const DuckDBError: duckdb_state = duckdb_state_DuckDBError; pub const DuckDBSuccess: duckdb_state = duckdb_state_DuckDBSuccess; diff --git a/crates/libduckdb-sys/src/string.rs b/crates/libduckdb-sys/src/string.rs new file mode 100644 index 00000000..d2f3515c --- /dev/null +++ b/crates/libduckdb-sys/src/string.rs @@ -0,0 +1,60 @@ +use std::{ + ffi::{c_char, CStr}, + ops::Deref, +}; + +use crate::duckdb_free; + +pub struct DuckDbString { + // Invariant: ptr[0..len+1] is valid C string, i.e. ptr[len] is NUL byte. + ptr: core::ptr::NonNull, + len: usize, +} + +impl DuckDbString { + /// Creates a `DuckDbString` from a raw pointer to a C string. + /// + /// # Safety + /// + /// The caller must ensure that the pointer is valid and points to a null-terminated C string. + /// The memory must remain valid for the lifetime of the returned `DuckDbString`. + pub unsafe fn from_ptr(ptr: *const c_char) -> Self { + let len = unsafe { CStr::from_ptr(ptr) }.to_bytes().len(); + unsafe { Self::from_raw_parts(ptr, len) } + } + + /// Creates a `DuckDbString` from raw parts. + /// + /// # Safety + /// + /// The caller must ensure that: + /// - `ptr` is a valid pointer to a null-terminated C string. + /// - `len` accurately represents the length of the string (excluding the null terminator). + /// - The memory referenced by `ptr` remains valid for the lifetime of the returned `DuckDbString`. + /// - The string data is not mutated for the lifetime of the returned `DuckDbString`. + pub unsafe fn from_raw_parts(ptr: *const c_char, len: usize) -> Self { + let ptr = unsafe { core::ptr::NonNull::new_unchecked(ptr as *mut c_char) }; + Self { ptr, len } + } + + fn to_bytes_with_nul(&self) -> &[u8] { + let ptr = self.ptr.as_ptr() as *const u8; + unsafe { core::slice::from_raw_parts(ptr, self.len + 1) } + } +} + +impl Deref for DuckDbString { + type Target = std::ffi::CStr; + + fn deref(&self) -> &Self::Target { + let bytes = self.to_bytes_with_nul(); + unsafe { CStr::from_bytes_with_nul_unchecked(bytes) } + } +} + +impl Drop for DuckDbString { + fn drop(&mut self) { + let ptr = self.ptr.as_ptr() as *mut core::ffi::c_void; + unsafe { duckdb_free(ptr) }; + } +}