Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(common): unify implementation of hash key #9671

Merged
merged 16 commits into from
May 11, 2023
Merged

refactor(common): unify implementation of hash key #9671

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
00940b7
initial commit
BugenZhao May 8, 2023
357d8d9
complete implementation
BugenZhao May 9, 2023
86edfb7
cleanup dead codes
BugenZhao May 9, 2023
1bf4f9c
extract key storage
BugenZhao May 9, 2023
df7f3c3
optionally dispatch by hash key size
BugenZhao May 9, 2023
3c04f3b
minor rename
BugenZhao May 9, 2023
81575e8
write serde all in the trait
BugenZhao May 9, 2023
38c5c9e
minor optimization
BugenZhao May 9, 2023
ecea436
refine
BugenZhao May 9, 2023
ade83c6
avoid matching
BugenZhao May 10, 2023
91a74d2
refactor serde impl
BugenZhao May 10, 2023
6acba19
add doc and fix clippy
BugenZhao May 10, 2023
0b83465
add more doc
BugenZhao May 10, 2023
76bbde5
refine implementation
BugenZhao May 10, 2023
4de4dff
fix decimal precision
BugenZhao May 11, 2023
15fdcac
fix more tests
BugenZhao May 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

80 changes: 40 additions & 40 deletions e2e_test/batch/duckdb/join/iejoin/test_iejoin.test.slt.part
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,46 +140,46 @@ WITH data_table AS (
GROUP BY bucket, low, high
ORDER BY bucket;
----
0 1577836800.000000 1578627360.000000 10
1 1578627360.000000 1579417920.000000 9
2 1579417920.000000 1580208480.000000 9
3 1580208480.000000 1580999040.000000 9
4 1580999040.000000 1581789600.000000 9
5 1581789600.000000 1582580160.000000 9
6 1582580160.000000 1583370720.000000 10
7 1583370720.000000 1584161280.000000 9
8 1584161280.000000 1584951840.000000 9
9 1584951840.000000 1585742400.000000 9
10 1585742400.000000 1586532960.000000 9
11 1586532960.000000 1587323520.000000 9
12 1587323520.000000 1588114080.000000 9
13 1588114080.000000 1588904640.000000 10
14 1588904640.000000 1589695200.000000 9
15 1589695200.000000 1590485760.000000 9
16 1590485760.000000 1591276320.000000 9
17 1591276320.000000 1592066880.000000 9
18 1592066880.000000 1592857440.000000 9
19 1592857440.000000 1593648000.000000 9
20 1593648000.000000 1594438560.000000 10
21 1594438560.000000 1595229120.000000 9
22 1595229120.000000 1596019680.000000 9
23 1596019680.000000 1596810240.000000 9
24 1596810240.000000 1597600800.000000 9
25 1597600800.000000 1598391360.000000 9
26 1598391360.000000 1599181920.000000 10
27 1599181920.000000 1599972480.000000 9
28 1599972480.000000 1600763040.000000 9
29 1600763040.000000 1601553600.000000 9
30 1601553600.000000 1602344160.000000 9
31 1602344160.000000 1603134720.000000 9
32 1603134720.000000 1603925280.000000 9
33 1603925280.000000 1604715840.000000 10
34 1604715840.000000 1605506400.000000 9
35 1605506400.000000 1606296960.000000 9
36 1606296960.000000 1607087520.000000 9
37 1607087520.000000 1607878080.000000 9
38 1607878080.000000 1608668640.000000 9
39 1608668640.000000 1609459200.000000 9
0 1577836800 1578627360 10
1 1578627360 1579417920 9
2 1579417920 1580208480 9
3 1580208480 1580999040 9
4 1580999040 1581789600 9
5 1581789600 1582580160 9
6 1582580160 1583370720 10
7 1583370720 1584161280 9
8 1584161280 1584951840 9
9 1584951840 1585742400 9
10 1585742400 1586532960 9
11 1586532960 1587323520 9
12 1587323520 1588114080 9
13 1588114080 1588904640 10
14 1588904640 1589695200 9
15 1589695200 1590485760 9
16 1590485760 1591276320 9
17 1591276320 1592066880 9
18 1592066880 1592857440 9
19 1592857440 1593648000 9
20 1593648000 1594438560 10
21 1594438560 1595229120 9
22 1595229120 1596019680 9
23 1596019680 1596810240 9
24 1596810240 1597600800 9
25 1597600800 1598391360 9
26 1598391360 1599181920 10
27 1599181920 1599972480 9
28 1599972480 1600763040 9
29 1600763040 1601553600 9
30 1601553600 1602344160 9
31 1602344160 1603134720 9
32 1603134720 1603925280 9
33 1603925280 1604715840 10
34 1604715840 1605506400 9
35 1605506400 1606296960 9
36 1606296960 1607087520 9
37 1607087520 1607878080 9
38 1607878080 1608668640 9
39 1608668640 1609459200 9


statement ok
Expand Down
14 changes: 7 additions & 7 deletions e2e_test/batch/tpch/q18.slt.part
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,11 +72,11 @@ Customer#000000070 70 4004 1993-05-07 220715.14 228
Customer#000000098 98 768 1996-08-20 220636.82 231
Customer#000000149 149 5606 1996-11-12 219959.08 231
Customer#000000055 55 484 1997-01-03 219920.62 224
Customer#000000140 140 4230 1992-03-04 219709.60 217
Customer#000000140 140 4230 1992-03-04 219709.6 217
Customer#000000082 82 39 1996-09-20 219707.84 231
Customer#000000037 37 2789 1998-03-14 219123.27 218
Customer#000000017 17 3269 1996-03-01 218697.85 220
Customer#000000149 149 3590 1995-05-13 218482.70 210
Customer#000000149 149 3590 1995-05-13 218482.7 210
Customer#000000134 134 614 1992-12-01 218116.21 204
Customer#000000092 92 4197 1996-08-13 217709.03 225
Customer#000000133 133 1156 1996-10-19 217682.81 218
Expand All @@ -97,9 +97,9 @@ Customer#000000052 52 5186 1996-08-03 208892.63 210
Customer#000000028 28 2050 1994-06-02 208517.98 217
Customer#000000076 76 2180 1996-09-14 208481.57 212
Customer#000000119 119 3588 1995-03-19 207925.83 212
Customer#000000134 134 1444 1994-12-06 207907.60 205
Customer#000000134 134 1444 1994-12-06 207907.6 205
Customer#000000103 103 742 1994-12-23 207632.55 198
Customer#000000017 17 4099 1992-08-21 207364.80 208
Customer#000000017 17 4099 1992-08-21 207364.8 208
Customer#000000109 109 1286 1993-05-14 207291.83 200
Customer#000000079 79 5633 1998-05-31 207119.83 203
Customer#000000062 62 2022 1992-03-15 206742.11 209
Expand All @@ -108,8 +108,8 @@ Customer#000000148 148 5185 1997-07-25 206179.68 198
Customer#000000044 44 3175 1994-07-15 205282.63 215
Customer#000000056 56 2565 1998-02-28 204438.57 201
Customer#000000149 149 3747 1996-08-20 204355.65 195
Customer#000000101 101 4964 1997-07-28 204163.10 197
Customer#000000062 62 4992 1992-05-10 203904.80 198
Customer#000000101 101 4964 1997-07-28 204163.1 197
Customer#000000062 62 4992 1992-05-10 203904.8 198
Customer#000000010 10 3751 1994-04-27 202917.72 204
Customer#000000076 76 2534 1996-07-17 202784.54 214
Customer#000000001 1 164 1992-10-21 202660.52 213
Expand All @@ -124,7 +124,7 @@ Customer#000000079 79 4672 1995-11-07 199593.71 203
Customer#000000131 131 930 1994-12-17 199102.23 204
Customer#000000118 118 4161 1993-08-21 198995.21 211
Customer#000000073 73 4069 1992-05-13 198816.13 199
Customer#000000142 142 5696 1995-05-04 198723.30 198
Customer#000000142 142 5696 1995-05-04 198723.3 198
Customer#000000134 134 3872 1996-09-06 198538.68 207
Customer#000000127 127 1059 1994-02-27 198360.22 194
Customer#000000103 103 4293 1996-08-20 198322.91 202
Expand Down
6 changes: 3 additions & 3 deletions e2e_test/streaming/tpch/q10.slt.part
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,13 @@ select * from tpch_q10;
130 Customer#000000130 31499.4100 5073.58 INDONESIA RKPx2OfZy0Vn 8wGWZ7F2EAvmMORl1k8iH 19-190-993-9281 ix slowly. express packages along the furiously ironic requests integrate daringly deposits. fur
136 Customer#000000136 359159.4900 -842.39 GERMANY QoLsJ0v5C1IQbh,DS1 17-501-210-4726 ackages sleep ironic, final courts. even requests above the blithely bold requests g
139 Customer#000000139 102799.6791 7897.78 INDONESIA 3ElvBwudHKL02732YexGVFVt 19-140-352-1403 nstructions. quickly ironic ideas are carefully. bold,
14 Customer#000000014 63310.3940 5266.30 ARGENTINA KXkletMlL2JQEA 11-845-129-3851 , ironic packages across the unus
14 Customer#000000014 63310.3940 5266.3 ARGENTINA KXkletMlL2JQEA 11-845-129-3851 , ironic packages across the unus
142 Customer#000000142 129732.1688 2209.81 INDONESIA AnJ5lxtLjioClr2khl9pb8NLxG2, 19-407-425-2584 . even, express theodolites upo
19 Customer#000000019 35730.0870 8914.71 CHINA uc,3bHIx84H,wdrmLOjVsiqXCq2tr 28-396-526-5053 nag. furiously careful packages are slyly at the accounts. furiously regular in
23 Customer#000000023 46655.3250 3332.02 CANADA OdY W13N7Be3OC5MpgfmcYss0Wn6TKT 13-312-472-8245 deposits. special deposits cajole slyly. fluffily special deposits about the furiously
25 Customer#000000025 80561.6934 7133.70 JAPAN Hp8GyFQgGHFYSilH5tBfe 22-603-468-3533 y. accounts sleep ruthlessly according to the regular theodolites. unusual instructions sleep. ironic, final
25 Customer#000000025 80561.6934 7133.7 JAPAN Hp8GyFQgGHFYSilH5tBfe 22-603-468-3533 y. accounts sleep ruthlessly according to the regular theodolites. unusual instructions sleep. ironic, final
4 Customer#000000004 68199.2151 2866.83 EGYPT XxVSJsLAGtn 14-128-190-5944 requests. final, regular ideas sleep final accou
40 Customer#000000040 44161.9506 1335.30 CANADA gOnGWAyhSV1ofv 13-652-915-8939 rges impress after the slyly ironic courts. foxes are. blithely
40 Customer#000000040 44161.9506 1335.3 CANADA gOnGWAyhSV1ofv 13-652-915-8939 rges impress after the slyly ironic courts. foxes are. blithely
46 Customer#000000046 46956.6000 5744.59 FRANCE eaTXWWm10L9 16-357-681-2007 ctions. accounts sleep furiously even requests. regular, regular accounts cajole blithely around the final pa
55 Customer#000000055 54376.8534 4572.11 IRAN zIRBR4KNEl HzaiV3a i9n6elrxzDEh8r8pDom 20-180-440-8525 ully unusual packages wake bravely bold packages. unusual requests boost deposits! blithely ironic packages ab
61 Customer#000000061 45467.8872 1536.24 PERU 9kndve4EAJxhg3veF BfXr7AqOsT39o gtqjaYE 27-626-559-8599 egular packages shall have to impress along the
Expand Down
14 changes: 7 additions & 7 deletions e2e_test/streaming/tpch/q18.slt.part
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ Customer#000000010 10 3751 1994-04-27 202917.72 204
Customer#000000010 10 4421 1997-04-04 258779.02 255
Customer#000000013 13 2438 1993-07-15 214494.39 210
Customer#000000017 17 3269 1996-03-01 218697.85 220
Customer#000000017 17 4099 1992-08-21 207364.80 208
Customer#000000017 17 4099 1992-08-21 207364.8 208
Customer#000000022 22 4583 1994-09-25 206495.43 197
Customer#000000028 28 2050 1994-06-02 208517.98 217
Customer#000000028 28 2306 1995-07-26 244704.23 235
Expand All @@ -34,7 +34,7 @@ Customer#000000052 52 5765 1994-12-15 249900.42 247
Customer#000000055 55 484 1997-01-03 219920.62 224
Customer#000000056 56 2565 1998-02-28 204438.57 201
Customer#000000062 62 2022 1992-03-15 206742.11 209
Customer#000000062 62 4992 1992-05-10 203904.80 198
Customer#000000062 62 4992 1992-05-10 203904.8 198
Customer#000000064 64 5895 1997-01-01 201419.83 200
Customer#000000067 67 3907 1992-08-19 240457.56 239
Customer#000000067 67 928 1995-03-02 228136.49 241
Expand Down Expand Up @@ -63,7 +63,7 @@ Customer#000000092 92 4197 1996-08-13 217709.03 225
Customer#000000094 94 2690 1996-03-31 224674.27 219
Customer#000000094 94 5413 1997-10-17 224382.57 212
Customer#000000098 98 768 1996-08-20 220636.82 231
Customer#000000101 101 4964 1997-07-28 204163.10 197
Customer#000000101 101 4964 1997-07-28 204163.1 197
Customer#000000103 103 4293 1996-08-20 198322.91 202
Customer#000000103 103 742 1994-12-23 207632.55 198
Customer#000000106 106 1761 1993-12-24 211925.95 218
Expand All @@ -84,20 +84,20 @@ Customer#000000131 131 930 1994-12-17 199102.23 204
Customer#000000133 133 1156 1996-10-19 217682.81 218
Customer#000000133 133 4613 1998-03-05 212339.55 214
Customer#000000133 133 710 1993-01-02 208974.42 196
Customer#000000134 134 1444 1994-12-06 207907.60 205
Customer#000000134 134 1444 1994-12-06 207907.6 205
Customer#000000134 134 3872 1996-09-06 198538.68 207
Customer#000000134 134 614 1992-12-01 218116.21 204
Customer#000000137 137 4900 1992-06-30 221320.76 227
Customer#000000140 140 4230 1992-03-04 219709.60 217
Customer#000000142 142 5696 1995-05-04 198723.30 198
Customer#000000140 140 4230 1992-03-04 219709.6 217
Customer#000000142 142 5696 1995-05-04 198723.3 198
Customer#000000142 142 5699 1992-07-30 226314.91 240
Customer#000000145 145 1575 1995-09-13 197031.52 204
Customer#000000145 145 518 1998-02-08 223537.09 214
Customer#000000145 145 807 1993-11-24 222392.53 216
Customer#000000146 146 4192 1998-04-19 197192.95 209
Customer#000000146 146 5925 1995-11-13 242588.87 242
Customer#000000148 148 5185 1997-07-25 206179.68 198
Customer#000000149 149 3590 1995-05-13 218482.70 210
Customer#000000149 149 3590 1995-05-13 218482.7 210
Customer#000000149 149 3619 1996-11-20 222274.54 221
Customer#000000149 149 3713 1998-05-07 215342.63 213
Customer#000000149 149 3747 1996-08-20 204355.65 195
Expand Down
1 change: 1 addition & 0 deletions src/common/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ strum = "0.24"
strum_macros = "0.24"
sysinfo = { version = "0.26", default-features = false }
thiserror = "1"
tinyvec = { version = "1", features = ["rustc_1_55", "grab_spare_slice"] }
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why need rustc_1_55 and grab_spare_slice? Maybe should leave some comments inline.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is because some interfaces of tinyvec are only available under these features. Should be documented by this crate.

tokio = { version = "0.2", package = "madsim-tokio", features = [
"rt",
"rt-multi-thread",
Expand Down
29 changes: 28 additions & 1 deletion src/common/src/array/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,12 @@ pub trait ArrayBuilder: Send + Sync + Sized + 'static {
self.append_n(1, value);
}

/// Append an owned value to builder.
fn append_owned(&mut self, value: Option<<Self::ArrayType as Array>::OwnedItem>) {
let value = value.as_ref().map(|s| s.as_scalar_ref());
self.append(value)
}

fn append_null(&mut self) {
self.append(None)
}
Expand Down Expand Up @@ -308,8 +314,9 @@ impl<A: Array> CompactableArray for A {
/// name, array type, builder type }` tuples. Refer to the following implementations as examples.
#[macro_export]
macro_rules! for_all_variants {
($macro:ident) => {
($macro:ident $(, $x:tt)*) => {
$macro! {
$($x, )*
{ Int16, int16, I16Array, I16ArrayBuilder },
{ Int32, int32, I32Array, I32ArrayBuilder },
{ Int64, int64, I64Array, I64ArrayBuilder },
Expand All @@ -332,6 +339,20 @@ macro_rules! for_all_variants {
};
}

macro_rules! do_dispatch {
($impl:expr, $type:ident, $inner:ident, $body:tt, $( { $variant_name:ident, $suffix_name:ident, $array:ty, $builder:ty } ),*) => {
match $impl {
$( $type::$variant_name($inner) => $body, )*
}
};
}

macro_rules! dispatch_all_variants {
($impl:expr, $type:ident, $scalar:ident, $body:tt) => {{
for_all_variants! { do_dispatch, $impl, $type, $scalar, $body }
}};
}

/// Define `ArrayImpl` with macro.
macro_rules! array_impl_enum {
( $( { $variant_name:ident, $suffix_name:ident, $array:ty, $builder:ty } ),*) => {
Expand Down Expand Up @@ -634,6 +655,12 @@ macro_rules! impl_array {
$( Self::$variant_name(inner) => ArrayBuilderImpl::$variant_name(inner.create_builder(capacity)), )*
}
}

pub fn data_type(&self) -> DataType {
match self {
$( Self::$variant_name(inner) => inner.data_type(), )*
}
}
}
}
}
Expand Down
Loading