chore: bump datafusion (#14)

Co-authored-by: Ruixiang Tan <tanruixiang0104@gmail.com>
CeresDB · Feb 4, 2024 · b9fb3ca · b9fb3ca
1 parent a905863
commit b9fb3ca
Show file tree

Hide file tree

Showing 30 changed files with 681 additions and 706 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,5 +1,6 @@
 [workspace]
 members = ["datafusion_util", "influxdb_influxql_parser", "iox_query_influxql", "observability_deps", "schema", "test_helpers"]
+resolver = "2"
 
 [workspace.package]
 version = "0.1.0"
@@ -8,6 +9,6 @@ edition = "2021"
 license = "MIT OR Apache-2.0"
 
 [workspace.dependencies]
-arrow = { version = "43.0.0" }
-datafusion = { git = "https://github.com/CeresDB/arrow-datafusion.git", rev = "9c3a537e25e5ab3299922864034f67fb2f79805d", default-features = false }
+arrow = { version = "49.0.0" }
+datafusion = { git = "https://github.com/CeresDB/arrow-datafusion.git", rev = "e21b03154", default-features = false }
 hashbrown = { version = "0.13.2" }
diff --git a/arrow_util/Cargo.toml b/arrow_util/Cargo.toml
@@ -8,8 +8,7 @@ license.workspace = true
 
 [dependencies]
 ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
-# need dyn_cmp_dict feature for comparing dictionary arrays
-arrow = { workspace = true, features = ["prettyprint", "dyn_cmp_dict"] }
+arrow = { workspace = true, features = ["prettyprint"] }
 # used by arrow anyway (needed for printing workaround)
 chrono = { version = "0.4", default-features = false }
 comfy-table = { version = "6.1", default-features = false }

diff --git a/arrow_util/src/display.rs b/arrow_util/src/display.rs
@@ -40,7 +40,7 @@ fn array_value_to_string(column: &ArrayRef, row: usize) -> Result<String> {
                 )
             })?;
             // treat as UTC
-            let ts = DateTime::<Utc>::from_utc(ts, Utc);
+            let ts = DateTime::<Utc>::from_naive_utc_and_offset(ts, Utc);
             // convert to string in preferred influx format
             let use_z = true;
             Ok(ts.to_rfc3339_opts(SecondsFormat::AutoSi, use_z))

diff --git a/arrow_util/src/test_util.rs b/arrow_util/src/test_util.rs
@@ -221,15 +221,15 @@ static REGEX_LINESEP: Lazy<Regex> = Lazy::new(|| Regex::new(r#"[+-]{6,}"#).expec
 ///
 ///   `        |`  -> `    |`
 ///   `         |` -> `    |`
-static REGEX_COL: Lazy<Regex> = Lazy::new(|| Regex::new(r#"\s+\|"#).expect("col regex"));
+static REGEX_COL: Lazy<Regex> = Lazy::new(|| Regex::new(r"\s+\|").expect("col regex"));
 
 /// Matches line like `metrics=[foo=1, bar=2]`
 static REGEX_METRICS: Lazy<Regex> =
-    Lazy::new(|| Regex::new(r#"metrics=\[([^\]]*)\]"#).expect("metrics regex"));
+    Lazy::new(|| Regex::new(r"metrics=\[([^\]]*)\]").expect("metrics regex"));
 
 /// Matches things like  `1s`, `1.2ms` and `10.2μs`
 static REGEX_TIMING: Lazy<Regex> =
-    Lazy::new(|| Regex::new(r#"[0-9]+(\.[0-9]+)?.s"#).expect("timing regex"));
+    Lazy::new(|| Regex::new(r"[0-9]+(\.[0-9]+)?.s").expect("timing regex"));
 
 /// Matches things like `FilterExec: .*` and `ParquetExec: .*`
 ///

diff --git a/datafusion_util/src/lib.rs b/datafusion_util/src/lib.rs
@@ -316,7 +316,7 @@ pub fn batch_filter(
         .evaluate(batch)
         .map(|v| v.into_array(batch.num_rows()))
         .and_then(|array| {
-            array
+            array?
                 .as_any()
                 .downcast_ref::<BooleanArray>()
                 .ok_or_else(|| {
@@ -397,7 +397,7 @@ mod tests {
         let ts_predicate_expr = make_range_expr(101, 202, "time");
         let expected_string =
             "TimestampNanosecond(101, None) <= time AND time < TimestampNanosecond(202, None)";
-        let actual_string = format!("{ts_predicate_expr:?}");
+        let actual_string = ts_predicate_expr.to_string();
 
         assert_eq!(actual_string, expected_string);
     }

diff --git a/influxdb_influxql_parser/src/string.rs b/influxdb_influxql_parser/src/string.rs
@@ -66,7 +66,7 @@ pub(crate) fn single_quoted_string(i: &str) -> ParseResult<&str, String> {
     let escaped = preceded(
         char('\\'),
         expect(
-            r#"invalid escape sequence, expected \\, \' or \n"#,
+            r"invalid escape sequence, expected \\, \' or \n",
             alt((char('\\'), char('\''), value('\n', char('n')))),
         ),
     );
@@ -277,10 +277,10 @@ mod test {
         );
 
         // escaped characters
-        let (_, got) = single_quoted_string(r#"'\n\''"#).unwrap();
+        let (_, got) = single_quoted_string(r"'\n\''").unwrap();
         assert_eq!(got, "\n'");
 
-        let (_, got) = single_quoted_string(r#"'\'hello\''"#).unwrap();
+        let (_, got) = single_quoted_string(r"'\'hello\''").unwrap();
         assert_eq!(got, "'hello'");
 
         // literal tab
@@ -307,8 +307,8 @@ mod test {
 
         // Invalid escape
         assert_expect_error!(
-            single_quoted_string(r#"'quick\idraw'"#),
-            r#"invalid escape sequence, expected \\, \' or \n"#
+            single_quoted_string(r"'quick\idraw'"),
+            r"invalid escape sequence, expected \\, \' or \n"
         );
     }
 
@@ -318,15 +318,15 @@ mod test {
         assert_eq!(got, "hello".into());
 
         // handle escaped delimiters "\/"
-        let (_, got) = regex(r#"/\/this\/is\/a\/path/"#).unwrap();
+        let (_, got) = regex(r"/\/this\/is\/a\/path/").unwrap();
         assert_eq!(got, "/this/is/a/path".into());
 
         // ignores any other possible escape sequence
-        let (_, got) = regex(r#"/hello\n/"#).unwrap();
+        let (_, got) = regex(r"/hello\n/").unwrap();
         assert_eq!(got, "hello\\n".into());
 
         // can parse possible escape sequence at beginning of regex
-        let (_, got) = regex(r#"/\w.*/"#).unwrap();
+        let (_, got) = regex(r"/\w.*/").unwrap();
         assert_eq!(got, "\\w.*".into());
 
         // Empty regex
@@ -344,6 +344,6 @@ mod test {
 
         // Single backslash fails, which matches Go implementation
         // See: https://go.dev/play/p/_8J1v5-382G
-        assert_expect_error!(regex(r#"/\/"#), "unterminated regex literal");
+        assert_expect_error!(regex(r"/\/"), "unterminated regex literal");
     }
 }
diff --git a/iox_query/src/exec/gapfill/algo.rs b/iox_query/src/exec/gapfill/algo.rs
@@ -6,8 +6,8 @@ mod interpolate;
 use std::{ops::Range, sync::Arc};
 
 use arrow::{
-    array::{Array, ArrayRef, TimestampNanosecondArray, UInt64Array},
-    compute::{kernels::take, SortColumn},
+    array::{Array, ArrayRef, PrimitiveArray, TimestampNanosecondArray, UInt64Array},
+    compute::kernels::take,
     datatypes::SchemaRef,
     record_batch::RecordBatch,
 };
@@ -151,13 +151,11 @@ impl GapFiller {
 
         let sort_columns = group_arr
             .iter()
-            .map(|(_, arr)| SortColumn {
-                values: Arc::clone(arr),
-                options: None,
-            })
+            .map(|(_, arr)| Arc::clone(arr))
             .collect::<Vec<_>>();
-        let mut ranges = arrow::compute::lexicographical_partition_ranges(&sort_columns)
-            .map_err(DataFusionError::ArrowError)?;
+        let mut ranges = arrow::compute::partition(&sort_columns)?
+            .ranges()
+            .into_iter();
 
         let mut series_ends = vec![];
         let mut cursor = self.cursor.clone_for_aggr_col(None)?;
@@ -941,7 +939,7 @@ impl StashedAggrBuilder<'_> {
     /// `input_aggr_array` at `offset` for use with the [`interleave`](arrow::compute::interleave)
     /// kernel.
     fn create_stash(input_aggr_array: &ArrayRef, offset: u64) -> Result<ArrayRef> {
-        let take_arr = vec![None, Some(offset)].into();
+        let take_arr: PrimitiveArray<_> = vec![None, Some(offset)].into();
         let stash =
             take::take(input_aggr_array, &take_arr, None).map_err(DataFusionError::ArrowError)?;
         Ok(stash)
@@ -1170,7 +1168,7 @@ mod tests {
         let arr = cursor
             .build_aggr_fill_null(&params, &[series], &input_times, &input_aggr_array)
             .unwrap();
-        insta::assert_yaml_snapshot!(array_to_lines(&time_arr, &arr), @r###"
+        insta::assert_yaml_snapshot!(array_to_lines(&time_arr, &arr), @r#"
         ---
         - +--------------------------------+------+
         - "| time                           | a0   |"
@@ -1183,7 +1181,7 @@ mod tests {
         - "| 1970-01-01T00:00:00.000001200Z | 12.0 |"
         - "| 1970-01-01T00:00:00.000001250Z |      |"
         - +--------------------------------+------+
-        "###);
+        "#);
 
         assert_cursor_end_state(&cursor, &input_times, &params);
     }
@@ -1215,7 +1213,7 @@ mod tests {
             .into();
         let arr =
             cursor.build_aggr_fill_null(&params, &[series], &input_times, &input_aggr_array)?;
-        insta::assert_yaml_snapshot!(array_to_lines(&time_arr, &arr), @r###"
+        insta::assert_yaml_snapshot!(array_to_lines(&time_arr, &arr), @r#"
         ---
         - +--------------------------------+------+
         - "| time                           | a0   |"
@@ -1230,7 +1228,7 @@ mod tests {
         - "| 1970-01-01T00:00:00.000001200Z | 12.0 |"
         - "| 1970-01-01T00:00:00.000001250Z |      |"
         - +--------------------------------+------+
-        "###);
+        "#);
 
         assert_cursor_end_state(&cursor, &input_times, &params);
 
@@ -1269,7 +1267,7 @@ mod tests {
         let arr = cursor
             .build_aggr_fill_prev(&params, &[series], &input_times, &input_aggr_array)
             .unwrap();
-        insta::assert_yaml_snapshot!(array_to_lines(&time_arr, &arr), @r###"
+        insta::assert_yaml_snapshot!(array_to_lines(&time_arr, &arr), @r#"
         ---
         - +--------------------------------+------+
         - "| time                           | a0   |"
@@ -1282,7 +1280,7 @@ mod tests {
         - "| 1970-01-01T00:00:00.000001200Z | 12.0 |"
         - "| 1970-01-01T00:00:00.000001250Z | 12.0 |"
         - +--------------------------------+------+
-        "###);
+        "#);
 
         assert_cursor_end_state(&cursor, &input_times, &params);
     }
@@ -1325,7 +1323,7 @@ mod tests {
         let arr = cursor
             .build_aggr_fill_prev(&params, &[series], &input_times, &input_aggr_array)
             .unwrap();
-        insta::assert_yaml_snapshot!(array_to_lines(&time_arr, &arr), @r###"
+        insta::assert_yaml_snapshot!(array_to_lines(&time_arr, &arr), @r#"
         ---
         - +--------------------------------+------+
         - "| time                           | a0   |"
@@ -1340,7 +1338,7 @@ mod tests {
         - "| 1970-01-01T00:00:00.000001200Z | 12.0 |"
         - "| 1970-01-01T00:00:00.000001250Z | 12.0 |"
         - +--------------------------------+------+
-        "###);
+        "#);
 
         assert_cursor_end_state(&cursor, &input_times, &params);
     }
@@ -1381,7 +1379,7 @@ mod tests {
         let arr = cursor
             .build_aggr_fill_null(&params, &series_ends, &input_times, &input_aggr_array)
             .unwrap();
-        insta::assert_yaml_snapshot!(array_to_lines(&time_arr, &arr), @r###"
+        insta::assert_yaml_snapshot!(array_to_lines(&time_arr, &arr), @r#"
         ---
         - +--------------------------------+------+
         - "| time                           | a0   |"
@@ -1395,7 +1393,7 @@ mod tests {
         - "| 1970-01-01T00:00:00.000001050Z | 11.0 |"
         - "| 1970-01-01T00:00:00.000001100Z |      |"
         - +--------------------------------+------+
-        "###);
+        "#);
 
         assert_cursor_end_state(&cursor, &input_times, &params);
     }
@@ -1445,7 +1443,7 @@ mod tests {
         let arr = cursor
             .build_aggr_fill_prev(&params, &series_ends, &input_times, &input_aggr_array)
             .unwrap();
-        insta::assert_yaml_snapshot!(array_to_lines(&time_arr, &arr), @r###"
+        insta::assert_yaml_snapshot!(array_to_lines(&time_arr, &arr), @r#"
         ---
         - +--------------------------------+------+
         - "| time                           | a0   |"
@@ -1459,7 +1457,7 @@ mod tests {
         - "| 1970-01-01T00:00:00.000001050Z | 21.0 |"
         - "| 1970-01-01T00:00:00.000001100Z | 21.0 |"
         - +--------------------------------+------+
-        "###);
+        "#);
 
         assert_cursor_end_state(&cursor, &input_times, &params);
     }
@@ -1534,7 +1532,7 @@ mod tests {
         let arr = cursor
             .build_aggr_fill_prev_stashed(&params, &series_ends, &input_times, &input_aggr_array)
             .unwrap();
-        insta::assert_yaml_snapshot!(array_to_lines(&time_arr, &arr), @r###"
+        insta::assert_yaml_snapshot!(array_to_lines(&time_arr, &arr), @r#"
         ---
         - +--------------------------------+-------+
         - "| time                           | a0    |"
@@ -1550,7 +1548,7 @@ mod tests {
         - "| 1970-01-01T00:00:00.000001100Z | 21.1  |"
         - "| 1970-01-01T00:00:00.000001150Z | 21.1  |"
         - +--------------------------------+-------+
-        "###);
+        "#);
 
         assert_cursor_end_state(&cursor, &input_times, &params);
     }

diff --git a/iox_query/src/exec/gapfill/algo/interpolate.rs b/iox_query/src/exec/gapfill/algo/interpolate.rs
@@ -365,7 +365,7 @@ mod test {
         let arr = cursor
             .build_aggr_fill_interpolate(&params, &series_ends, &input_times, &input_aggr_array)
             .unwrap();
-        insta::assert_yaml_snapshot!(array_to_lines(&time_arr, &arr), @r###"
+        insta::assert_yaml_snapshot!(array_to_lines(&time_arr, &arr), @r#"
         ---
         - +--------------------------------+------+
         - "| time                           | a0   |"
@@ -382,7 +382,7 @@ mod test {
         - "| 1970-01-01T00:00:00.000001900Z | 0    |"
         - "| 1970-01-01T00:00:00.000002Z    |      |"
         - +--------------------------------+------+
-        "###);
+        "#);
 
         assert_cursor_end_state(&cursor, &input_times, &params);
     }
@@ -437,7 +437,7 @@ mod test {
         let arr = cursor
             .build_aggr_fill_interpolate(&params, &series_ends, &input_times, &input_aggr_array)
             .unwrap();
-        insta::assert_yaml_snapshot!(array_to_lines(&time_arr, &arr), @r###"
+        insta::assert_yaml_snapshot!(array_to_lines(&time_arr, &arr), @r#"
         ---
         - +--------------------------------+------+
         - "| time                           | a0   |"
@@ -454,7 +454,7 @@ mod test {
         - "| 1970-01-01T00:00:00.000001900Z | 0    |"
         - "| 1970-01-01T00:00:00.000002Z    |      |"
         - +--------------------------------+------+
-        "###);
+        "#);
 
         assert_cursor_end_state(&cursor, &input_times, &params);
     }
@@ -509,7 +509,7 @@ mod test {
         let arr = cursor
             .build_aggr_fill_interpolate(&params, &series_ends, &input_times, &input_aggr_array)
             .unwrap();
-        insta::assert_yaml_snapshot!(array_to_lines(&time_arr, &arr), @r###"
+        insta::assert_yaml_snapshot!(array_to_lines(&time_arr, &arr), @r#"
         ---
         - +--------------------------------+--------+
         - "| time                           | a0     |"
@@ -526,7 +526,7 @@ mod test {
         - "| 1970-01-01T00:00:00.000001900Z | 0.0    |"
         - "| 1970-01-01T00:00:00.000002Z    |        |"
         - +--------------------------------+--------+
-        "###);
+        "#);
 
         assert_cursor_end_state(&cursor, &input_times, &params);
     }