fix ndjson error

databendlabs · Jun 24, 2023 · 06b0efc · 06b0efc
1 parent 19a2e69
commit 06b0efc
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 9 deletions.
diff --git a/src/query/pipeline/sources/src/input_formats/input_format_text.rs b/src/query/pipeline/sources/src/input_formats/input_format_text.rs
@@ -412,20 +412,24 @@ impl RowBatchTrait for RowBatch {
  if !self.row_ends.is_empty() {
  self.row_ends.truncate(1);
  self.data.truncate(self.row_ends[0]);
- self.num_fields.truncate(1);
- self.field_ends.truncate(self.num_fields[0]);
+ // for ndjson, it's always empty
+ if self.num_fields.len() > 0 && self.field_ends.len() > 0 {
+ self.num_fields.truncate(1);
+ self.field_ends.truncate(self.num_fields[0]);
+ }
  }
  }
 
  fn remove_first_line(&mut self) {
  if !self.row_ends.is_empty() {
- let first_row_size = self.row_ends[0];
-
- for _ in 0..self.num_fields[0] {
- self.field_ends.remove(0);
+ // for ndjson, it's always empty
+ if self.num_fields.len() > 0 && self.field_ends.len() > 0 {
+ for _ in 0..self.num_fields[0] {
+ self.field_ends.remove(0);
+ }
+ self.num_fields.remove(0);
  }
- self.num_fields.remove(0);
-
+ let first_row_size = self.row_ends[0];
  for _ in 0..self.row_ends[0] {
  self.data.remove(0);
  }

diff --git a/tests/suites/1_stateful/00_copy/00_0011_dirtributed_copy_from_stage_for_text_file.result b/tests/suites/1_stateful/00_copy/00_0011_dirtributed_copy_from_stage_for_text_file.result
@@ -11,4 +11,4 @@ null
 103 12-pack drill bits 12-pack of drill bits with sizes ranging from #40 to #3
 104 hammer 12oz carpenter's hammer
 105 hammer 14oz carpenter's hammer
-11
+11
diff --git a/tests/suites/1_stateful/00_copy/00_0011_dirtributed_copy_from_stage_for_text_file.sh b/tests/suites/1_stateful/00_copy/00_0011_dirtributed_copy_from_stage_for_text_file.sh
@@ -3,6 +3,7 @@
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../../../shell_env.sh
 
+echo "drop table if exists products;" | $MYSQL_CLIENT_CONNECT
 echo "drop stage if exists s1;" | $MYSQL_CLIENT_CONNECT
 echo "CREATE STAGE s1 FILE_FORMAT = (TYPE = CSV);" | $MYSQL_CLIENT_CONNECT
 echo "create table products (id int, name string, description string);" | $MYSQL_CLIENT_CONNECT