Add session variable trim_tailing_spaces_for_external_table_query

apache · May 17, 2022 · 52c4715 · 52c4715
1 parent 31e718e
commit 52c4715
Show file tree

Hide file tree

Showing 4 changed files with 30 additions and 5 deletions.
diff --git a/be/src/exec/broker_scanner.cpp b/be/src/exec/broker_scanner.cpp
@@ -364,9 +364,11 @@ void BrokerScanner::split_line(const Slice& line) {
  if (p1 == _value_separator_length) {
  // Match a separator
  non_space = curpos;
- // Trim trailing spaces. Be consistent with hive and trino's behavior.
- while (non_space > start && *(value + non_space - 1) == ' ') {
- non_space--;
+ // Trim tailing spaces. Be consistent with hive and trino's behavior.
+ if (_state->trim_tailing_spaces_for_external_table_query()) {
+ while (non_space > start && *(value + non_space - 1) == ' ') {
+ non_space--;
+ }
  }
  _split_values.emplace_back(value + start, non_space - start);
  start = curpos + _value_separator_length;
@@ -379,8 +381,10 @@ void BrokerScanner::split_line(const Slice& line) {
 
  CHECK(curpos == line.size) << curpos << " vs " << line.size;
  non_space = curpos;
- while (non_space > start && *(value + non_space - 1) == ' ') {
- non_space--;
+ if (_state->trim_tailing_spaces_for_external_table_query()) {
+ while (non_space > start && *(value + non_space - 1) == ' ') {
+ non_space--;
+ }
  }
  _split_values.emplace_back(value + start, non_space - start);
  }

diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h
@@ -326,6 +326,10 @@ class RuntimeState {
 
  bool enable_vectorized_exec() const { return _query_options.enable_vectorized_engine; }
 
+ bool trim_tailing_spaces_for_external_table_query() const {
+ return _query_options.trim_tailing_spaces_for_external_table_query;
+ }
+
  bool return_object_data_as_binary() const {
  return _query_options.return_object_data_as_binary;
  }

diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -180,6 +180,8 @@ public class SessionVariable implements Serializable, Writable {
 
  public static final String ENABLE_PROJECTION = "enable_projection";
 
+ public static final String TRIM_TAILING_SPACES_FOR_EXTERNAL_TABLE_QUERY = "trim_tailing_spaces_for_external_table_query";
+
  // session origin value
  public Map<Field, String> sessionOriginValue = new HashMap<Field, String>();
  // check stmt is or not [select /*+ SET_VAR(...)*/ ...]
@@ -442,6 +444,9 @@ public class SessionVariable implements Serializable, Writable {
  @VariableMgr.VarAttr(name = ENABLE_PROJECTION)
  private boolean enableProjection = false;
 
+ @VariableMgr.VarAttr(name = TRIM_TAILING_SPACES_FOR_EXTERNAL_TABLE_QUERY)
+ private boolean trimTailingSpacesForExternalTableQuery = false;
+
  public String getBlockEncryptionMode() {
  return blockEncryptionMode;
  }
@@ -906,6 +911,14 @@ public boolean isEnableProjection() {
  return enableProjection;
  }
 
+ public boolean isTrimTailingSpacesForExternalTableQuery() {
+ return trimTailingSpacesForExternalTableQuery;
+ }
+
+ public void setTrimTailingSpacesForExternalTableQuery(boolean trimTailingSpacesForExternalTableQuery) {
+ this.trimTailingSpacesForExternalTableQuery = trimTailingSpacesForExternalTableQuery;
+ }
+
  // Serialize to thrift object
  // used for rest api
  public TQueryOptions toThrift() {
@@ -923,6 +936,7 @@ public TQueryOptions toThrift() {
  tResult.setCodegenLevel(codegenLevel);
  tResult.setEnableVectorizedEngine(enableVectorizedEngine);
  tResult.setReturnObjectDataAsBinary(returnObjectDataAsBinary);
+ tResult.setTrimTailingSpacesForExternalTableQuery(trimTailingSpacesForExternalTableQuery);
 
  tResult.setBatchSize(batchSize);
  tResult.setDisableStreamPreaggregations(disableStreamPreaggregations);

diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift
@@ -160,6 +160,9 @@ struct TQueryOptions {
  // show bitmap data in result, if use this in mysql cli may make the terminal
  // output corrupted character
  43: optional bool return_object_data_as_binary = false
+
+ // trim tailing spaces while querying external table and stream load
+ 44: optional bool trim_tailing_spaces_for_external_table_query = false
 }