From 4ffb70c5d97690809bf77bf50cab49a5081a8d15 Mon Sep 17 00:00:00 2001 From: blackfox1983 Date: Thu, 7 May 2020 15:16:05 +0000 Subject: [PATCH 1/4] fix bug of query failed when column not exist --- be/src/exec/es/es_scan_reader.cpp | 7 +++-- be/src/exec/es/es_scan_reader.h | 4 ++- be/src/exec/es/es_scroll_parser.cpp | 40 +++++++++++++++++++++++++++-- be/src/exec/es/es_scroll_parser.h | 4 ++- be/src/exec/es/es_scroll_query.cpp | 6 ++++- be/src/exec/es/es_scroll_query.h | 3 ++- be/src/exec/es_http_scan_node.cpp | 6 +++-- be/src/exec/es_http_scanner.cpp | 6 +++-- be/src/exec/es_http_scanner.h | 5 +++- 9 files changed, 68 insertions(+), 13 deletions(-) diff --git a/be/src/exec/es/es_scan_reader.cpp b/be/src/exec/es/es_scan_reader.cpp index 33a7a3bdb58ee9..39ade23b1c9466 100644 --- a/be/src/exec/es/es_scan_reader.cpp +++ b/be/src/exec/es/es_scan_reader.cpp @@ -35,7 +35,10 @@ const std::string REQUEST_SEPARATOR = "/"; const std::string REQUEST_SEARCH_FILTER_PATH = "filter_path=hits.hits._source,hits.total,_id,hits.hits._source.fields,hits.hits.fields"; -ESScanReader::ESScanReader(const std::string& target, const std::map& props) : _scroll_keep_alive(config::es_scroll_keepalive), _http_timeout_ms(config::es_http_timeout_ms) { +ESScanReader::ESScanReader(const std::string& target, const std::map& props, bool use_doc_value) : + _scroll_keep_alive(config::es_scroll_keepalive), + _http_timeout_ms(config::es_http_timeout_ms), + _use_doc_value(use_doc_value) { _target = target; _index = props.at(KEY_INDEX); _type = props.at(KEY_TYPE); @@ -142,7 +145,7 @@ Status ESScanReader::get_next(bool* scan_eos, std::unique_ptr& scr } } - scroll_parser.reset(new ScrollParser()); + scroll_parser.reset(new ScrollParser(_use_doc_value)); VLOG(1) << "get_next request ES, returned response: " << response; Status status = scroll_parser->parse(response, _exactly_once); if (!status.ok()){ diff --git a/be/src/exec/es/es_scan_reader.h b/be/src/exec/es/es_scan_reader.h index 60137c1bd085f1..af99a6e905ec12 100644 --- a/be/src/exec/es/es_scan_reader.h +++ b/be/src/exec/es/es_scan_reader.h @@ -40,7 +40,7 @@ class ESScanReader { static constexpr const char* KEY_QUERY = "query"; static constexpr const char* KEY_BATCH_SIZE = "batch_size"; static constexpr const char* KEY_TERMINATE_AFTER = "limit"; - ESScanReader(const std::string& target, const std::map& props); + ESScanReader(const std::string& target, const std::map& props, bool use_doc_value); ~ESScanReader(); // launch the first scroll request, this method will cache the first scroll response, and return the this cached response when invoke get_next @@ -94,6 +94,8 @@ class ESScanReader { int _http_timeout_ms; bool _exactly_once; + + bool _use_doc_value; }; } diff --git a/be/src/exec/es/es_scroll_parser.cpp b/be/src/exec/es/es_scroll_parser.cpp index fbbcfb7ffb71da..48a06124e51ea5 100644 --- a/be/src/exec/es/es_scroll_parser.cpp +++ b/be/src/exec/es/es_scroll_parser.cpp @@ -198,11 +198,12 @@ static Status get_float_value(const rapidjson::Value &col, PrimitiveType type, v return Status::OK(); } -ScrollParser::ScrollParser() : +ScrollParser::ScrollParser(bool use_doc_value) : _scroll_id(""), _total(0), _size(0), - _line_index(0) { + _line_index(0), + _use_doc_value(use_doc_value) { } ScrollParser::~ScrollParser() { @@ -247,6 +248,16 @@ Status ScrollParser::parse(const std::string& scroll_result, bool exactly_once) } VLOG(1) << "es_scan_reader parse scroll result: " << scroll_result; + if (!outer_hits_node.HasMember(FIELD_INNER_HITS)) { + // this is caused by query some columns which are not exit, e.g. + // A Index has fields: k1,k2,k3. and we put some rows into this Index (some fields dose NOT contain any data) + // e.g. + // put index/_doc/1 {"k2":"123"} + // put index/_doc/2 {"k3":"123} + // then we use sql `select k1 from table` + // what E.S. return is like this: {hits: {total:2} + return Status::OK(); + } const rapidjson::Value &inner_hits_node = outer_hits_node[FIELD_INNER_HITS]; if (!inner_hits_node.IsArray()) { LOG(WARNING) << "exception maybe happend on es cluster, reponse:" << scroll_result; @@ -275,7 +286,32 @@ int ScrollParser::get_total() { Status ScrollParser::fill_tuple(const TupleDescriptor* tuple_desc, Tuple* tuple, MemPool* tuple_pool, bool* line_eof, const std::map& docvalue_context) { *line_eof = true; + if (_size <= 0 || _line_index >= _size) { + // _source is fetched from E.S. + if (!_use_doc_value) { + return Status::OK(); + } + + // _fields(doc_value) is fetched from E.S. + if (_total <= 0 || _line_index >= _total) { + return Status::OK(); + } + + + // here is operations for `enable_doc_value_scan`. + // This indicates that the fields does not exist(e.g. never assign values to these fields), but other fields have values. + // so, number of rows is >= 0, we need fill `NULL` to these fields that does not exist. + _line_index++; + tuple->init(tuple_desc->byte_size()); + for (int i = 0; i < tuple_desc->slots().size(); ++i) { + const SlotDescriptor* slot_desc = tuple_desc->slots()[i]; + if (slot_desc->is_materialized()) { + tuple->set_null(slot_desc->null_indicator_offset()); + } + } + + *line_eof = false; return Status::OK(); } diff --git a/be/src/exec/es/es_scroll_parser.h b/be/src/exec/es/es_scroll_parser.h index 85ac92b4a534e8..3e6c4c3ea60932 100644 --- a/be/src/exec/es/es_scroll_parser.h +++ b/be/src/exec/es/es_scroll_parser.h @@ -30,7 +30,7 @@ class Status; class ScrollParser { public: - ScrollParser(); + ScrollParser(bool use_doc_value); ~ScrollParser(); Status parse(const std::string& scroll_result, bool exactly_once = false); @@ -50,5 +50,7 @@ class ScrollParser { rapidjson::Document _document_node; rapidjson::Value _inner_hits_node; + + bool _use_doc_value; }; } diff --git a/be/src/exec/es/es_scroll_query.cpp b/be/src/exec/es/es_scroll_query.cpp index e9b88eec676e8c..6ae4dcb4737109 100644 --- a/be/src/exec/es/es_scroll_query.cpp +++ b/be/src/exec/es/es_scroll_query.cpp @@ -63,7 +63,8 @@ std::string ESScrollQueryBuilder::build_clear_scroll_body(const std::string& scr std::string ESScrollQueryBuilder::build(const std::map& properties, const std::vector& fields, - std::vector& predicates, const std::map& docvalue_context) { + std::vector& predicates, const std::map& docvalue_context, + bool* use_doc_value) { rapidjson::Document es_query_dsl; rapidjson::Document::AllocatorType &allocator = es_query_dsl.GetAllocator(); es_query_dsl.SetObject(); @@ -86,6 +87,9 @@ std::string ESScrollQueryBuilder::build(const std::map } } } + + *use_doc_value = pure_docvalue; + rapidjson::Value source_node(rapidjson::kArrayType); if (pure_docvalue) { for (auto& select_field : fields) { diff --git a/be/src/exec/es/es_scroll_query.h b/be/src/exec/es/es_scroll_query.h index 959ec455dd85d7..1bd4e7f36d68ad 100644 --- a/be/src/exec/es/es_scroll_query.h +++ b/be/src/exec/es/es_scroll_query.h @@ -35,6 +35,7 @@ class ESScrollQueryBuilder { // @note: predicates should processed before pass it to this method, // tie breaker for predicate wheather can push down es can reference the push-down filters static std::string build(const std::map& properties, - const std::vector& fields, std::vector& predicates, const std::map& docvalue_context); + const std::vector& fields, std::vector& predicates, const std::map& docvalue_context, + bool* use_doc_value); }; } diff --git a/be/src/exec/es_http_scan_node.cpp b/be/src/exec/es_http_scan_node.cpp index cb5eeb2ca6c3d3..d7c303160e9154 100644 --- a/be/src/exec/es_http_scan_node.cpp +++ b/be/src/exec/es_http_scan_node.cpp @@ -443,14 +443,16 @@ void EsHttpScanNode::scanner_worker(int start_idx, int length, std::promisebatch_size()) { properties[ESScanReader::KEY_TERMINATE_AFTER] = std::to_string(limit()); } + + bool use_doc_value = false; properties[ESScanReader::KEY_QUERY] - = ESScrollQueryBuilder::build(properties, _column_names, _predicates, _docvalue_context); + = ESScrollQueryBuilder::build(properties, _column_names, _predicates, _docvalue_context, &use_doc_value); // start scanner to scan std::unique_ptr scanner(new EsHttpScanner( _runtime_state, runtime_profile(), _tuple_id, - properties, scanner_expr_ctxs, &counter)); + properties, scanner_expr_ctxs, &counter, use_doc_value)); status = scanner_scan(std::move(scanner), scanner_expr_ctxs, &counter); if (!status.ok()) { LOG(WARNING) << "Scanner[" << start_idx << "] process failed. status=" diff --git a/be/src/exec/es_http_scanner.cpp b/be/src/exec/es_http_scanner.cpp index 4eecaa0066942b..ca822f6567a71b 100644 --- a/be/src/exec/es_http_scanner.cpp +++ b/be/src/exec/es_http_scanner.cpp @@ -36,7 +36,8 @@ EsHttpScanner::EsHttpScanner( TupleId tuple_id, const std::map& properties, const std::vector& conjunct_ctxs, - EsScanCounter* counter) : + EsScanCounter* counter, + bool use_doc_value) : _state(state), _profile(profile), _tuple_id(tuple_id), @@ -56,6 +57,7 @@ EsHttpScanner::EsHttpScanner( _counter(counter), _es_reader(nullptr), _es_scroll_parser(nullptr), + _use_doc_value(use_doc_value), _rows_read_counter(nullptr), _read_timer(nullptr), _materialize_timer(nullptr) { @@ -74,7 +76,7 @@ Status EsHttpScanner::open() { } const std::string& host = _properties.at(ESScanReader::KEY_HOST_PORT); - _es_reader.reset(new ESScanReader(host, _properties)); + _es_reader.reset(new ESScanReader(host, _properties, _use_doc_value)); if (_es_reader == nullptr) { return Status::InternalError("Es reader construct failed."); } diff --git a/be/src/exec/es_http_scanner.h b/be/src/exec/es_http_scanner.h index 23ebe7c1c6b661..a780b17150e8d0 100644 --- a/be/src/exec/es_http_scanner.h +++ b/be/src/exec/es_http_scanner.h @@ -62,7 +62,8 @@ class EsHttpScanner { TupleId tuple_id, const std::map& properties, const std::vector& conjunct_ctxs, - EsScanCounter* counter); + EsScanCounter* counter, + bool use_doc_value); ~EsHttpScanner(); Status open(); @@ -94,6 +95,8 @@ class EsHttpScanner { std::unique_ptr _es_reader; std::unique_ptr _es_scroll_parser; + bool _use_doc_value; + // Profile RuntimeProfile::Counter* _rows_read_counter; RuntimeProfile::Counter* _read_timer; From 89b5f93fc62287a7eefccc64062bf58f5199de75 Mon Sep 17 00:00:00 2001 From: blackfox1983 Date: Fri, 8 May 2020 14:56:26 +0000 Subject: [PATCH 2/4] fix complier failed of ut --- be/test/exec/es_scan_reader_test.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/be/test/exec/es_scan_reader_test.cpp b/be/test/exec/es_scan_reader_test.cpp index cba77356d60ae7..3a40b19d91298c 100644 --- a/be/test/exec/es_scan_reader_test.cpp +++ b/be/test/exec/es_scan_reader_test.cpp @@ -226,8 +226,9 @@ TEST_F(MockESServerTest, workflow) { props[ESScanReader::KEY_BATCH_SIZE] = "1"; std::vector predicates; std::map docvalue_context; - props[ESScanReader::KEY_QUERY] = ESScrollQueryBuilder::build(props, fields, predicates, docvalue_context); - ESScanReader reader(target, props); + bool use_doc_value = false; + props[ESScanReader::KEY_QUERY] = ESScrollQueryBuilder::build(props, fields, predicates, docvalue_context, &use_doc_value); + ESScanReader reader(target, props, use_doc_value); auto st = reader.open(); ASSERT_TRUE(st.ok()); bool eos = false; From ad34153beb641e3dc3d299bfde3da6d31d3c9a37 Mon Sep 17 00:00:00 2001 From: blackfox1983 Date: Sat, 9 May 2020 06:45:11 +0000 Subject: [PATCH 3/4] fixed style --- be/src/exec/es/es_scan_reader.cpp | 6 +++--- be/src/exec/es/es_scan_reader.h | 4 ++-- be/src/exec/es/es_scroll_parser.cpp | 6 +++--- be/src/exec/es/es_scroll_parser.h | 4 ++-- be/src/exec/es/es_scroll_query.cpp | 4 ++-- be/src/exec/es/es_scroll_query.h | 2 +- be/src/exec/es_http_scan_node.cpp | 6 +++--- be/src/exec/es_http_scanner.cpp | 6 +++--- be/src/exec/es_http_scanner.h | 4 ++-- be/test/exec/es_scan_reader_test.cpp | 6 +++--- 10 files changed, 24 insertions(+), 24 deletions(-) diff --git a/be/src/exec/es/es_scan_reader.cpp b/be/src/exec/es/es_scan_reader.cpp index 39ade23b1c9466..4d6673815ccd8e 100644 --- a/be/src/exec/es/es_scan_reader.cpp +++ b/be/src/exec/es/es_scan_reader.cpp @@ -35,10 +35,10 @@ const std::string REQUEST_SEPARATOR = "/"; const std::string REQUEST_SEARCH_FILTER_PATH = "filter_path=hits.hits._source,hits.total,_id,hits.hits._source.fields,hits.hits.fields"; -ESScanReader::ESScanReader(const std::string& target, const std::map& props, bool use_doc_value) : +ESScanReader::ESScanReader(const std::string& target, const std::map& props, bool doc_value_mode) : _scroll_keep_alive(config::es_scroll_keepalive), _http_timeout_ms(config::es_http_timeout_ms), - _use_doc_value(use_doc_value) { + _doc_value_mode(doc_value_mode) { _target = target; _index = props.at(KEY_INDEX); _type = props.at(KEY_TYPE); @@ -145,7 +145,7 @@ Status ESScanReader::get_next(bool* scan_eos, std::unique_ptr& scr } } - scroll_parser.reset(new ScrollParser(_use_doc_value)); + scroll_parser.reset(new ScrollParser(_doc_value_mode)); VLOG(1) << "get_next request ES, returned response: " << response; Status status = scroll_parser->parse(response, _exactly_once); if (!status.ok()){ diff --git a/be/src/exec/es/es_scan_reader.h b/be/src/exec/es/es_scan_reader.h index af99a6e905ec12..3bd64cf88aca71 100644 --- a/be/src/exec/es/es_scan_reader.h +++ b/be/src/exec/es/es_scan_reader.h @@ -40,7 +40,7 @@ class ESScanReader { static constexpr const char* KEY_QUERY = "query"; static constexpr const char* KEY_BATCH_SIZE = "batch_size"; static constexpr const char* KEY_TERMINATE_AFTER = "limit"; - ESScanReader(const std::string& target, const std::map& props, bool use_doc_value); + ESScanReader(const std::string& target, const std::map& props, bool doc_value_mode); ~ESScanReader(); // launch the first scroll request, this method will cache the first scroll response, and return the this cached response when invoke get_next @@ -95,7 +95,7 @@ class ESScanReader { bool _exactly_once; - bool _use_doc_value; + bool _doc_value_mode; }; } diff --git a/be/src/exec/es/es_scroll_parser.cpp b/be/src/exec/es/es_scroll_parser.cpp index 48a06124e51ea5..102a5f2359ccb6 100644 --- a/be/src/exec/es/es_scroll_parser.cpp +++ b/be/src/exec/es/es_scroll_parser.cpp @@ -198,12 +198,12 @@ static Status get_float_value(const rapidjson::Value &col, PrimitiveType type, v return Status::OK(); } -ScrollParser::ScrollParser(bool use_doc_value) : +ScrollParser::ScrollParser(bool doc_value_mode) : _scroll_id(""), _total(0), _size(0), _line_index(0), - _use_doc_value(use_doc_value) { + _doc_value_mode(doc_value_mode) { } ScrollParser::~ScrollParser() { @@ -289,7 +289,7 @@ Status ScrollParser::fill_tuple(const TupleDescriptor* tuple_desc, if (_size <= 0 || _line_index >= _size) { // _source is fetched from E.S. - if (!_use_doc_value) { + if (!_doc_value_mode) { return Status::OK(); } diff --git a/be/src/exec/es/es_scroll_parser.h b/be/src/exec/es/es_scroll_parser.h index 3e6c4c3ea60932..94fdce8f8f938a 100644 --- a/be/src/exec/es/es_scroll_parser.h +++ b/be/src/exec/es/es_scroll_parser.h @@ -30,7 +30,7 @@ class Status; class ScrollParser { public: - ScrollParser(bool use_doc_value); + ScrollParser(bool doc_value_mode); ~ScrollParser(); Status parse(const std::string& scroll_result, bool exactly_once = false); @@ -51,6 +51,6 @@ class ScrollParser { rapidjson::Document _document_node; rapidjson::Value _inner_hits_node; - bool _use_doc_value; + bool _doc_value_mode; }; } diff --git a/be/src/exec/es/es_scroll_query.cpp b/be/src/exec/es/es_scroll_query.cpp index 6ae4dcb4737109..0c4f581e513486 100644 --- a/be/src/exec/es/es_scroll_query.cpp +++ b/be/src/exec/es/es_scroll_query.cpp @@ -64,7 +64,7 @@ std::string ESScrollQueryBuilder::build_clear_scroll_body(const std::string& scr std::string ESScrollQueryBuilder::build(const std::map& properties, const std::vector& fields, std::vector& predicates, const std::map& docvalue_context, - bool* use_doc_value) { + bool* doc_value_mode) { rapidjson::Document es_query_dsl; rapidjson::Document::AllocatorType &allocator = es_query_dsl.GetAllocator(); es_query_dsl.SetObject(); @@ -88,7 +88,7 @@ std::string ESScrollQueryBuilder::build(const std::map } } - *use_doc_value = pure_docvalue; + *doc_value_mode = pure_docvalue; rapidjson::Value source_node(rapidjson::kArrayType); if (pure_docvalue) { diff --git a/be/src/exec/es/es_scroll_query.h b/be/src/exec/es/es_scroll_query.h index 1bd4e7f36d68ad..ed5e7e0dd7cab6 100644 --- a/be/src/exec/es/es_scroll_query.h +++ b/be/src/exec/es/es_scroll_query.h @@ -36,6 +36,6 @@ class ESScrollQueryBuilder { // tie breaker for predicate wheather can push down es can reference the push-down filters static std::string build(const std::map& properties, const std::vector& fields, std::vector& predicates, const std::map& docvalue_context, - bool* use_doc_value); + bool* doc_value_mode); }; } diff --git a/be/src/exec/es_http_scan_node.cpp b/be/src/exec/es_http_scan_node.cpp index d7c303160e9154..fa7b895805b38a 100644 --- a/be/src/exec/es_http_scan_node.cpp +++ b/be/src/exec/es_http_scan_node.cpp @@ -444,15 +444,15 @@ void EsHttpScanNode::scanner_worker(int start_idx, int length, std::promise scanner(new EsHttpScanner( _runtime_state, runtime_profile(), _tuple_id, - properties, scanner_expr_ctxs, &counter, use_doc_value)); + properties, scanner_expr_ctxs, &counter, doc_value_mode)); status = scanner_scan(std::move(scanner), scanner_expr_ctxs, &counter); if (!status.ok()) { LOG(WARNING) << "Scanner[" << start_idx << "] process failed. status=" diff --git a/be/src/exec/es_http_scanner.cpp b/be/src/exec/es_http_scanner.cpp index ca822f6567a71b..f04408088fd800 100644 --- a/be/src/exec/es_http_scanner.cpp +++ b/be/src/exec/es_http_scanner.cpp @@ -37,7 +37,7 @@ EsHttpScanner::EsHttpScanner( const std::map& properties, const std::vector& conjunct_ctxs, EsScanCounter* counter, - bool use_doc_value) : + bool doc_value_mode) : _state(state), _profile(profile), _tuple_id(tuple_id), @@ -57,7 +57,7 @@ EsHttpScanner::EsHttpScanner( _counter(counter), _es_reader(nullptr), _es_scroll_parser(nullptr), - _use_doc_value(use_doc_value), + _doc_value_mode(doc_value_mode), _rows_read_counter(nullptr), _read_timer(nullptr), _materialize_timer(nullptr) { @@ -76,7 +76,7 @@ Status EsHttpScanner::open() { } const std::string& host = _properties.at(ESScanReader::KEY_HOST_PORT); - _es_reader.reset(new ESScanReader(host, _properties, _use_doc_value)); + _es_reader.reset(new ESScanReader(host, _properties, _doc_value_mode)); if (_es_reader == nullptr) { return Status::InternalError("Es reader construct failed."); } diff --git a/be/src/exec/es_http_scanner.h b/be/src/exec/es_http_scanner.h index a780b17150e8d0..f94eded5ed9742 100644 --- a/be/src/exec/es_http_scanner.h +++ b/be/src/exec/es_http_scanner.h @@ -63,7 +63,7 @@ class EsHttpScanner { const std::map& properties, const std::vector& conjunct_ctxs, EsScanCounter* counter, - bool use_doc_value); + bool doc_value_mode); ~EsHttpScanner(); Status open(); @@ -95,7 +95,7 @@ class EsHttpScanner { std::unique_ptr _es_reader; std::unique_ptr _es_scroll_parser; - bool _use_doc_value; + bool _doc_value_mode; // Profile RuntimeProfile::Counter* _rows_read_counter; diff --git a/be/test/exec/es_scan_reader_test.cpp b/be/test/exec/es_scan_reader_test.cpp index 3a40b19d91298c..28976b22eafd52 100644 --- a/be/test/exec/es_scan_reader_test.cpp +++ b/be/test/exec/es_scan_reader_test.cpp @@ -226,9 +226,9 @@ TEST_F(MockESServerTest, workflow) { props[ESScanReader::KEY_BATCH_SIZE] = "1"; std::vector predicates; std::map docvalue_context; - bool use_doc_value = false; - props[ESScanReader::KEY_QUERY] = ESScrollQueryBuilder::build(props, fields, predicates, docvalue_context, &use_doc_value); - ESScanReader reader(target, props, use_doc_value); + bool doc_value_mode = false; + props[ESScanReader::KEY_QUERY] = ESScrollQueryBuilder::build(props, fields, predicates, docvalue_context, &doc_value_mode); + ESScanReader reader(target, props, doc_value_mode); auto st = reader.open(); ASSERT_TRUE(st.ok()); bool eos = false; From f3fa2fb1f53f44f5583449b9031f88630afd03ec Mon Sep 17 00:00:00 2001 From: blackfox1983 Date: Sat, 9 May 2020 06:55:36 +0000 Subject: [PATCH 4/4] add a todo comment --- be/src/exec/es/es_scroll_parser.cpp | 6 +++--- be/src/exec/es/es_scroll_parser.h | 6 +++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/be/src/exec/es/es_scroll_parser.cpp b/be/src/exec/es/es_scroll_parser.cpp index 102a5f2359ccb6..1fa9a036f89dd9 100644 --- a/be/src/exec/es/es_scroll_parser.cpp +++ b/be/src/exec/es/es_scroll_parser.cpp @@ -255,7 +255,7 @@ Status ScrollParser::parse(const std::string& scroll_result, bool exactly_once) // put index/_doc/1 {"k2":"123"} // put index/_doc/2 {"k3":"123} // then we use sql `select k1 from table` - // what E.S. return is like this: {hits: {total:2} + // what ES return is like this: {hits: {total:2} return Status::OK(); } const rapidjson::Value &inner_hits_node = outer_hits_node[FIELD_INNER_HITS]; @@ -288,12 +288,12 @@ Status ScrollParser::fill_tuple(const TupleDescriptor* tuple_desc, *line_eof = true; if (_size <= 0 || _line_index >= _size) { - // _source is fetched from E.S. + // _source is fetched from ES if (!_doc_value_mode) { return Status::OK(); } - // _fields(doc_value) is fetched from E.S. + // _fields(doc_value) is fetched from ES if (_total <= 0 || _line_index >= _total) { return Status::OK(); } diff --git a/be/src/exec/es/es_scroll_parser.h b/be/src/exec/es/es_scroll_parser.h index 94fdce8f8f938a..b97e6bc80aa2cb 100644 --- a/be/src/exec/es/es_scroll_parser.h +++ b/be/src/exec/es/es_scroll_parser.h @@ -50,7 +50,11 @@ class ScrollParser { rapidjson::Document _document_node; rapidjson::Value _inner_hits_node; - + + // todo(milimin): ScrollParser should be divided into two classes: SourceParser and DocValueParser, + // including remove some variables in the current implementation, e.g. pure_doc_value. + // All above will be done in the DOE refactoring projects. + // Current bug fixes minimize the scope of changes to avoid introducing other new bugs. bool _doc_value_mode; }; }