diff --git a/be/src/olap/rowset/segment_v2/row_ranges.h b/be/src/olap/rowset/segment_v2/row_ranges.h index 56756ee60f981d..c98320d069a84f 100644 --- a/be/src/olap/rowset/segment_v2/row_ranges.h +++ b/be/src/olap/rowset/segment_v2/row_ranges.h @@ -225,13 +225,15 @@ class RowRanges { return _ranges[_ranges.size() - 1].to(); } - size_t range_size() { return _ranges.size(); } + size_t range_size() const { return _ranges.size(); } - int64_t get_range_from(size_t range_index) { return _ranges[range_index].from(); } + RowRange get_range(size_t index) const { return _ranges[index]; } - int64_t get_range_to(size_t range_index) { return _ranges[range_index].to(); } + int64_t get_range_from(size_t range_index) const { return _ranges[range_index].from(); } - size_t get_range_count(size_t range_index) { return _ranges[range_index].count(); } + int64_t get_range_to(size_t range_index) const { return _ranges[range_index].to(); } + + size_t get_range_count(size_t range_index) const { return _ranges[range_index].count(); } std::string to_string() { std::string result; diff --git a/be/src/vec/exec/format/parquet/level_decoder.cpp b/be/src/vec/exec/format/parquet/level_decoder.cpp index 79c18c7af08449..033ca5cbdd1d84 100644 --- a/be/src/vec/exec/format/parquet/level_decoder.cpp +++ b/be/src/vec/exec/format/parquet/level_decoder.cpp @@ -87,6 +87,7 @@ doris::Status doris::vectorized::LevelDecoder::init_v2(const doris::Slice& level } size_t doris::vectorized::LevelDecoder::get_levels(doris::vectorized::level_t* levels, size_t n) { + // toto template. if (_encoding == tparquet::Encoding::RLE) { n = std::min((size_t)_num_levels, n); auto num_decoded = _rle_decoder.get_values(levels, n); diff --git a/be/src/vec/exec/format/parquet/parquet_common.cpp b/be/src/vec/exec/format/parquet/parquet_common.cpp index 4a25102e4fe041..95920ef2a16006 100644 --- a/be/src/vec/exec/format/parquet/parquet_common.cpp +++ b/be/src/vec/exec/format/parquet/parquet_common.cpp @@ -118,6 +118,107 @@ Status FilterMap::generate_nested_filter_map(const std::vector& rep_lev return Status::OK(); } +Status ColumnSelectVector::init(const std::vector& run_length_null_map, size_t num_values, + NullMap* null_map, FilterMap* filter_map, size_t filter_map_index, + const std::unordered_set* skipped_indices) { + _num_values = num_values; + _num_nulls = 0; + _read_index = 0; + size_t map_index = 0; + bool is_null = false; + _has_filter = filter_map->has_filter(); + + if (filter_map->has_filter()) { + // No run length null map is generated when _filter_all = true + // DCHECK(!filter_map->filter_all()); + _data_map.resize(num_values); + for (auto& run_length : run_length_null_map) { + if (is_null) { + _num_nulls += run_length; + for (int i = 0; i < run_length; ++i) { + _data_map[map_index++] = FILTERED_NULL; + } + } else { + for (int i = 0; i < run_length; ++i) { + _data_map[map_index++] = FILTERED_CONTENT; + } + } + is_null = !is_null; + } + + size_t num_read = 0; + size_t i = 0; + size_t valid_count = 0; + + while (valid_count < num_values) { + DCHECK_LT(filter_map_index + i, filter_map->filter_map_size()); + + if (skipped_indices != nullptr && skipped_indices->count(filter_map_index + i) > 0) { + ++i; + continue; + } + + if (filter_map->filter_map_data()[filter_map_index + i]) { + _data_map[valid_count] = + _data_map[valid_count] == FILTERED_NULL ? NULL_DATA : CONTENT; + num_read++; + } + ++valid_count; + ++i; + } + + _num_filtered = num_values - num_read; + + if (null_map != nullptr && num_read > 0) { + NullMap& map_data_column = *null_map; + auto null_map_index = map_data_column.size(); + map_data_column.resize(null_map_index + num_read); + + if (_num_nulls == 0) { + memset(map_data_column.data() + null_map_index, 0, num_read); + } else if (_num_nulls == num_values) { + memset(map_data_column.data() + null_map_index, 1, num_read); + } else { + for (i = 0; i < num_values; ++i) { + if (_data_map[i] == CONTENT) { + map_data_column[null_map_index++] = (UInt8) false; + } else if (_data_map[i] == NULL_DATA) { + map_data_column[null_map_index++] = (UInt8) true; + } + } + } + } + } else { + _num_filtered = 0; + _run_length_null_map = &run_length_null_map; + if (null_map != nullptr) { + NullMap& map_data_column = *null_map; + auto null_map_index = map_data_column.size(); + map_data_column.resize(null_map_index + num_values); + + for (auto& run_length : run_length_null_map) { + if (is_null) { + memset(map_data_column.data() + null_map_index, 1, run_length); + null_map_index += run_length; + _num_nulls += run_length; + } else { + memset(map_data_column.data() + null_map_index, 0, run_length); + null_map_index += run_length; + } + is_null = !is_null; + } + } else { + for (auto& run_length : run_length_null_map) { + if (is_null) { + _num_nulls += run_length; + } + is_null = !is_null; + } + } + } + return Status::OK(); +} + ParsedVersion::ParsedVersion(std::string application, std::optional version, std::optional app_build_hash) : _application(std::move(application)), diff --git a/be/src/vec/exec/format/parquet/parquet_common.h b/be/src/vec/exec/format/parquet/parquet_common.h index b39faf1c107371..fe5139ded7a812 100644 --- a/be/src/vec/exec/format/parquet/parquet_common.h +++ b/be/src/vec/exec/format/parquet/parquet_common.h @@ -104,105 +104,7 @@ class ColumnSelectVector { Status init(const std::vector& run_length_null_map, size_t num_values, NullMap* null_map, FilterMap* filter_map, size_t filter_map_index, - const std::unordered_set* skipped_indices = nullptr) { - _num_values = num_values; - _num_nulls = 0; - _read_index = 0; - size_t map_index = 0; - bool is_null = false; - _has_filter = filter_map->has_filter(); - - if (filter_map->has_filter()) { - // No run length null map is generated when _filter_all = true - DCHECK(!filter_map->filter_all()); - _data_map.resize(num_values); - for (auto& run_length : run_length_null_map) { - if (is_null) { - _num_nulls += run_length; - for (int i = 0; i < run_length; ++i) { - _data_map[map_index++] = FILTERED_NULL; - } - } else { - for (int i = 0; i < run_length; ++i) { - _data_map[map_index++] = FILTERED_CONTENT; - } - } - is_null = !is_null; - } - - size_t num_read = 0; - size_t i = 0; - size_t valid_count = 0; - - while (valid_count < num_values) { - DCHECK_LT(filter_map_index + i, filter_map->filter_map_size()); - - if (skipped_indices != nullptr && - skipped_indices->count(filter_map_index + i) > 0) { - ++i; - continue; - } - - if (filter_map->filter_map_data()[filter_map_index + i]) { - _data_map[valid_count] = - _data_map[valid_count] == FILTERED_NULL ? NULL_DATA : CONTENT; - num_read++; - } - ++valid_count; - ++i; - } - - _num_filtered = num_values - num_read; - - if (null_map != nullptr && num_read > 0) { - NullMap& map_data_column = *null_map; - auto null_map_index = map_data_column.size(); - map_data_column.resize(null_map_index + num_read); - - if (_num_nulls == 0) { - memset(map_data_column.data() + null_map_index, 0, num_read); - } else if (_num_nulls == num_values) { - memset(map_data_column.data() + null_map_index, 1, num_read); - } else { - for (i = 0; i < num_values; ++i) { - if (_data_map[i] == CONTENT) { - map_data_column[null_map_index++] = (UInt8) false; - } else if (_data_map[i] == NULL_DATA) { - map_data_column[null_map_index++] = (UInt8) true; - } - } - } - } - } else { - _num_filtered = 0; - _run_length_null_map = &run_length_null_map; - if (null_map != nullptr) { - NullMap& map_data_column = *null_map; - auto null_map_index = map_data_column.size(); - map_data_column.resize(null_map_index + num_values); - - for (auto& run_length : run_length_null_map) { - if (is_null) { - memset(map_data_column.data() + null_map_index, 1, run_length); - null_map_index += run_length; - _num_nulls += run_length; - } else { - memset(map_data_column.data() + null_map_index, 0, run_length); - null_map_index += run_length; - } - is_null = !is_null; - } - } else { - for (auto& run_length : run_length_null_map) { - if (is_null) { - _num_nulls += run_length; - } - is_null = !is_null; - } - } - } - return Status::OK(); - } + const std::unordered_set* skipped_indices = nullptr); size_t num_values() const { return _num_values; } diff --git a/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.cpp index 5bcf4abf91981f..74398e6da4aa5b 100644 --- a/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.cpp +++ b/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.cpp @@ -47,77 +47,110 @@ struct IOContext; namespace doris::vectorized { #include "common/compile_check_begin.h" -ColumnChunkReader::ColumnChunkReader(io::BufferedStreamReader* reader, - tparquet::ColumnChunk* column_chunk, FieldSchema* field_schema, - const tparquet::OffsetIndex* offset_index, - cctz::time_zone* ctz, io::IOContext* io_ctx) + +template +ColumnChunkReader::ColumnChunkReader( + io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk, + FieldSchema* field_schema, const tparquet::OffsetIndex* offset_index, size_t total_rows, + io::IOContext* io_ctx) : _field_schema(field_schema), _max_rep_level(field_schema->repetition_level), _max_def_level(field_schema->definition_level), _stream_reader(reader), _metadata(column_chunk->meta_data), _offset_index(offset_index), - // _ctz(ctz), + _total_rows(total_rows), _io_ctx(io_ctx) {} -Status ColumnChunkReader::init() { +template +Status ColumnChunkReader::init() { size_t start_offset = has_dict_page(_metadata) ? _metadata.dictionary_page_offset : _metadata.data_page_offset; size_t chunk_size = _metadata.total_compressed_size; // create page reader - _page_reader = create_page_reader(_stream_reader, _io_ctx, start_offset, chunk_size, - _metadata.num_values, _offset_index); + _page_reader = create_page_reader( + _stream_reader, _io_ctx, start_offset, chunk_size, _total_rows, _offset_index); // get the block compression codec RETURN_IF_ERROR(get_block_compression_codec(_metadata.codec, &_block_compress_codec)); - if (has_dict_page(_metadata)) { - // seek to the directory page - _page_reader->seek_to_page(_metadata.dictionary_page_offset); - // Parse dictionary data when reading - // RETURN_IF_ERROR(_page_reader->next_page_header()); - // RETURN_IF_ERROR(_decode_dict_page()); - } else { - // seek to the first data page - _page_reader->seek_to_page(_metadata.data_page_offset); - } _state = INITIALIZED; + RETURN_IF_ERROR(_parse_first_page_header()); return Status::OK(); } -Status ColumnChunkReader::next_page() { - if (_state == HEADER_PARSED) { - return Status::OK(); - } - if (UNLIKELY(_state == NOT_INIT)) { - return Status::Corruption("Should initialize chunk reader"); +template +Status ColumnChunkReader::skip_nested_values( + const std::vector& def_levels) { + size_t no_value_cnt = 0; + size_t value_cnt = 0; + + for (size_t idx = 0; idx < def_levels.size(); idx++) { + level_t def_level = def_levels[idx]; + if (IN_COLLECTION && def_level < _field_schema->repeated_parent_def_level) { + no_value_cnt++; + } else if (def_level < _field_schema->definition_level) { + no_value_cnt++; + } else { + value_cnt++; + } } - if (UNLIKELY(_remaining_num_values != 0)) { - return Status::Corruption("Should skip current page"); + + RETURN_IF_ERROR(skip_values(value_cnt, true)); + RETURN_IF_ERROR(skip_values(no_value_cnt, false)); + return Status::OK(); +} + +template +Status ColumnChunkReader::_parse_first_page_header() { + RETURN_IF_ERROR(parse_page_header()); + + const tparquet::PageHeader* header = nullptr; + RETURN_IF_ERROR(_page_reader->get_page_header(header)); + if (header->type == tparquet::PageType::DICTIONARY_PAGE) { + // the first page maybe directory page even if _metadata.__isset.dictionary_page_offset == false, + // so we should parse the directory page in next_page() + RETURN_IF_ERROR(_decode_dict_page()); + // parse the real first data page + RETURN_IF_ERROR(_page_reader->dict_next_page()); + _state = INITIALIZED; } - RETURN_IF_ERROR(_page_reader->next_page_header()); - - if (!_dict_checked) { - _dict_checked = true; - const tparquet::PageHeader* header; - RETURN_IF_ERROR(_page_reader->get_page_header(header)); - if (header->type == tparquet::PageType::DICTIONARY_PAGE) { - // the first page maybe directory page even if _metadata.__isset.dictionary_page_offset == false, - // so we should parse the directory page in next_page() - RETURN_IF_ERROR(_decode_dict_page()); - // parse the real first data page - return next_page(); - } + return Status::OK(); +} + +template +Status ColumnChunkReader::parse_page_header() { + if (_state == HEADER_PARSED || _state == DATA_LOADED) { + return Status::OK(); } + RETURN_IF_ERROR(_page_reader->parse_page_header()); - RETURN_IF_ERROR(_page_reader->get_num_values(_remaining_num_values)); - _chunk_parsed_values += _remaining_num_values; + const tparquet::PageHeader* header = nullptr; + ; + RETURN_IF_ERROR(_page_reader->get_page_header(header)); + int32_t page_num_values = _page_reader->is_header_v2() ? header->data_page_header_v2.num_values + : header->data_page_header.num_values; + _remaining_rep_nums = page_num_values; + _remaining_def_nums = page_num_values; + _remaining_num_values = page_num_values; + + // no offset will parse all header. + if constexpr (OFFSET_INDEX == false) { + _chunk_parsed_values += _remaining_num_values; + } _state = HEADER_PARSED; + return Status::OK(); +} +template +Status ColumnChunkReader::next_page() { + _state = INITIALIZED; + RETURN_IF_ERROR(_page_reader->next_page()); return Status::OK(); } -void ColumnChunkReader::_get_uncompressed_levels(const tparquet::DataPageHeaderV2& page_v2, - Slice& page_data) { +template +void ColumnChunkReader::_get_uncompressed_levels( + const tparquet::DataPageHeaderV2& page_v2, Slice& page_data) { int32_t rl = page_v2.repetition_levels_byte_length; int32_t dl = page_v2.definition_levels_byte_length; _v2_rep_levels = Slice(page_data.data, rl); @@ -126,12 +159,16 @@ void ColumnChunkReader::_get_uncompressed_levels(const tparquet::DataPageHeaderV page_data.size -= dl + rl; } -Status ColumnChunkReader::load_page_data() { - // TODO: remove checking HEADER_PARSED or change name +template +Status ColumnChunkReader::load_page_data() { + if (_state == DATA_LOADED) { + return Status::OK(); + } if (UNLIKELY(_state != HEADER_PARSED)) { return Status::Corruption("Should parse page header"); } - const tparquet::PageHeader* header; + + const tparquet::PageHeader* header = nullptr; RETURN_IF_ERROR(_page_reader->get_page_header(header)); int32_t uncompressed_size = header->uncompressed_page_size; @@ -171,22 +208,22 @@ Status ColumnChunkReader::load_page_data() { SCOPED_RAW_TIMER(&_statistics.decode_level_time); if (header->__isset.data_page_header_v2) { RETURN_IF_ERROR(_rep_level_decoder.init_v2(_v2_rep_levels, _max_rep_level, - _remaining_num_values)); + _remaining_rep_nums)); } else { RETURN_IF_ERROR(_rep_level_decoder.init( &_page_data, header->data_page_header.repetition_level_encoding, _max_rep_level, - _remaining_num_values)); + _remaining_rep_nums)); } } if (_max_def_level > 0) { SCOPED_RAW_TIMER(&_statistics.decode_level_time); if (header->__isset.data_page_header_v2) { RETURN_IF_ERROR(_def_level_decoder.init_v2(_v2_def_levels, _max_def_level, - _remaining_num_values)); + _remaining_def_nums)); } else { RETURN_IF_ERROR(_def_level_decoder.init( &_page_data, header->data_page_header.definition_level_encoding, _max_def_level, - _remaining_num_values)); + _remaining_def_nums)); } } auto encoding = header->__isset.data_page_header_v2 ? header->data_page_header_v2.encoding @@ -204,8 +241,6 @@ Status ColumnChunkReader::load_page_data() { RETURN_IF_ERROR(Decoder::get_decoder(_metadata.type, encoding, page_decoder)); // Set type length page_decoder->set_type_length(_get_type_length()); - // Initialize the time convert context - // page_decoder->init(_field_schema, _ctz); _decoders[static_cast(encoding)] = std::move(page_decoder); _page_decoder = _decoders[static_cast(encoding)].get(); } @@ -216,8 +251,9 @@ Status ColumnChunkReader::load_page_data() { return Status::OK(); } -Status ColumnChunkReader::_decode_dict_page() { - const tparquet::PageHeader* header; +template +Status ColumnChunkReader::_decode_dict_page() { + const tparquet::PageHeader* header = nullptr; RETURN_IF_ERROR(_page_reader->get_page_header(header)); DCHECK_EQ(tparquet::PageType::DICTIONARY_PAGE, header->type); SCOPED_RAW_TIMER(&_statistics.decode_dict_time); @@ -253,8 +289,6 @@ Status ColumnChunkReader::_decode_dict_page() { Decoder::get_decoder(_metadata.type, tparquet::Encoding::RLE_DICTIONARY, page_decoder)); // Set type length page_decoder->set_type_length(_get_type_length()); - // Initialize the time convert context - // page_decoder->init(_field_schema, _ctz); // Set the dictionary data RETURN_IF_ERROR(page_decoder->set_dict(dict_data, uncompressed_size, header->dictionary_page_header.num_values)); @@ -264,14 +298,17 @@ Status ColumnChunkReader::_decode_dict_page() { return Status::OK(); } -void ColumnChunkReader::_reserve_decompress_buf(size_t size) { +template +void ColumnChunkReader::_reserve_decompress_buf(size_t size) { if (size > _decompress_buf_size) { _decompress_buf_size = BitUtil::next_power_of_two(size); _decompress_buf = make_unique_buffer(_decompress_buf_size); } } -Status ColumnChunkReader::skip_values(size_t num_values, bool skip_data) { +template +Status ColumnChunkReader::skip_values(size_t num_values, + bool skip_data) { if (UNLIKELY(_remaining_num_values < num_values)) { return Status::IOError("Skip too many values in current page. {} vs. {}", _remaining_num_values, num_values); @@ -285,24 +322,10 @@ Status ColumnChunkReader::skip_values(size_t num_values, bool skip_data) { } } -void ColumnChunkReader::insert_null_values(MutableColumnPtr& doris_column, size_t num_values) { - SCOPED_RAW_TIMER(&_statistics.decode_value_time); - doris_column->insert_many_defaults(num_values); - _remaining_num_values -= num_values; -} - -size_t ColumnChunkReader::get_rep_levels(level_t* levels, size_t n) { - DCHECK_GT(_max_rep_level, 0); - return _rep_level_decoder.get_levels(levels, n); -} - -size_t ColumnChunkReader::get_def_levels(level_t* levels, size_t n) { - DCHECK_GT(_max_def_level, 0); - return _def_level_decoder.get_levels(levels, n); -} - -Status ColumnChunkReader::decode_values(MutableColumnPtr& doris_column, DataTypePtr& data_type, - ColumnSelectVector& select_vector, bool is_dict_filter) { +template +Status ColumnChunkReader::decode_values( + MutableColumnPtr& doris_column, DataTypePtr& data_type, ColumnSelectVector& select_vector, + bool is_dict_filter) { if (select_vector.num_values() == 0) { return Status::OK(); } @@ -317,7 +340,159 @@ Status ColumnChunkReader::decode_values(MutableColumnPtr& doris_column, DataType return _page_decoder->decode_values(doris_column, data_type, select_vector, is_dict_filter); } -int32_t ColumnChunkReader::_get_type_length() { +template +Status ColumnChunkReader::seek_to_nested_row(size_t left_row) { + if constexpr (OFFSET_INDEX) { + while (true) { + if (_page_reader->start_row() <= left_row && left_row < _page_reader->end_row()) { + break; + } else if (has_next_page()) { + RETURN_IF_ERROR(next_page()); + _current_row = _page_reader->start_row(); + } else [[unlikely]] { + return Status::InternalError("no match seek row {}, current row {}", left_row, + _current_row); + } + }; + + RETURN_IF_ERROR(parse_page_header()); + RETURN_IF_ERROR(load_page_data()); + RETURN_IF_ERROR(_skip_nested_rows_in_page(left_row - _current_row)); + _current_row = left_row; + } else { + while (true) { + RETURN_IF_ERROR(parse_page_header()); + if (_page_reader->is_header_v2()) { + if (_page_reader->start_row() <= left_row && left_row < _page_reader->end_row()) { + RETURN_IF_ERROR(load_page_data()); + // this page contain this row. + RETURN_IF_ERROR(_skip_nested_rows_in_page(left_row - _current_row)); + _current_row = left_row; + break; + } + + _current_row = _page_reader->end_row(); + if (has_next_page()) [[likely]] { + RETURN_IF_ERROR(next_page()); + } else { + return Status::InternalError("no match seek row {}, current row {}", left_row, + _current_row); + } + } else { + RETURN_IF_ERROR(load_page_data()); + std::vector rep_levels; + std::vector def_levels; + bool cross_page = false; + + size_t result_rows = 0; + RETURN_IF_ERROR(load_page_nested_rows(rep_levels, left_row - _current_row, + &result_rows, &cross_page)); + RETURN_IF_ERROR(fill_def(def_levels)); + RETURN_IF_ERROR(skip_nested_values(def_levels)); + bool need_load_next_page = true; + while (cross_page) { + need_load_next_page = false; + rep_levels.clear(); + def_levels.clear(); + RETURN_IF_ERROR(load_cross_page_nested_row(rep_levels, &cross_page)); + RETURN_IF_ERROR(fill_def(def_levels)); + RETURN_IF_ERROR(skip_nested_values(def_levels)); + } + if (left_row == _current_row) { + break; + } + if (need_load_next_page) { + if (has_next_page()) [[likely]] { + RETURN_IF_ERROR(next_page()); + } else { + return Status::InternalError("no match seek row {}, current row {}", + left_row, _current_row); + } + } + } + }; + } + + return Status::OK(); +} + +template +Status ColumnChunkReader::_skip_nested_rows_in_page(size_t num_rows) { + if (num_rows == 0) { + return Status::OK(); + } + + std::vector rep_levels; + std::vector def_levels; + + bool cross_page = false; + size_t result_rows = 0; + RETURN_IF_ERROR(load_page_nested_rows(rep_levels, num_rows, &result_rows, &cross_page)); + RETURN_IF_ERROR(fill_def(def_levels)); + RETURN_IF_ERROR(skip_nested_values(def_levels)); + DCHECK(cross_page == false); + if (num_rows != result_rows) [[unlikely]] { + return Status::InternalError("no match skip rows, expect {} vs. real {}", num_rows, + result_rows); + } + return Status::OK(); +} + +template +Status ColumnChunkReader::load_page_nested_rows( + std::vector& rep_levels, size_t max_rows, size_t* result_rows, bool* cross_page) { + if (_state != DATA_LOADED) [[unlikely]] { + return Status::IOError("Should load page data first to load nested rows"); + } + *cross_page = false; + *result_rows = 0; + rep_levels.reserve(rep_levels.size() + _remaining_rep_nums); + while (_remaining_rep_nums) { + level_t rep_level = _rep_level_decoder.get_next(); + if (rep_level == 0) { // rep_level 0 indicates start of new row + if (*result_rows == max_rows) { // this page contain max_rows, page no end. + _current_row += max_rows; + _rep_level_decoder.rewind_one(); + return Status::OK(); + } + (*result_rows)++; + } + _remaining_rep_nums--; + rep_levels.emplace_back(rep_level); + } + _current_row += *result_rows; + + auto need_check_cross_page = [&]() -> bool { + return !OFFSET_INDEX && _remaining_rep_nums == 0 && !_page_reader->is_header_v2() && + has_next_page(); + }; + *cross_page = need_check_cross_page(); + return Status::OK(); +}; + +template +Status ColumnChunkReader::load_cross_page_nested_row( + std::vector& rep_levels, bool* cross_page) { + RETURN_IF_ERROR(next_page()); + RETURN_IF_ERROR(parse_page_header()); + RETURN_IF_ERROR(load_page_data()); + + *cross_page = has_next_page(); + while (_remaining_rep_nums) { + level_t rep_level = _rep_level_decoder.get_next(); + if (rep_level == 0) { // rep_level 0 indicates start of new row + *cross_page = false; + _rep_level_decoder.rewind_one(); + break; + } + _remaining_rep_nums--; + rep_levels.emplace_back(rep_level); + } + return Status::OK(); +} + +template +int32_t ColumnChunkReader::_get_type_length() { switch (_field_schema->physical_type) { case tparquet::Type::INT32: [[fallthrough]]; @@ -355,5 +530,10 @@ bool has_dict_page(const tparquet::ColumnMetaData& column) { column.dictionary_page_offset < column.data_page_offset; } +template class ColumnChunkReader; +template class ColumnChunkReader; +template class ColumnChunkReader; +template class ColumnChunkReader; + #include "common/compile_check_end.h" } // namespace doris::vectorized diff --git a/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h b/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h index db0530da5974c3..4022eac702b1bd 100644 --- a/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h +++ b/be/src/vec/exec/format/parquet/vparquet_column_chunk_reader.h @@ -34,9 +34,6 @@ #include "vec/exec/format/parquet/parquet_common.h" #include "vparquet_page_reader.h" -namespace cctz { -class time_zone; -} // namespace cctz namespace doris { class BlockCompressionCodec; @@ -54,9 +51,29 @@ template class ColumnStr; using ColumnString = ColumnStr; +struct ColumnChunkReaderStatistics { + int64_t decompress_time = 0; + int64_t decompress_cnt = 0; + int64_t decode_header_time = 0; + int64_t decode_value_time = 0; + int64_t decode_dict_time = 0; + int64_t decode_level_time = 0; + int64_t skip_page_header_num = 0; + int64_t parse_page_header_num = 0; +}; + /** * Read and decode parquet column data into doris block column. - *

Usage:

+ *

Usage:

struct ColumnChunkReaderStatistics { + int64_t decompress_time = 0; + int64_t decompress_cnt = 0; + int64_t decode_header_time = 0; + int64_t decode_value_time = 0; + int64_t decode_dict_time = 0; + int64_t decode_level_time = 0; + int64_t skip_page_header_num = 0; + int64_t parse_page_header_num = 0; + }; * // Create chunk reader * ColumnChunkReader chunk_reader(BufferedStreamReader* reader, * tparquet::ColumnChunk* column_chunk, @@ -74,58 +91,27 @@ using ColumnString = ColumnStr; * chunk_reader.decode_values(slice, num_values); * } */ +template class ColumnChunkReader { public: - struct Statistics { - int64_t decompress_time = 0; - int64_t decompress_cnt = 0; - int64_t decode_header_time = 0; - int64_t decode_value_time = 0; - int64_t decode_dict_time = 0; - int64_t decode_level_time = 0; - int64_t skip_page_header_num = 0; - int64_t parse_page_header_num = 0; - }; - ColumnChunkReader(io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk, FieldSchema* field_schema, const tparquet::OffsetIndex* offset_index, - cctz::time_zone* ctz, io::IOContext* io_ctx); + size_t total_row, io::IOContext* io_ctx); ~ColumnChunkReader() = default; // Initialize chunk reader, will generate the decoder and codec. Status init(); // Whether the chunk reader has a more page to read. - bool has_next_page() const { return _chunk_parsed_values < _metadata.num_values; } - - // Deprecated - // Seek to the specific page, page_header_offset must be the start offset of the page header. - // _end_offset may exceed the actual data area, so we can only use the number of parsed values - // to determine whether there are remaining pages to read. That's to say we can't use the - // PageLocation in parquet metadata to seek to the specified page. We should call next_page() - // and skip_page() to skip pages one by one. - // todo: change this interface to seek_to_page(int64_t page_header_offset, size_t num_parsed_values) - // and set _chunk_parsed_values = num_parsed_values - // [[deprecated]] - void seek_to_page(int64_t page_header_offset) { - _remaining_num_values = 0; - _page_reader->seek_to_page(page_header_offset); - _state = INITIALIZED; - } - - // Seek to next page. Only read and parse the page header. - Status next_page(); - - // Skip current page(will not read and parse) if the page is filtered by predicates. - Status skip_page() { - Status res = Status::OK(); - _remaining_num_values = 0; - if (_state == HEADER_PARSED) { - res = _page_reader->skip_page(); + bool has_next_page() const { + if constexpr (OFFSET_INDEX) { + return _page_reader->has_next_page(); + } else { + // no offset need parse all page header. + return _chunk_parsed_values < _metadata.num_values; } - _state = PAGE_SKIPPED; - return res; } + // Skip some values(will not read and parse) in current page if the values are filtered by predicates. // when skip_data = false, the underlying decoder will not skip data, // only used when maintaining the consistency of _remaining_num_values. @@ -142,16 +128,6 @@ class ColumnChunkReader { } // The remaining number of values in current page(including null values). Decreased when reading or skipping. uint32_t remaining_num_values() const { return _remaining_num_values; } - // null values are generated from definition levels - // the caller should maintain the consistency after analyzing null values from definition levels. - void insert_null_values(MutableColumnPtr& doris_column, size_t num_values); - // Get the raw data of current page. - Slice& get_page_data() { return _page_data; } - - // Get the repetition levels - size_t get_rep_levels(level_t* levels, size_t n); - // Get the definition levels - size_t get_def_levels(level_t* levels, size_t n); // Decode values in current page into doris column. Status decode_values(MutableColumnPtr& doris_column, DataTypePtr& data_type, @@ -170,7 +146,7 @@ class ColumnChunkReader { // Get page decoder Decoder* get_page_decoder() { return _page_decoder; } - Statistics& statistics() { + ColumnChunkReaderStatistics& statistics() { _statistics.decode_header_time = _page_reader->statistics().decode_header_time; _statistics.skip_page_header_num = _page_reader->statistics().skip_page_header_num; _statistics.parse_page_header_num = _page_reader->statistics().parse_page_header_num; @@ -187,33 +163,70 @@ class ColumnChunkReader { ->convert_dict_column_to_string_column(dict_column); } + size_t page_start_row() const { return _page_reader->start_row(); } + + size_t page_end_row() const { return _page_reader->end_row(); } + + Status parse_page_header(); + Status next_page(); + + Status seek_to_nested_row(size_t left_row); + Status skip_nested_values(const std::vector& def_levels); + Status fill_def(std::vector& def_values) { + auto before_sz = def_values.size(); + auto append_sz = _remaining_def_nums - _remaining_rep_nums; + def_values.resize(before_sz + append_sz, 0); + if (max_def_level() != 0) { + auto ptr = def_values.data() + before_sz; + _def_level_decoder.get_levels(ptr, append_sz); + } + _remaining_def_nums -= append_sz; + return Status::OK(); + } + + Status load_page_nested_rows(std::vector& rep_levels, size_t max_rows, + size_t* result_rows, bool* cross_page); + Status load_cross_page_nested_row(std::vector& rep_levels, bool* cross_page); + private: enum ColumnChunkReaderState { NOT_INIT, INITIALIZED, HEADER_PARSED, DATA_LOADED, PAGE_SKIPPED }; + // for check dict page. + Status _parse_first_page_header(); Status _decode_dict_page(); + void _reserve_decompress_buf(size_t size); int32_t _get_type_length(); + void _get_uncompressed_levels(const tparquet::DataPageHeaderV2& page_v2, Slice& page_data); + Status _skip_nested_rows_in_page(size_t num_rows); ColumnChunkReaderState _state = NOT_INIT; FieldSchema* _field_schema = nullptr; - level_t _max_rep_level; - level_t _max_def_level; - tparquet::LogicalType _parquet_logical_type; + const level_t _max_rep_level; + const level_t _max_def_level; io::BufferedStreamReader* _stream_reader = nullptr; tparquet::ColumnMetaData _metadata; - const tparquet::OffsetIndex* _offset_index; - // cctz::time_zone* _ctz; + const tparquet::OffsetIndex* _offset_index = nullptr; + size_t _current_row = 0; + size_t _total_rows = 0; io::IOContext* _io_ctx = nullptr; - std::unique_ptr _page_reader; + std::unique_ptr> _page_reader; BlockCompressionCodec* _block_compress_codec = nullptr; LevelDecoder _rep_level_decoder; LevelDecoder _def_level_decoder; size_t _chunk_parsed_values = 0; + // this page remaining rep/def nums + // if max_rep_level = 0 / max_def_level = 0, this value retail hava value. + uint32_t _remaining_rep_nums = 0; + uint32_t _remaining_def_nums = 0; + // this page remaining values to be processed (for read/skip). + // need parse this page header. uint32_t _remaining_num_values = 0; + Slice _page_data; DorisUniqueBufferPtr _decompress_buf; size_t _decompress_buf_size = 0; @@ -225,7 +238,7 @@ class ColumnChunkReader { // Map: encoding -> Decoder // Plain or Dictionary encoding. If the dictionary grows too big, the encoding will fall back to the plain encoding std::unordered_map> _decoders; - Statistics _statistics; + ColumnChunkReaderStatistics _statistics; }; #include "common/compile_check_end.h" diff --git a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp index 215f3ec26194fa..51840dca0db9bf 100644 --- a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp +++ b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp @@ -103,30 +103,27 @@ static void fill_array_offset(FieldSchema* field, ColumnArray::Offsets64& offset } } -Status ParquetColumnReader::create(io::FileReaderSPtr file, FieldSchema* field, - const tparquet::RowGroup& row_group, - const std::vector& row_ranges, cctz::time_zone* ctz, - io::IOContext* io_ctx, - std::unique_ptr& reader, - size_t max_buf_size, const tparquet::OffsetIndex* offset_index) { +Status ParquetColumnReader::create( + io::FileReaderSPtr file, FieldSchema* field, const tparquet::RowGroup& row_group, + const std::vector& row_ranges, cctz::time_zone* ctz, io::IOContext* io_ctx, + std::unique_ptr& reader, size_t max_buf_size, + std::unordered_map& col_offsets, bool in_collection) { + size_t total_rows = row_group.num_rows; if (field->data_type->get_primitive_type() == TYPE_ARRAY) { std::unique_ptr element_reader; RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx, - element_reader, max_buf_size)); - element_reader->set_nested_column(); - auto array_reader = ArrayColumnReader::create_unique(row_ranges, ctz, io_ctx); + element_reader, max_buf_size, col_offsets, true)); + auto array_reader = ArrayColumnReader::create_unique(row_ranges, total_rows, ctz, io_ctx); RETURN_IF_ERROR(array_reader->init(std::move(element_reader), field)); reader.reset(array_reader.release()); } else if (field->data_type->get_primitive_type() == TYPE_MAP) { std::unique_ptr key_reader; std::unique_ptr value_reader; RETURN_IF_ERROR(create(file, &field->children[0].children[0], row_group, row_ranges, ctz, - io_ctx, key_reader, max_buf_size)); + io_ctx, key_reader, max_buf_size, col_offsets, true)); RETURN_IF_ERROR(create(file, &field->children[0].children[1], row_group, row_ranges, ctz, - io_ctx, value_reader, max_buf_size)); - key_reader->set_nested_column(); - value_reader->set_nested_column(); - auto map_reader = MapColumnReader::create_unique(row_ranges, ctz, io_ctx); + io_ctx, value_reader, max_buf_size, col_offsets, true)); + auto map_reader = MapColumnReader::create_unique(row_ranges, total_rows, ctz, io_ctx); RETURN_IF_ERROR(map_reader->init(std::move(key_reader), std::move(value_reader), field)); reader.reset(map_reader.release()); } else if (field->data_type->get_primitive_type() == TYPE_STRUCT) { @@ -135,29 +132,55 @@ Status ParquetColumnReader::create(io::FileReaderSPtr file, FieldSchema* field, for (int i = 0; i < field->children.size(); ++i) { std::unique_ptr child_reader; RETURN_IF_ERROR(create(file, &field->children[i], row_group, row_ranges, ctz, io_ctx, - child_reader, max_buf_size)); - child_reader->set_nested_column(); + child_reader, max_buf_size, col_offsets, in_collection)); child_readers[field->children[i].name] = std::move(child_reader); } - auto struct_reader = StructColumnReader::create_unique(row_ranges, ctz, io_ctx); + auto struct_reader = StructColumnReader::create_unique(row_ranges, total_rows, ctz, io_ctx); RETURN_IF_ERROR(struct_reader->init(std::move(child_readers), field)); reader.reset(struct_reader.release()); } else { - const tparquet::ColumnChunk& chunk = row_group.columns[field->physical_column_index]; - auto scalar_reader = - ScalarColumnReader::create_unique(row_ranges, chunk, offset_index, ctz, io_ctx); - RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size)); - reader.reset(scalar_reader.release()); + auto physical_index = field->physical_column_index; + const tparquet::OffsetIndex* offset_index = + col_offsets.find(physical_index) != col_offsets.end() ? &col_offsets[physical_index] + : nullptr; + + const tparquet::ColumnChunk& chunk = row_group.columns[physical_index]; + + if (in_collection) { + if (offset_index == nullptr) { + auto scalar_reader = ScalarColumnReader::create_unique( + row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); + + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size)); + reader.reset(scalar_reader.release()); + } else { + auto scalar_reader = ScalarColumnReader::create_unique( + row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); + + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size)); + reader.reset(scalar_reader.release()); + } + } else { + if (offset_index == nullptr) { + auto scalar_reader = ScalarColumnReader::create_unique( + row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); + + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size)); + reader.reset(scalar_reader.release()); + } else { + auto scalar_reader = ScalarColumnReader::create_unique( + row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); + + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size)); + reader.reset(scalar_reader.release()); + } + } } return Status::OK(); } void ParquetColumnReader::_generate_read_ranges(int64_t start_index, int64_t end_index, std::list& read_ranges) { - if (_nested_column) { - read_ranges.emplace_back(start_index, end_index); - return; - } int index = _row_range_index; while (index < _row_ranges.size()) { const RowRange& read_range = _row_ranges[index]; @@ -176,7 +199,10 @@ void ParquetColumnReader::_generate_read_ranges(int64_t start_index, int64_t end } } -Status ScalarColumnReader::init(io::FileReaderSPtr file, FieldSchema* field, size_t max_buf_size) { +template +Status ScalarColumnReader::init(io::FileReaderSPtr file, + FieldSchema* field, + size_t max_buf_size) { _field_schema = field; auto& chunk_meta = _chunk_meta.meta_data; int64_t chunk_start = has_dict_page(chunk_meta) ? chunk_meta.dictionary_page_offset @@ -189,13 +215,14 @@ Status ScalarColumnReader::init(io::FileReaderSPtr file, FieldSchema* field, siz } _stream_reader = std::make_unique(file, chunk_start, chunk_len, prefetch_buffer_size); - _chunk_reader = std::make_unique(_stream_reader.get(), &_chunk_meta, field, - _offset_index, _ctz, _io_ctx); + _chunk_reader = std::make_unique>( + _stream_reader.get(), &_chunk_meta, field, _offset_index, _total_rows, _io_ctx); RETURN_IF_ERROR(_chunk_reader->init()); return Status::OK(); } -Status ScalarColumnReader::_skip_values(size_t num_values) { +template +Status ScalarColumnReader::_skip_values(size_t num_values) { if (num_values == 0) { return Status::OK(); } @@ -218,7 +245,7 @@ Status ScalarColumnReader::_skip_values(size_t num_values) { LOG(WARNING) << ss.str(); return Status::InternalError("Failed to decode definition level."); } - if (def_level == 0) { + if (def_level < _field_schema->definition_level) { null_size += loop_skip; } else { nonnull_size += loop_skip; @@ -237,9 +264,12 @@ Status ScalarColumnReader::_skip_values(size_t num_values) { return Status::OK(); } -Status ScalarColumnReader::_read_values(size_t num_values, ColumnPtr& doris_column, - DataTypePtr& type, FilterMap& filter_map, - bool is_dict_filter) { +template +Status ScalarColumnReader::_read_values(size_t num_values, + ColumnPtr& doris_column, + DataTypePtr& type, + FilterMap& filter_map, + bool is_dict_filter) { if (num_values == 0) { return Status::OK(); } @@ -271,7 +301,11 @@ Status ScalarColumnReader::_read_values(size_t num_values, ColumnPtr& doris_colu LOG(WARNING) << ss.str(); return Status::InternalError("Failed to decode definition level."); } - bool is_null = def_level == 0; + + for (int i = 0; i < loop_read; i++) { + _def_levels.emplace_back(def_level); + } + bool is_null = def_level < _field_schema->definition_level; if (!(prev_is_null ^ is_null)) { null_map.emplace_back(0); } @@ -285,11 +319,14 @@ Status ScalarColumnReader::_read_values(size_t num_values, ColumnPtr& doris_colu prev_is_null = is_null; has_read += loop_read; } + } else { + _def_levels.resize(_def_levels.size() + num_values, 0); } } else { if (_chunk_reader->max_def_level() > 0) { return Status::Corruption("Not nullable column has null values in parquet file"); } + _def_levels.resize(_def_levels.size() + num_values, 0); data_column = doris_column->assume_mutable(); } if (null_map.size() == 0) { @@ -316,93 +353,15 @@ Status ScalarColumnReader::_read_values(size_t num_values, ColumnPtr& doris_colu * A row of complex type may be stored across two(or more) pages, and the parameter `align_rows` indicates that * whether the reader should read the remaining value of the last row in previous page. */ -Status ScalarColumnReader::_read_nested_column(ColumnPtr& doris_column, DataTypePtr& type, - FilterMap& filter_map, size_t batch_size, - size_t* read_rows, bool* eof, bool is_dict_filter, - bool align_rows) { - std::unique_ptr nested_filter_map; - - FilterMap* current_filter_map = &filter_map; - size_t origin_size = 0; - if (align_rows) { - origin_size = _rep_levels.size(); - // just read the remaining values of the last row in previous page, - // so there's no a new row should be read. - batch_size = 0; - /* - * Since the function is repeatedly called to fetch data for the batch size, - * it causes `_rep_levels.resize(0); _def_levels.resize(0);`, resulting in the - * definition and repetition levels of the reader only containing the latter - * part of the batch (i.e., missing some parts). Therefore, when using the - * definition and repetition levels to fill the null_map for structs and maps, - * the function should not be called multiple times before filling. - * todo: - * We may need to consider reading the entire batch of data at once, as this approach - * would be more user-friendly in terms of function usage. However, we must consider that if the - * data spans multiple pages, memory usage may increase significantly. - */ - } else { - _rep_levels.resize(0); - _def_levels.resize(0); - if (_nested_filter_map_data) { - _nested_filter_map_data->resize(0); - } - } - size_t parsed_rows = 0; - size_t remaining_values = _chunk_reader->remaining_num_values(); - bool has_rep_level = _chunk_reader->max_rep_level() > 0; - bool has_def_level = _chunk_reader->max_def_level() > 0; - - // Handle repetition levels (indicates nesting structure) - if (has_rep_level) { - LevelDecoder& rep_decoder = _chunk_reader->rep_level_decoder(); - // Read repetition levels until batch is full or no more values - while (parsed_rows <= batch_size && remaining_values > 0) { - level_t rep_level = rep_decoder.get_next(); - if (rep_level == 0) { // rep_level 0 indicates start of new row - if (parsed_rows == batch_size) { - rep_decoder.rewind_one(); - break; - } - parsed_rows++; - } - _rep_levels.emplace_back(rep_level); - remaining_values--; - } - - // Generate nested filter map - if (filter_map.has_filter() && (!filter_map.filter_all())) { - if (_nested_filter_map_data == nullptr) { - _nested_filter_map_data.reset(new std::vector()); - } - RETURN_IF_ERROR(filter_map.generate_nested_filter_map( - _rep_levels, *_nested_filter_map_data, &nested_filter_map, - &_orig_filter_map_index, origin_size)); - // Update current_filter_map to nested_filter_map - current_filter_map = nested_filter_map.get(); - } - } else if (!align_rows) { - // case : required child columns in struct type - parsed_rows = std::min(remaining_values, batch_size); - remaining_values -= parsed_rows; - _rep_levels.resize(parsed_rows, 0); - } - - // Process definition levels (indicates null values) - size_t parsed_values = _chunk_reader->remaining_num_values() - remaining_values; - _def_levels.resize(origin_size + parsed_values); - if (has_def_level) { - // if parsed_values is 0, we don't need to decode levels - if (parsed_values != 0) { - _chunk_reader->def_level_decoder().get_levels(&_def_levels[origin_size], parsed_values); - } - } else { - std::fill(_def_levels.begin() + origin_size, _def_levels.end(), 0); - } +template +Status ScalarColumnReader::_read_nested_column( + ColumnPtr& doris_column, DataTypePtr& type, FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof, bool is_dict_filter) { + _rep_levels.clear(); + _def_levels.clear(); // Handle nullable columns MutableColumnPtr data_column; - std::vector null_map; NullMap* map_data_column = nullptr; if (doris_column->is_nullable()) { SCOPED_RAW_TIMER(&_decode_null_map_time); @@ -417,138 +376,85 @@ Status ScalarColumnReader::_read_nested_column(ColumnPtr& doris_column, DataType data_column = doris_column->assume_mutable(); } - // Process definition levels to build null map - size_t has_read = origin_size; - size_t ancestor_nulls = 0; - size_t null_size = 0; - size_t nonnull_size = 0; - null_map.emplace_back(0); - bool prev_is_null = false; + std::vector null_map; std::unordered_set ancestor_null_indices; + std::vector nested_filter_map_data; - while (has_read < origin_size + parsed_values) { - level_t def_level = _def_levels[has_read++]; - size_t loop_read = 1; - while (has_read < origin_size + parsed_values && _def_levels[has_read] == def_level) { - has_read++; - loop_read++; - } - - if (def_level < _field_schema->repeated_parent_def_level) { - for (size_t i = 0; i < loop_read; i++) { - ancestor_null_indices.insert(has_read - loop_read + i); - } - ancestor_nulls += loop_read; - continue; - } - - bool is_null = def_level < _field_schema->definition_level; - if (is_null) { - null_size += loop_read; - } else { - nonnull_size += loop_read; - } - - if (prev_is_null == is_null && (USHRT_MAX - null_map.back() >= loop_read)) { - null_map.back() += loop_read; - } else { - if (!(prev_is_null ^ is_null)) { - null_map.emplace_back(0); - } - size_t remaining = loop_read; - while (remaining > USHRT_MAX) { - null_map.emplace_back(USHRT_MAX); - null_map.emplace_back(0); - remaining -= USHRT_MAX; - } - null_map.emplace_back((u_short)remaining); - prev_is_null = is_null; + auto read_and_fill_data = [&](size_t before_rep_level_sz, size_t filter_map_index) { + RETURN_IF_ERROR(_chunk_reader->fill_def(_def_levels)); + std::unique_ptr nested_filter_map = std::make_unique(); + if (filter_map.has_filter()) { + RETURN_IF_ERROR(gen_filter_map(filter_map, filter_map_index, before_rep_level_sz, + _rep_levels.size(), nested_filter_map_data, + &nested_filter_map)); } - } - size_t num_values = parsed_values - ancestor_nulls; + null_map.clear(); + ancestor_null_indices.clear(); + RETURN_IF_ERROR(gen_nested_null_map(before_rep_level_sz, _rep_levels.size(), null_map, + ancestor_null_indices)); - // Handle filtered values - if (current_filter_map->filter_all()) { - // Skip all values if everything is filtered - if (null_size > 0) { - RETURN_IF_ERROR(_chunk_reader->skip_values(null_size, false)); - } - if (nonnull_size > 0) { - RETURN_IF_ERROR(_chunk_reader->skip_values(nonnull_size, true)); - } - if (ancestor_nulls != 0) { - RETURN_IF_ERROR(_chunk_reader->skip_values(ancestor_nulls, false)); - } - } else { ColumnSelectVector select_vector; { SCOPED_RAW_TIMER(&_decode_null_map_time); - RETURN_IF_ERROR( - select_vector.init(null_map, num_values, map_data_column, current_filter_map, - _nested_filter_map_data ? origin_size : _filter_map_index, - &ancestor_null_indices)); + RETURN_IF_ERROR(select_vector.init( + null_map, + _rep_levels.size() - before_rep_level_sz - ancestor_null_indices.size(), + map_data_column, nested_filter_map.get(), 0, &ancestor_null_indices)); } RETURN_IF_ERROR( _chunk_reader->decode_values(data_column, type, select_vector, is_dict_filter)); - if (ancestor_nulls != 0) { - RETURN_IF_ERROR(_chunk_reader->skip_values(ancestor_nulls, false)); - } - } - *read_rows += parsed_rows; - _filter_map_index += parsed_values; - - // Handle cross-page reading - if (_chunk_reader->remaining_num_values() == 0) { - if (_chunk_reader->has_next_page()) { - RETURN_IF_ERROR(_chunk_reader->next_page()); - RETURN_IF_ERROR(_chunk_reader->load_page_data()); - return _read_nested_column(doris_column, type, filter_map, 0, read_rows, eof, - is_dict_filter, true); - } else { - *eof = true; - } - } - - // Apply filtering to repetition and definition levels - if (current_filter_map->has_filter()) { - if (current_filter_map->filter_all()) { - _rep_levels.resize(0); - _def_levels.resize(0); - } else { - std::vector filtered_rep_levels; - std::vector filtered_def_levels; - filtered_rep_levels.reserve(_rep_levels.size()); - filtered_def_levels.reserve(_def_levels.size()); - - const uint8_t* filter_map_data = current_filter_map->filter_map_data(); - - for (size_t i = 0; i < _rep_levels.size(); i++) { - if (filter_map_data[i]) { - filtered_rep_levels.push_back(_rep_levels[i]); - filtered_def_levels.push_back(_def_levels[i]); + if (ancestor_null_indices.size() != 0) { + RETURN_IF_ERROR(_chunk_reader->skip_values(ancestor_null_indices.size(), false)); + } + if (filter_map.has_filter()) { + auto new_rep_sz = before_rep_level_sz; + for (size_t idx = before_rep_level_sz; idx < _rep_levels.size(); idx++) { + if (nested_filter_map_data[idx - before_rep_level_sz]) { + _rep_levels[new_rep_sz] = _rep_levels[idx]; + _def_levels[new_rep_sz] = _def_levels[idx]; + new_rep_sz++; } } - - _rep_levels = std::move(filtered_rep_levels); - _def_levels = std::move(filtered_def_levels); + _rep_levels.resize(new_rep_sz); + _def_levels.resize(new_rep_sz); + } + return Status::OK(); + }; + + while (_current_range_idx < _row_ranges.size()) { + size_t left_row = std::max(_current_row_index, _row_ranges[_current_range_idx].first_row); + size_t right_row = std::min(left_row + batch_size - *read_rows, + (size_t)_row_ranges[_current_range_idx].last_row); + _current_row_index = left_row; + RETURN_IF_ERROR(_chunk_reader->seek_to_nested_row(left_row)); + size_t load_rows = 0; + bool cross_page = false; + size_t before_rep_level_sz = _rep_levels.size(); + RETURN_IF_ERROR(_chunk_reader->load_page_nested_rows(_rep_levels, right_row - left_row, + &load_rows, &cross_page)); + RETURN_IF_ERROR(read_and_fill_data(before_rep_level_sz, _filter_map_index)); + _filter_map_index += load_rows; + while (cross_page) { + before_rep_level_sz = _rep_levels.size(); + RETURN_IF_ERROR(_chunk_reader->load_cross_page_nested_row(_rep_levels, &cross_page)); + RETURN_IF_ERROR(read_and_fill_data(before_rep_level_sz, _filter_map_index - 1)); + } + *read_rows += load_rows; + _current_row_index += load_rows; + _current_range_idx += (_current_row_index == _row_ranges[_current_range_idx].last_row); + if (*read_rows == batch_size) { + break; } } - - // Prepare for next row - ++_orig_filter_map_index; - - if (_rep_levels.size() > 0) { - // make sure the rows of complex type are aligned correctly, - // so the repetition level of first element should be 0, meaning a new row is started. - DCHECK_EQ(_rep_levels[0], 0); - } + *eof = _current_range_idx == _row_ranges.size(); return Status::OK(); } -Status ScalarColumnReader::read_dict_values_to_column(MutableColumnPtr& doris_column, - bool* has_dict) { +template +Status ScalarColumnReader::read_dict_values_to_column( + MutableColumnPtr& doris_column, bool* has_dict) { bool loaded; RETURN_IF_ERROR(_try_load_dict_page(&loaded, has_dict)); if (loaded && *has_dict) { @@ -556,28 +462,24 @@ Status ScalarColumnReader::read_dict_values_to_column(MutableColumnPtr& doris_co } return Status::OK(); } - -MutableColumnPtr ScalarColumnReader::convert_dict_column_to_string_column( +template +MutableColumnPtr +ScalarColumnReader::convert_dict_column_to_string_column( const ColumnInt32* dict_column) { return _chunk_reader->convert_dict_column_to_string_column(dict_column); } -Status ScalarColumnReader::_try_load_dict_page(bool* loaded, bool* has_dict) { - *loaded = false; - *has_dict = false; - if (_chunk_reader->remaining_num_values() == 0) { - if (!_chunk_reader->has_next_page()) { - *loaded = false; - return Status::OK(); - } - RETURN_IF_ERROR(_chunk_reader->next_page()); - *loaded = true; - *has_dict = _chunk_reader->has_dict(); - } +template +Status ScalarColumnReader::_try_load_dict_page(bool* loaded, + bool* has_dict) { + // _chunk_reader init will load first page header to check whether has dict page + *loaded = true; + *has_dict = _chunk_reader->has_dict(); return Status::OK(); } -Status ScalarColumnReader::read_column_data( +template +Status ScalarColumnReader::read_column_data( ColumnPtr& doris_column, DataTypePtr& type, const std::shared_ptr& root_node, FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter) { @@ -597,32 +499,33 @@ Status ScalarColumnReader::read_column_data( doris_column, type, is_dict_filter); DataTypePtr& resolved_type = _converter->get_physical_type(); - do { - if (_chunk_reader->remaining_num_values() == 0) { - if (!_chunk_reader->has_next_page()) { - *eof = true; - *read_rows = 0; - return Status::OK(); - } - RETURN_IF_ERROR(_chunk_reader->next_page()); - } - if (_nested_column) { - RETURN_IF_ERROR(_chunk_reader->load_page_data_idempotent()); - RETURN_IF_ERROR(_read_nested_column(resolved_column, resolved_type, filter_map, - batch_size, read_rows, eof, is_dict_filter, false)); - break; - } + _def_levels.clear(); + _rep_levels.clear(); + *read_rows = 0; + + if constexpr (IN_COLLECTION) { + RETURN_IF_ERROR(_read_nested_column(resolved_column, resolved_type, filter_map, batch_size, + read_rows, eof, is_dict_filter)); + return _converter->convert(resolved_column, _field_schema->data_type, type, doris_column, + is_dict_filter); + } + + int64_t right_row = 0; + if constexpr (OFFSET_INDEX == false) { + RETURN_IF_ERROR(_chunk_reader->parse_page_header()); + right_row = _chunk_reader->page_end_row(); + } else { + right_row = _chunk_reader->page_end_row(); + } + auto before_filter_map_index = _filter_map_index; + do { // generate the row ranges that should be read std::list read_ranges; - _generate_read_ranges(_current_row_index, - _current_row_index + _chunk_reader->remaining_num_values(), - read_ranges); + _generate_read_ranges(_current_row_index, right_row, read_ranges); if (read_ranges.size() == 0) { // skip the whole page - _current_row_index += _chunk_reader->remaining_num_values(); - RETURN_IF_ERROR(_chunk_reader->skip_page()); - *read_rows = 0; + _current_row_index = right_row; } else { bool skip_whole_batch = false; // Determining whether to skip page or batch will increase the calculation time. @@ -637,12 +540,8 @@ Status ScalarColumnReader::read_column_data( filter_map.can_filter_all(remaining_num_values, _filter_map_index)) { // We can skip the whole page if the remaining values is filtered by predicate columns _filter_map_index += remaining_num_values; - _current_row_index += _chunk_reader->remaining_num_values(); - RETURN_IF_ERROR(_chunk_reader->skip_page()); + _current_row_index = right_row; *read_rows = remaining_num_values; - if (!_chunk_reader->has_next_page()) { - *eof = true; - } break; } skip_whole_batch = batch_size <= remaining_num_values && @@ -652,6 +551,7 @@ Status ScalarColumnReader::read_column_data( } } // load page data to decode or skip values + RETURN_IF_ERROR(_chunk_reader->parse_page_header()); RETURN_IF_ERROR(_chunk_reader->load_page_data_idempotent()); size_t has_read = 0; for (auto& range : read_ranges) { @@ -669,18 +569,34 @@ Status ScalarColumnReader::read_column_data( filter_map, is_dict_filter)); } has_read += read_values; + *read_rows += read_values; _current_row_index += read_values; if (has_read == batch_size) { break; } } - *read_rows = has_read; } + } while (false); - if (_chunk_reader->remaining_num_values() == 0 && !_chunk_reader->has_next_page()) { + if (right_row == _current_row_index) { + if (!_chunk_reader->has_next_page()) { *eof = true; + } else { + RETURN_IF_ERROR(_chunk_reader->next_page()); } - } while (false); + } + + if (filter_map.has_filter()) { + size_t new_rep_sz = 0; + for (size_t idx = before_filter_map_index; idx < _filter_map_index; idx++) { + if (filter_map.filter_map_data()[idx]) { + _def_levels[new_rep_sz] = _def_levels[idx - before_filter_map_index]; + new_rep_sz++; + } + } + _def_levels.resize(new_rep_sz); + } + _rep_levels.resize(_def_levels.size(), 0); return _converter->convert(resolved_column, _field_schema->data_type, type, doris_column, is_dict_filter); @@ -888,7 +804,7 @@ Status StructColumnReader::read_column_data( field_rows += loop_rows; } DCHECK_EQ(*read_rows, field_rows); - DCHECK_EQ(*eof, field_eof); + // DCHECK_EQ(*eof, field_eof); } } @@ -924,6 +840,12 @@ Status StructColumnReader::read_column_data( return Status::OK(); } + +template class ScalarColumnReader; +template class ScalarColumnReader; +template class ScalarColumnReader; +template class ScalarColumnReader; + #include "common/compile_check_end.h" }; // namespace doris::vectorized diff --git a/be/src/vec/exec/format/parquet/vparquet_column_reader.h b/be/src/vec/exec/format/parquet/vparquet_column_reader.h index 80f13f629bca89..83d548ed4cf1c9 100644 --- a/be/src/vec/exec/format/parquet/vparquet_column_reader.h +++ b/be/src/vec/exec/format/parquet/vparquet_column_reader.h @@ -69,7 +69,7 @@ class ParquetColumnReader { skip_page_header_num(0), parse_page_header_num(0) {} - Statistics(io::BufferedStreamReader::Statistics& fs, ColumnChunkReader::Statistics& cs, + Statistics(io::BufferedStreamReader::Statistics& fs, ColumnChunkReaderStatistics& cs, int64_t null_map_time) : read_time(fs.read_time), read_calls(fs.read_calls), @@ -116,9 +116,9 @@ class ParquetColumnReader { } }; - ParquetColumnReader(const std::vector& row_ranges, cctz::time_zone* ctz, - io::IOContext* io_ctx) - : _row_ranges(row_ranges), _ctz(ctz), _io_ctx(io_ctx) {} + ParquetColumnReader(const std::vector& row_ranges, size_t total_rows, + cctz::time_zone* ctz, io::IOContext* io_ctx) + : _row_ranges(row_ranges), _total_rows(total_rows), _ctz(ctz), _io_ctx(io_ctx) {} virtual ~ParquetColumnReader() = default; virtual Status read_column_data(ColumnPtr& doris_column, DataTypePtr& type, const std::shared_ptr& root_node, @@ -138,8 +138,9 @@ class ParquetColumnReader { const tparquet::RowGroup& row_group, const std::vector& row_ranges, cctz::time_zone* ctz, io::IOContext* io_ctx, std::unique_ptr& reader, - size_t max_buf_size, const tparquet::OffsetIndex* offset_index = nullptr); - void set_nested_column() { _nested_column = true; } + size_t max_buf_size, + std::unordered_map& col_offsets, + bool in_collection = false); virtual const std::vector& get_rep_level() const = 0; virtual const std::vector& get_def_level() const = 0; virtual Statistics statistics() = 0; @@ -152,10 +153,9 @@ class ParquetColumnReader { std::list& read_ranges); FieldSchema* _field_schema = nullptr; - // When scalar column is the child of nested column, we should turn off the filtering by page index and lazy read. - bool _nested_column = false; const std::vector& _row_ranges; - cctz::time_zone* _ctz = nullptr; + size_t _total_rows = 0; + const cctz::time_zone* _ctz = nullptr; io::IOContext* _io_ctx = nullptr; int64_t _current_row_index = 0; int _row_range_index = 0; @@ -164,14 +164,15 @@ class ParquetColumnReader { size_t _filter_map_index = 0; }; +template class ScalarColumnReader : public ParquetColumnReader { ENABLE_FACTORY_CREATOR(ScalarColumnReader) public: - ScalarColumnReader(const std::vector& row_ranges, + ScalarColumnReader(const std::vector& row_ranges, size_t total_rows, const tparquet::ColumnChunk& chunk_meta, const tparquet::OffsetIndex* offset_index, cctz::time_zone* ctz, io::IOContext* io_ctx) - : ParquetColumnReader(row_ranges, ctz, io_ctx), + : ParquetColumnReader(row_ranges, total_rows, ctz, io_ctx), _chunk_meta(chunk_meta), _offset_index(offset_index) {} ~ScalarColumnReader() override { close(); } @@ -197,11 +198,78 @@ class ScalarColumnReader : public ParquetColumnReader { private: tparquet::ColumnChunk _chunk_meta; - const tparquet::OffsetIndex* _offset_index; + const tparquet::OffsetIndex* _offset_index = nullptr; std::unique_ptr _stream_reader; - std::unique_ptr _chunk_reader; + std::unique_ptr> _chunk_reader; + // rep def levels buffer. std::vector _rep_levels; std::vector _def_levels; + + size_t _current_range_idx = 0; + + Status gen_nested_null_map(size_t level_start_idx, size_t level_end_idx, + std::vector& null_map, + std::unordered_set& ancestor_null_indices) { + size_t has_read = level_start_idx; + null_map.emplace_back(0); + bool prev_is_null = false; + + while (has_read < level_end_idx) { + level_t def_level = _def_levels[has_read++]; + size_t loop_read = 1; + while (has_read < _def_levels.size() && _def_levels[has_read] == def_level) { + has_read++; + loop_read++; + } + + if (def_level < _field_schema->repeated_parent_def_level) { + for (size_t i = 0; i < loop_read; i++) { + ancestor_null_indices.insert(has_read - level_start_idx - loop_read + i); + } + continue; + } + + bool is_null = def_level < _field_schema->definition_level; + + if (prev_is_null == is_null && (USHRT_MAX - null_map.back() >= loop_read)) { + null_map.back() += loop_read; + } else { + if (!(prev_is_null ^ is_null)) { + null_map.emplace_back(0); + } + size_t remaining = loop_read; + while (remaining > USHRT_MAX) { + null_map.emplace_back(USHRT_MAX); + null_map.emplace_back(0); + remaining -= USHRT_MAX; + } + null_map.emplace_back((u_short)remaining); + prev_is_null = is_null; + } + } + return Status::OK(); + } + + Status gen_filter_map(FilterMap& filter_map, size_t filter_loc, size_t level_start_idx, + size_t level_end_idx, std::vector& nested_filter_map_data, + std::unique_ptr* nested_filter_map) { + nested_filter_map_data.resize(level_end_idx - level_start_idx); + for (size_t idx = level_start_idx; idx < level_end_idx; idx++) { + if (idx != level_start_idx && _rep_levels[idx] == 0) { + filter_loc++; + } + nested_filter_map_data[idx - level_start_idx] = + filter_map.filter_map_data()[filter_loc]; + } + + auto new_filter = std::make_unique(); + RETURN_IF_ERROR(new_filter->init(nested_filter_map_data.data(), + nested_filter_map_data.size(), false)); + *nested_filter_map = std::move(new_filter); + + return Status::OK(); + } + std::unique_ptr _converter = nullptr; std::unique_ptr> _nested_filter_map_data = nullptr; size_t _orig_filter_map_index = 0; @@ -210,17 +278,17 @@ class ScalarColumnReader : public ParquetColumnReader { Status _read_values(size_t num_values, ColumnPtr& doris_column, DataTypePtr& type, FilterMap& filter_map, bool is_dict_filter); Status _read_nested_column(ColumnPtr& doris_column, DataTypePtr& type, FilterMap& filter_map, - size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter, - bool align_rows); + size_t batch_size, size_t* read_rows, bool* eof, + bool is_dict_filter); Status _try_load_dict_page(bool* loaded, bool* has_dict); }; class ArrayColumnReader : public ParquetColumnReader { ENABLE_FACTORY_CREATOR(ArrayColumnReader) public: - ArrayColumnReader(const std::vector& row_ranges, cctz::time_zone* ctz, - io::IOContext* io_ctx) - : ParquetColumnReader(row_ranges, ctz, io_ctx) {} + ArrayColumnReader(const std::vector& row_ranges, size_t total_rows, + cctz::time_zone* ctz, io::IOContext* io_ctx) + : ParquetColumnReader(row_ranges, total_rows, ctz, io_ctx) {} ~ArrayColumnReader() override { close(); } Status init(std::unique_ptr element_reader, FieldSchema* field); Status read_column_data(ColumnPtr& doris_column, DataTypePtr& type, @@ -245,9 +313,9 @@ class ArrayColumnReader : public ParquetColumnReader { class MapColumnReader : public ParquetColumnReader { ENABLE_FACTORY_CREATOR(MapColumnReader) public: - MapColumnReader(const std::vector& row_ranges, cctz::time_zone* ctz, - io::IOContext* io_ctx) - : ParquetColumnReader(row_ranges, ctz, io_ctx) {} + MapColumnReader(const std::vector& row_ranges, size_t total_rows, + cctz::time_zone* ctz, io::IOContext* io_ctx) + : ParquetColumnReader(row_ranges, total_rows, ctz, io_ctx) {} ~MapColumnReader() override { close(); } Status init(std::unique_ptr key_reader, @@ -286,9 +354,9 @@ class MapColumnReader : public ParquetColumnReader { class StructColumnReader : public ParquetColumnReader { ENABLE_FACTORY_CREATOR(StructColumnReader) public: - StructColumnReader(const std::vector& row_ranges, cctz::time_zone* ctz, - io::IOContext* io_ctx) - : ParquetColumnReader(row_ranges, ctz, io_ctx) {} + StructColumnReader(const std::vector& row_ranges, size_t total_rows, + cctz::time_zone* ctz, io::IOContext* io_ctx) + : ParquetColumnReader(row_ranges, total_rows, ctz, io_ctx) {} ~StructColumnReader() override { close(); } Status init( diff --git a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp index 37b6b898b5c976..585d993df4dcfc 100644 --- a/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp +++ b/be/src/vec/exec/format/parquet/vparquet_group_reader.cpp @@ -123,17 +123,11 @@ Status RowGroupReader::init( std::min(MAX_COLUMN_BUF_SIZE, MAX_GROUP_BUF_SIZE / _read_table_columns.size()); for (const auto& read_table_col : _read_table_columns) { auto read_file_col = _table_info_node_ptr->children_file_column_name(read_table_col); - auto* field = const_cast(schema.get_column(read_file_col)); - auto physical_index = field->physical_column_index; std::unique_ptr reader; - // TODO : support rested column types - const tparquet::OffsetIndex* offset_index = - col_offsets.find(physical_index) != col_offsets.end() ? &col_offsets[physical_index] - : nullptr; RETURN_IF_ERROR(ParquetColumnReader::create(_file_reader, field, _row_group_meta, _read_ranges, _ctz, _io_ctx, reader, - max_buf_size, offset_index)); + max_buf_size, col_offsets, false)); if (reader == nullptr) { VLOG_DEBUG << "Init row group(" << _row_group_id << ") reader failed"; return Status::Corruption("Init row group reader failed"); diff --git a/be/src/vec/exec/format/parquet/vparquet_page_index.cpp b/be/src/vec/exec/format/parquet/vparquet_page_index.cpp index 0640d3293c573a..6d940bed74f261 100644 --- a/be/src/vec/exec/format/parquet/vparquet_page_index.cpp +++ b/be/src/vec/exec/format/parquet/vparquet_page_index.cpp @@ -41,7 +41,7 @@ struct FieldSchema; namespace doris::vectorized { #include "common/compile_check_begin.h" -Status PageIndex::create_skipped_row_range(tparquet::OffsetIndex& offset_index, +Status PageIndex::create_skipped_row_range(const tparquet::OffsetIndex& offset_index, int64_t total_rows_of_group, int page_idx, RowRange* row_range) { const auto& page_locations = offset_index.page_locations; diff --git a/be/src/vec/exec/format/parquet/vparquet_page_index.h b/be/src/vec/exec/format/parquet/vparquet_page_index.h index 92d9e2147641b7..f64b39e2fd4051 100644 --- a/be/src/vec/exec/format/parquet/vparquet_page_index.h +++ b/be/src/vec/exec/format/parquet/vparquet_page_index.h @@ -44,7 +44,7 @@ class PageIndex { public: PageIndex() = default; ~PageIndex() = default; - Status create_skipped_row_range(tparquet::OffsetIndex& offset_index, + Status create_skipped_row_range(const tparquet::OffsetIndex& offset_index, int64_t total_rows_of_group, int page_idx, RowRange* row_range); bool check_and_get_page_index_ranges(const std::vector& columns); Status parse_column_index(const tparquet::ColumnChunk& chunk, const uint8_t* buff, diff --git a/be/src/vec/exec/format/parquet/vparquet_page_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_page_reader.cpp index cfdb7cea52b7a6..3734dc217f5657 100644 --- a/be/src/vec/exec/format/parquet/vparquet_page_reader.cpp +++ b/be/src/vec/exec/format/parquet/vparquet_page_reader.cpp @@ -40,23 +40,33 @@ namespace doris::vectorized { #include "common/compile_check_begin.h" static constexpr size_t INIT_PAGE_HEADER_SIZE = 128; -std::unique_ptr create_page_reader(io::BufferedStreamReader* reader, - io::IOContext* io_ctx, uint64_t offset, - uint64_t length, int64_t num_values, - const tparquet::OffsetIndex* offset_index) { - if (offset_index) { - return std::make_unique(reader, io_ctx, offset, length, - num_values, offset_index); - } else { - return std::make_unique(reader, io_ctx, offset, length); +template +PageReader::PageReader(io::BufferedStreamReader* reader, + io::IOContext* io_ctx, uint64_t offset, + uint64_t length, size_t total_rows, + const tparquet::OffsetIndex* offset_index) + : _reader(reader), + _io_ctx(io_ctx), + _offset(offset), + _start_offset(offset), + _end_offset(offset + length), + _total_rows(total_rows), + _offset_index(offset_index) { + _next_header_offset = _offset; + _state = INITIALIZED; + + if constexpr (OFFSET_INDEX) { + _end_row = _offset_index->page_locations.size() >= 2 + ? _offset_index->page_locations[1].first_row_index + : _total_rows; } } -PageReader::PageReader(io::BufferedStreamReader* reader, io::IOContext* io_ctx, uint64_t offset, - uint64_t length) - : _reader(reader), _io_ctx(io_ctx), _start_offset(offset), _end_offset(offset + length) {} - -Status PageReader::_parse_page_header() { +template +Status PageReader::parse_page_header() { + if (_state == HEADER_PARSED) { + return Status::OK(); + } if (UNLIKELY(_offset < _start_offset || _offset >= _end_offset)) { return Status::IOError("Out-of-bounds Access"); } @@ -94,6 +104,14 @@ Status PageReader::_parse_page_header() { header_size <<= 2; } + if constexpr (OFFSET_INDEX == false) { + if (is_header_v2()) { + _end_row = _start_row + _cur_page_header.data_page_header_v2.num_rows; + } else if constexpr (!IN_COLLECTION) { + _end_row = _start_row + _cur_page_header.data_page_header.num_values; + } + } + _statistics.parse_page_header_num++; _offset += real_header_size; _next_header_offset = _offset + _cur_page_header.compressed_page_size; @@ -101,16 +119,8 @@ Status PageReader::_parse_page_header() { return Status::OK(); } -Status PageReader::skip_page() { - if (UNLIKELY(_state != HEADER_PARSED)) { - return Status::IOError("Should generate page header first to skip current page"); - } - _offset = _next_header_offset; - _state = INITIALIZED; - return Status::OK(); -} - -Status PageReader::get_page_data(Slice& slice) { +template +Status PageReader::get_page_data(Slice& slice) { if (UNLIKELY(_state != HEADER_PARSED)) { return Status::IOError("Should generate page header first to load current page data"); } @@ -120,9 +130,14 @@ Status PageReader::get_page_data(Slice& slice) { slice.size = _cur_page_header.compressed_page_size; RETURN_IF_ERROR(_reader->read_bytes(slice, _offset, _io_ctx)); _offset += slice.size; - _state = INITIALIZED; + _state = DATA_LOADED; return Status::OK(); } + +template class PageReader; +template class PageReader; +template class PageReader; +template class PageReader; #include "common/compile_check_end.h" } // namespace doris::vectorized diff --git a/be/src/vec/exec/format/parquet/vparquet_page_reader.h b/be/src/vec/exec/format/parquet/vparquet_page_reader.h index cc162841a9ca3b..c33a7ca8cdbfa6 100644 --- a/be/src/vec/exec/format/parquet/vparquet_page_reader.h +++ b/be/src/vec/exec/format/parquet/vparquet_page_reader.h @@ -24,6 +24,17 @@ #include "common/cast_set.h" #include "common/status.h" +#include "util/block_compression.h" +#include "vec/exec/format/parquet/parquet_common.h" +namespace doris { +class BlockCompressionCodec; + +namespace io { +class BufferedStreamReader; +struct IOContext; +} // namespace io + +} // namespace doris namespace doris { namespace io { @@ -38,6 +49,8 @@ namespace doris::vectorized { /** * Use to deserialize parquet page header, and get the page data in iterator interface. */ + +template class PageReader { public: struct Statistics { @@ -47,146 +60,118 @@ class PageReader { }; PageReader(io::BufferedStreamReader* reader, io::IOContext* io_ctx, uint64_t offset, - uint64_t length); - virtual ~PageReader() = default; + uint64_t length, size_t total_rows, + const tparquet::OffsetIndex* offset_index = nullptr); + ~PageReader() = default; - // Deprecated - // Parquet file may not be standardized, - // _end_offset may exceed the actual data area. - // ColumnChunkReader::has_next_page() use the number of parsed values for judgment - // [[deprecated]] - bool has_next_page() const { return _offset < _end_offset; } + bool has_next_page() const { + if constexpr (OFFSET_INDEX) { + return _page_index + 1 != _offset_index->page_locations.size(); + } else { + // Deprecated + // Parquet file may not be standardized, + // _end_offset may exceed the actual data area. + // ColumnChunkReader::has_next_page() use the number of parsed values for judgment + // ref:https://github.com/duckdb/duckdb/issues/10829 + // [[deprecated]] + LOG(FATAL) << "has_next_page should not be called when no offset index"; + return _offset < _end_offset; + } + } - virtual Status next_page_header() { return _parse_page_header(); } + Status parse_page_header(); - virtual Status get_page_header(const tparquet::PageHeader*& page_header) { - if (UNLIKELY(_state != HEADER_PARSED)) { - return Status::InternalError("Page header not parsed"); + Status next_page() { + _statistics.skip_page_header_num += _state == INITIALIZED; + if constexpr (OFFSET_INDEX) { + _page_index++; + _start_row = _offset_index->page_locations[_page_index].first_row_index; + if (_page_index + 1 < _offset_index->page_locations.size()) { + _end_row = _offset_index->page_locations[_page_index + 1].first_row_index; + } else { + _end_row = _total_rows; + } + int64_t next_page_offset = _offset_index->page_locations[_page_index].offset; + _offset = next_page_offset; + _next_header_offset = next_page_offset; + _state = INITIALIZED; + } else { + if (UNLIKELY(_offset == _start_offset)) { + return Status::Corruption("should parse first page."); + } + + if (is_header_v2()) { + _start_row += _cur_page_header.data_page_header_v2.num_rows; + } else if constexpr (!IN_COLLECTION) { + _start_row += _cur_page_header.data_page_header.num_values; + } + + _offset = _next_header_offset; + _state = INITIALIZED; } - page_header = &_cur_page_header; + return Status::OK(); } - virtual Status get_num_values(uint32_t& num_values) { - if (_state != HEADER_PARSED) { - return Status::InternalError("Page header not parsed"); - } - if (_cur_page_header.type == tparquet::PageType::DATA_PAGE_V2) { - num_values = _cur_page_header.data_page_header_v2.num_values; + Status dict_next_page() { + if constexpr (OFFSET_INDEX) { + _state = INITIALIZED; + return Status::OK(); } else { - num_values = _cur_page_header.data_page_header.num_values; + return next_page(); } - return Status::OK(); } - virtual Status skip_page(); + Status get_page_header(const tparquet::PageHeader*& page_header) { + if (UNLIKELY(_state != HEADER_PARSED)) { + return Status::InternalError("Page header not parsed"); + } + page_header = &_cur_page_header; + return Status::OK(); + } - virtual Status get_page_data(Slice& slice); + Status get_page_data(Slice& slice); Statistics& statistics() { return _statistics; } - void seek_to_page(int64_t page_header_offset) { - _offset = page_header_offset; - _next_header_offset = page_header_offset; - _state = INITIALIZED; - } + bool is_header_v2() { return _cur_page_header.__isset.data_page_header_v2; } -protected: - enum PageReaderState { INITIALIZED, HEADER_PARSED }; - PageReaderState _state = INITIALIZED; - tparquet::PageHeader _cur_page_header; - Statistics _statistics; + size_t start_row() const { return _start_row; } - Status _parse_page_header(); + size_t end_row() const { return _end_row; } private: + enum PageReaderState { INITIALIZED, HEADER_PARSED, DATA_LOADED }; + PageReaderState _state = INITIALIZED; + Statistics _statistics; + io::BufferedStreamReader* _reader = nullptr; io::IOContext* _io_ctx = nullptr; + // current reader offset in file location. uint64_t _offset = 0; - uint64_t _next_header_offset = 0; + // this page offset in file location. uint64_t _start_offset = 0; uint64_t _end_offset = 0; -}; - -class PageReaderWithOffsetIndex : public PageReader { -public: - PageReaderWithOffsetIndex(io::BufferedStreamReader* reader, io::IOContext* io_ctx, - uint64_t offset, uint64_t length, int64_t num_values, - const tparquet::OffsetIndex* offset_index) - : PageReader(reader, io_ctx, offset, length), - _num_values(num_values), - _offset_index(offset_index) {} - - Status next_page_header() override { - // lazy to parse page header in get_page_header - return Status::OK(); - } - - Status get_page_header(const tparquet::PageHeader*& page_header) override { - if (_state != HEADER_PARSED) { - RETURN_IF_ERROR(_parse_page_header()); - } - page_header = &_cur_page_header; - return Status::OK(); - } - - Status get_num_values(uint32_t& num_values) override { - if (UNLIKELY(_page_index >= _offset_index->page_locations.size())) { - return Status::IOError("End of page"); - } - - if (_page_index < _offset_index->page_locations.size() - 1) { - num_values = cast_set( - _offset_index->page_locations[_page_index + 1].first_row_index - - _offset_index->page_locations[_page_index].first_row_index); - } else { - num_values = cast_set( - _num_values - _offset_index->page_locations[_page_index].first_row_index); - } - return Status::OK(); - } - - Status skip_page() override { - if (UNLIKELY(_page_index >= _offset_index->page_locations.size())) { - return Status::IOError("End of page"); - } - - if (_state != HEADER_PARSED) { - _statistics.skip_page_header_num++; - } - - seek_to_page(_offset_index->page_locations[_page_index].offset + - _offset_index->page_locations[_page_index].compressed_page_size); - _page_index++; - return Status::OK(); - } - - Status get_page_data(Slice& slice) override { - if (_page_index >= _offset_index->page_locations.size()) { - return Status::IOError("End of page"); - } - if (_state != HEADER_PARSED) { - RETURN_IF_ERROR(_parse_page_header()); - } - - // dirctionary page is not in page location - if (LIKELY(_cur_page_header.type != tparquet::PageType::DICTIONARY_PAGE)) { - _page_index++; - } - - return PageReader::get_page_data(slice); - } - -private: + uint64_t _next_header_offset = 0; + // current page row range + size_t _start_row = 0; + size_t _end_row = 0; + // total rows in this column chunk + size_t _total_rows = 0; + // for page index size_t _page_index = 0; - int64_t _num_values = 0; const tparquet::OffsetIndex* _offset_index; + + tparquet::PageHeader _cur_page_header; }; -std::unique_ptr create_page_reader(io::BufferedStreamReader* reader, - io::IOContext* io_ctx, uint64_t offset, - uint64_t length, int64_t num_values = 0, - const tparquet::OffsetIndex* offset_index = nullptr); +template +std::unique_ptr> create_page_reader( + io::BufferedStreamReader* reader, io::IOContext* io_ctx, uint64_t offset, uint64_t length, + size_t total_rows, const tparquet::OffsetIndex* offset_index = nullptr) { + return std::make_unique>(reader, io_ctx, offset, length, + total_rows, offset_index); +} #include "common/compile_check_end.h" } // namespace doris::vectorized diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_reader.cpp index 1959dad3059262..8ef0f32c1b4b5e 100644 --- a/be/src/vec/exec/format/parquet/vparquet_reader.cpp +++ b/be/src/vec/exec/format/parquet/vparquet_reader.cpp @@ -1042,17 +1042,80 @@ Status ParquetReader::_process_page_index(const tparquet::RowGroup& row_group, } }; - if (!_enable_filter_by_min_max || _lazy_read_ctx.has_complex_type || - _lazy_read_ctx.conjuncts.empty()) { - read_whole_row_group(); - return Status::OK(); - } PageIndex page_index; if (!config::enable_parquet_page_index || !_has_page_index(row_group.columns, page_index) || _colname_to_slot_id == nullptr) { read_whole_row_group(); return Status::OK(); } + + std::vector parquet_col_ids; + for (size_t idx = 0; idx < _read_table_columns.size(); idx++) { + const auto& read_table_col = _read_table_columns[idx]; + const auto& read_file_col = _read_file_columns[idx]; + if (!_colname_to_slot_id->contains(read_table_col)) { + continue; + } + const auto* field = _file_metadata->schema().get_column(read_file_col); + + std::function f = [&](const FieldSchema* field) { + if (field->data_type->get_primitive_type() == TYPE_ARRAY) { + f(&field->children[0]); + } else if (field->data_type->get_primitive_type() == TYPE_MAP) { + f(&field->children[0].children[0]); + f(&field->children[0].children[1]); + } else if (field->data_type->get_primitive_type() == TYPE_STRUCT) { + for (int i = 0; i < field->children.size(); ++i) { + f(&field->children[i]); + } + } else { + int parquet_col_id = field->physical_column_index; + if (parquet_col_id >= 0) { + parquet_col_ids.push_back(parquet_col_id); + } + } + }; + + f(field); + } + + auto parse_offset_index = [&]() -> Status { + std::vector off_index_buff(page_index._offset_index_size); + Slice res(off_index_buff.data(), page_index._offset_index_size); + size_t bytes_read = 0; + { + SCOPED_RAW_TIMER(&_statistics.read_page_index_time); + RETURN_IF_ERROR(_tracing_file_reader->read_at(page_index._offset_index_start, res, + &bytes_read, _io_ctx)); + } + _column_statistics.read_bytes += bytes_read; + _column_statistics.page_index_read_calls++; + _col_offsets.clear(); + + for (auto parquet_col_id : parquet_col_ids) { + auto& chunk = row_group.columns[parquet_col_id]; + if (chunk.offset_index_length == 0) [[unlikely]] { + continue; + } + tparquet::OffsetIndex offset_index; + SCOPED_RAW_TIMER(&_statistics.parse_page_index_time); + RETURN_IF_ERROR( + page_index.parse_offset_index(chunk, off_index_buff.data(), &offset_index)); + _col_offsets[parquet_col_id] = offset_index; + } + return Status::OK(); + }; + + // from https://github.com/apache/doris/pull/55795 + RETURN_IF_ERROR(parse_offset_index()); + + // Check if page index is needed for min-max filter. + if (!_enable_filter_by_min_max || _push_down_simple_expr.empty()) { + read_whole_row_group(); + return Status::OK(); + } + + // read column index. std::vector col_index_buff(page_index._column_index_size); size_t bytes_read = 0; Slice result(col_index_buff.data(), page_index._column_index_size); @@ -1062,18 +1125,9 @@ Status ParquetReader::_process_page_index(const tparquet::RowGroup& row_group, &bytes_read, _io_ctx)); } _column_statistics.read_bytes += bytes_read; + _column_statistics.page_index_read_calls++; + std::vector skipped_row_ranges; - std::vector off_index_buff(page_index._offset_index_size); - Slice res(off_index_buff.data(), page_index._offset_index_size); - { - SCOPED_RAW_TIMER(&_statistics.read_page_index_time); - RETURN_IF_ERROR(_tracing_file_reader->read_at(page_index._offset_index_start, res, - &bytes_read, _io_ctx)); - } - _column_statistics.read_bytes += bytes_read; - // read twice: parse column index & parse offset index - _column_statistics.page_index_read_calls += 2; - SCOPED_RAW_TIMER(&_statistics.parse_page_index_time); for (size_t idx = 0; idx < _read_table_columns.size(); idx++) { const auto& read_table_col = _read_table_columns[idx]; @@ -1095,10 +1149,6 @@ Status ParquetReader::_process_page_index(const tparquet::RowGroup& row_group, if (chunk.offset_index_length == 0) { continue; } - tparquet::OffsetIndex offset_index; - RETURN_IF_ERROR(page_index.parse_offset_index(chunk, off_index_buff.data(), &offset_index)); - _col_offsets[parquet_col_id] = offset_index; - if (!_push_down_simple_expr.contains(slot_id)) { continue; } @@ -1113,6 +1163,7 @@ Status ParquetReader::_process_page_index(const tparquet::RowGroup& row_group, if (num_of_pages <= 0) { continue; } + const tparquet::OffsetIndex& offset_index = _col_offsets[parquet_col_id]; std::vector skipped_page_range; const std::vector& encoded_min_vals = column_index.min_values; diff --git a/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp b/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp index 8d8738604a95f7..fbc1c7309548e0 100644 --- a/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp +++ b/be/test/vec/exec/format/parquet/parquet_thrift_test.cpp @@ -179,7 +179,7 @@ static int fill_nullable_column(ColumnPtr& doris_column, level_t* definitions, s static Status get_column_values(io::FileReaderSPtr file_reader, tparquet::ColumnChunk* column_chunk, FieldSchema* field_schema, ColumnPtr& doris_column, - DataTypePtr& data_type, level_t* definitions) { + DataTypePtr& data_type, level_t* definitions, size_t total_rows) { tparquet::ColumnMetaData chunk_meta = column_chunk->meta_data; size_t start_offset = has_dict_page(chunk_meta) ? chunk_meta.dictionary_page_offset : chunk_meta.data_page_offset; @@ -199,12 +199,12 @@ static Status get_column_values(io::FileReaderSPtr file_reader, tparquet::Column io::BufferedFileStreamReader stream_reader(file_reader, start_offset, chunk_size, 1024); - ColumnChunkReader chunk_reader(&stream_reader, column_chunk, field_schema, nullptr, &ctz, - nullptr); + ColumnChunkReader chunk_reader(&stream_reader, column_chunk, field_schema, + nullptr, total_rows, nullptr); // initialize chunk reader static_cast(chunk_reader.init()); // seek to next page header - static_cast(chunk_reader.next_page()); + static_cast(chunk_reader.parse_page_header()); // load page data into underlying container static_cast(chunk_reader.load_page_data()); int rows = chunk_reader.remaining_num_values(); @@ -212,7 +212,7 @@ static Status get_column_values(io::FileReaderSPtr file_reader, tparquet::Column if (field_schema->definition_level == 0) { // required field std::fill(definitions, definitions + rows, 1); } else { - chunk_reader.get_def_levels(definitions, rows); + chunk_reader._def_level_decoder.get_levels(definitions, rows); } MutableColumnPtr data_column; if (src_column->is_nullable()) { @@ -242,10 +242,11 @@ static Status get_column_values(io::FileReaderSPtr file_reader, tparquet::Column if (definitions[i] != level_type) { if (level_type == 0) { // null values - chunk_reader.insert_null_values(data_column, num_values); + data_column->insert_many_defaults(num_values); } else { std::vector null_map = {(u_short)num_values}; - RETURN_IF_ERROR(run_length_map.init(null_map, rows, nullptr, &filter_map, 0)); + RETURN_IF_ERROR( + run_length_map.init(null_map, num_values, nullptr, &filter_map, 0)); RETURN_IF_ERROR(chunk_reader.decode_values(data_column, resolved_type, run_length_map, false)); } @@ -257,10 +258,10 @@ static Status get_column_values(io::FileReaderSPtr file_reader, tparquet::Column } if (level_type == 0) { // null values - chunk_reader.insert_null_values(data_column, num_values); + data_column->insert_many_defaults(num_values); } else { std::vector null_map = {(u_short)num_values}; - RETURN_IF_ERROR(run_length_map.init(null_map, rows, nullptr, &filter_map, 0)); + RETURN_IF_ERROR(run_length_map.init(null_map, num_values, nullptr, &filter_map, 0)); RETURN_IF_ERROR( chunk_reader.decode_values(data_column, resolved_type, run_length_map, false)); } @@ -415,7 +416,7 @@ static void read_parquet_data_and_check(const std::string& parquet_file, static_cast( get_column_values(reader, &t_metadata.row_groups[0].columns[c], const_cast(schema_descriptor.get_column(c)), - data_column, data_type, defs.data())); + data_column, data_type, defs.data(), rows)); } // `date_v2_col` date, // 14 - 13, DATEV2 { @@ -425,7 +426,7 @@ static void read_parquet_data_and_check(const std::string& parquet_file, static_cast( get_column_values(reader, &t_metadata.row_groups[0].columns[13], const_cast(schema_descriptor.get_column(13)), - data_column, data_type, defs.data())); + data_column, data_type, defs.data(), rows)); } // `timestamp_v2_col` timestamp, // 15 - 9, DATETIMEV2 { @@ -435,7 +436,7 @@ static void read_parquet_data_and_check(const std::string& parquet_file, static_cast( get_column_values(reader, &t_metadata.row_groups[0].columns[9], const_cast(schema_descriptor.get_column(9)), - data_column, data_type, defs.data())); + data_column, data_type, defs.data(), rows)); } io::FileReaderSPtr result; diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run80.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run80.hql index 7f374dacfc0128..21f381cfa70eef 100644 --- a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run80.hql +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run80.hql @@ -21,3 +21,24 @@ CREATE TABLE `parquet_topn_lazy_mat_table`( msck repair table orc_topn_lazy_mat_table; msck repair table parquet_topn_lazy_mat_table; + + + +CREATE TABLE `parquet_topn_lazy_complex_table`( + id INT, + col1 STRING, + col2 STRUCT>, + col3 MAP> +) STORED AS PARQUET LOCATION + '/user/doris/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/'; + +CREATE TABLE `parquet_topn_lazy_complex_table_multi_pages`( + id INT, + col1 STRING, + col2 STRUCT>, + col3 MAP> +) STORED AS PARQUET LOCATION + '/user/doris/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/'; + +msck repair table parquet_topn_lazy_complex_table; +msck repair table parquet_topn_lazy_complex_table_multi_pages; \ No newline at end of file diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_1.parquet b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_1.parquet new file mode 100644 index 00000000000000..7bac48aa2c123f Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_1.parquet differ diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_2.parquet b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_2.parquet new file mode 100644 index 00000000000000..fd8d8852078a7a Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_2.parquet differ diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_3.parquet b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_3.parquet new file mode 100644 index 00000000000000..881fc9e7304e5b Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_3.parquet differ diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_4.parquet b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_4.parquet new file mode 100644 index 00000000000000..ddde09317ba38b Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table/data_part_4.parquet differ diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_1.parquet b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_1.parquet new file mode 100644 index 00000000000000..1d9a3707dc2d5e Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_1.parquet differ diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_2.parquet b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_2.parquet new file mode 100644 index 00000000000000..307da5f60b0d22 Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_2.parquet differ diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_3.parquet b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_3.parquet new file mode 100644 index 00000000000000..1ea7d9e84619e2 Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_3.parquet differ diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_4.parquet b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_4.parquet new file mode 100644 index 00000000000000..4edcc8841ba438 Binary files /dev/null and b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_topn_lazy_complex_table_multi_pages/data_part_4.parquet differ diff --git a/regression-test/data/external_table_p0/hive/test_hive_topn_lazy_mat.out b/regression-test/data/external_table_p0/hive/test_hive_topn_lazy_mat.out index 248bf4b90f794a..0d4e28fd9f097c 100644 --- a/regression-test/data/external_table_p0/hive/test_hive_topn_lazy_mat.out +++ b/regression-test/data/external_table_p0/hive/test_hive_topn_lazy_mat.out @@ -1240,6 +1240,720 @@ 1 user1 1.0 false 0.5 1 8 user8 8.0 true 4.0 1 1 user1 1.0 false 0.5 1 9 user9 9.0 false 4.5 1 +-- !complex_1 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} + +-- !complex_2 -- +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} + +-- !complex_3 -- +text_24 {24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null} +text_23 {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]} +text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]} + +-- !complex_4 -- +{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]} +{8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]} +{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]} + +-- !complex_5 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} + +-- !complex_6 -- +{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]} +{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]} +{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null} + +-- !complex_7 -- +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} + +-- !complex_8 -- +3 text_3 +7 text_7 +12 text_12 + +-- !complex_9 -- +8 {8:["c", null], 9:["d1", "d2"], 10:null} text_8 + +-- !complex_10 -- +{"a":1, "b":[1, 2]} 1 +{"a":10, "b":null} 10 +{"a":11, "b":[11, 12]} 11 + +-- !complex_11 -- +2 text_2 +10 text_10 +20 text_20 + +-- !complex_12 -- +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} + +-- !complex_13 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} + +-- !complex_1 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} + +-- !complex_2 -- +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null} + +-- !complex_3 -- +text_24 {24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null} +text_23 {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]} +text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]} +text_21 {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null} +text_20 {20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]} +text_19 {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]} +text_18 {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null} +text_17 {17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]} + +-- !complex_4 -- +{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]} +{8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]} +{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]} +{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {"a":6, "b":null} +{5:["m", "n"], 6:null, 7:["x1", "x2"]} {"a":null, "b":[5, 6]} +{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {"a":4, "b":[4, 5]} +{3:["p", "q"], 4:null, 5:["z1", "z2"]} {"a":3, "b":null} +{24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null} + +-- !complex_5 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} + +-- !complex_6 -- +{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]} +{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]} +{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null} +{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]} +{15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null} +{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]} +{17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]} +{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null} + +-- !complex_7 -- +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} + +-- !complex_8 -- +3 text_3 +7 text_7 +12 text_12 + +-- !complex_9 -- +8 {8:["c", null], 9:["d1", "d2"], 10:null} text_8 + +-- !complex_10 -- +{"a":1, "b":[1, 2]} 1 +{"a":10, "b":null} 10 +{"a":11, "b":[11, 12]} 11 +{"a":null, "b":[12, 13]} 12 +{"a":13, "b":null} 13 +{"a":14, "b":[14, 15]} 14 +{"a":null, "b":null} 15 +{"a":16, "b":[16, 17]} 16 + +-- !complex_11 -- +2 text_2 +10 text_10 +20 text_20 + +-- !complex_12 -- +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} +14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} +16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} + +-- !complex_13 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} +15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} + +-- !complex_1 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} +13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} +14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} +15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} +16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} +17 text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null} +18 text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} +19 text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} +20 text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]} +21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} +22 text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]} +23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} +24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null} + +-- !complex_2 -- +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null} +23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} +22 text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]} +21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} +20 text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]} +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +19 text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} +18 text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} +17 text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null} +16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} +15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} +14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} +13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} +12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} +11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} + +-- !complex_3 -- +text_24 {24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null} +text_23 {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]} +text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]} +text_21 {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null} +text_20 {20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]} +text_19 {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]} +text_18 {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null} +text_17 {17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]} +text_16 {16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]} +text_15 {15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null} +text_14 {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]} +text_13 {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null} +text_12 {12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]} +text_11 {11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]} +text_10 {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {"a":10, "b":null} +text_9 {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]} +text_8 {8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]} +text_7 {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]} +text_6 {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {"a":6, "b":null} +text_5 {5:["m", "n"], 6:null, 7:["x1", "x2"]} {"a":null, "b":[5, 6]} +text_4 {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {"a":4, "b":[4, 5]} +text_3 {3:["p", "q"], 4:null, 5:["z1", "z2"]} {"a":3, "b":null} +text_2 {2:["s1", null], 3:["v1", "v2"], 4:null} {"a":null, "b":[2, 3]} +text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]} + +-- !complex_4 -- +{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]} +{8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]} +{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]} +{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {"a":6, "b":null} +{5:["m", "n"], 6:null, 7:["x1", "x2"]} {"a":null, "b":[5, 6]} +{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {"a":4, "b":[4, 5]} +{3:["p", "q"], 4:null, 5:["z1", "z2"]} {"a":3, "b":null} +{24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null} +{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]} +{22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]} +{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null} +{20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]} +{2:["s1", null], 3:["v1", "v2"], 4:null} {"a":null, "b":[2, 3]} +{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]} +{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null} +{17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]} +{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]} +{15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null} +{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]} +{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null} +{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]} +{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]} +{10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {"a":10, "b":null} +{1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]} + +-- !complex_5 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} + +-- !complex_6 -- +{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]} +{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]} +{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null} +{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]} +{15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null} +{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]} +{17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]} +{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null} +{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]} +{20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]} +{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null} +{22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]} +{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]} +{24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null} + +-- !complex_7 -- +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} +13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} +14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} +15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} + +-- !complex_8 -- +3 text_3 +7 text_7 +12 text_12 + +-- !complex_9 -- +8 {8:["c", null], 9:["d1", "d2"], 10:null} text_8 + +-- !complex_10 -- +{"a":1, "b":[1, 2]} 1 +{"a":10, "b":null} 10 +{"a":11, "b":[11, 12]} 11 +{"a":null, "b":[12, 13]} 12 +{"a":13, "b":null} 13 +{"a":14, "b":[14, 15]} 14 +{"a":null, "b":null} 15 +{"a":16, "b":[16, 17]} 16 +{"a":null, "b":[17, 18]} 17 +{"a":18, "b":null} 18 +{"a":19, "b":[1, null]} 19 + +-- !complex_11 -- +2 text_2 +10 text_10 +20 text_20 + +-- !complex_12 -- +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} +14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} +16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} +18 text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} +20 text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]} +22 text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]} +24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null} + +-- !complex_13 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} +15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} +17 text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null} +19 text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} +21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} +23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} + +-- !complex_1 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} + +-- !complex_2 -- +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} + +-- !complex_3 -- +text_24 {24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null} +text_23 {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]} +text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]} + +-- !complex_4 -- +{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]} +{8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]} +{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]} + +-- !complex_5 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} + +-- !complex_6 -- +{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]} +{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]} +{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null} + +-- !complex_7 -- +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} + +-- !complex_8 -- +3 text_3 +7 text_7 +12 text_12 + +-- !complex_9 -- +8 {8:["c", null], 9:["d1", "d2"], 10:null} text_8 + +-- !complex_10 -- +{"a":1, "b":[1, 2]} 1 +{"a":10, "b":null} 10 +{"a":11, "b":[11, 12]} 11 + +-- !complex_11 -- +2 text_2 +10 text_10 +20 text_20 + +-- !complex_12 -- +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} + +-- !complex_13 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} + +-- !complex_1 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} + +-- !complex_2 -- +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null} + +-- !complex_3 -- +text_24 {24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null} +text_23 {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]} +text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]} +text_21 {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null} +text_20 {20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]} +text_19 {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]} +text_18 {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null} +text_17 {17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]} + +-- !complex_4 -- +{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]} +{8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]} +{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]} +{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {"a":6, "b":null} +{5:["m", "n"], 6:null, 7:["x1", "x2"]} {"a":null, "b":[5, 6]} +{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {"a":4, "b":[4, 5]} +{3:["p", "q"], 4:null, 5:["z1", "z2"]} {"a":3, "b":null} +{24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null} + +-- !complex_5 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} + +-- !complex_6 -- +{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]} +{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]} +{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null} +{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]} +{15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null} +{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]} +{17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]} +{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null} + +-- !complex_7 -- +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} + +-- !complex_8 -- +3 text_3 +7 text_7 +12 text_12 + +-- !complex_9 -- +8 {8:["c", null], 9:["d1", "d2"], 10:null} text_8 + +-- !complex_10 -- +{"a":1, "b":[1, 2]} 1 +{"a":10, "b":null} 10 +{"a":11, "b":[11, 12]} 11 +{"a":null, "b":[12, 13]} 12 +{"a":13, "b":null} 13 +{"a":14, "b":[14, 15]} 14 +{"a":null, "b":null} 15 +{"a":16, "b":[16, 17]} 16 + +-- !complex_11 -- +2 text_2 +10 text_10 +20 text_20 + +-- !complex_12 -- +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} +14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} +16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} + +-- !complex_13 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} +15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} + +-- !complex_1 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} +13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} +14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} +15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} +16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} +17 text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null} +18 text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} +19 text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} +20 text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]} +21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} +22 text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]} +23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} +24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null} + +-- !complex_2 -- +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null} +23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} +22 text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]} +21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} +20 text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]} +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +19 text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} +18 text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} +17 text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null} +16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} +15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} +14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} +13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} +12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} +11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} + +-- !complex_3 -- +text_24 {24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null} +text_23 {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]} +text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]} +text_21 {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null} +text_20 {20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]} +text_19 {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]} +text_18 {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null} +text_17 {17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]} +text_16 {16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]} +text_15 {15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null} +text_14 {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]} +text_13 {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null} +text_12 {12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]} +text_11 {11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]} +text_10 {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {"a":10, "b":null} +text_9 {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]} +text_8 {8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]} +text_7 {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]} +text_6 {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {"a":6, "b":null} +text_5 {5:["m", "n"], 6:null, 7:["x1", "x2"]} {"a":null, "b":[5, 6]} +text_4 {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {"a":4, "b":[4, 5]} +text_3 {3:["p", "q"], 4:null, 5:["z1", "z2"]} {"a":3, "b":null} +text_2 {2:["s1", null], 3:["v1", "v2"], 4:null} {"a":null, "b":[2, 3]} +text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]} + +-- !complex_4 -- +{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]} +{8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]} +{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]} +{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {"a":6, "b":null} +{5:["m", "n"], 6:null, 7:["x1", "x2"]} {"a":null, "b":[5, 6]} +{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {"a":4, "b":[4, 5]} +{3:["p", "q"], 4:null, 5:["z1", "z2"]} {"a":3, "b":null} +{24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null} +{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]} +{22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]} +{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null} +{20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]} +{2:["s1", null], 3:["v1", "v2"], 4:null} {"a":null, "b":[2, 3]} +{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]} +{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null} +{17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]} +{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]} +{15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null} +{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]} +{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null} +{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]} +{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]} +{10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {"a":10, "b":null} +{1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]} + +-- !complex_5 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} + +-- !complex_6 -- +{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]} +{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]} +{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null} +{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]} +{15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null} +{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]} +{17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]} +{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null} +{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]} +{20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]} +{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null} +{22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]} +{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]} +{24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null} + +-- !complex_7 -- +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} +13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} +14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} +15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} + +-- !complex_8 -- +3 text_3 +7 text_7 +12 text_12 + +-- !complex_9 -- +8 {8:["c", null], 9:["d1", "d2"], 10:null} text_8 + +-- !complex_10 -- +{"a":1, "b":[1, 2]} 1 +{"a":10, "b":null} 10 +{"a":11, "b":[11, 12]} 11 +{"a":null, "b":[12, 13]} 12 +{"a":13, "b":null} 13 +{"a":14, "b":[14, 15]} 14 +{"a":null, "b":null} 15 +{"a":16, "b":[16, 17]} 16 +{"a":null, "b":[17, 18]} 17 +{"a":18, "b":null} 18 +{"a":19, "b":[1, null]} 19 + +-- !complex_11 -- +2 text_2 +10 text_10 +20 text_20 + +-- !complex_12 -- +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} +14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} +16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} +18 text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} +20 text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]} +22 text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]} +24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null} + +-- !complex_13 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} +15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} +17 text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null} +19 text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} +21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} +23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} + -- !1 -- 1 user1 1.0 false 0.5 1 2 user2 2.0 true 1.0 1 @@ -2481,3 +3195,716 @@ 1 user1 1.0 false 0.5 1 8 user8 8.0 true 4.0 1 1 user1 1.0 false 0.5 1 9 user9 9.0 false 4.5 1 +-- !complex_1 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} + +-- !complex_2 -- +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} + +-- !complex_3 -- +text_24 {24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null} +text_23 {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]} +text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]} + +-- !complex_4 -- +{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]} +{8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]} +{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]} + +-- !complex_5 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} + +-- !complex_6 -- +{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]} +{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]} +{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null} + +-- !complex_7 -- +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} + +-- !complex_8 -- +3 text_3 +7 text_7 +12 text_12 + +-- !complex_9 -- +8 {8:["c", null], 9:["d1", "d2"], 10:null} text_8 + +-- !complex_10 -- +{"a":1, "b":[1, 2]} 1 +{"a":10, "b":null} 10 +{"a":11, "b":[11, 12]} 11 + +-- !complex_11 -- +2 text_2 +10 text_10 +20 text_20 + +-- !complex_12 -- +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} + +-- !complex_13 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} + +-- !complex_1 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} + +-- !complex_2 -- +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null} + +-- !complex_3 -- +text_24 {24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null} +text_23 {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]} +text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]} +text_21 {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null} +text_20 {20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]} +text_19 {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]} +text_18 {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null} +text_17 {17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]} + +-- !complex_4 -- +{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]} +{8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]} +{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]} +{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {"a":6, "b":null} +{5:["m", "n"], 6:null, 7:["x1", "x2"]} {"a":null, "b":[5, 6]} +{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {"a":4, "b":[4, 5]} +{3:["p", "q"], 4:null, 5:["z1", "z2"]} {"a":3, "b":null} +{24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null} + +-- !complex_5 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} + +-- !complex_6 -- +{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]} +{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]} +{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null} +{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]} +{15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null} +{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]} +{17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]} +{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null} + +-- !complex_7 -- +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} + +-- !complex_8 -- +3 text_3 +7 text_7 +12 text_12 + +-- !complex_9 -- +8 {8:["c", null], 9:["d1", "d2"], 10:null} text_8 + +-- !complex_10 -- +{"a":1, "b":[1, 2]} 1 +{"a":10, "b":null} 10 +{"a":11, "b":[11, 12]} 11 +{"a":null, "b":[12, 13]} 12 +{"a":13, "b":null} 13 +{"a":14, "b":[14, 15]} 14 +{"a":null, "b":null} 15 +{"a":16, "b":[16, 17]} 16 + +-- !complex_11 -- +2 text_2 +10 text_10 +20 text_20 + +-- !complex_12 -- +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} +14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} +16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} + +-- !complex_13 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} +15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} + +-- !complex_1 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} +13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} +14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} +15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} +16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} +17 text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null} +18 text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} +19 text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} +20 text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]} +21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} +22 text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]} +23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} +24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null} + +-- !complex_2 -- +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null} +23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} +22 text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]} +21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} +20 text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]} +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +19 text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} +18 text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} +17 text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null} +16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} +15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} +14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} +13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} +12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} +11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} + +-- !complex_3 -- +text_24 {24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null} +text_23 {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]} +text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]} +text_21 {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null} +text_20 {20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]} +text_19 {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]} +text_18 {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null} +text_17 {17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]} +text_16 {16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]} +text_15 {15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null} +text_14 {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]} +text_13 {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null} +text_12 {12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]} +text_11 {11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]} +text_10 {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {"a":10, "b":null} +text_9 {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]} +text_8 {8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]} +text_7 {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]} +text_6 {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {"a":6, "b":null} +text_5 {5:["m", "n"], 6:null, 7:["x1", "x2"]} {"a":null, "b":[5, 6]} +text_4 {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {"a":4, "b":[4, 5]} +text_3 {3:["p", "q"], 4:null, 5:["z1", "z2"]} {"a":3, "b":null} +text_2 {2:["s1", null], 3:["v1", "v2"], 4:null} {"a":null, "b":[2, 3]} +text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]} + +-- !complex_4 -- +{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]} +{8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]} +{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]} +{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {"a":6, "b":null} +{5:["m", "n"], 6:null, 7:["x1", "x2"]} {"a":null, "b":[5, 6]} +{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {"a":4, "b":[4, 5]} +{3:["p", "q"], 4:null, 5:["z1", "z2"]} {"a":3, "b":null} +{24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null} +{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]} +{22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]} +{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null} +{20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]} +{2:["s1", null], 3:["v1", "v2"], 4:null} {"a":null, "b":[2, 3]} +{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]} +{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null} +{17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]} +{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]} +{15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null} +{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]} +{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null} +{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]} +{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]} +{10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {"a":10, "b":null} +{1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]} + +-- !complex_5 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} + +-- !complex_6 -- +{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]} +{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]} +{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null} +{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]} +{15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null} +{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]} +{17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]} +{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null} +{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]} +{20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]} +{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null} +{22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]} +{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]} +{24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null} + +-- !complex_7 -- +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} +13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} +14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} +15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} + +-- !complex_8 -- +3 text_3 +7 text_7 +12 text_12 + +-- !complex_9 -- +8 {8:["c", null], 9:["d1", "d2"], 10:null} text_8 + +-- !complex_10 -- +{"a":1, "b":[1, 2]} 1 +{"a":10, "b":null} 10 +{"a":11, "b":[11, 12]} 11 +{"a":null, "b":[12, 13]} 12 +{"a":13, "b":null} 13 +{"a":14, "b":[14, 15]} 14 +{"a":null, "b":null} 15 +{"a":16, "b":[16, 17]} 16 +{"a":null, "b":[17, 18]} 17 +{"a":18, "b":null} 18 +{"a":19, "b":[1, null]} 19 + +-- !complex_11 -- +2 text_2 +10 text_10 +20 text_20 + +-- !complex_12 -- +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} +14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} +16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} +18 text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} +20 text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]} +22 text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]} +24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null} + +-- !complex_13 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} +15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} +17 text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null} +19 text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} +21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} +23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} + +-- !complex_1 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} + +-- !complex_2 -- +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} + +-- !complex_3 -- +text_24 {24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null} +text_23 {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]} +text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]} + +-- !complex_4 -- +{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]} +{8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]} +{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]} + +-- !complex_5 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} + +-- !complex_6 -- +{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]} +{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]} +{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null} + +-- !complex_7 -- +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} + +-- !complex_8 -- +3 text_3 +7 text_7 +12 text_12 + +-- !complex_9 -- +8 {8:["c", null], 9:["d1", "d2"], 10:null} text_8 + +-- !complex_10 -- +{"a":1, "b":[1, 2]} 1 +{"a":10, "b":null} 10 +{"a":11, "b":[11, 12]} 11 + +-- !complex_11 -- +2 text_2 +10 text_10 +20 text_20 + +-- !complex_12 -- +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} + +-- !complex_13 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} + +-- !complex_1 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} + +-- !complex_2 -- +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null} + +-- !complex_3 -- +text_24 {24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null} +text_23 {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]} +text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]} +text_21 {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null} +text_20 {20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]} +text_19 {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]} +text_18 {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null} +text_17 {17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]} + +-- !complex_4 -- +{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]} +{8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]} +{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]} +{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {"a":6, "b":null} +{5:["m", "n"], 6:null, 7:["x1", "x2"]} {"a":null, "b":[5, 6]} +{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {"a":4, "b":[4, 5]} +{3:["p", "q"], 4:null, 5:["z1", "z2"]} {"a":3, "b":null} +{24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null} + +-- !complex_5 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} + +-- !complex_6 -- +{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]} +{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]} +{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null} +{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]} +{15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null} +{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]} +{17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]} +{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null} + +-- !complex_7 -- +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} + +-- !complex_8 -- +3 text_3 +7 text_7 +12 text_12 + +-- !complex_9 -- +8 {8:["c", null], 9:["d1", "d2"], 10:null} text_8 + +-- !complex_10 -- +{"a":1, "b":[1, 2]} 1 +{"a":10, "b":null} 10 +{"a":11, "b":[11, 12]} 11 +{"a":null, "b":[12, 13]} 12 +{"a":13, "b":null} 13 +{"a":14, "b":[14, 15]} 14 +{"a":null, "b":null} 15 +{"a":16, "b":[16, 17]} 16 + +-- !complex_11 -- +2 text_2 +10 text_10 +20 text_20 + +-- !complex_12 -- +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} +14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} +16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} + +-- !complex_13 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} +15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} + +-- !complex_1 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} +13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} +14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} +15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} +16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} +17 text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null} +18 text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} +19 text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} +20 text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]} +21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} +22 text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]} +23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} +24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null} + +-- !complex_2 -- +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null} +23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} +22 text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]} +21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} +20 text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]} +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +19 text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} +18 text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} +17 text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null} +16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} +15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} +14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} +13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} +12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} +11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} + +-- !complex_3 -- +text_24 {24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null} +text_23 {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]} +text_22 {22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]} +text_21 {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null} +text_20 {20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]} +text_19 {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]} +text_18 {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null} +text_17 {17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]} +text_16 {16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]} +text_15 {15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null} +text_14 {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]} +text_13 {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null} +text_12 {12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]} +text_11 {11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]} +text_10 {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {"a":10, "b":null} +text_9 {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]} +text_8 {8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]} +text_7 {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]} +text_6 {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {"a":6, "b":null} +text_5 {5:["m", "n"], 6:null, 7:["x1", "x2"]} {"a":null, "b":[5, 6]} +text_4 {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {"a":4, "b":[4, 5]} +text_3 {3:["p", "q"], 4:null, 5:["z1", "z2"]} {"a":3, "b":null} +text_2 {2:["s1", null], 3:["v1", "v2"], 4:null} {"a":null, "b":[2, 3]} +text_1 {1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]} + +-- !complex_4 -- +{9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} {"a":9, "b":[9, 10]} +{8:["c", null], 9:["d1", "d2"], 10:null} {"a":null, "b":[8, 9]} +{7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} {"a":7, "b":[7, 8]} +{6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} {"a":6, "b":null} +{5:["m", "n"], 6:null, 7:["x1", "x2"]} {"a":null, "b":[5, 6]} +{4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} {"a":4, "b":[4, 5]} +{3:["p", "q"], 4:null, 5:["z1", "z2"]} {"a":3, "b":null} +{24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null} +{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]} +{22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]} +{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null} +{20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]} +{2:["s1", null], 3:["v1", "v2"], 4:null} {"a":null, "b":[2, 3]} +{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]} +{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null} +{17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]} +{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]} +{15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null} +{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]} +{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null} +{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]} +{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]} +{10:["q1", null], 11:["w1", "w2"], 12:["x", null]} {"a":10, "b":null} +{1:["a", null], 2:["x", "y"], 3:["k1", null]} {"a":1, "b":[1, 2]} + +-- !complex_5 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} + +-- !complex_6 -- +{11:["x", null], 12:["u1", "u2"], 13:["l", null]} {"a":11, "b":[11, 12]} +{12:["p", null], 13:["k1", "k2"], 14:null} {"a":null, "b":[12, 13]} +{13:["d", null], 14:["e1", "e2"], 15:["f1", null]} {"a":13, "b":null} +{14:["f", null], 15:["g1", "g2"], 16:["h1", null]} {"a":14, "b":[14, 15]} +{15:["z", null], 16:["y1", "y2"], 17:null} {"a":null, "b":null} +{16:["i", null], 17:["j1", "j2"], 18:["k", null]} {"a":16, "b":[16, 17]} +{17:["t", null], 18:["u1", "u2"], 19:null} {"a":null, "b":[17, 18]} +{18:["v", null], 19:["w1", "w2"], 20:["aa", null]} {"a":18, "b":null} +{19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} {"a":19, "b":[1, null]} +{20:["y1", null], 21:null, 22:["qq", "rr"]} {"a":null, "b":[2, 3]} +{21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} {"a":21, "b":null} +{22:["a", null], 23:["b", "c"], 24:["dd", null]} {"a":22, "b":[4, 5]} +{23:["k", null], 24:["q1", "q2"], 25:["tt", null]} {"a":null, "b":[6, 7]} +{24:["u1", null], 25:["u2", "u3"], 26:null} {"a":24, "b":null} + +-- !complex_7 -- +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} +13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} +14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} +15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} + +-- !complex_8 -- +3 text_3 +7 text_7 +12 text_12 + +-- !complex_9 -- +8 {8:["c", null], 9:["d1", "d2"], 10:null} text_8 + +-- !complex_10 -- +{"a":1, "b":[1, 2]} 1 +{"a":10, "b":null} 10 +{"a":11, "b":[11, 12]} 11 +{"a":null, "b":[12, 13]} 12 +{"a":13, "b":null} 13 +{"a":14, "b":[14, 15]} 14 +{"a":null, "b":null} 15 +{"a":16, "b":[16, 17]} 16 +{"a":null, "b":[17, 18]} 17 +{"a":18, "b":null} 18 +{"a":19, "b":[1, null]} 19 + +-- !complex_11 -- +2 text_2 +10 text_10 +20 text_20 + +-- !complex_12 -- +2 text_2 {"a":null, "b":[2, 3]} {2:["s1", null], 3:["v1", "v2"], 4:null} +4 text_4 {"a":4, "b":[4, 5]} {4:["k1", null], 5:["a1", "a2"], 6:["t1", null]} +6 text_6 {"a":6, "b":null} {6:["z1", null], 7:["t1", "t2"], 8:["c1", null]} +8 text_8 {"a":null, "b":[8, 9]} {8:["c", null], 9:["d1", "d2"], 10:null} +10 text_10 {"a":10, "b":null} {10:["q1", null], 11:["w1", "w2"], 12:["x", null]} +12 text_12 {"a":null, "b":[12, 13]} {12:["p", null], 13:["k1", "k2"], 14:null} +14 text_14 {"a":14, "b":[14, 15]} {14:["f", null], 15:["g1", "g2"], 16:["h1", null]} +16 text_16 {"a":16, "b":[16, 17]} {16:["i", null], 17:["j1", "j2"], 18:["k", null]} +18 text_18 {"a":18, "b":null} {18:["v", null], 19:["w1", "w2"], 20:["aa", null]} +20 text_20 {"a":null, "b":[2, 3]} {20:["y1", null], 21:null, 22:["qq", "rr"]} +22 text_22 {"a":22, "b":[4, 5]} {22:["a", null], 23:["b", "c"], 24:["dd", null]} +24 text_24 {"a":24, "b":null} {24:["u1", null], 25:["u2", "u3"], 26:null} + +-- !complex_13 -- +1 text_1 {"a":1, "b":[1, 2]} {1:["a", null], 2:["x", "y"], 3:["k1", null]} +3 text_3 {"a":3, "b":null} {3:["p", "q"], 4:null, 5:["z1", "z2"]} +5 text_5 {"a":null, "b":[5, 6]} {5:["m", "n"], 6:null, 7:["x1", "x2"]} +7 text_7 {"a":7, "b":[7, 8]} {7:["h", null], 8:["b1", "b2"], 9:["aa", "bb"]} +9 text_9 {"a":9, "b":[9, 10]} {9:["aa", null], 10:["bb", "cc"], 11:["mm", null]} +11 text_11 {"a":11, "b":[11, 12]} {11:["x", null], 12:["u1", "u2"], 13:["l", null]} +13 text_13 {"a":13, "b":null} {13:["d", null], 14:["e1", "e2"], 15:["f1", null]} +15 text_15 {"a":null, "b":null} {15:["z", null], 16:["y1", "y2"], 17:null} +17 text_17 {"a":null, "b":[17, 18]} {17:["t", null], 18:["u1", "u2"], 19:null} +19 text_19 {"a":19, "b":[1, null]} {19:["x1", null], 20:["x2", "x3"], 21:["pp", null]} +21 text_21 {"a":21, "b":null} {21:["z1", "z2"], 22:["m1", null], 23:["k1", "k2"]} +23 text_23 {"a":null, "b":[6, 7]} {23:["k", null], 24:["q1", "q2"], 25:["tt", null]} diff --git a/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy b/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy index 6ec5438a518892..7d6a3181ceff39 100644 --- a/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy +++ b/regression-test/suites/external_table_p0/hive/test_hive_topn_lazy_mat.groovy @@ -156,9 +156,28 @@ suite("test_hive_topn_lazy_mat", "p0,external,hive,external_docker,external_dock order by a.id,b.id limit 100; """ } - } - + for (String table : ["parquet_topn_lazy_complex_table", "parquet_topn_lazy_complex_table_multi_pages"]) { + for (int limit : limitValues) { + qt_complex_1 """ select * from ${table} order by id asc limit ${limit}; """ + qt_complex_2 """ select * from ${table} order by col1 desc limit ${limit}; """ + qt_complex_3 """ select col1,col3,col2 from ${table} order by id desc limit ${limit}; """ + qt_complex_4 """ select col3,col2 from ${table} order by col1 desc limit ${limit}; """ + + qt_complex_5 """ select * from ${table} where id = 1 order by id limit ${limit}; """ + qt_complex_6 """ select col3,col2 from ${table} where id > 10 order by id limit ${limit}; """ + qt_complex_7 """ select * from ${table} where id between 5 and 15 order by id limit ${limit}; """ + qt_complex_8 """ select id, col1 from ${table} where id in (3, 7, 12) order by id limit ${limit}; """ + + qt_complex_9 """ select id,col3,col1 from ${table} where col1 = 'text_8' order by id limit ${limit}; """ + qt_complex_10 """ select col2,id from ${table} where col1 like 'text_1%' order by id limit ${limit}; """ + qt_complex_11 """ select id, col1 from ${table} where col1 in ('text_2', 'text_10', 'text_20') order by id limit ${limit}; """ + + qt_complex_12 """ select * from ${table} where id%2 = 0 order by id limit ${limit}; """ + qt_complex_13 """ select * from ${table} where id%2 = 1 order by id limit ${limit}; """ + } + } + } for (String hivePrefix : ["hive2"]) { String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort")