From c0034cf730bb16f6d762d66f5aa184c9d621ef55 Mon Sep 17 00:00:00 2001 From: daidai <2017501503@qq.com> Date: Tue, 23 Jul 2024 15:50:12 +0800 Subject: [PATCH 1/3] [fix](serde)fix bug in DataTypeNullableSerDe.deserialize_column_from_fixed_json --- be/src/vec/data_types/serde/data_type_nullable_serde.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/be/src/vec/data_types/serde/data_type_nullable_serde.cpp b/be/src/vec/data_types/serde/data_type_nullable_serde.cpp index 98ff1eb7f81b9b..2adca5f4eb6ce3 100644 --- a/be/src/vec/data_types/serde/data_type_nullable_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_nullable_serde.cpp @@ -138,8 +138,10 @@ Status DataTypeNullableSerDe::deserialize_column_from_fixed_json( auto& null_map = col.get_null_map_data(); auto& nested_column = col.get_nested_column(); - null_map.resize_fill( - rows, null_map.back()); // data_type_nullable::insert_column_last_value_multiple_times() + uint8_t val = null_map.back(); + size_t new_sz = null_map.size() + rows - 1; + null_map.resize_fill(new_sz, + val); // data_type_nullable::insert_column_last_value_multiple_times() if (rows - 1 != 0) { nested_serde->insert_column_last_value_multiple_times(nested_column, rows - 1); } From 8903bcf24367f25ba9853db4ed5788adea9231bc Mon Sep 17 00:00:00 2001 From: daidai <2017501503@qq.com> Date: Mon, 29 Jul 2024 11:44:40 +0800 Subject: [PATCH 2/3] append test --- .../create_preinstalled_scripts/run65.hql | 28 +++++ .../partition_col2=1/000000_0_copy_10 | Bin 0 -> 738 bytes .../partition_col2=1/000000_0_copy_11 | Bin 0 -> 806 bytes .../partition_col2=1/000000_0_copy_12 | Bin 0 -> 936 bytes .../partition_col2=1/000000_0_copy_13 | Bin 0 -> 1153 bytes .../partition_col2=1/000000_0_copy_10 | Bin 0 -> 3193 bytes .../partition_col2=1/000000_0_copy_11 | Bin 0 -> 5703 bytes .../partition_col2=1/000000_0_copy_9 | Bin 0 -> 1687 bytes .../hive/test_hive_opt_fill_partition.out | 101 ++++++++++++++++++ .../hive/test_hive_opt_fill_partition.groovy | 74 +++++++++++++ 10 files changed, 203 insertions(+) create mode 100644 docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run65.hql create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_partition_multi_stripe/partition_col1=hello/partition_col2=1/000000_0_copy_10 create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_partition_multi_stripe/partition_col1=hello/partition_col2=1/000000_0_copy_11 create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_partition_multi_stripe/partition_col1=hello/partition_col2=1/000000_0_copy_12 create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_partition_multi_stripe/partition_col1=hello/partition_col2=1/000000_0_copy_13 create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_partition_multi_row_group/partition_col1=hello/partition_col2=1/000000_0_copy_10 create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_partition_multi_row_group/partition_col1=hello/partition_col2=1/000000_0_copy_11 create mode 100644 docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_partition_multi_row_group/partition_col1=hello/partition_col2=1/000000_0_copy_9 create mode 100644 regression-test/data/external_table_p0/hive/test_hive_opt_fill_partition.out create mode 100644 regression-test/suites/external_table_p0/hive/test_hive_opt_fill_partition.groovy diff --git a/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run65.hql b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run65.hql new file mode 100644 index 00000000000000..2c17d743d5cdd8 --- /dev/null +++ b/docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run65.hql @@ -0,0 +1,28 @@ +use default; + + +CREATE TABLE orc_partition_multi_stripe ( + col1 STRING, + col2 INT, + col3 DOUBLE +) PARTITIONED BY ( + partition_col1 STRING, + partition_col2 INT +) +STORED AS ORC +LOCATION '/user/doris/preinstalled_data/orc_table/orc_partition_multi_stripe'; +; +msck repair table orc_partition_multi_stripe; + +CREATE TABLE parquet_partition_multi_row_group ( + col1 STRING, + col2 INT, + col3 DOUBLE +) PARTITIONED BY ( + partition_col1 STRING, + partition_col2 INT +) +STORED AS PARQUET +LOCATION '/user/doris/preinstalled_data/parquet_table/parquet_partition_multi_row_group'; +; +msck repair table parquet_partition_multi_row_group; diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_partition_multi_stripe/partition_col1=hello/partition_col2=1/000000_0_copy_10 b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_partition_multi_stripe/partition_col1=hello/partition_col2=1/000000_0_copy_10 new file mode 100644 index 0000000000000000000000000000000000000000..46ebfc96e7eef00b78dc0e0a41d48db86cc50afd GIT binary patch literal 738 zcmeYdau#M_;9?hI<=D#@z+l6`@OaYGq_l(t0ink?POms}{%Fv=_$f*(YN~Ms>;?=B z`as2^T+9p%3_^TB9YX9JECMVNUswYeoPb&{JxEGQ5D0p7Pnd9>BQw@?7e~^(`|u;^@KF0OF0g+Bu}NYtZKe+ zM(C20j|`8Kj)J8^H|G&%2AKa2Goks<;>;2x|D|<+T!`Yo!%R^B^>Cs3&*F>7VQCP>*E5Y7=+zEg(!H96>eV~Ikm`Dp^z_DQi30~z z8JA4?u`}Y-=kB1E151u@zhh|Q5Y%H_&?8{Y-p$Z`L_BN(hxK%>02i-W#=JZ;WLOe; zJmr%WF03nKST*SxlVt4-&YUlaY9@=*I~;oL&ajy7YUj=Z3aoMSZtgM5XO~>d#vs7R z@Hj!t;<(Pyvj=r1=q4^pSif<4U|0KNL*)t2IiAR-1x9Hp_O*Y@Bg+LF3V`sf!y`8xDFgd1!tt-Z(15+^3p2BSkh3@dRx|X} literal 0 HcmV?d00001 diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_partition_multi_stripe/partition_col1=hello/partition_col2=1/000000_0_copy_11 b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_partition_multi_stripe/partition_col1=hello/partition_col2=1/000000_0_copy_11 new file mode 100644 index 0000000000000000000000000000000000000000..77c18939ffc18abce676158909ca57f990e0fe96 GIT binary patch literal 806 zcmeYdau#M_;9?hI<=D#@z+l6`@OaYGq_l(t0ink?POms}{%Fv=_$f*(YN~Ms>;?=B z`as2^T+9p%3_^TB9YX9JECMVNUswYeoPb&{JxEGQ5D0p7Pnd9>BQw@?7e~^(`|u;^@KF0OF0g+Bu}NYtZKe+ zM(C20j|`8Kj)J8^H|G&%2AKa2v!VIV;>ybkBpT!w9g#S|R z*DEBu;d8$JBBt6K|RI= zJp$J3-3;AF#KRVFSWo8)aPgXD%*!)Fh9!~5Q$AVY!n!hsRg<1EN!HHb&H0jOX0kYa z2E$YjhF-ffY^J;1xik*(n*1`54qN>DA2Z`)n+X>K83Y&^9w&&sNSyfSS%TP!X$HX? zt^?E0(Ze5S2%cCST5tszO)4i3Djzv>;)u@CNpt2WbEx$mSh7(_nB7{ynAvc)BXhf` zwmY*sL-SOY2|i;*+0lx8~~P@2{r%# literal 0 HcmV?d00001 diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_partition_multi_stripe/partition_col1=hello/partition_col2=1/000000_0_copy_12 b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/orc_table/orc_partition_multi_stripe/partition_col1=hello/partition_col2=1/000000_0_copy_12 new file mode 100644 index 0000000000000000000000000000000000000000..e1327b620f2d0e55a7d96a029c1bc668962c8319 GIT binary patch literal 936 zcmeYdau#M_;9?hI<=D#^z+l6`@OaYGq_l(t0ink?POm(2{%Fv=_$f*(YN~M+>;?=B z`as2^T+9p%3_^TB9YX9JECMVNUsMAa+<;myJxEGQ5D0p7~RI$-2GADM2hWPQ5U*^o0 zRDO4?c-gVoCjXZ;OIOAV87(;-C;KYa%05~mSN7)>r#O39&>m((3tEda%aDRLZ3YW^ z&>m)k25pa20D}`!&{~{PLj-Ng{d$FjInO}>D+mo(pjwnb1&ad%)rq_S1;#42z{#*Q zh~n#+#taOn4qbMJ8lVG{9w!JVOnC5+>0*yiSHdh$CoUgj?-dN0ZVN2TC!{f5%5j(_ z>9M)_=*$C;dK^jvoqS|?oOBc{6}mZ(Ff)7r8qoD9>FJHrk_Qf`GA^0&V`s#v&)q>S z2bLV+e$UXxA*si>phv)(y_=!?hdcAOk$RMuQ0Ay zHFKi2P{y}Jvkf=fZ(cYvFOexknbG98f%KI(a+(V`%%^h&u&lmW`@gZFQ0~MV4F)kr zhQ|qF8R=6WJxmZgF*V}&iU$ieP8UA0`?$&k-N4lkRus>MoLP8SwhKY4NF6i(;FEbCLJMO zJ_kWf?a&1sQ%usOBn&J$4oOV9nQ%;P!m8ape8PWhBp5jQ7zG$58Wfb67X-s7#OU7%x`?CAXbyW&&JsnCs#aIJP2_m4nC$z0rRy~lEVUeF3t+GTrIx3V z)biNk%(5fr;i)BU1}L&o63byWXkzJ+3Se+TN-P#<)Q}R({d$FjInMzT$345toVc7__D1Ct&n2q#Q<@Q~?Zk5E^_ zEKescA7k$o44G~VEX*gQF4QMAH+$>6$S*4^*&91_nP&-5`dNHH=zP7vF1RPE^DgE|w`FCJL)V8O<4p%b@H zC{54}T>W50;S68TNzLvZGbeUNcBxw09~bbEYV=H2RpTpDNMlPfYi48B4(nrUWnk0x zR9K2s5J*YQV3JVsS-MJb^jj lYE)V~XPq1m1IGkL0Y-@i1tlg19*uef_NFh)%>F^n;sAClhbI64 literal 0 HcmV?d00001 diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_partition_multi_row_group/partition_col1=hello/partition_col2=1/000000_0_copy_10 b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_partition_multi_row_group/partition_col1=hello/partition_col2=1/000000_0_copy_10 new file mode 100644 index 0000000000000000000000000000000000000000..fbe3d0ce52a024f4d405e6bfc4950f9f8dbe65dc GIT binary patch literal 3193 zcmeH~J#X4T7{~97<5;{XMB%Z{k_VPpsYIz?AfXaNvsG1wsQmzy0k>G%LDJC(TYL%rD*5mWtbM{}ppP&0nK0kf00}olqo&g6u zLNw4-*KGiT9OMcx#&lW$Q6apa4q6EvEsf|C33`q(rYF5y1aU6L(ew%-KTUYQ^@YL_ zO*|0_-|Exj!M%x!iGQZZqDMs+{V=;;e7ofBSKbzOu(O}9uXkJeA@y2r zJ}?j@=Uo#?15s|tRaI0VEtf@Jkb?&RNnNd1Ww;FiRU zK=T5KJ?hLU6aoKybGg6M>gyAn>v=5j^ZO5IpQ}3c>Y36akTbU>fT6K}-U} zT$BKjzGE8dFc%ZRJwyQzX@qI0_wbMFpClxLWQAm}T&Hil#lF+Fi{H8(n`BOVj`iw8 z)9QR~S`H%bzF6+p3%mca?dC6Sx94;_dA+D>MJ<0+yKq{Kyk_dw@o~ANX|HXwRxva~ k({#hqDus~~V0Is1Q?V{hRCd8X$(}qI z;$cr_55$8Z8=t^oF7*8yZU_22X!R%CAK?}d^#2!nAGM%>HFdwGV39HK=bu8d8 z4zFMy3kcDI_Va~xjBo@;;`oxIhy7SoWx;N4ZzUA#+M`$a==+l6=$oD`V(}r2r_DWr z>}FZ8w{}oxk2X9B>U^t>`xpPt87BTaBtO<@=wrRp+CSXuviB={1G{}^Kfg=7d)YY; zb|x%u`toLM?;*9}J(uqnNIe!EFIbAuaTUkpe->Z%d_*{8pX->epQQj(S652UA^kLo zrLT|4yt48>n6viG&&6!c+%m8q;m|BE3)%dpK1-L1{KHNX;lM*Rg`|V9@>|JhqKZSx zyde6O_5~P8mz;EB7fs9n%TpyzWdzLIOHum+A?+8ZV!kUZ*$zAAq+^aAq-lMndZbF?mAUGOeBslQ{1SfvC z2pWR`L1U1SphbsN-%-%A#}vybI0yg)2LZPTYC(XY7Gxx7gaCp@h>_rS2q3r}a*Loc z3=mX?83}4(fS?v;B)A>{2(CxmA}Eal1f@|%f=UD+s6-eE&M`o6j@=??N&rDqVk9WZ z06|G+Bq%EYL0NH&pgsl=)W;YJn&SXLbDWXjP6Y_=RJRDK697SVf{~y;2@upL83}Hp z0KrYvErQc2fZ%kBk)S#a5LBlb39e=Uf~y&~2o5zT0g--kv}lJKqd?eb`R%Bl&`E+sV{5zC=6aN*bYo#*K26A5GqtvqBuPT_q@gdR gQ|avdQgUs5F`ZpZ8LyO#icq_tKS6|6>0iiy0h|A$mjD0& literal 0 HcmV?d00001 diff --git a/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_partition_multi_row_group/partition_col1=hello/partition_col2=1/000000_0_copy_9 b/docker/thirdparties/docker-compose/hive/scripts/preinstalled_data/parquet_table/parquet_partition_multi_row_group/partition_col1=hello/partition_col2=1/000000_0_copy_9 new file mode 100644 index 0000000000000000000000000000000000000000..f3c7d4fe72d98e234d13874e2a0689cb79fea691 GIT binary patch literal 1687 zcmd^=&2G~`5XWcNZt9jI<cYpH!XKbGwy>(E170Lv0G<6kY(5VO^EI1Z5)%JOE`+ z_2JjWU*%!i?}`^Y468RI`mvP@LWK(5WD_z@+`X)eQal!f_yNLrA{ zrRklKxpDmZNjtg--}jsCuoGYbaA%L}YO{Ch>`_Vb>A(ftp1K~0&#K`YzNMMCW7G<& zW~3Vs#F&YpG)wbWyaq;@lu?n(l9x9C>l-tX&%xXjcFn{!qe-GIRcZlk#7*3uqj@Rq zhKUy30+)_n7E`nHmVPw{=tf+Nr;C?GFKrBX#%(~zD%kxhZ5rbjs#mgibgh^!( zb;8Q0UN_XA9`&Q(#k+RU{n!p7(BH&Ca&{gLo_CVcMVR!XUbo~_9J^wd_8aHXX{%&= xPH=EgtJ?O<&}-CP+qG@S4V-$_t2S$Ox6wMRHV?hvnc Date: Wed, 31 Jul 2024 14:07:47 +0800 Subject: [PATCH 3/3] move code. --- .../data_types/serde/data_type_nullable_serde.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/be/src/vec/data_types/serde/data_type_nullable_serde.cpp b/be/src/vec/data_types/serde/data_type_nullable_serde.cpp index 2adca5f4eb6ce3..014e8b0d5ebd24 100644 --- a/be/src/vec/data_types/serde/data_type_nullable_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_nullable_serde.cpp @@ -135,14 +135,14 @@ Status DataTypeNullableSerDe::deserialize_column_from_fixed_json( if (!st.ok()) { return st; } - auto& null_map = col.get_null_map_data(); - auto& nested_column = col.get_nested_column(); - - uint8_t val = null_map.back(); - size_t new_sz = null_map.size() + rows - 1; - null_map.resize_fill(new_sz, - val); // data_type_nullable::insert_column_last_value_multiple_times() if (rows - 1 != 0) { + auto& null_map = col.get_null_map_data(); + auto& nested_column = col.get_nested_column(); + + uint8_t val = null_map.back(); + size_t new_sz = null_map.size() + rows - 1; + null_map.resize_fill(new_sz, + val); // data_type_nullable::insert_column_last_value_multiple_times() nested_serde->insert_column_last_value_multiple_times(nested_column, rows - 1); } *num_deserialized = rows;