From eba2e8f9b66c55033e821ae72e3af75727e71d6e Mon Sep 17 00:00:00 2001 From: ion-elgreco <15728914+ion-elgreco@users.noreply.github.com> Date: Sat, 9 Mar 2024 13:29:01 +0100 Subject: [PATCH] read only checkpoints that match _last_updated version --- crates/core/src/kernel/snapshot/log_segment.rs | 2 +- .../_delta_log/00000000000000000000.json | 4 ++++ .../00000000000000000001.checkpoint.parquet | Bin 0 -> 22972 bytes .../_delta_log/00000000000000000001.json | 2 ++ .../_delta_log/00000000000000000002.json | 2 ++ .../00000000000000000003.checkpoint.parquet | Bin 0 -> 23521 bytes .../_delta_log/00000000000000000003.json | 2 ++ .../_delta_log/_last_checkpoint | 1 + ...8-4d2b-87a0-89be205c338b-c000.snappy.parquet | Bin 0 -> 1432 bytes ...9-463c-bd2d-f68333e6d03d-c000.snappy.parquet | Bin 0 -> 1432 bytes ...2-41a5-aca6-c2d1f2c43873-c000.snappy.parquet | Bin 0 -> 1432 bytes ...4-4c54-810d-25ba3f0a4016-c000.snappy.parquet | Bin 0 -> 1432 bytes python/tests/test_table_read.py | 6 ++++++ 13 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 crates/test/tests/data/table_failed_last_checkpoint_update/_delta_log/00000000000000000000.json create mode 100644 crates/test/tests/data/table_failed_last_checkpoint_update/_delta_log/00000000000000000001.checkpoint.parquet create mode 100644 crates/test/tests/data/table_failed_last_checkpoint_update/_delta_log/00000000000000000001.json create mode 100644 crates/test/tests/data/table_failed_last_checkpoint_update/_delta_log/00000000000000000002.json create mode 100644 crates/test/tests/data/table_failed_last_checkpoint_update/_delta_log/00000000000000000003.checkpoint.parquet create mode 100644 crates/test/tests/data/table_failed_last_checkpoint_update/_delta_log/00000000000000000003.json create mode 100644 crates/test/tests/data/table_failed_last_checkpoint_update/_delta_log/_last_checkpoint create mode 100644 crates/test/tests/data/table_failed_last_checkpoint_update/part-00001-6791b37e-f318-4d2b-87a0-89be205c338b-c000.snappy.parquet create mode 100644 crates/test/tests/data/table_failed_last_checkpoint_update/part-00001-9c90a84d-6999-463c-bd2d-f68333e6d03d-c000.snappy.parquet create mode 100644 crates/test/tests/data/table_failed_last_checkpoint_update/part-00001-bea93a33-9112-41a5-aca6-c2d1f2c43873-c000.snappy.parquet create mode 100644 crates/test/tests/data/table_failed_last_checkpoint_update/part-00001-fed6d112-d244-4c54-810d-25ba3f0a4016-c000.snappy.parquet diff --git a/crates/core/src/kernel/snapshot/log_segment.rs b/crates/core/src/kernel/snapshot/log_segment.rs index 8a88c6a600..323d52cd82 100644 --- a/crates/core/src/kernel/snapshot/log_segment.rs +++ b/crates/core/src/kernel/snapshot/log_segment.rs @@ -449,7 +449,7 @@ async fn list_log_files_with_checkpoint( let checkpoint_files = files .iter() .filter_map(|f| { - if f.location.is_checkpoint_file() { + if f.location.is_checkpoint_file() && f.location.commit_version() == Some(cp.version) { Some(f.clone()) } else { None diff --git a/crates/test/tests/data/table_failed_last_checkpoint_update/_delta_log/00000000000000000000.json b/crates/test/tests/data/table_failed_last_checkpoint_update/_delta_log/00000000000000000000.json new file mode 100644 index 0000000000..ae3330f9ab --- /dev/null +++ b/crates/test/tests/data/table_failed_last_checkpoint_update/_delta_log/00000000000000000000.json @@ -0,0 +1,4 @@ +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"98c9faeb-7940-43eb-9898-50b2a99c0a7e","name":null,"description":null,"format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"price\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"sold\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"deleted\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"createdTime":1709986334419,"configuration":{}}} +{"add":{"path":"part-00001-9c90a84d-6999-463c-bd2d-f68333e6d03d-c000.snappy.parquet","partitionValues":{},"size":1432,"modificationTime":1709986334424,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"deleted\":false,\"id\":\"1\",\"sold\":0,\"price\":0},\"maxValues\":{\"price\":4,\"id\":\"5\",\"deleted\":false,\"sold\":4},\"nullCount\":{\"deleted\":0,\"id\":0,\"price\":0,\"sold\":0}}","tags":null,"deletionVector":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"commitInfo":{"timestamp":1709986334424,"operation":"WRITE","operationParameters":{"mode":"Append"},"clientVersion":"delta-rs.0.17.1"}} \ No newline at end of file diff --git a/crates/test/tests/data/table_failed_last_checkpoint_update/_delta_log/00000000000000000001.checkpoint.parquet b/crates/test/tests/data/table_failed_last_checkpoint_update/_delta_log/00000000000000000001.checkpoint.parquet new file mode 100644 index 0000000000000000000000000000000000000000..86af3c9c473427bb72a454098095369455ba8b8c GIT binary patch literal 22972 zcmeG^4Qy29mNR8qDxizTdoz7$T&;MwE0oSmJGIkApLaU#45gH|w4HVobviR`hfZhe z%=8yAtk1`~K4Vyxu!Ln<)@50jHN+T02rlc! z?|1LqduKWYeTxYLoqqTGIsfN;=lkw~Ub){bID}rIw_B(XY!BFM>uffguQlcyizY@J zp71rh8k(D+2sYsX;eic8wXjyG5~^xjs_NI3v(8!8to!c= zGFISEwar!`tf}$Zs@B%jS5GDK(LgjGtw_cNn^-NLtAM{!O0xY7_(3iT>z)Oe|lwp>8ahNX2t?ZO`ngL)u^{8Ha9dS~5Kj&*_;|Dmt2iaafqx zP>0fvgS6|~cJ1E1fv=s;CSwV0c`CD<#kovMhdPRT^-9Nq7Y99lFE*C zz%CxJOB=vkH6U6`tg7drwO&MG+T9)!9@1@=6|j|8+hDh}tyl|R>Yf?m8~WPmXf~h3 zEv`M2nwd)H9I0e3zY?PTR3e=xS9(@_w)kdRE7Y3MsriE$+r6<&dMr6UlZ_(Am4G3e zOypKfCKd=&M2r;2v`ogVQB}@HQ!^|_Ft8;GqN?hZASn>bi!%ujfGOKrsSG*QhNkN^ zx}2$%i@0hmlbwp@tEaP>*G#Xs+ zaW{-5;w^Ev+tU#DG&eUi$GpuAt!`Jm!Q&l`HjTNW%`SIKLky;F%%y?Yg^h~)7{K^q zK3BB0Io{CX^Z6Q@Tbg1Gqj68XVXURKsi`T^5_dJlt-ye`9uywrQmSon)VKcx)H(G) zZk0C8R*u>1E40(pifBCUn2zQrC=%?i6!%w3A5>r{xWkAq+yM_&g6834au7#-wL0eH zEj{x#hrmq@dCw&mSq#?idS(z}QNTdh!@|R=MXvfEwl#mQZP7-Xs8;7VtJPV00#Q49 z3fE3$;>odOjJ7GhN1w2ZC+yND+M~_zkXmyu`87Yn+SrlZQE#_7?KV9HAk~`3Y*jV& zXdJ8X@Yg;OO^-7%b%|Y)Z4+1rUj_oWS_ywu5u5?Nz&UYdsy`9SWbw%9g=1tgJ%rmG zJngDe4{&W`(Nqr3nCvjw=GuTq(zZG`9OUST*yh>|E2Hz;N@ZrVGSkavLQf%7Scym3 z_RLH=k2n!BAW-1yW<#fnfA{X?({7|#2W@HE3rSX~Yc|r(rzsNBk*uz9?Z(5c(bUe^ zf2oRaE|>H>M-O^eGKSj9Tt1r5QLO!NjridjXlvf5iaLhRAqCx%MlDW1{ZB14XRbljYq|3q9A056wxi`&*- zO$)6ko%<*_LZUsCh~+cc)j9Zzjwc4-_LJOfx!12tz`qZ3rES^G=~ObE2+ZCx3gzHn z#X?nPY%GVzy<50EM(Rs|UX9t#=CpqypI|}gR-~gOCsw4djzzQaWICEk=GiIQ0AF{9 zp(JpY8Js%itX5lEOucH;GB_e(4TaR_!5MRj_$n>!Ao=N&Iu(`%71JbhFQrKiVU@5- ze=KxcY_E2=)9xnH=vKpa57MT*3Lw`lpzR)>1F3H=@?u_+Sp4Up3iDjdaq(k;99}1| zkCV5Iq$rvyMM@=YBqS?#4ZyB_%&<{b=3Yfsx??NOv7ig?OcsdJ%3siJDShgPC|Oc(;3%&^f+%)Mqa!8sPxIP&(M+@_BI$O`L`A%`ndl-P8ZLe;*e~K??9!Blm*UFe#~syfZo{Gfoa7dD6A?VLZv1Vt#5D^#H=WWmoHmiFVZheubz6!6(CZ;m83GH&paFJIIEXCOC6$nU|AXKA56^W z6X|#&-kD4#wkmJPi549s2xQ`jzl2e$S}#$T;P9vLS`nC66h?;&AcH z+MfWCF4_O7P!${|_$!;A1$q(cYwoY8DX)yB%xRFei&eIo`|UMw8;3vGQEk-$h@ZaQ znuXg3F{Kg=a{WqtGB?qVa>c0a9~Z^v1#esTJudCS<`Yr zm~O8cRQgQPzf(F;?CM!j*Hr0-VY?fDbR9f#f%$4SqZ`It6s-||>AFqov3mioS2u$v z*Z`MS7XqLp7vSQMZj}ixr^%YUpKth1QK^xRS}`wtZWlppHMQbr4#A@|uT}EqrPadm zdz9|)9}*^%=BY=7r5aU3-X*w{ z-pg*@-0u}GRVdxp+c*(tpA>eglYcFADb0OB;Z&8ktsg+V201V_9CDA zeU8t)I>RSioafE=cJbzcmwEHsJ$5d(S6|WEhks)iHY&IezpAxA{;f9W%3iy$NtyHh zYg+sD*R}Tf-zjZyw3}N)Yogg~X6|`b0Vb@nc`KZ}20~a1so7%{PB@)CX6vp+=fp=U z6xDa=OFi}d1$AZWfCC`t>vs^Nsf-4ydaJrO0Yb0CNhLKx1;eeD)+o?mZDFMfEUf6> zz!s{v+kykv+MJkmsZ<~zEKkxPDU~K{I*387IMHUP)gZTn1;J9?1>&gyGEDUsT6aMy z1PTgE4OIGA@q)?_id8K_pqX&dEIgnn3~zw-6Dehf;wQpxcvlPO3Fy`xHy8}|8b}j` zF-N~0kx*`YI{*wf>R`sCaQ)4?%03!`;YM5?LS-m7*{A}WQ-Al2(t;k{Z81u@;I>qN zLmP@zccY7Qv`bW+VDKlL&u$Z#*6EziRG^-K84UH_<~CsP+ZL5j@M}h~N&!-%P=HN6 z15`)sM7G>q?wX4GBhlX zJ-R`BmigO}zxu3NGZ^ld#gLbOT~G}v9;5Y_U_DsF zhg%DkYOenZN_%9~>fHh`sLw2;+>_Trj86L>(?N-(VY!T zN5c%l4T5l)L3nSZ1Ox*W*M_A_!wkm0x6vnVG8k7P9EN2LkW~fNyAkO?guyyNu#WyS z3Fxh7OTsdoEh(B~PJQ(m>9c1TxDN>2M-1H8JNTwxMVhg)>huoj@D2v%4+Q2419SBG zl3?^FSx(Zo&q<#?$6$R;uwFev9RBeI9(5CurJx43~IcpNjym)_x=mX25zM2qqUG3b1yOT5n+BCFat9` zCb?!yVb)&oW4+Pwq_igqxdS3#-+)f{L})G%ny&y2Bq!G<)or5`7WGDe)P=8mPfQkW zCEpUv14l_{-%OR%gL4y~QK`k2+Upa}YJfs#0JO?mgOTRo@N@)CANYw+YwzxVYots&%*qMd0(3*60 zab9|Bez};h66WK_SO9pbEXvh?-t z<>Edrv zY?0+TxqV#j!Y=qN%YN#IM|hUy&G-ae>Th@&grC45&h3&j@HGc(+ha03)6(|9LStj= z9IgxC7k>KO%`}d^9peBN;K6*4+=1(1nLmRR<#8?!$a1cAwy`nX+uhhWPtRMi6F&Zq z5dzInVNb^*;A_ma!dD2Nfgk15GeQj_-Gxm=EQgT3^)ZCsiO+m~M<=IOZr_UIew0!u zGBFw&O5x0Hy#qiGiB6?Q_V;moavtfO!y$h*GCV2!aVR)66AyJwM~1gD`Goo>B2CKj z-rlA8>m5LP2YoQLEdKmE7aJRW9Vn0D{K+mHH}coK1b@RwZwdYa$ZyC$5li<^$AhWa z(G-(iY_5F{@qxBMG^9^xbhHzFp@0kM#S!*lmPu1%o>WVOo&)0+`~)?91^W1SaHu&B zGLHm@Ci|ySxyYb96%Te!jtuut5CZ>Bz}JdKth3J%u1{H}aqtUSjs!bhBZEDek+3(7 zbo7&aa71RN1>DFJL|}(+tg(@ij>AO0+<_gz{?ur2$Q2FyW`=^Pd}O%CH4=7{(Ttnr z?bq8=49DjXh<^^~2WP_%qV_LD!tUAl6tK}ffc!M$2$x4+(3C0mMf{ zgJZEz-)4$0oxM9je`+TS|IB~2`*(soT8aNyQ~zvisxuuK>i5QigG|c${Rngy5Bk7D{TY-m zvtQNTJNkDb5zN;~G{eYWBjHCU(Bix7CjF5s1;`a>{y5HWSwwmY`R&v3+ePCPs^U|f zxp)|-`Zuo!Ooj1F*5zk`4y2b%DVR?s`eaItA`myhFH&HOWGb-lr_nCq)3+Yt-BSGd z(LP!Ro&b8w;%^JicMq8LspqeQ#tr-h=1TMD&mp}f_@k{8>|+7)ES6i4?9e^~rRnLW zaiRz8VLo{xPAM!YaM}xoW<6}GVS0a88>!VG6iXE}J zKnD3U_`g2hj^H@S1M=O`rpPp$uM|sxJxTUYK@N}~!Tqg0fc#^pwm4nfRPe9c91?Qz99G|28!$Y|ZP6hh~zGxA+ zF8MtGr{_X`3*@&I@s*Gtnc|4YHv=&&9Utz3m z8mD{(Y-(|&scSkG!abo`_ADWv1srGRW61H8{e$bf8AEk}zJtUo_uoz$q&}S2r(~Td zHRXJPV?nw<=-~GU@{>>6oHg(-d+wAp&$ H|84$1fFDJs literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/table_failed_last_checkpoint_update/_delta_log/00000000000000000001.json b/crates/test/tests/data/table_failed_last_checkpoint_update/_delta_log/00000000000000000001.json new file mode 100644 index 0000000000..b4d6ce3d67 --- /dev/null +++ b/crates/test/tests/data/table_failed_last_checkpoint_update/_delta_log/00000000000000000001.json @@ -0,0 +1,2 @@ +{"add":{"path":"part-00001-fed6d112-d244-4c54-810d-25ba3f0a4016-c000.snappy.parquet","partitionValues":{},"size":1432,"modificationTime":1709986334474,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"deleted\":false,\"price\":0,\"id\":\"1\",\"sold\":0},\"maxValues\":{\"price\":4,\"id\":\"5\",\"sold\":4,\"deleted\":false},\"nullCount\":{\"id\":0,\"sold\":0,\"deleted\":0,\"price\":0}}","tags":null,"deletionVector":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"commitInfo":{"timestamp":1709986334475,"operation":"WRITE","operationParameters":{"mode":"Append"},"clientVersion":"delta-rs.0.17.1"}} \ No newline at end of file diff --git a/crates/test/tests/data/table_failed_last_checkpoint_update/_delta_log/00000000000000000002.json b/crates/test/tests/data/table_failed_last_checkpoint_update/_delta_log/00000000000000000002.json new file mode 100644 index 0000000000..6df3799b4b --- /dev/null +++ b/crates/test/tests/data/table_failed_last_checkpoint_update/_delta_log/00000000000000000002.json @@ -0,0 +1,2 @@ +{"add":{"path":"part-00001-6791b37e-f318-4d2b-87a0-89be205c338b-c000.snappy.parquet","partitionValues":{},"size":1432,"modificationTime":1709986423857,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"deleted\":false,\"price\":0,\"id\":\"1\",\"sold\":0},\"maxValues\":{\"sold\":4,\"id\":\"5\",\"price\":4,\"deleted\":false},\"nullCount\":{\"id\":0,\"price\":0,\"sold\":0,\"deleted\":0}}","tags":null,"deletionVector":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"commitInfo":{"timestamp":1709986423857,"operation":"WRITE","operationParameters":{"mode":"Append"},"clientVersion":"delta-rs.0.17.1"}} \ No newline at end of file diff --git a/crates/test/tests/data/table_failed_last_checkpoint_update/_delta_log/00000000000000000003.checkpoint.parquet b/crates/test/tests/data/table_failed_last_checkpoint_update/_delta_log/00000000000000000003.checkpoint.parquet new file mode 100644 index 0000000000000000000000000000000000000000..cf46a2973e6777bed24da0984f6fbe067f0c1a2b GIT binary patch literal 23521 zcmeHP3v66fdY*A6i8nE9h`3kdl|{jXTbIP~JUoskf`vOCd&Z9A*v8LkfZlo79(x{U zX6%U#p;;EOY&OeMHce?7rBRyFZA#NLO;Lna6+){tilQi@D2fn5C_*cQ5JCt=QG`(c z|3CLN_ud(g6IcZ)10J7y9^d~w|M}0k$LSXb-JFB#=lXlO8qT)fX4_!1+5GJ>|Adr? zHgEB_x|&Ym zb=*2tUYR=wG?2o2Wsx9%53N-cRTMW#%cyUxqGRwRF{uUCDPIpg&a&Mnyc6E;;-K&yzgKU z>W*Hm^?JwQEJH4HI9NDfqM=*%RbG%~e)u1U*%QJHcP=G1hS}iv@m4aq# z$TrlRsa;j^s)=kaEfs2KbJ_W1Jdq<&57_wwcEQ8KlY&$NS-oYQI?Ik3$*#^)0+Tl> z+aP~w%2ig`%c50Y+qGP+qHUN{R+Gws^-lWM6b$ZM!__F02^3n$#11XwA%SW~ z61lvRtv$y-Cv3p^q(g#vqKtxbQP`DPDb3Dyvy{A-?=3%htdW$kgh;FBb-FWPopEqyAk(d-guDZEfkC6$ z1s`{FG$HxDlGoepce_2!t!~NJEXAa@=9nk$p76w4z3p4P%`xb@IiCS$7dFe&#{ko| z#qW-Kw`1}UWVbo(Ha!cV=K6o&r3VwSY!0`?KG^0aGb6Yh!_%|4G2YtfhV3M6RJXb|H7Z-qtrJoz z54&Qv{oU%?4I`yp+DKzRLh=EUJfKyg~t7re2kKRh+7LH(Od~jGNKBQQMknje57>y_99e7fe2*b^~6K zo{nW@4ET)2LU6|ZL!AK5rC3*WBG1&@#}4`sW?J-C<_l6G&sM$l{K@sgo;u9O9ZgHg zYnIQb0VM(${m$5rkiNrM^#eDRY0p!3dwFDyn!to{N%}FSS+tL33xjF9b8~ykJsa%R z(J8;xDNX6C7RXVMCe53+??{Rg@M>ANIEL?EscNSkt#CK_H|)Sh60t%yw>A%7(qv)? zj>T!3E%#i03H-Y;Yu%CS&ZLr=L~#D5Nys}&D-l*@CnoZE4sa816_EQ1&}%X8?9K!h z3JDg3ZbmzDR%pfg+L)A!Co@tiSzyO227KM&mYhmk=F&==vsNi-LG{ZYYq>PBP)8y4 zX_$=pM0|~w?~(rW>9vBTp<;e&uBH6c!L8xe=+6t>u)f~Q-CMHJvb&vjH?c;?UAFET z;7xuKQAA#Xn??)0bbjn zS!^#0ix`LZ1I}Iy0QJ_lQgv1AhSP-vO z+MH_1$Vvq_QCKPCCT47O6LYQEO|Xvzw+^g#(zlVz3T~osY2_vwAi9b6L@cRCtvBf= z3gM;QL>Kwcaq(jzewi7@PSaUA=6^A>mppVeKGus-2c?ntc4CDPgXgm%nlIN_}ZOLq1q8S5S;*TeHCSoYJiYFt*Y zUO;xp)!`~lQyLEM!H?>I7CTCkfsCg~oB7Qc+41T|RpakX>?$NO@kG2UnM!PzuTBsz zIwa#I5NmZAMAS|OE2`@!+)va@M3Iq6s%FPwS=J@DDP#Ot`BVJwfRHZP|E@@tY$nuq zdEX0q8Sd-vsj08-ep8j*AZ_Q@*y``G*TXR={$NF&RRs`!{8lfs+&V;BVHns^Uli+# zO7qh4n0f%B@_JG$RgVY5>5ZHGOBsmOI7Q@NO0|i)sR9N*u6v_$Q!hhGA#UcSOb!`# zfIk2`>MI6aQQxc_=tHw|{H|-(+)IW7Z_>6+=3{cp1Ei8;bMmH25KEq3J*e(6Bp>OWHY*E%tg@~@t3aJWFET@aDBQSYJv^8G`bXkl3c*W zCfz6#uF8`&d*5I4?P5|R9}RqgyJF`-ZS@WO`wq?{7r$Dm7B8>mj{Qik{`x*{N-myw zfO}jn?t75?sa*VcgIaq@;No)a)h4y}?T6Lk;Z59_+~KXwYVoX#bIG+IxYgo`E!>qF zx%$APDihZq<5F_%!2s7I7vI^zom?fC&+k-ud;KYu^yg2jwZ|m2_G(lW`eJLTwePv2+V|L^+V_p;)V@cb zS3B(anO!Y^@^h_x_9d-+_?LE7b8qd{$`^iR=QhjaFYMFG-~3wZv;Q}C?h(1qA79nV zzkf|DU;nLKhPn35_Hdn)%VmpCv)h5(8k?`isosy_)mBdXb;(X8*qN&BV8ia zSLm++>+4JI94ZYCK+xaqLW-sKoCC+&oS1kiT%a5*Rnm~;TS%-ss6n}nr1em43f&47grz$?%P)|5(=GMyw!$V+wN6OmaVjNL7yeEe(1yt*H91MfK3DQ(y zOw(^gC6pfD3V`8I9o(1W`mMTTr7rFh(tx9GG&o zOr^R*UF@Tsqhbfcd_ohj0|jPv+Nv`XD4Sr0p&Z@Z0)~3vqA&{cnsKaffZWI&kUz<( z6mlT%&bYhFxRiOXYvD-fS(bZI#s+gsAa?sW7hhN3Q2&irxKPueoRpXQQ;&jB`#t=r z2mkJyc8=!{{jHrlVYdk{t+e2y^shSLM)w&9{aputg~pygVCNcb!m(8rL-nDX4g~NG z$X6?c%PSee7liORL%4CT1wu)XCkS@2{PJ4ih1IB#J2%Vk+$_hx>#6$BVeqJ%fAQb# zoRfd$@9f;+8k_LppEal#($w@spGA=4vA1Vm-zS{AZ<%+*`9naC=imH$JNHwY@WlhF zpD+6w_=yLE*B)SV=M-Un2ADAW&Oc~@S(>{j`HFf#gs-yqJt%zH$nee)-oAr`cjTd} z@pMN*c%%8_4Zp`L<}0ly~fOMt!ACLH|> z6?=Ktudg-5?81Bw}`++Ah6jcywXyJKuMaX5_oH~@Y-e|pxuBr--qTe5P@$P zfom>RDOOjX&$@);E++U@8g~2;>2j|}n_QOnoIh|2|Knx@uhYN_Y~V{yHVC9s_F1Ihc0a72hVAYiX+eo~WIdsRbI8w+H1eq8<#C+(hNyqwy~v zA#eC{xYVeoD`4;J5Z>vya|ew6($sv__WQmzNe2IvJoq>m3_bYeCsm8l-2h|uY`LzS z-zl8f$>#K)ljyAHj*|JWJXIZ-A*jCol<>`y4DJBIea7I9|5YiR^_vh}=stg1xb!qb zc#{zJ{Sz_tiBti>z{N31cv)f??-IrdhOs}UVpv`SvbuzIH7cBqGOR0vb%|lUAFqgI z*jrLG$DI0TTzDzY;JzTZuNd6H33XAhV$FED>eGa9A;Dl?{7*F7{$r%2OOq90^gCIV zrNfiL3zH1%AYr}5uwG57@yd!N{T^7w`e;gccZy-1B&;h8>%dGYR>_80w^1e_mcN~t z7G9W!5U8A9Yb6YU=31H&XNl+yAPV{0#k6XoN|$Andk1)}X-``CAca9qz1YNmNRVg# znR#JWm7@}*>2AP{jBqgn!oZ0XeY8^f+X4JnMEL`t48okusg|uoS$oHiSB@?}Bb<5$ zatEZzzJZ+XiFEezDRiLM-=O5=SV37fDp66+B*S*s}CKuk{V_z=uqd?rr zUuMKFzE~Y`&0b7wIF=XA{CwH8-zM7o|CLSIUsOlC#EOhOya^BvzO-!e?-BX;82O98 z1o9QHwZJF(zNwEj3;I}9Ss2(2SSrohR z396Le@H7lR!C~y%BWB^N2xB{9B0SU3j^IK|OM4N=1@Q|%1MXI8$J)+GfCW70?-4t3 zJPZqDk)s0k#RgH#x6ik9^T)-w0!+T$nUTpx>hA$V0y8o#ovkg zC@-Jr!giy4txL!^hWu8LFNpGn15>fg;A}jUnvbTK?qbD`BGLnEgKEg0Q0VL+{=z{Q z@QW?%!#tC^#yqLE2t5ZUE#wJl_6iQD?V*v@IOseQ8krePr}B|ucPbw0ni(G(oFW2& zX`t7RPOS6Kan+upNbN8$L@^TTa*Yr7WyeQ-8RTP-^n)!Tb1jfYoFWE0{Sz%MjCE`# z_Qg)D2o0v9p%Irf>Yp14r3#U;KG*oDo1A9StnYx{pJLcvM36uc_=m}cA0!=Eh>W`D z<7p71cL?Qa#THc`1Kt@_a|aIX9SEW4uI%_g-f}z+ z3kMdY(LRa~-m>F6Vpiix^iT?QMXHWwM)J|lzWMm5kK$FRKMMLJ-W2}~1Y=-G*}*bc z3emF`-vZMfoc~DAF#mP)MRZ}iQGZdSXRZIdADn`Wj5KBcCVT4imnl0w5X12%`x*4N zNaIW8)z61{Y%d}gf!PRrlYfWYQ;9Am&LD3=uZ8~+>%?yl`ZKII@ae-Z=6@pc*V_+H zSQ_60fl0;x!PMAZjKP?|Ok{Ky#Fdd5v1 zqr2R=E`iRnC_l|AV|?B=-D(k^w;@O1uZ7|C$aK`(GZOb2F9P{Ej(3e6x z!Kb(p;t37X#gi`bBV;{DPMpJ@VJtFHiRs`F&~F*u2&ymTPh{&yN6=@%UoS$9d%6}P zq0}76IU5hph#mboqkoGmeg_EO=-(Y^`f%TDB0bU*o$2v`y%)vps1Iv_#$f%CI{gbzjQa9muhd`|uzHplAD9Mxw3GZX z@8Ena-Ia-q4EkcBVWwsMdIYwMhy387fh_8m`LE*dorBZJ1e@!mnlY5Gh3MlX(Bix3 zCi~G;3er@d{*&0hZ4vn?m3KfVZx6LosEVh%^6^pZ8tC2#luF~5sLRiSoyae_QmBwh z42YB(MIdg%yhwpBlB>YFpFzKb&%j2AcT33^K>uhP+5-GmCEqse?;bMSQ!ig9wHxFM z7Awma$RocMbq07UIfRno%RPI z7!&FWN};Z~hzD#HAbnZJKX*S2GR7~O+JQpUGdLT87zVLoHtO*WOQUXDL&3_Ze@nd1 zen@SNc$AL>7NxOSHolw2m-n|YwUd9){37>Yt~@?EI3xB_d#U{g+NnI6OPO)KlI;cYXAI_+3-}(@cnAC*qHl>eQW@`w$Y|G$36FvZ zu5W($cV!_r0}{ACw7~ko)@wCB^*urPD9JaDbH$5h*;ecT{nCDO2R`>vema0|0u7LK z22uW43&fAo{ZwaP8@7{aJIKqZe_#lfM!$&tL)5O)H^s+3l-EB5Q@gtOF^c_L5x;`{ z!_;nwAL1m0N5g#8$HxwmuR{DF(WBk~#g14pm__*v^Isot$FZIC0r_s!8<~aumFy|7 zCW(PG%(9uMyew(A$twCz|<*#O?IaJXE`WQ;0HVf131IH_%P`2f0CSaIRBi@rx*9J5vkRLw?c+d&@3YWOR_lC~`y5cWhq+MbBpmKfhd?Ub*8Pc4pnduC%{ToYPF&l37sz;?Dj zh8#~`KU90SVyG^$cZg(F=WiD^QXTdikh4zIn!LZjwh)~kbgJhE;-in+ob~WmKKL&% kHs|v9%l|ap{XW-b_z!L@!fox_^MBi}%{E&<{NvvL4**Q{!2kdN literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/table_failed_last_checkpoint_update/_delta_log/00000000000000000003.json b/crates/test/tests/data/table_failed_last_checkpoint_update/_delta_log/00000000000000000003.json new file mode 100644 index 0000000000..2ea49db4b5 --- /dev/null +++ b/crates/test/tests/data/table_failed_last_checkpoint_update/_delta_log/00000000000000000003.json @@ -0,0 +1,2 @@ +{"add":{"path":"part-00001-bea93a33-9112-41a5-aca6-c2d1f2c43873-c000.snappy.parquet","partitionValues":{},"size":1432,"modificationTime":1709986423962,"dataChange":true,"stats":"{\"numRecords\":5,\"minValues\":{\"id\":\"1\",\"sold\":0,\"deleted\":false,\"price\":0},\"maxValues\":{\"deleted\":false,\"id\":\"5\",\"price\":4,\"sold\":4},\"nullCount\":{\"id\":0,\"price\":0,\"sold\":0,\"deleted\":0}}","tags":null,"deletionVector":null,"baseRowId":null,"defaultRowCommitVersion":null,"clusteringProvider":null}} +{"commitInfo":{"timestamp":1709986423962,"operation":"WRITE","operationParameters":{"mode":"Append"},"clientVersion":"delta-rs.0.17.1"}} \ No newline at end of file diff --git a/crates/test/tests/data/table_failed_last_checkpoint_update/_delta_log/_last_checkpoint b/crates/test/tests/data/table_failed_last_checkpoint_update/_delta_log/_last_checkpoint new file mode 100644 index 0000000000..6869291ae5 --- /dev/null +++ b/crates/test/tests/data/table_failed_last_checkpoint_update/_delta_log/_last_checkpoint @@ -0,0 +1 @@ +{"size":6,"size_in_bytes":23521,"version":1} \ No newline at end of file diff --git a/crates/test/tests/data/table_failed_last_checkpoint_update/part-00001-6791b37e-f318-4d2b-87a0-89be205c338b-c000.snappy.parquet b/crates/test/tests/data/table_failed_last_checkpoint_update/part-00001-6791b37e-f318-4d2b-87a0-89be205c338b-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..81742b70491fc8a617b5d690ec4c0f33f6680565 GIT binary patch literal 1432 zcmb7EPfrs;6o0ebwo3&Aq%+wydmtukI3NPjgqUhF1*KroEeW;|5>v~jB2}mP>@~INfa2Nz| z`WXOBuw;Icm}^9)h~zkM!7q)XT|ZCplv`yvn@tQ1pzgwmfi%MsiPyJ*_c}R9^*gq) zvCa*V5pS7z&ph9Qm{8HSQeHYQ&60MQ-T`yZ zS-q-f#a2&z=;2v$lIEAeJ50Fz+&wumyc70cn<5T-xDanFeiOW3Nf#^Bzl6WXyr1-r z%zqAAe5Od>EItO_LE1fcrpKw+P{j{&dewuKi7%$ec$WBTW@!|5uUuNXSGlhHZ2vT_ z(&;n`Qpz$ubnZymkuwdsKq7ogNm@E(yh}MxEQ+ExDe#%;kbITMED?E&NZrTyyDIb| z-SHF(4P~k>E2Qt#NxZpQ*=?@R-QSc8k5M?SoaU4jlJfjAvEyXF7YQR|WLkLnaB_E3 zX%?0L3h@dgG39wmLiiB}*3r=_m4p-;t}KwCSlFvo?lwc|GG&=cCNJALob3>dt>YAP;Pt-KMt+==@+v2f8yhR F@^2v5-QNHJ literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/table_failed_last_checkpoint_update/part-00001-9c90a84d-6999-463c-bd2d-f68333e6d03d-c000.snappy.parquet b/crates/test/tests/data/table_failed_last_checkpoint_update/part-00001-9c90a84d-6999-463c-bd2d-f68333e6d03d-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..0d502583fa4e604a410d339fa9ed6cc1938174bd GIT binary patch literal 1432 zcmb7EPfrs;6o0ebwo3&Aq%+wudmtukI3NPjgqUhF1*KroZ4;yr5>v~jB2}wk{K$>BR#M`^Tdu0w%{n9oz z*0~`v;yn`|nd`X_bHSt@&qql|0QwlPJ`0RKE1$qvbj;ud=Mr0y#)8}&G*)*>0;i-{Rv7zhQ9sB#bEfpo-?!J8qwpX6&Hdz|7$ zV$or(*jsD0wmOg6^(X$b8nQ!AUwtC`I56_B|G`|Y&VZz}T?#Yqip0NMg6mi(Yg?MN2+u;65I#^-wOE^2`{iJ(j z{&UdcD@FQY@iA}@($0l5Jx;}jDt?jEn;xu8d^1JHwZwNbOQW!RRi*M^`DV~(`?qnO zPNz{&N|o`Ub5E&`nrWy765(4Z($Xp8U8#9uQ53yNfzM2bK?}5)u9*Z zj;By)Xj6SzA$_M#;?33N-RAn-!%emD6ou2;X--=qDbFtvJ5Kg{kuX9=riIs!CwDis zW>NdE5U)WJQ=X?JgdcHW9i2d>laL~Ss|qA27WQh(_nV<~iL%T#lo!;AenCHU{Ge2{ ztYC9aPnk?~{fhcG*CbrGz?R#!){B?^&b3y1=#}4UZ)`mu%8ifVN79;~ej$thCm#MM F{{v~jB2}mP>@~INfa2Nz| z`WXOBuw;Icm}^9)h~zkM!7q)XT|ZCplv`yvn@tQ1pzgwmfi%MsiPyJ*_c}R9^*gq) zvCa*V5pS7z&ph9Qm{8HSQeHYQ&60MQ-T`yZ zS-q-f#a2&z=;2v$lIEAeJ50Fz+&wumyc70cn<5T-xDanFeiOW3Nf#^Bzl6WXyr1-r z%zqAAe5Od>EItO_LE1fcrpKw+P{j{&dewuKi7%$ec$WBTW@!|5uUuNXSGlhHZ2vT_ z(&;n`Qpz$ubnZymkuwdsKq7ogNm@E(yh}MxEQ+ExDe#%;kbITMED?E&NZrTyyDIb| z-SHF(4P~k>E2Qt#NxZpQ*=?@R-QSc8k5M?SoaU4jlJfjAvEyXF7YQR|WLkLnaB_E3 zX%?0L3h@dgG39wmLiiB}*3r=_m4p-;t}KwCSlFvo?lwc|GG&=cCNJALob3>dt>YAP;Pt-KMt+==@+v2f8yhR F@^2v5-QNHJ literal 0 HcmV?d00001 diff --git a/crates/test/tests/data/table_failed_last_checkpoint_update/part-00001-fed6d112-d244-4c54-810d-25ba3f0a4016-c000.snappy.parquet b/crates/test/tests/data/table_failed_last_checkpoint_update/part-00001-fed6d112-d244-4c54-810d-25ba3f0a4016-c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..81742b70491fc8a617b5d690ec4c0f33f6680565 GIT binary patch literal 1432 zcmb7EPfrs;6o0ebwo3&Aq%+wydmtukI3NPjgqUhF1*KroEeW;|5>v~jB2}mP>@~INfa2Nz| z`WXOBuw;Icm}^9)h~zkM!7q)XT|ZCplv`yvn@tQ1pzgwmfi%MsiPyJ*_c}R9^*gq) zvCa*V5pS7z&ph9Qm{8HSQeHYQ&60MQ-T`yZ zS-q-f#a2&z=;2v$lIEAeJ50Fz+&wumyc70cn<5T-xDanFeiOW3Nf#^Bzl6WXyr1-r z%zqAAe5Od>EItO_LE1fcrpKw+P{j{&dewuKi7%$ec$WBTW@!|5uUuNXSGlhHZ2vT_ z(&;n`Qpz$ubnZymkuwdsKq7ogNm@E(yh}MxEQ+ExDe#%;kbITMED?E&NZrTyyDIb| z-SHF(4P~k>E2Qt#NxZpQ*=?@R-QSc8k5M?SoaU4jlJfjAvEyXF7YQR|WLkLnaB_E3 zX%?0L3h@dgG39wmLiiB}*3r=_m4p-;t}KwCSlFvo?lwc|GG&=cCNJALob3>dt>YAP;Pt-KMt+==@+v2f8yhR F@^2v5-QNHJ literal 0 HcmV?d00001 diff --git a/python/tests/test_table_read.py b/python/tests/test_table_read.py index 6372b10e2f..e0ca4467ee 100644 --- a/python/tests/test_table_read.py +++ b/python/tests/test_table_read.py @@ -740,3 +740,9 @@ def test_encode_partition_value(input_value: Any, expected: str) -> None: assert [encode_partition_value(val) for val in input_value] == expected else: assert encode_partition_value(input_value) == expected + + +def test_read_table_last_checkpoint_not_updated(): + dt = DeltaTable("../crates/test/tests/data/table_failed_last_checkpoint_update") + + assert dt.version() == 3