From 9594e8ad48fda6f93ca4402a41bbac2f77f64415 Mon Sep 17 00:00:00 2001 From: Stephen Best Date: Thu, 9 Nov 2023 18:34:12 +0100 Subject: [PATCH] New forking implementation (first draft) * Set config value `forks` instead of `parallel_workers` * Supervisor process forks worker `n` processes * Supervisor terminates workers when it is stopped * Workers monitor supervisor and terminate themselves if supervisor exits to prevent zombie processes * Supervisor runs pre-fork hook before forking * Workers run post-fork hook immediately after forking --- lib/racecar.rb | 3 + lib/racecar/config.rb | 13 + lib/racecar/forking_runner.rb | 131 ++++++++++ racecar-2.10.beta.3.3ad7680.gem | Bin 0 -> 51200 bytes .../cooperative_sticky_assignment_spec.rb | 11 +- spec/integration/forking_spec.rb | 247 ++++++++++++++++++ spec/support/integration_helper.rb | 27 +- 7 files changed, 423 insertions(+), 9 deletions(-) create mode 100644 lib/racecar/forking_runner.rb create mode 100644 racecar-2.10.beta.3.3ad7680.gem create mode 100644 spec/integration/forking_spec.rb diff --git a/lib/racecar.rb b/lib/racecar.rb index 6c8b379e..934c5870 100644 --- a/lib/racecar.rb +++ b/lib/racecar.rb @@ -8,6 +8,7 @@ require "racecar/consumer_set" require "racecar/runner" require "racecar/parallel_runner" +require "racecar/forking_runner" require "racecar/producer" require "racecar/config" require "racecar/version" @@ -74,6 +75,8 @@ def self.runner(processor) if config.parallel_workers && config.parallel_workers > 1 ParallelRunner.new(runner: runner, config: config, logger: logger) + elsif config.forks && config.forks > 0 + ForkingRunner.new(runner: runner, config: config, logger: logger) else runner end diff --git a/lib/racecar/config.rb b/lib/racecar/config.rb index 17039f09..ffd84310 100644 --- a/lib/racecar/config.rb +++ b/lib/racecar/config.rb @@ -188,11 +188,24 @@ class Config < KingKonf::Config desc "Strategy for switching topics when there are multiple subscriptions. `exhaust-topic` will only switch when the consumer poll returns no messages. `round-robin` will switch after each poll regardless.\nWarning: `round-robin` will be the default in Racecar 3.x" string :multi_subscription_strategy, allowed_values: %w(round-robin exhaust-topic), default: "exhaust-topic" + desc "How many worker processes to fork" + integer :forks, default: 0 + # The error handler must be set directly on the object. attr_reader :error_handler attr_accessor :subscriptions, :logger, :parallel_workers + attr_accessor :prefork, :postfork + + def prefork + @prefork ||= lambda { |*_| } + end + + def postfork + @postfork ||= lambda { |*_| } + end + def statistics_interval_ms if Rdkafka::Config.statistics_callback statistics_interval * 1000 diff --git a/lib/racecar/forking_runner.rb b/lib/racecar/forking_runner.rb new file mode 100644 index 00000000..541e19d5 --- /dev/null +++ b/lib/racecar/forking_runner.rb @@ -0,0 +1,131 @@ +# frozen_string_literal: true + +module Racecar + class ForkingRunner + def initialize(runner:, config:, logger:, liveness_monitor: LivenessMonitor.new) + @runner = runner + @config = config + @logger = logger + @pids = [] + @liveness_monitor = liveness_monitor + @running = false + end + + attr_reader :config, :runner, :logger, :pids, :liveness_monitor + private :config, :runner, :logger, :pids, :liveness_monitor + + def run + config.prefork.call + @running = true + + @pids = config.forks.times.map do |n| + pid = fork do + liveness_monitor.child_post_fork + config.postfork.call + + liveness_monitor.on_exit do + logger.warn "Supervisor dead, exiting." + runner.stop + end + + runner.run + end + logger.info "Racecar forked consumer process #{pid}" + + pid + end + + liveness_monitor.parent_post_fork + + install_signal_handlers + + wait_for_child_processes + end + + def stop + @running = false + $stdout.puts "Racecar::ForkingRunner runner stopping #{Process.pid}" + terminate_workers + end + + def running? + !!@running + end + + private + + def terminate_workers + pids.each do |pid| + begin + Process.kill("TERM", pid) + rescue Errno::ESRCH + end + end + end + + def check_workers + pids.each do |pid| + unless worker_running?(pid) + $stdout.puts("A forker worker has exited. Shuttin' it down ...") + stop + return + end + end + end + + def worker_running?(pid) + _, status = Process.waitpid2(pid, Process::WNOHANG) + status.nil? + rescue Errno::ECHILD + false + end + + def wait_for_child_processes + pids.each do |pid| + begin + Process.wait(pid) + rescue Errno::ECHILD + end + end + end + + def install_signal_handlers + Signal.trap("CHLD") do |sid| + # Received when child process terminates + # $stderr.puts "👼👼👼👼👼👼👼 SIGCHLD" + check_workers if running? + end + Signal.trap("TERM") do |sid| + stop + end + Signal.trap("INT") do |sid| + stop + end + end + + class LivenessMonitor + def initialize(pipe_ends = IO.pipe) + @read_end, @write_end = pipe_ends + @monitor_thread = nil + end + + attr_reader :read_end, :write_end, :monitor_thread + private :read_end, :write_end, :monitor_thread + + def parent_post_fork + read_end.close + end + + def child_post_fork + write_end.close + end + + def on_exit(&block) + monitor_thread = Thread.new do + IO.select([read_end]) + block.call + end + end + end + end +end diff --git a/racecar-2.10.beta.3.3ad7680.gem b/racecar-2.10.beta.3.3ad7680.gem new file mode 100644 index 0000000000000000000000000000000000000000..ea2cf5598ed09b32f8f25d099ac03183b785ecb8 GIT binary patch literal 51200 zcmeFXQ;;r95H_~9ZQHhOpS5kiYumPM+s0WNXKmZw&G#o4N&Ppu3aXO6>6z-8sh*my zetPO@J2QZhsS&`4!NLpZ|0-enPuSSlfc}U6PyT1k!otK1#LULd#KOeR&cewK#Kg?X z#>xRi#PolaL;uHfU0nc1&i}dOZfRy_`+qe2PvieH`~P;g|NV0R)As+jP6?x-fO@r| z%s@b|HMbp(*pU3^42(V&%gH{SXe7ZXCtGPEoP}ubfkYAyr|q{~ueSc3y9(d6vnn(h zYy%vuV5?=?uSNM-+x&lG`4642!me0rnmuisaFG}~16ZW%Xpx*rz}*+mgdR~)!gK@n zu(m+7lPLJN{jB}~o5&U`v*C$ha}mTAGtUVMdh$cjjBbw@N}DV~0oUsp%~J@E&vv90 z{A;BmGlVdfXmSwRy;HG%aj{d9<$uP98~W5HArq!>WneaPgbV}%R8n=6MEZa7k9U%8 z6%ShCA@o6o*RLBUu>CF=-kbD8aH|!#%!`Nw8+mJ4z(#Y|){vV>a4lvope6y7WajJQ zQ>s(5VmPO*1Q?;uZ_jQyCFO@X-I+Ik(!LN#wc#VTdq^~)S2re5q?Cw>sP@uEQ0wSw zdXnsgxR8$cBgcHkC|Uh1vmh57zZN}im~N#3>X*8}0=NqfeO_p1hs1>f`tXm(hZ0s{ zXFjRJ^i-;@0l$#`_&(>}T|Gmd88gJy<&du*d*mNB4V+8k7I!{)g3j*B%luSRih;e1 zb7>3=;j`#}4~J2{$^LnxTqoxO2inFk;mz$sW?YM*BD!!JjAb0FR7XjLAtL5l= z%ekn7!Em{R5uMcYwyUQ8iaa`7#=AMd~ z;C_(9t3b>#-AT_f%p7k|XIGLyojH0mjYCW?mF_hUAlNMIb%w!7;0<@Derw?Py1w}=I)WaTyA4J zRl?b(bogqj1rjSBT;_PTLRQ(m2~+{f((0vu%`v$ymfZzG3)O*NEo&)f`L~*0a<~+Z zZiV0nl_LlyyLNQDjYSjJLM>FPX|wI1v`GMU&3twlfc9X!(bXI&R>{J5+Zqx>TL-Qs zby?v45pL+p-2R8oZ3AucL_j|Sg@%O6DAXlY{-6WiSfFK0FGG(&og6&3heIZBv5+=E zuZ?6C+>4`#OdG@wd4mG_^RYevBuW+&vg%O+g{^Uot*}3~PtQ%7_9n{QM|)5tqgn9|HbAhX2g0EUe5N|1FPJIjh)3SwMXz_lg9onpUj-jkFkO$U)`OFk?!2OCg;^bw)Ys9BIVYzbvU;B9G zdK;NealfVNQ0~MQ1@p$Pc1WdM_zh8ZxaPP1_zM1ORPHdwwB+N9_PvO<2mEzAvk(7%6}{GivlEK_Iz7ujEhxh(s(rr*(DmsL@ja9K*J@LAsI;NuJ&h&^k8Wb zZHiC4rON#~*Xw;EY**i7QOjeo_OftA&HSH;3V8K9a3Vv$R(NZy{i4=={3g8TT((;w zj04B>tdYv1&3m^ldb1Vm=~w(_TuTS2txh8WnM;%LuJ1lHe@U%5&ngZ$6LomdIW>R2 zxYHT9^x1ydqUFvzxUHW&BGjQHrI6+F_|%G)ZAKPk=|h9 z0NtwK?IEckuD~d~%MRaxqdz6CE<_UKD@$||EuNs9B4XJ199qvuo+^{PUDdtda|@H-+XVA|m0ZYiP;)tSPI zA9=d%>YG>eohPf~TV5vq=KmH7M!)_7IAV&wJ}^x|2o8QZyZ`+7moL4K|N0Mn{>IF0 zeX7XQkrv#(l5~I+d4G2HE(H>e(!(?NZjhMub?F#@(UFW1@wPw>Nr(j_P5v-L7lqtH zojkBJ^Z9>&b^oMG)-$kG&$*YRfn(@zW-s=2@%8du5)$~j_&By)*px%R+zBH77FUza zEZ1|6iN5WwpQ+vXn-upxBL51G%yrGh40+jY=R;!Ov9vx5{DQa8l9$~p+3{DYt^m*jbIU15&hL3RZSgSQpPD?m)!b!{KE5T^?x{Cm|9}|mVVpZ z^S3DPR$ckE1N@Ijg?#rsz}JPGNR3*m*1n!Vy>(5h)CBK@v1y&_m94G^I@#X5>s~%< zyS#n3^RI|12%dQvp2w9_7K|GaH3N0XD&4m&VR1;II6Rw_Eq@!MRap)9CdEU0rX6i% zu@>}z|MISYNtG`W^K@TZIKv@OC{DESdIfPPdcey-VVvykrlWxo#~%hJ*T;xirV?rn znI;N<;T(J$^C4ftpDpUJ6pzG|9OxLJ{W7fJD+t?JYc8Bb$&iSvdgaO`)tmP1ebW0E zI@enda^&KCOEqOUaZc9{v{-ppUvnk>eXDzNaeX;hwd2p_uUs+NYp?7ZMjRBuP8Aj=!HDN%cZG=Vcl+llC6ldw z1<})DqdUOQ&+Eq9#vu>z+SX^#%@Ad^qhnxEI89iZ>`GrfkycxL;*O8QiQ`s?zf-#< zyS&|$dNBULU&lQoB0g7VsM-q+g?^WEi^<0g7H}ECn_cEf052`ft^$TP`_pFhZ_RWm z@DSUan6O|qJgymV;ChGUSb-2!1UbN+t#2kA!6Xj;6b3Pm;&%zM{E%ncGnZ%k zkAFRIg5TUthmS(>F|qmb4L@)D=5yK})8e)B)rTYVgkVpeEmQc{$2&Zx-OHXyZf$o} zcKxK3h$AM*iYexxLI|&fRuE?nKS$O)nHYq=2#XY>PK=3W4d31=Jh+iYV$Jv44XBrL zM+urbW51?M4jT$RYez5;!W)=$oXkTOcD_qq?23p6iV@TSj`k%C7~oJ~_LvCSREcPM z3GBGee!8$a-}EhT+BkFSgiwNtOq4ILtWX3#Z#0n7+e*1zs~1KcE6|VvpIS?3 zeXGN#?Q6Qj6=A#;qD1T&LK=js<^Zefa7I1k(}i9~N)#g%hf;8^g*4c@@BKFo27cC4 z$D98>v|?-^yS%yFF-Zev79Z#QyyxH^(Q~ueEJ2|te?3p0?k=YVLarA zJu1))q#aSvCXKWHj}RA7O+%;LFnt0&;0$02QfW?&4JriZ986{C0#UBDGsqF9fS%av zFrW%Y6gxzC3ZNPh_l6mSHf{x&9n=!R0r~;o^qC~E!USv(%~yE+OOX^q?;^46aah*e zo{1WWI0q;#hH&bspyAPM?XZ3PE4N(}$nI?k?7~^AvR!XUrpRgjZM-(hP+%wIhRC;y zJxL%!Y(k@-M+?USt7^`1<&ddp%aqqv$sr+KyaVn!jW;APpIfF$(4Nq1Ngp3+8JQg; zrmn$G8%403ov}|jhvH35B6U3q^=M5%VbpE!oFJh&O~;^#$;-*(96roG8$KU6S_V0( zz}g^NtO>9-$m$Q<0OSR|u6bym_8UVWekL_M%y<=12gul8W+bM6nMzm;nyyUIfdkvr z|Nd#kx!{b?3S0PI)x=W?%ojlGfb(Z6gggMHrUzhQkfOW59Tn-(ZI1I+f;vu49u!kwSnd5U0JrsoH zDK9T?{H~{EkL@Y+Wr950G(scy`0ZYrD~5p8BjqU(=%A>B5CqIq!eWd%QHfgUD#W4)N;)Um5w$ZvA!hzRka zBOd3%FYj;gAI?W&lU+1M`M`$B+ysg`w5FbP;K;&&0;1E|LpCUIVh1z^KG7K2AtEs% zIu&0=d#rm%9!Uri zd5?4r-&REs9=R(Hk{1u85Mnzuq}O*Kh!L-&W+Q<~l8siMKNc3tu@6DW45;lAx6%~2 z6^)+9K59FJkF>dr~}-<%|hYpu%Fe0whV}wS(*h zqZ$4)hA870`oS4~u67!Gpueg}u7R}(Lw)Bh^^5nrm=@r?@AKH~m9Jp49gHA}S6OV3 zC``gNonh4akxm@88(ir@ zI}lq7!d1PJVm5}@V!|OOOm`R}Q#47Bsl8P=;j|#(bT?n8`$ehFnumg3@QY39xyj?<3iuwBUSHGUL$SWgR%c8TY|0v+RnYQKp`^d1qUbx&&=o90&c7t zp&kgvt^)9*1Txc~=z9Bv_1k!g{Ui-*eX!xtQz%@hh3`U~!z30pA&E$nG}CF;vKt}VabgB{D8VGJx9|?h zSc-tNQ0KGrgPMR~h50wt$x0~4nBd7JIejVTxFsmebMyG zmugocuVL4hYCXnhp-r(3Uh|k*LSE$e{vN(xW7GvWY*i;x4B$tCS(}&O4k(HFNPVhz9t+{bRpF zTlpiif9c;l%mmY{j*2sjiDz}_leCnO;}T-Q0@I>2=+2|GYXi*qJE;F~Nbgr-ZEa6MTIe@qcuOkC!}Xu-aD!^E`i_a z5@DljI_rK#3WX2YjJ+tcE%LQFw@#SDo`o%aHkN%S z@Nev&efW{d>Tjjw<%LM!9(L?kawxYV`4Ge~OPB@D{X2&Ygc*jh9$WinfNKM)(r$w! zlNr9Sd5R$ZH9R`_{zJ={K|F+hPy;Dy=Rl9px_^71WGh)xiY-w?vR3$`XIUSLgPe{H zoz^_)#i`k7AvhXaG45OegGVYD)B*mut;w;QyRo>iAM|%gPx8Y748*5n=%%sc)Nb?9 z*xP|VqI_+z3EOT(>ao&+PnqL7UMZc?V)-(+M4TT(lz@k}>{%SYt3&g#K#MAxNjdwIo1HMmV1! z*QYtB2oC~$e61myI>$aL{wqF?>i1=i7@RR?-}n>$ThP4?<4dqRH~xo^@Zfax{hZbT z&8%26mc)7CrZ5rrX7F*1Qb>Jh8MaLPGD^g0UZ&*@&kRV+IWD+zrH`Ng_v7wuOH`oW z&C%;6_tab9l~S1cVN~+z``W)xE!hC`oiOlMCpWh0_teeelk~?Q>z6SB#ZO6^usAol z+7B*AHH>*04N+ri3Hw4n-&*TWxc{e>+cTO6+^vX@_{scyFsI_9N(YPc?ojC48$Lwu zx&Qv9gEFgb#0W86%**D#`&};^DodHqF@w>K@3wJ}<%yvXNo^V^5wx-KM6gaA`cx88 zVP^MtxK(ilRrZ+X)jX`t4M$9*^3w8#!}i)PQGyRHhg+-4{iU+kr{A~i_Smu1@>W5X zpU(=d@7O#4;_bRIe>+<{zn)*gaYUGH_Gxz}Y1oxu?~TVT4}H7Z%OXy+5APtNM&uCQ z&2A!oml|}MqfbsM#TsCS1Qe0}#VAG};HFcEzDi%ur%7x_Y$Gj3#iE#On+QAJpD+SY zF)(GC1Po=xqVw--Z27WkT!~!01jBRcf^W<`PiA5#C@%oy?!G&WGS^=q7AvpD_vjbfovwN% z2R4~>vzEbB8>F~JvYdTwe_j*Da`WIdIswpy#jS!J|12kZayAvE`gzN zbnDV}&?`k2gmGhRu1E&8M7$jbl2xN%yI4OAMC&jne{B~b#SQ4Q1@#5c^(gJ@_sitr zbjk3Y_P=6}3#co*pHv~=@jsrPK8^rqrz1><=v}FwOMBZ}TZWNeTL+(`UxAVM*ZH|0 zG=8I=^aUyM&Z_OD2VCT>h^SvooTNRHo3_ntT0xN$M34UNZeGCxqlnLF&kCPhmPcIz z;Kt_{MTFkqrR3qmsl%mYSVXFZ*N=mW^IXE*+1&1ytg^48@$(a;0MTC}o zGrbJ&L2!9YXMZrr#x96I_vD21z#HAg^cdxg(-J~+^cL^yL^o5WLE3;@y|(0K6lR}j zNG^qxEPt~outksKGbm=_>d(xg>w?a31f)^DMUyoU)(O|mWkEl7VDATVW}B$JRW9|| zA?xNU7;-W3 zrbcSwIdcSJag}y&X1BzlAt7pW%}IFP&5i6#o* zu5cV)qQidC^eicPWFW~6Y!W7buqx+22us`X>^I?1yrc+b_M_=Y+NI`HhE=LM>I^v( zXAso-GNn(rD@EmG>|SuwsHN_(?NHHF@pe4RP~4=oYNP-Yx?NDMUJN!X#HW#P;P-s8 zkSO9fVtqWhYDX#_h7oY+IQ9`B1lZalUfcWo~aWY2t zUYhC|hh#V5fT5L~6mY({hB8PoUEVth|0=**SLC-$V_RUIKg67y_d?%3RxT4vIIm-r zsAR8io`r&jR5FXIvoID{0(ua-%b)G3P?c($3u}RV0dOPLTnU>zw5@#pFYg05$+`0| zJ`FcW^+%f3@~hOeP{F%tQKTpRjMk&y`avF&dZZL2_1cToViNEFuSn-d(J*$$wdZWzYNcyo(<|3A7~1Ik-U7FNJ7QW2_-uKw8GYl)bY zoXXJ+_Z7Z($NXwjl2jo=9|rfGUYXCE-AO{)4a;e*Y9D#qvuwdKZPr<6M>=7f=zNOK zy+n&)bRkSG^*+YZBwd6ArTPPUjNX%>?&Db;t?q8kd>sdr(A0AaUhIRb67E88*fi{n z@5bw@>rj(ynt9iHXy3CK?IpY3#?%A<$wXxNxtB4qcE|{E`hnfzSa15x8C_G4W}3_L zM8yow^DYPeAt40rbb1uo0BD-{gR0uLYTmMZ5K+niZ%2IGhRkZHQ*5Cd`PUYBEm^N{ zFJ(5kBcpvRGpL`$@XCG(8Ra2#Q};-4lAdT0!v~m)L!3*$0FajBqR;I%moo+o)a41{ za#GDvPC2C6;ha5nf~zez{V)_ETjx?l?<0i!U&RM51^5LkZyt-QjWDdJT&7b=v0>Z| z<2-KVDTln`0Z2HljUThyp=b~ZA2Lhc9n(-aXnZj$p*b^_WpK$uhA3myC5*@DW7`Ag zClat{D43vePDH1;VOyok*`YxqUvaj(VubG}H&5Q+bX6`ni-qKxuqD|ib*2 z*vajvlt(|I@V24(?sdiNHs{6(W_1~5^Z$l${ds`@YnDo^+$^V!g)X-c6gJA-`M{M*#NhiG>*=mmE{VN57(6Z`KdLpT`SU`l9i z>7#SkVi6?|COGj#2FBU)qMdjVT0!30xkLnxGK4-9ec^kCuWOf9->~d!Y5@I!QaX{O z)8xwH6eWfZwIl28xfO(0oLl`hLTUj`zdM4zYosUixNZ*vZ2?(P^f{bz7 zR&+O>rT*N4bEIv9Y5Ewt95Dn4{@i&(!7!ceKFR5cEz0s-p*s!}2F*AZgdC&?y?wLG z&In%vm0gMtR<}`z#*xzG67m*F6g%Vj$xJJ92=Px|Vb7sR$x?J)*VViUQ{hB+yt3j= zmQhakNtd< zm(dD9Z_f&a3O#jfzVFZBJ$XcPNd`B zSjpr?JexTP`eFu7GFY1tLE`mD5j9E2(FDH5YfQe{6@_`lbb!>~umgKv?oOthG&EW; zSIHb@`Yx#;$qEcMrI<+s(%KsHsTn8xsdK`v;Rt3gccYL_mW;=#t;&J9Q8MU7Rv3hJ zWj_a8GpR>;q{%cuAu$BA8r)_B6m@IvT5Zj-u7a&1s>0qz%^(e-J7Q{_V%sl zhJUx8t8YF#+E28+o{@R&=>2En_X#`ad#QM2Kt^K!pTPA_xN@C-{kfR2=exF9c-r6hBg)@eEPry zt8~f4Wtvnrz%e*FYw^r5{XsfL4V`d8BOJ^x?TG73X^rI$V?S_a?~HRUy7J_SipiK3 z)4QrEiFJjqjUJ*K-l;?MzoZ|H*Oy2-MP7O`q5x9J4|&`Z*i)99RLCHe+K8(X*J*-z zuxX|V7SkI=b~GSevBa@)Z!Vc+&&r3~_l?C4&1vd|i!3aj&?^)e`bYni0PaO}WQuPP zM<~ZHy9P;P_LMk^7FNLy0%#coBW2ga7x1gq1F3NVIv#J4;h2sMJU<;t34nvU1S}9y z=32wV@^EHwR#mgMWggPo-&jwwt0DueI4#1)rr)GfxTJZf;$Y-JZdxNRwf371Q-EJ+ zIE8sRrCwuzz5Ch)i!yxvfy;Y39r=1I^Ge-cvp3zC z0cvqzZ`s?e7yy9WvKnx5d5ejE3OpLq=2a1SY}CAqXdZ!y2jWs`;DvrJLA>nJ^Q|K^ zBOW>?{O*kSV@08pu4t@IeYVzt&yIaCN%ciNrZ>RAht*FC%4LdZRnuV$emwY_&xw+< zD7>pk!!FTUA*k8+CS)aPF^fKtOO5JivJ16k_j8t6b7eYQ53+3E?=sr8r6YUA7^m|wbv&^RC1Jr{7r&a@e|{ZI{W`I!aM8sjAOrk zHNP)`2ma*#y0vAhF2-E7oUyRC3np#UXlL_T-S=DtS$>QYxzn0s9oEA%&)5JoOH$77=g>qtPa*x|IYG{}inR zO`g-hq;gSHDe%i}!MmjwiEXk~Cd$uyhmQVaKp^^#B$xAGzPs*~v>Yge=88anTx@V+ ztH&k~dU%Qv#M$1{njl@$&{hfx9V^vKoEOCi!5|I`u>Eo3iM zqU0-iWM&C^>~4DC9g9G>F9IRYL+;vX%aS4L?X zR8ea*=KqbpQ)=C?GIy>PE9sqa4h{NauWel470KWmtTy*_UMo%(z*nk5Cj^rr!qaT0 z7PeA@%dy14zU{6^X`O~a^$Z#eyxs5k{XC|v6vw_#|X<(5QkXr^K0Eiy6{qPBW_Py1yYi2Wu)Ja#! z^C*QOTWco!Sc8M6JP`?w#GA^=$~(@qg2GB!PKk0r)>|16+Z@?A%w~aS*lG=#JE#y_ zAiqhCUgtow?i)wo$ZlHf(~X9v;1%xvk9(B^;&LE&BX|;Mjv3K1s__f5k-4x+!1yApuvSyewPM8V9&TxOGfa$6@CqXp9? zbS`?9HfHOB8tY5Y&j#mkx+)>$b9l~>Yaf;;GD=C+b1QG?$|-nS)w#UHYWa`lP~x*9 zM^o+`lp>lgH4weL;ZcOMbSqJ#II$}f*tEFJ&qteKLLelb%;|$JJ@ZKcKyP6k_KUO1 zS;r9mAVA%$B&uE_29pW@b*VBozjKmT&_m$?6W`;1{JA{%@0*P}nd*0~YP=s-SC%78 zniDA)m*fcF?=>7OX7ug7IeE%l*hp=t82`AZZUo?Bx@Z6L?{bREMbXRY%3x${OSyEG z8z5ic&hK^yDqCA_9~fWnk~F|SXiv zmN?u%aY$iw%@h%H7z~_gLhv=@LG#urc1yXI1R%)ku|`^?E%sJ#FTgqMc~e@!V#;=Z z%wc1+WJFtmV&|-~8OE{EU=0g>RX|q(8^N!?&(j8B%z`kCszJJD@dv?*#f`HF07}%v{jb5wK}25gCoN|cgWPL zTZWjFZbleEZT&+*tER0)@ka;D*w&$=ERb3c2Hs#@Ksje5w@BNHiM)eDTh7iF*Bkp1 zZ$YzNDpEhI&X*e8HMGkUf!qmMxtnpm^~f7@ye`28C^oxfgsJV{!|bTs*{ZxsvkPg} zT#H9$!Z#%21BT5qvLQ_<^aL%8xK>xE@}ZsJX|@Z8*}6jbTq;??7Qd9KLONk#yIXl4 z3g;MwQhoFBjyBg{h!Y@dPm2%O-e)yI+Gmceoita^28-S}b1zdTmt)aS zE_F4Zq${~<$#rWl-2Q1Q`r9BZU`@{M@a+cZ=h-3HwVZ??jx)r>kZDG+Gs~EcF&2)Q zH*kIHkqMbW`q&70)?)hZZ7(eex4&w4)VrI|1&c8?g>C@46yjd6==M5}jmw*29}*r+ zNZ2Y-$@eOr03WO06^}|s5=-H$AeM=+_B%?J$ql|JfE|?D;yHyu$J}#eUgp)ot$XX? zOB)o{xj7|I_1enXrPqnww)gdCpFwYImnY*DKD{5++;KveL}6A6}uTs(|xPa zv`N5!=mZ2g>N@8xX8&c3ya6Q|yV z7UF;AmB(W@xZ8co;2hQX(Ex0-Rb#?!?uwZO0xJ)&!4_ zF6m!T1h;Gy)haPQz>u|SuA*8#&5RKq(;|8qsGpLSFwIqOFf*~gaxLBbTP2c)k1-Gx zWqcklj=;OAQ&G-Sx=?pKT##C0(3sN3QxSIHqNptn5>s8XCZ1di5rZ+nwO8Gu3~IAEXOn9Wn)Pwb0?x#S*QP`}Z58@Krct+M z9=XXF%pU$d7)yhJ+>#dhT!o!t4I;C_R-CWAF<8V)5HP2h_dWBuQ0*`7-}Z7z@YTC+ zM-qJ4>w=XHSv8!JiacnC6N-NCS#dX&Z3!4=*CKn7Tm}$hf=3Wm2%Es;NCd3Psle%s zIP*mqHgk;!s2f#M;ErqdAyu9g!yFuB!3vvSQ0F}AU1TO{(`_mlv7H%^OOQisuK!M= zshmG|#*y7Wu>!;x#BS4El!^!H7o?YZ>%3KqJWcLX%)yhcV%2A{zauPKY0v7T;+Com z{RNP;m>LJ@Iz>Xkn(*JBYoLVR#LOG8fJ2_y0lh-{5H=4g#9A3V4%p?_Os&mT)WQpo>&JCxufGr#3jlHsKIBD(s>d z2!}x!Q;jQe$c{v!d6dOS3SlhA75#ZOt3l$bUoMZ4L-Q+K*xg@X# zKX_6ms4#}7^um$;ZdHOC(L*%yC{%w|#N@dOt@PnYv=Pi!o@G#L9H~{{{2ffiC{3px zt*&#dnH}S?c@GlMgEEe7< zA*REpQ*%*fB*uWzNYhNxGBj76ycEtqxMf+vIr1DiTV2E=9+r@od=&3Jiz?xg<%Zdg zDg;Z`#OLEIjRVpy88@`7!JH4{XlWGobVgx`aeJQxk%J|E-7__&j+4>62n?oLXBM$Y4$O?AUNC2~YwWwRV*4)miW1WXLpi`tabcNbB9+TdTsYJFX;4n8OV-nn- zHW?7Nz!e26^EBqp?@ly2_c|9R{$T+gigZ9@^rcb?Y4x_l!yZG~5xcDC*WW0|={;|V zA^$0irBblK`k1Od@-JPxqre9v;NP6-j}XfHM3x<`e7=XIG)3jA?E}LwWp}fWYP$Z# zuISuD_f05NVI5?x16``pJC~TYFO+VozBP97fp3gtiMZ)`%w^>1k-fVl29m0TXdj@{ z)8L_E>7PwY;=V15-qPck_8nEsOOA^b@xQhwuPjZ7+EY3;Tw>ZTx??q?RU5QL;X3CI zuA2Yav0%cp*`pC9j?*O0jQxwFQ;y-&+v#=@rg>m6 ziO|dwLHEF7!m`_7v{Gs)%oe74F+i#kA{4#LtP2x1>&JbKYWh)B90QpPau`qKXh1e( z*Pi#AujP(~mM88W@OcVrlQ-xSD?c7K=Y>xx^D=a~scq8gRYwq*n0!Cy=fZ+sAs-$F zdO$~Z+xa(7gLZvhnNUgiL6f971^T}M@Dt}FMS1+9y|V#O!qlf(@J6fOFqUvnFc5IN ziNrRS?mA^@@e_1Kl;Ds6Z zb3at33*QV(6b@;#9YcZ8j$l)<;d8_ZL8CUhhn3=*6x1{nEuf6*v&f6eyuFPWsMzdK z&Qd4?R@IJ*0#dQJ8%}BEUL)Za5gj>O5MDrsupGnPtBGQ^sIYo=o{H|g1DWIeeNMj5 zLP$1?FGyQejkZHDz{TfT?fBsxzp1V=nZK!8B56iUipyd1Rg7#biQ#uv-ugi@=T2p- zw;&bto-#TYKIhEkR)W$&t@ngz>iv+FD#L)y{p&z3oQz3QrIy*Tfkw?_A}?N%s|U2@ z)GTwKp#BibB$q{5WuDb=&BtDffzrmp6%_9-uam+-AZc*>Pxjn8N=P_ySd9RO8mO?j zCmOjGEaLAMrz4kwG8dJ$S*B;Qm@llgv<+&xNt0VyKIo#u-|IB2J&n-b!pYAL;@mD+ z9)831RHB?ps-A)`S02Pk;X-&~G+3a2{DVL&h~JBsC|C>?<^xlG>;?$)%HiOe9%0d) zNo8!*6}8mVH!rt_=&BuJ097;I0>`_S%^knJcJLZMzs3ve7PpuAp9xCLGrrd|9&^eK z1dTxdI}2!v3*ADmgrCGTDK%yR(Q8&5Vu?3#VJ?J{=&^}`PCvQwDV|IuU!HQhv@w$6iY|b$*6f^>2*z`b&C!P1Kxg(X_z(bO8Ek`Tfg9 zZlKCV`gp~8c|fn&Ky1u(IV>dsm_19ZCRgGMrj?d8_N#eV^JnsXSrz0wC@rt(-ZYAw`fL|0y(8SnrkfH5JJMEA zS}j^UcCZ$t+3C~Q*VAS3W3&a}Eq6!XyMD7|w|n*$_lfr+%RubN)|h_}V{RVEV#F0q z%Rne=ylPjF+se9oDJV`M@lZxB!sf3*<8VcX!&xSsu8;432`|@MMOEe#8|UMVvE?z4 zEIxu^WN;vSyu4$k%R5SIN-l{8rQn^_00E?q-|5fb04kk6zXd|`tQ5se07Ck;j8#4Yik0}p%7$7~yu}eyMXpGm`DEikVk9PQUcq+Yx9+|^?7<})BSgdz z8jGl;L6g#*lI}SQ#R$=kB+?1Bldlr#m}(pBHFo7aF!gw<7!Z*`=xpOWrVX*HM*7&G zAdKq)Ow=%D~eJH@?4-g73Aa!q{Z=f}t;Hl?!Y_!N$0m#r~7#4Hm+60XK2_ObmU z7e01}*rS}6k~A`;swS?Ei{$W1|HkK#ABP7)OwH@E&=&1;S@O(;9Mmcwmaz-!1v}NX zK|Xbc0CJ%PYa8X^?7g)vcppH4I~n6FyF!t?upIFBqkJ$Hq;H4N^7<`EnSAn?r;08i zaB{V;LQV`7^gT^dOS5?l-AUAn9&5BGP2+_Xpk-%$*=dA!1v@ZZ7<_Ytr{gR<;#jc7 ziD4U+QhX0wmpE`<`*-);gPVZWbp+3w_uS_}-1-i=-7Z1wzQD>@im4#DLBb*STMi(A zo!MW`Dj{dNS&RUzk@zD6l+EMR8cG`5Wy^EQu8AZiSz!u^q{Ys^PU|$Fk>rEhhQ{yBdb+f?e#Nkg-D%jbLU` zC!lD+u#jeCT%tP`U)TnAM+MwkG>~Qq^JCR?VB(?O)Ss>7Mb3HHb`9f+j+&WSrriSt z>Er)?3!)Z^uvfAu1*9U z`%r}jrS5m7VQ2rnO-^DMf+X{)5HT;C+s3#mXtqVeV-oy%=WFIV{e;vLs2)2$aKl&5R{vWnXXccQAXEAe%a+aP& z5ix_gA!YL&oP`643FOQwH%aI!cOz`4kskQ&==rDejG0a!p$_O2W{eeN(m0GuG+H3rv&T`voW7m560(r30we{Fbf| zZg|sHBR6#STkF7JnSEZc2M87zm&GR-j`8hBLBZ${SWq1EmbTv^u`d3uyh|f23YNxW zM?e&VYnr#0PcxH<@8R#OpECutmh!@}Oc zCMvCa5wc;V-Dk!zMVXNSOgeMA$7EqGj%uN0D9+3#Wp#4ABOLsPSYub6K0z?NV%qK} z_?cXD+2LC7gO<@`-psV%ptV+PX^i~O%+VI(G^m^c^zc=^G0rlxnMWh3E))UeJLW|n zCD9Wbgs~7d(?%98`m2476a&){xh2j^h{9mjU=JXNxtNh@3GG2q?C`GVX8%>yfq!2D zFpvv~6HS>+RH8T=+x*WBkxK{xsZMlVywJMTB$1a6asVku0@i5mREmFs`Udzh1H@D9 z`v>bBy*9I1C!GQ)zgheAG{|ggXQrHgNdzeBMnhVrQQYV(JgMay{L)=-O6CMv5mZ>? zvMo9!kfCNl5E>P!Kd~5CU!V)lAQ{@mz1?=tDxz3$GEm-))2E4x)MPpi>^D_WiBkqs z^{2%y3Mj@(VP(CXaad{lhKU4qERIc2d&9(SFtm3Sj~iaZNrXTGO)i4qK^-yDl78=) z$iTdbphY&>3_)X6WQV$B!r&Qh%adCqi`a{qt_CU!JI|2eN&En_wWP)M*Z#frp|wiv zBZvga13LJ0ss1GWw%TrG{l~oHb_tS#5wz$m)#l7#cE_(#786%Ui~sg}luDxC`g=8o z2*6MQ;6xZm`%{fuE^+c@l_cZ+3E#o?E@|2U3|8m0iowq=Q3gZAmZ?~Eh{JS=XJVJa zGH~A6m)Hgymh2+PP8TD+V*oB8YbFAF5f;A$-l<1{S9Lj3@m|Ah5hjj8 zc1%WoqMvmi8Z8@(jK$)4c|lh4xtcmiMy-VV6W6-`M9ML0Wkjz%8DN3=({=@}#~ zUF<`-SC5~j#qjqXZ0OcwV%c4Hx zQ;e5Nfmo3nPe-}r_F~Pdg3svJnR-iYP9vdqLN_&Hz3B?cmj3h=ILYfSDjBPd+REVV;ktQ(&CQ-SdBafQP7;xb?W(62M*Kq%L=97uO^p zTDJzj>B9*?Zu2p0cSn)pcwbX;id>eu)wxa8Idr%C8v&Rw+lxH=yZ;LSW{oh}S%#)Jt1iA!@>9A*ukT44-%2z|g`ODoT`BAwY{FE=PA^hXnc zkQXx`#`>j4FWF3h)yznR+w=zi@uO(7F=Mrl{k|LU^+eGNA(yRkEi0llm%6fK`l5b> zXO}L-Ev-yNQ1QHC>S|sD!!n+jb@NNH;l0L-i%S|>Mr=$u5^_~6_KcJrne9Ha&A=?h z=~W~s`AY5@t$vzBF>szNW*7+=hKJ<<$BI5j1&li_ax)HEp>atL)aXaL?1NPXGa}&r zW5X81h3{K9Gu@VPox0VJeonV0#&5yZ^pf~K?JGyjF6>ZXst9Xlo7k5AqARMX7W#~> zHm^Wt)e02wuCC?=S9Q{Kr43-^R6>pff);$M110Ivu}|Yv&^}EtXtf+tm?cyr8=N{+ zM)he^Afw*X{?!LDDNF(V2a{O_|LfY{?5lBd+X?i)=CKeos=#qEpA6HO%bi^amW8ef zW^qK?IT2*~PHMD1ErK`lc&znTO|u0$B$Tu2n==vlP}j#Kx01(h&eadm6t^Y*VN?^-e9(Mgk(voQ1eP#FB{&N)q!-rnl^X^4eraV{{=2(7Kjeqk|02%6;dEhBJX)6l zGz)E8M(WzfE&k`Up*s!gcgu~;ySLdzfuCYzGD zq#!XOq3978I>i(GR`TXA?hxrVxv95Sf)$Dt9yENnQ?}k|2B8*oK}8A)*=wzl>N{=axX=*D96^P`zV4dv1VQdSLNW1AO>+JcZ@I#{nTE}GTb)Q zj*Hcwl4!m$1`+*9806Q(6rU^>Yqx-w1$REnmq?~18zlkU4+><;{6-tB*C*sTw+P=6)Z|na| zuakcWTp>9Uw><+?&sNM4)1%Z%P5p%p<{hRm-8H?Ngwc90myF)V>1ah3%KF3C@; zQ$!&Ch!>w{64tJYz!RYny_DV@1XHZE5hSTWmyvmtu{yB9B`_-qZqP=oQ?6AVsq#u1 zvM4tZUz97UuidezuGVdor&-^IkvWdkoXavHdm3_72mml5(1kZIFa$D4>?IS1m~S(a zi^h-A^)WXAtfvHpq*ce2AvGN%Q(vTMEH%rR9GNKK9I=@rmMat0mED={GJ9Ck&0?&Z z_Dj^&9h@9LHM#1v4YxH7=Hzj@Ja@mO2sx8R2J>|CH?B>!TaMRf3vIu40LW6eY|qRCwoEXoC}$Z-vhwzK=71vn>qG z116j`xlKnlR~c=f-Yq)zH_V*TxYDn~kd_iN7isI7T3~`aJUxL8YqnQGj`gD@vtsUo z80j@JENLzgp178QfDV2!_-h54Woxu1pA3Q-ChJU1>P?gCm0))w{!tTCh%xtGPZ{0; zpw2P;8*tGgUTa(hZbv(R4d4YbcATy+e$c2Se<$C9NNn8}*NxoZ9^Th8klhntW^Cda zgW#n=MsyP}ZJVZ`oGo!GnY1~tv|hoD`2gM3Xp z(og0sjb(N!@HP&H3i$^qI_5U4;WTTWNlVxBHoYg_7p;~F;|$nf#L{zQ{jgE~2G%l; zgvHr(j2WEtj5NtSDrHi9=}cpBBA(Fza0?_#J-bQeNs%*)$)bV-qsJvGT~H|@V>?X|Gu^|$eMDmXS3bgDU(Gey ztAFp-Sx#>r%I zdUbr+KR&rUI{#+>9Dm=F%T4WgHJ^QO41d1n|*`( zKrwI@ucsl;6$W{V2I!l4Ym|yF{VjC3Pw6xXk8ArlX=K@B3PmBe3uaU_MRRP|N; z?zt|>ISC#rTPRdm($%;cy_B*?myM6)qGugaq}B#rB)njA3b5kKQZ|6ekesVT+C`lS z2=FLrX0jtQ+7TR95^|U9o_eQ!$*P|&uyX)6y@wpU(YJeKGh+4aw)D-mJX^YhmcKn$ zlD7xV#^$E*EZuc>7#7d$ z-qU_4?&(R>z#F3(v*c+8b39EXO$2S}#qpO|6md{7jp&?asP?=@7gWw>MG?2EQ1+ZL zXnkgYhE`3{1C+sUGEpHG-^D;>^8CfakSq}7Muo0wh)N|#Zt_}1ii@47P&y^4kGE<9 zV27Qg(p*oXX%_$O<2_PzlFBIQQ<+4LM%5bTaR#9wM3KW?@0US0(XBVX0 z1YsqLxS02+xtt9ZmwB=%O~sJ^l`iREb*B!AsSf1MulJs^ljifw62tx6zh-XD-D)L? z?5ZRqR7HapU+=&8dcXhr{3Tj=`H#!<*B6&Zhy8=2^Go|wf>-4slA*;H{jZO{ z8~A0HA%~s1RxyTCakUA?XQKx$kTwZG-1s#c#R%X6pWDqf;53}eF8H-oTU--c zkW1-SSb&~TZu@eW>(ZhkUE>h9T%tM_Rp2O28m;_d7`0PMfqQpj z()(ec&X*KDi)WcyR>XF(>n>mLmD)0uxK!;uhaxh&%yX1<5KpfIaxLJ*-E81Q zMiL}uSh0Frs3hkqU1eRbc0KUR`Etz~^z==(Z8E}Q1-4epG#W71YSqg+rvcp-_hu_} z23n@YcS`tJ^GWp_7SG<75K7mbx)dcJV%kA~%4rm0yLzLV!q7?1Kpm0mm1OpU;o%Y` zHH1xQxERs9kTbqi>&3}KDU&cocAD-c%S#UUCucQyf+UiS)1_7A_T0i|$(|*7hs!%~ zc6``a6&{HhU$KY}C88~y9Gh^>4~)Z~)xh6(%qWyle=Al~tL2MgHDYdGIY5FuMT@{BxSI0>mk)$Hd1bgGcv}g>>5DDVEb?oK->usRM)Ua& z^(hj0Y+v((cRhHd-qmSNq$5YRn(~;6NeEbjfR%FW&_sz>^)8WZ(fu3asTHRl!}2Ni z{K_PETY!RQ`L^ciad>YWubvTtb1C?j=C%;< z-E?bD4d;fBUKSNwP4>iGw1kU^eWjZH9@&{FnW@zml$htOv4!Xv-cFLd4!F30SQkt7 z@wCJ!!}QrI5-aBuBU zi4%R{)J#Vkan6cU-bVJrjnnj|T9&GI)b!qT+ud%f({8o*E<2rGcemHw^Y?bPw>EdZ zC!MYDy`%H<({s$5*#q$k2>jN1r`_oB9C(^U3B;w7G2+P=EKDtm{=MFX+mOC}Ru>W2 z$6%>q-Y0V(tGIp2D^36Hqxn@~cByQa<-iSHsMd2WYsE-Vp)SbqsKZa8zbgiBbd+;0F!GD9C zJGO#Y`=QQpwEXHrL?Xc#q+b0s$l}?lMok%q_ zV*>IxT-^6wVPM3f!YDUV5c#?h5<8}d+|(H4e#B25=2Mu8D}ghG)YEn&jI0)aOg=f< zf2q=nPJd*$br(#v>QU2RbS2JKmvWA;Ui0gV^NwA!Rl$@FAL9m1@&soLIVclJ!IXs* z9PAlA#ahpB&gm$!#UhsJb#z_Zv?OZLHPg%)Jt_9X@orD-; z=rWpV`9xsWdP!;}5v=-YHHb3I@Un-I26jd?M z?g_MG3hRTGlUtddaZxD}H+I@!k84VoOM5?=77ValFo3Ko!rR#j*y1|iSO<7acIS|U z)mjCm^XwTspkrJbmP5cTwtHDuIsGT67xm~dG)7!4PYn>f5lfW@s;(=ASU zaO_Q^sfyvb?;Ua&C(>V?;gxpl4z|((Oc*(yg|=ja zuD|1V*`rgZ0pv2*?3X=c#aUu(ZKUkCgG+7=yp1hWim6YvHj7l4Kt3bRyYJ934V=6w z82!%OQtj^7iK$qCq@93ipmi!%dA)eX@a^+VEI>ldLH^+SVGAejwL*+GJ(BuCq&fiZ z3`roRs#*FO&b!q@^C?3(L8a4+ks--3x|D0;(fx7gj)P9?)nJ`L*^t+AG`e;?GS18r z$?y?)w`n}1`lG3FzM`9vJ`<-3Qy>m+523o5LPI8{QJ{z)%Kgf`-t0;QJ4rw^BbOaY zZ^HVUI~7xKD4GDjjEb*j!^ar?8ou{JVcJH_<_|M?LnV)Ji5D>mttyx}QH%g^EzXCN z<7=O7b-+SO>?}?V?U2`0E;TPqMi@$B`zbB#LQGWH_RyFd-B65B653MH_u=fyTqNwV zMeBq0a2ozcnKn6L|9V1|W;mqAskTJpFrT`l&$U zbe@syhsy$i0n6%P#VGx_+wSamhd~knFkJw>#XJZw)@R=T=A)?9Uh+qwlSFBMiuo@+ zdDHa7&jyHPTg-3rs@||w+3^;$UWiew)sRS*i34LVCc{Tb7mn*xIz$GA7~QE(h_0Ja z=6DVR!{G3MX6UEc)w(P?Uw?UgaCCBU)Ph;oYOfQ@PbrKSX>CNJLV+a??#R}0l?jW8 z3?2u_BXt-Tcar8y&d%K&Js~vOISo?M9oP3SyyJ_y_k922_@e23dwls7hw@9X1}LlaUDYy~lc82yFYh8KrcOX$J1>VPei zAezTOyk@9@y+RD78+VHCH|#ec&@qV$k@Oap@Q?e_NO<|l3b zss7vE*xdT0v$@sowzsyrJ6q7cyR)dxZKo|?V z*25VzYBe0$>*)+J&;I7oi`pk2e^&iCveB=K|2tb-EAfA;vsuCajm_=NkNE#V@c+~G zS)Q#U7^viy&~ypblU!b3@D?K^H}ZCf1ts4WR$rl(VP^|=*9-7f7-IGw)`!>!+r*+DzlL`^=@ak)5HJNr7oQbA<~Wh_vz`^ zM@MHzrij+r>G@^P!*3Q^Tpk~M4gYy{1n*(1H=JQt3)B*TQB3#ur4o*G@F*pl9OKY7W>1DVus!~bJhcanJMgsB{^jw-<v6Wq z%cK8w*)zX2&2Rm$PA{AnrH=Oxzd1U;Jia(OG-F)2-L0?7*3-6B`&?f)wXbN4%Ai8& z3@-YYr)S3p{qv)DJ+JP`<@x@}1-!u)7nl2&NBx(lU-n-epY)F} z9@+TeLB2dceSOwHJ~_TT-hT=0pYOliKRGz+A0EBj|E~WE)?SA9|9bVm1Xb|3ORwOQ zBN&&L#{p+>iYV7Vy&v)c+FL(308cvyEiiZ!GrhDVuRzNs76$Hvcp4>Y1&F(NB5`@4 zI3!7Ded%19RA|rx-_+Q>wd~hlvJaYkJ0xkaEv9J-M@Mc;3N}e(VL65T5%0)#87^9^6-k})S-q!Xos>ou62jkA(v0*x7Z**d!_xyKG8eB8PaXJs!tsCTX)#32Q`MT) z^zIZ#bj8Z}O0a30gK?>vnPYb}eUhmSKYuE&G@AIcLVuZMs$}Hiew^ekcoR@c7c{|B z3Kwj^W+rDB8A#Bm36O#YWbqN4vr;r~HUd;sh(Cx=~%)gQ6>SHpkffcVQ(fd}vZ?W+BEXA@q3 z#Q)z5{+k*SADjwsp2|v!Xg@>8_*_z!=<*a<f}*>e8lVCh?JcEXRMP{j`27`%hhtySAfi=|8a zOJ0O3%Y2RqhZhG~kYB5iqPO)2TnCl{e)Bcec=z84yu>Q%HyDSgeDpY6tN(XWZ@e>^ zq$+`i6!p>9VR@(#Thlq?WyKxI(Qnu5ZUp7tXX+c>Vvb95)~jZzw}Er6KJ)&wj*}*M zabnZubl?SkXE&O^0=U9uEqmN*wSY~Xap-2xgFlxyTroW!uTeQm{oxXMvi8Kgpo}M^ zvQwzP!eqR9n;GKw&%gYpQTwCN|4|YDCiK7E-Kx_6&CW;q|2wDuMGUlsc)6PAFoQ+R zbe%z8NS%3I%6d_zb@Yk|fiEl@5d=zt+4;z|{d)Mny3)~y(f@30Z0=O$zwMp&NBsXF zKR?!ssEAd)S6_&^rD#bI-s(Co6mCsucUY=r(FBRxDS@L@T%oO?-_c1v)v>Zoo zRR~YzPZ}Ejj03pK3*Cq!xTslmLc87VHUR7Hs)_7v71!vNKSJAotd=CJ))nI^$p=*p zYS!~>m|(1~V4vzej()H6pH!6?`fb}OQ<`j*MJQX>bs!Q=@L%vYxV`FSmftHC2+{?Q zeO}_rWW3su?-dKs{WFP$%Z=4Mv zGrItGJCD6`hLs8Zmf*+P+kdCm4VRGp16()+0z12VI}eba-|xx))@0}RaMX{wWVSZ8 zx_h0@Zf6&SqrL9leZ0j7K5kgsqmTRN@fClbv)+BoS$DR#Hrso<-Of&XYkQ;9eGDJT zXHG}=>mM$=g<)5}@QF@m*%T|L{gG&N_x{7&+3D39>o6AOo3XD(=(w2Rxg+_m^Dgl6n{~3H;$X8${rg{@D!_n` zhoSX{Ao9I>!zD@$L7KS);ImTWeU|_o!T>1@`tb0NT*rHL35%hs22`sZ=bsZI{NeBn z|H^JuhQRdw^4qfeAe+9^>9n^y-Oau2&F)rbtFsF4F0V7JTQr>&-e=y+;Fo99+nsKw zyR*5wnn%;fex(!9{IVn*W`^dt=vJ*Qg*i^>`LpoDb^alE)zBkvix_&aBpF};by11wF3hG*3NsM zvnCY3({uh)pYWsD?2VnRjlG@St@lS^3?2LZpKKGSNym%D=+^#R)cGN5~oJkOlYMdbtWlj%{E1UN+r?3-=h1xNc`-cX#DhU1CyyWKz}uz_GkNCZHAmxA6fKNcagz zFjqep8<0c!lj#v2=7Kw$yF0t>oz0ES&8_X-oxMkpI>|o%nfE0n#=R(lVH8Kj{8zG= zK4wasIR67s+_G@T!~Q(R7(_M;c}7BW5d1&yoq#sa{Hjy<(_HtDW7s!3?aiHy?Z?8K zM+J4y9!)KOK}KddRg^>1XJ}8g!pXcJp?hqv%lM;a-9)Shai8GLhcJA`yfu=*VF%T2?Pje7}8L>;41p4KQ51{@ z6^_%VbMgsymt6q&$gY1*<`JY-qY^Dy@0HFmqf!=$zvz&hLJNf1X@xRbFOuTpis_gybRxtT8$TJQpL z!SK&~fj`9Rb+&ffo9&IAy}hmWZm07g$PH(lPYO$NqmA!F9IR_fGzZHQ@vUdb0%7@s z`|~`)H2si3x7XR;*xB0IYj?Zdot^Ic9WJ0#!@HUOd*}Nbv;0e;g}fb>Lkr(Qe@>8j zoaootZtrYtb%5;eYy#1Lh~6`(>}^dOe-1d2U$hE|-DAo=!~Y}bOZy?EKkl-cOz~Rn z>k5s$x8e~5+eD#TR0;w8fM{n$UoZtGC3-8Z5ccFF%?p;k{M`M(v~a#+TweW+ym!9P zju9(9;MY!jBkZsK#x}vdS;b#Pc^|X*_0uGt8{kjUA{x*8m=zY!VigOA_*tLdjaol! zY`3nud-ZpJg80uk8m_AZYcch&gB%k8QUU4@n*VQWV`Cxz-^R!M&%cNKKTkZw6iV4H zMYFk}-WOii-=um&To8)NZ*w(8sQ_H7&A2$zkMI0U<2Q5K+6u$=aCfV-ySIy;e%;Y7z)5=??zBUNqS0_`D^SCA9$$uEN5JMMef^$)&S25OX{n7iW-{e$7KY5>WHlieo|L(n8%n|X2 z!GE~l73scnQ?ktz;xq654T|s}{)H0b-U;04;q$aG|AY9y-R@TKzq8Z+i2r{${6A)(c>w)?JdSa;sIIKG;+DWN^Zw&+@0aKks~fafXn#1TZ~dHe_l4Scf=QoB`90r#%+>kZ<9~%> z{+jr|U6ua<|2ID3{|CDNFRvBV$iUl>C0neH9>?AgWclkrmsN%bP7RRU8_o(Ua00Ec z7I7AZp-TK3*0#VZBA@XuBwC~PuOEdyua0zvSOQd~~kO7Wi>mA84{kF4VK!B45g zQ^%jorGB@qIVA78_q^MvzWZSKuTlM%&HDdn{NLzqRO^3iwmTp3|M$TEo4k*4dUd6+ z;OjyrNE8*b8RDZ_mMAirff)Hf8_jFp^Qxb#V##Z)|FiTv^yZ9X757)S5S!ETL4g=9 zc_G#EoHt8!=d1G~bNb6KFwi0igDj-KYewKOS`-*uH?1_GY|1JgOCDdMAO8^A{nJm+ z>eIeztkvFtk1XXzF?t3y!QvjnU|293B!t_diH7YWHrJtjjQ?^3csd!K!_ z^3~r=iNZ5^<;;?2-jfwPG}VQi;{4gidQHEn{O9EK|5bRv`}x04ySr7*|GT-h^RfQh z?~4C}D+vfguIh^{p?QG1*I5D5-7-cEr9M|l+mF*AT*`fgftr+fEd2s+u>>QDfHV%( zYBuX2jvTGDTa1DUu`WE3H_Y{fq>P>j*{-Trq@{lvmTIH@$k2+Qi9Jh;$sOP~W8gFqcUZ#fuBD>szQb zK?M!alE07&`pJ)_dA{>~->EDCC(C>%>hNFEvHEzrpL*f~AU6wETtKfJf`z}=Ebkz? z|0pm1zE;)CXTy49k&E!x%gvUk?w>%nz&uHLO#BlDBvY}>bpYqc9?KAIdl_e@B zclx4#rHZxE;v04VT&vccke?pwN|p@CWj&n{nad5retGn@@}Fib`2zUG2pA{zX>Eb= z0&FbVJ{C^*C7>(iw@CRoosZtxN*TIY#bz0KnTnT z=EEhoK-%%+C7>@YGsPBCY56cck?&S) zynmNrU@q?6!dWf3g^AkYk@SIWpc@lT&QzeopHmXM&-k*2!=z%f!1&}pnU&yQ{xA^O zuD@pcm0UvJtGTh8y%8Dyd$ZN@ahj~vTNxUlTP=N^U0j|YpL~hW&gDtq0b`y8*<4q) z$~6?YmrK1tQ-r2=h5!vOPxlY|XZx35`LiUyj>g5>VFbW&7dG^Tyto(6_e(d5*g@My z&m4xfMsRy@?pkABE1a^kP9X9Km{jdsGcX=PfnE~BguARVw6JcdbmNuAcdeFubBd3& zEhDTSxr4y8H7E9;FzF{h>PBpyDAIoJU*)sm+SC5iruVeo(Oii2cMW%0rJc2Bt5@5k&Kt6+=o(oX*`1kLsaN`W;yvdVUJgtm(8T=zXYX6r z+c>TS|MMwY;KT$JuzArfK}<`GM92IrQ5sURC(Cf5CeR={1keb&A)28$-#*R0z`nsg z%09`SOI=P?R|AwyEAkBU%~(Wt)v42UId$#_7-2XC^D-YN6o@do1h#!TgUka0o2uQ=!_M%VT7J|&KV-^x^$ z57d#IA!&92%;g^pOa1a=zE*}7Iv!|^&A<`;&G*Uxc*0!O&2+!mz7 zxmYy7)hlzle%L!a+JAlE9D>vqy<3d&kSW739Ze0=ngy-`oHZb zSnSW$|It5p`oAwo|5uKuz`RTQ?>RO_@S9TML|AvIY=zhhAgJ(M{jrE?jOC~e}2?`v46Dt=a+jgx~d3hjGJG*o>%#s^1?Yl}OI z;l6__ctD|=lX8Ul&@c>UoZ;7F=ICge7ZhZt;157nfREyrU2Y08BuodeHDau-O@EMl z0IN#6543P5(J%%1<}?LpI;8@Bc@^*;Df+yK{K8_~y@hL13ETPU#rdN;V z-fHXM!{;fFhy$PstOG!F?y4@mr>Imh?lzovIr5Lg3={b4fW!#HP|wJ13Kt{+ur#x< zqV0&%n880N*27T&ta8-5=2P^@gG01l0Q-A>8At2up=hZTcEd0mjWFD4)+awAj+z=L zxG6WDzdrbO|1a+jVXMDBh{t_*TNo6(7R9bc(cLPDrA5-&GhQV>%^{I*SER(?Hmoxx z*up3>KtEdBS-aIw;?t#(kjLw!imNtc|4VVzNYvs=8#wtX3-4bdH)n7pO*&h|75C3X zl?d^+@gT)|1G>L+Y~1bps&Z&fFH0IRRkSZs0euCH86F1?4RwkR41izXvV&w370{oKE*;87ue%sBpU=x$qV^+3 zV7_5Wj^e0#xyd{!R7nO!Et?x;mHo^qeu32p>zrU%1&m?Yaf?cT8g3bo5$0uq6VJK3}nOL;^>kU^kCnvqijr!>R ze5xXIne=UaCdL9zIqmeb>;v&{(*;NREQ!9wJrk(}9D&uZ(CN*Ge|YrdvBv}}CYtzA zT6PsSfFI|U@g?i3$ARj+5sDZ0_dWN^`id`7h@YVpq2A?#$P1|QLA&UpYM4~9bWx)A z*=&Yln9oOjROq1vf%#Auq*g6^SRxcQTq1rr=~^X*>~{qOTP-CYl}!eD!-r~kLe1j$ zQ#golfu@S&a8_c$PnxX;C*FPnuZqS_k*cKJy^AHP@M zaXCKCM}kL0PK6F&+UKOQMJM`Qc4hlV6;RQ5K7;?;*8l3qgUN@jg#W7^U5i=jRN%-N zdo#<+ya$-+*{aYSMHSr7fO9v8nYu)Ql==rhW;#mnWc&n-dNGQMq!@L@Bm4=^uO&KY z??aZ|#{Qw@1o3*l8}%@Y&VW#B7A@K|Kr=vi(}o;Ibq(ltdg*Q-7^mAyx+tFDo5g?!H<-`fhi9`_W_Z>(SP>b4b2V zXCO)B=z#!yUp57a+mdF!%4hQlkSe%^FrNtLQ9_h9P%VOOpB?V)zI+ux_gum57Qnn@ zz0|SrGl>N%X%?#nl|_I}OXu9r;g;1+fb9Z~o+uIW;tZx8{Z4zZ0L___%KaKC_ACXO z(qmU4G8>UBrLIxbWaZ=&qn*Qlcg4qA0pOdxaLI?BrA41XICgvE;At__0voDKPU-fH zqjxCp6mZzmu_n(b72y(5IL3m(2uw(-B+eEXp)rWfnc{{FrwiTS0|ziAL-gOkumf+X=7@3|9NVE*nZJ>DbWOR{S78HV- zl13m1n-yu&hFEh{$jgeoP2iS5#hRf)E*}|P0VX`b&CauN{oJr`7Ovz zmXA%?m&YB<@s(Ys#}XQQtBBN73D*rSI1E~5s7xld|1zJ?q%O^oP2*@+Hb)H|o=;(Q zAJ1Xp0Tc3y(CWE3qN~Do5i5wGHmo}#C;8cL(1&{SRE4%Rmt^xjf)AV$Saf!!~XS)X%)l^It_ zYQsjkNI#J22U!dbhW^JqgFQ6{N;SKrvot|9mVb%anF!f2>NIFfR7S2p=V>}jK4gGp zv&`o`zHP{#&f8I8x6wsgOEU)v-wv8|Q*gK%3w>8nIig^DBCtCmdHthwBqie2EA&lc z>*BSxM6X+T0-bkA&nX?9^09U&q7z|}Y+^6uW;VQqX$?z80qjA$(ku zp7O`-sI(l~jM35YVI4oFOZwGTc4Bv2SO?HUCML#@E^vm;VF|#B?z4Fz;BZxIe_iCl zGQHQxSp7?WplywSCW<9S;*a zpJT_cce|?(j=tV;S9S4VZs+&rfT7lwZS#}py#w!k7>kN+10y5vEw&{rnTQI=_e{mc z$vKKwa6zQZ7c8~m0Dbbj_k)MzCUGTluleyW?gQCWt6HnB|K`odk8abIV1EoCK3=h2 zm(R5!7B4uIlGqjt1cdd)9n;oRB)3-tlvkJE0()$ouy?+R#+T~@e74@R*^5G9KkVO2 zuDu(_$@)J}j(6Ap$H_(uGk(H-_7QI?`1g48x2S9#;y@T}rY{GQS)_4Blttd-Gw)a!zvO|9el-VhGBlIF`7uB2&IE1%W1kb;hC z(^?RLCy9=CyjAZjGg3o0h_!*Pf!BA{Y-f8gD-;*SC>H9Rm{c^{yg@JmdK)n=c28n$ zi!nWf@y%~N8wHvRTbkEio2#mwDz>q@p2h%F>8aoNIEo3W;u^4URYwk@yQqB#$sm3S zbg+7FYV8`HytZ1~A1f0N@oKhT&;nEo!&q|y0L;7E-eg3Hd*gnfw=uBP_a>`Is4Kmw z?KjRQR1-aBokgsy905Qz_7ESkHegr?`ibAtT~uk6&@HcHnJND+VBF_yI!24`3i>PX0_z;z$N7$7Gj=h9KR&pKoq z>0;DGmzeemS<8+T<^u1Q`-?y(UJB5qxW1xP|9=McLZ+ zeXnJ5@sQYF(JYd-USoi^u&qyd#ucPB5+D#`_)CwPL?1KpHfaqAo2cMv(Oxr`KB}Le< z4cW9-c09}fuVV1&fL(>n=BbQav&o|4eOCOR5LE4=TXB!i**TXG663mq!WN1Wn(4~$ zm7N?NV?mk|s~jT$=uCnP91JDrP0i#RZ&cf|GEW_=lD9&r+NdVlpMJ~0boYn>p^(&xh?1g7VUko!kpcuM zERdS%J=DQ&(J;@?d07EFqF}tSREFhL%zZwwgBB`&C+#Zpq-$) zsG0)R`|dJLW~k@=lYP7z{TXvyr|NS#s>*D#o)WB7D_ zI?9S6TBP>}3aDRC*m#i-*bgAFi7iwK$QP<5CyYh+GPFUEl{-~HEieF*T0aL2z(BPM z+Z|0%ODqhQ%BZb<%k>sqjhbnE7DqTK>^Sd6Qe{OkPg_WbRd$b_cFD9(@<^x)SOcn4&c&Jysx9JV zBJ5nALdsDfs-9`z#0pXw?M?tD73~yyL}!nRyq(Q8)zxqgYwbE5yi^I*K@B0~2X2ZE zS3j!hlDZ>EBgrl_WITtBWVj|YkZC7A6)&Y%_#!DP5S{&RW~`0D5y)UjeoR8vaUHO; zVSOu5jg5Nu6n*J?<#+Unoz9N{>@CBh;}J{u7;<9(%GctEjtW#0;USN9#a*qf2tdPV zwVhCoMg^>{h!Lw6@wjM_VGoX9N?tT?hO-1+9r_v8%q6#;wzC#%)rs}L6e0c{PZ35O z%*GT-8drMZIaiaaPkQnpr%=0WP+gAG6vJ--@)=dcVY6xlXm}#pTrhwsZRV+ZBXaG1 zLy%@c7iF=_wr$(&vbt>Bw%uhLUAAr8wr!i=)Wj?%Vitp#Si(B$i_%{==AM+wq}#az3Bqk6m}R zJlQ@)TwT<*9BTe-U342+Og~SF?01PuI;c8oI|Ti#5$spRF}16l2X?Ni7}l|xXFhK#`&XvvOF~vYR5Mhs<)$H~0_B^4gE)FRKfEsF{n*d(cho&2k^yl%#TS{Oe!gRTK6&W#{gu+VZ z84#G-YfK9jDgtA(jPv5U^y!%eY3NsL@jx9nKR269qBbV9^Hl2ckVxS_nj9SkCOYQE zC|G0~`i*Nu)wtU&`6a~xzUX~*dK#YTnaT1N>rbhfC&yZg1kZ3j4DY*@%9O$*-&pPw zEJvT#9sN;qh<2jnOjvj)sWu0bEXXzP33{t5+NnB1kWSQMGdRX&WF7OfRM#%w#hDH? zOJooRm+3hiJyh`0?%esKY2-98=$dIJ(oPFQ;pz^G*8JRB~39*TNND1zT9g)eBN& zIwxI?vn{L1wq5rf7n*$0%eqURC1qyz&V=?M|3-PC_aaHGMz0=y{2h8G7Z)-mLDg)b zYxoOo`ojPFKu5psvB`RVTeTX;diznQGOtZ+%&%zW@BEZ=P}jy>O8Mb2dbedtSpTF$uUR%{_mo>lI?T-4N(E3k9f$BgicKze7f*UkR~w(=Kf10LJ0pf zqLdZRdgBpw`7TRYWGFooe9;@L_V%{k^2%QRg{5<5p1#E2abmc?YG6Cs!6!3w{LWf;q*bK|d>P}%Ty1k`QDl()r zcJD6b8OW?+`07ZHjUxt47tVFtCEAp{Xf7a2Dd1p~)_nJ^;HN1sEUgI0$#Ql)boH|mUMr9FOeAdIQfqdTzYQ(I2GBF!< zD<(+vs4fN3UHx$rW`k}uc1DcGlA-?`BhX}it%-w&9pc)-@H*9$oQb$!v_3%xX`-@uWoKwa!EoNl z@(KHU1C<53HaWe$sRbduFQxwWadY95F^VhnC%KvxI(|NOe>_Bq!!Agq66a(l}~pc>-+VQOR86*%w&t8vtDMB zXuG8@+Xg~6n2E{R{)K>%$Swac5g~h<7o%Gp5Zw)Ddlp-lW zlGIJTFxwZ(r5;r#vLJE&NKhu+2#2@!FOYz?eED7VgWwn`Ht#MggBafTtchm-*Z7vT-a*3<)8F3gO-)lPj)E(yY%V z8t^MmZF?a?g2Q+8&Ir1^eY~|wb}JiwdzVhEB);ZLm8cTEnV_KS7@b7uZV0K~!{tk; zB2i5eNKpYxFM=QXI-U=))wzU)^I$NxwPo7(O6!|ijSNc`JHLv^X`-Zsqa1O>^y9ft z&H>Wf+`nFIC0};nSu@sypb6@3#?NZz;@JDduo7f+X!%1 z5>+Lk4>26Cyyv-h-rMbh!$!OhEE{Pl?sz2BC+7MW#e#6k-ekUxL6{9P`IV{|>QUKm zd9)sl$^;C3?;3^b2DArtDWyz2*A$-N2s=accLtw^G#XJ46BRpD=E!(eGe<0g=O)xJ zKCj;%%8m7oE=fsi+;plCOH?_{%VE(8mC^j5>adwQsvj+l1#(={DDQXee=|yoVo6bU=)|bka<6q&2LkB zD`==Yln=^Ml%hL#S$*^041w)#s4*@SU}|uhE0wzFRtnajO5kPJhhHYUD^1G*tcDfJ zLBi=Cb`X+f!e&LGE&caEX3fQaYhV5{6XX(4Ddb}LJ#E0yF-q|vK}L!0EOriYuX5sJ zp!%1ZF>2e3kbPTt)wVjwh_~l$>oXQCnt}(({S%93O^;@t%z*RuHcFbDzO`clDt7%% z(PuFkegw_rFQ!T{wWrNY}ZO}Qr|NrE+6jG?3%|)z^W|3jh9ci#yLyc zuPYxti^&exdR2m_Lc{kpRsj=Inj0sJp1Zpeed|SHC)^tKJ5d{H&e~1U=~uD3>xzSJ ze&$`FgInH0m1azFI|Pym@-JRS{%1M(Z)(hP-h~pb#RT~RW~e71m#i@XZ9}eB#!SM;T{zi44WzJ3(WZ}*_$*$eP4C150D!T?=?BLdYUaG z*0Nt#{M-OFj?N!LfRuMX8-CwM_lXC9=c~Kb#WdhTIs!0VLZ-3+zjQt@%PP9h84mlt z{cGa=tZ}-Bx3bfi>JW|xqK7#@Ac{&2YLO=iIi6%vD@MZKW7?krw7I%Ai|53CC;Zkz z=BOdt^Yn4u#dG((6{1ZRs76|IWX{P~*%@2csd)CN6S8}>Bw%D5r5&l>t$$5)50+nx zXwLCod2S9}?!Gvl@lzOucDQ7xiG+(Gxl)ccD9v2tN-|ZJGqoN>xSke3#fzV^c;ol*U=0_3CLso* z_8l*)ihUY+VP&sPiNf+a`M!j0G)yiP(rQ9&9%VAndbBR5xR9X z_sZ_tShX3X2CrixufQdjIhN2lb7ce6-;JG>@0aDQ(Ff{cJas<13bPgiRkzXECBzfv zCfl;oXyYYGqNlwhukIoelOc3HFCjW+u`s?FXyH;N(g&)b*W~bM)=7*0R zt6PCYGlnW7Ph<5Xu69FcVY$Cl;8!lP41B8F!p4EXT~d!EzSGTy<$L6~iCW(<3nQM( z3%JY(ZOYR{rV(-L>%uO#eW3N?VzA2Eszxu%;YPdVP!e#t)clD*Juk6@bt-5ey`EnH z)@~TxeB^%PlG3VLEeuqTH|YLJX^xS*>X52FL=~ZwM}oN}Ynru&2N)O<8j=>4RpV#- z_ozf7In@2pk{=xXD^chWE4O=g?5p7}YC?eI`tH7%rEWjxdkOiiY3SS!pH1cQX`rJ{Y zQ30%C4jv!I0SNef9;YXM8@~6ua^C>QgPydts+7`OXaTPLti1?~?IuFe3ngZ>d1R6IFS>HT{H!t`sDJvyIDW`+8KUhn=p!8ZB4m*sAct$Z`iCFZr@GzN= zpg&(nU!cI?8c}`cx?Y&=X zsBdzZ$^8B((e8hX3K}bMSBj)?%)}O-($S6MJ=n3?aO_^-^?)aUb2a;F+%UAbc%ALHxw7F||i^rly0Ae8vLe%*ol~^8tW>&-?Z@2Jiv$DnJK|Gu5_Ef+KMm!LR9fikX{X^d|(NgJZQ@ zf}%nS$&v{53A#O(qk$Jkf=6)A`!3a6#ue&oyW7`!y7?V%|#yj2`e<4Ha&FGzA(f52ZLUo!0?IxO7&IB z5-ur-J9jR{H_Sl=Rv*d5tAnYcDqXAdYbK!$UCRSBqer(b9Quhe7D&>NDXcj-JnAbX zD=dh9LN{&@9ctM+(GHUsNRaYkDn~DJ&ur)gxbnz;+D+KV2~9WPM4MP8V?#FIQdP(F zNT&F&S!4)k5ykNYAYtT+ga3iU{8Y)L(&X5&7^lngx@yfuhov$Kn7V$Yb`=f|5fiaY zVg+jJ3fv7&wD7+ONt)LS3A|rX zquoQjrbg9-2Ruw0SRETI@Pr|m*%`U!;rZ6M#>7N7&usThg_WSPTFvt#E~3aPY^BV! z^>D2VWDQ2Zd1k-}f3T|N;WIEh0Eta8gq>7K6)~NDIzh8OD>O(Crf_oulo&SNsxp3V zSYL-$mu1`431a7`y#?l=>bGn}(CxA)qnMHKXJh#A?)t_go{DsJY>bVT8JDmUCkwPk z=ax~~|BWIa235X*kH&W)-v$E})wMWc?3 z@-rykbv!xiPXanO+o5(Qr55u^KHWCPWuaaqKQV=y36HvU>Sch|#)Ru28BUvRV89Xx z)F}tc0La!1+mPXC4%Y~~^V3ASp+J7z#XraS0!B<815`N?Y*rQ67drvl#)77%<#FNq z$UC@c`@3@$m)DD7I54G%8@!{p)K;_KkC+N8 zHcB8xm=o&-I;7m18?2?az~)`9@Lp1-Vy;5GDT<0J1t9#;n+`2 z0&Ncy5XeUJz#mor(CoOad5dF%I{3kRwi+5VJVU8&`Z+0!UG`p?mTvSO3_On=st=ZH;3 z8Z6^DPE!5U|>c!rmE{Q^uqo%-I~EOYBrhaFd-e92Y2xYp#0a6k%Y|V&st~d^|)=Dw-%rezdN4bY}Zf zn?dOk>3AKz3RpmcoF<|UEYv+ zADiUlRngwUn<}}uz3#k6X<+J`im<(J#EL>^PF-ign2v$zm&T;!d)#s*u1g;KQH^NE z`r3?W^-%kW^!|GCyYBk&S12Zu$e~4H9-&mO*6aKIc{juSJt@4e6Euvj}l;%AFhEk1yLFe6*vWdztUtx93Ie>vd~y0_}{-FMej4%caRN?nObztY1WOF&&!?Qqjny z2-zb*u8LZypwvdTtOu|5wGXCKs;w&1E!golr@hN;UtM%PTMi}5w2E2PyQ<7`e=27p zx5Oc7GndT5*k5{2V&9}tRT;HBy{uM;h3*-KEfu`WfHp^!9IGkZe3(FtbYf<82)ZiU ztYcjELa8R`9+Bk4LYRIWskbCe5=JG8f5L>by3;zf9;pZC13Knne}cUQ{y9qFMiG$d zW!zZ^5M9nCYQFzEM+qhSyO?u$;nf)$FLlR?SCic+eAV;KEvz1(R(GLCrAx?dkcyHB z4)6Y2**Pe2$$WE^rc^z_7Ft(^UW|4L%TO!G3!NpMyUGSB z9O%-<1!lcf?%foT?3Rjwup7=8UW2r9=Y?6{P**25eAfyTUua$3v?3f6p8~AG)SV)O z8tA;M=L+C?YZK7=8kw4+_|r`NL_n}J1n`Z&>#0!pvQ3`X?4uemJ{-;RGV$dCz`{rp zA}AnS9L2!)IO5HQc&fD)mXeRKUuOb|Y?y*lBPnfuJwN2tGCg!O$PWNa!v+MgK?j-Lt@&ZQ*P_ZDeTQFw# zIGkLhpvcKj$2QEbhxV|7R7r*PXGtj8QJBpE8mfR%U~n8VrMTvH(cz|B<4mhK2>CG> z`Dw@aY8i;f_JE7C{wO~f3L_-s^6KJ-piZEjpq1W(8t?PUbCQ9+H_9(qMu6?3@6-5Y z?8QrT^1EHNHgz*;$~w$*miH4HfUtmo9P&j`i!Fq5ma!D6=_PnGiLCYII$8hi>_wXt zOU0Rr#&&Ez%;>%ZdEu)?lH`3OLZGUnQFO1gq6((gcv@s1_xjO?zF}kDuEBFCs)K0}hpXngPE6p2NkVrp3TD_v{#;zTw!VCQ`=2 z(yp1LL>i(@cY;HEO*wxXWkJ5bw;!B-ar|tkaq*Nt@CcV@hMLkvS)nwB(rsD{{8$+)Dd!i`kwkfY)>X1HkL)Ee`0B`O;(Lp<)E|g6IGKd#@_*D{Q<`lAl6I ztgm1fn2Y@JM2xr~=nvvx1UX66xGSYHi@=%T{TQI+#Y{?;kUsz&ILnM7K}WX9TIUDe zwi=?%9(;kk;=|mbOO{5L_>jaeImdei=U3)fOHKJ2J}f|uJw37t8pancoR)wwRHkim z!3=Iy>q;L)C1Rs9jx*vbC+F0dHAQiX%P@YNfah?>!P>P$>{UUBC zF$7X4NU~pG8p3>6i6)j;p?F;{j>eUb{2<3xD5=WZ6=faG$bo@PuGt+-{+O5bI46U- zjzg80N$cL)edH%^jjMFd+847R!Bd`|H*;0_o7>b4pptUj;Q3@9AV1S0p1Q_JVGt*4 zZjjt=lYhl(iTL^q4^?P{A}J=l=Pm4l)!#X&;kd&T_QT?bl_gd zKl2Jrv);hD=F~~7AOBkzfwB)#+T+itSYVbiKSLZR>RaKiV!x4$?;?rOIAo+V&W`XE zUtT5EYjAX;`?JI!woD}pPk8M-=IukV{tYjZpb z7rvYgTG>|Bn)`Dz@^NW!Yi*NMzpm%-<_X8oHdAg2*mA{RP4%Tr2p4j18DN8ihcT4!10 z%VH z=lG<(ES`beZECf0Dp&B>8Zevleh{uH!NIHDCoQ0WnEJ0279ZE*AN*f7(SeR!+^DK) za09&rD&`LnJ*ERoA6_R~*Spi==K7DXfFi$6i>W>DIIB28v3~vE<)2`#IQcHP5omZz z{yJbV%s-453(6?`_34{zAK*rwX=OMJ7uYmFhd6?|(oy@X@+_Tstbf6r2NV$O5Nln0RR;$g zc{Pi~&m~*x|5k98nVdnc-%Z#Xw$dwpPU;`!_zNyhG31-GsmOsYN;^2|WHi}MtFQ$4 z^Qo5&mKa_089#;f=nUWTRot@l_!A;~2#;$fl;F-&abBP^O|n(`X}3^9&;=+-njguZ z!%IoIXb{!)1>&VH;MxkeN4#e9!IvY_IuvN<<%P9fT7b@A0M=p^2EbUU=|b&0TbqPa zuGEnN6&RdL*mG7K9K-v-sL0h!4Xv$u*t<@DvV;T*A<5Z`lsqgT#Yvd2zk8S|s`y=d zH*e|jWW&hy(QTf~P!C(A=O(Um>*9`K?REYfp7yVeeZN zH2viE3@6Nd;vypFO<2F>{i?U8nV)IdMxgD0o$&fqrHQ5X2W3JV&LSV9DmBo=3}ZEr zptqz^Bh6#SAse25BKcYt^3F@j*?t95VlCc+o%b){Am!Q@SB!bn762_NMfCno6XM?o zvJ=D4-5t2np5%TNmcB|Y#gye-6x#y5J^P(6ZmyVx@Jk~WguN_7ms!$p8SBH_tb!R3 z!17eFP2nBeTa16)aG3+8he(>#XS(%m^kw>$f-|;Nj)_Ybb!hC0E=>&47lUgE%`8d( z`*aAC;hyU{@Lv}gGYXX|)B|yv`_kzE6zq~ekIb_E22jM>Zi3jl&v(6ircG!vDu2&t ze)XLTq47ibzcgw*l!+tb?25`^A=_l5^sq9~SQomvGHq~o@Pt+AF?JT@_xHoZM_V-k9>%2kfIJd#A1%w3;O9 zE)N%36UyJFr)3Z_HSGGSbxf<~s)NNgA`cxjSg2mlX2j{qacXx7M0=!TXqN9NOd}I>`wDB}n zkO=3Um-ak=joVk)VmVXKrQal=6GU{zcn+^8&vF2Aa5Tl?jY8uBD$>HKNZ3ai4Sglv zH~)_L2_<~7{O@rG4bZAhu;Vby%x^(ywlL{Qasr>uSM|PIE$i8l^@DlZiN3AiCVIuB z07gd_A+l_oJne{zIh@s?B6dGUuqOT9Is7i4Iyd()kp7N>fvQ#azP?Y6Tgd`&Nbc&k zbFIC9>4t>mOoDHjGws5rW{Jj+@d5V;V9?@*S+cX#Ayx<_GV!PJ{{VmHu{UozSn^u^ZYn{AN59?ic~&2E zssmsz+hjzXubw8yhLY6jY|fYAOQ1zjN{?1pnc9Asmsn-TF4M(Im3fPiIHT)7aE=$` zA3xFRm<`aq!C_)%!GmR6&yA=?9Pi^CELbx)qYsW-J0SNj8N)+jZ`8hQbfiCBt-Z?X?NN0mP*W}Gs&JmHOa^q?`AIIs>}ojzhp z4Si~hvwK-QyAVNPUBIS+(bl|Z+2^+8DJar`Qr9z!1JaUR#wTr6#$k$+QywtdL$Xv( z-#s(F*cGo>dGI$*nG$bsanfsOvl;;s%5NCd z5FsqO9W&mhQj+llAkCWS`UXfrVgm*(7cqyB5_GwB6qT?#6BsiO-lN0l{%%a1|KV3FW11b&yUV0)*YfCuh zBew0uEs@zqs}T@um6T6Rxde|fS<>Jlw&G6+YL*QCn&<<|`2l+Dp4vKNmjWT;k0ZfK zpnEZRULZe(@?0LwvBJa0GBDS+dM9ES;k zg9Fj+&vCVg&1Wv!MW2Lk){qakz^M*Q*ldJfcroToj(7?WjXSbZC334QT&}?`%4}K# zpoG!`eq)e|ABvyza3G`lt~2Kgf?Vn%$GV6nc(5j7>5nkJBpYA+inTUz6^;Un1f(pr ztptD%ykxl{8jk4u#V(Tt^Es&D#5qA9B^fRMGOE(4Avou83k8-n;KRW%s9d6gvEzJs zdG%~-(>>~xR98fn?xis$8~dJ2F4czJks6W(iNSx0GKkR}1DJ-#!>6F=K5;2muh6-B^xA^NU>kVOgJFOKQbO z9Md=m(_2+vCR+gRuPeF+AnAR9TfpZB;4bK@;{)*HZPdag0o(6U(%C5Fns@nQNQbd` zz$@Y69b!4{yua3P*D2!%V)+0K5QQ7vAMbEQR8mY58caj+=A58N%hRH)Jf&`biITVl zJ1yb@*7aloQnopuSgE(f(;I4_#22EfO$R7QS$`AJ)n!LeRXwq=t}ju7#$yF_&-MSs zNVjV%KxH$r8>J@3vB1s1KhH_t8i#ors=BSDwdipc&EbRjoj2av?d$lY>VrU>c+hRN zd^kSkYoBpq5-@($>d!;)WcmqmXu$!Bh303)z7AJniX^CSm(O{@p*_M0(?{Cy;%O*n z_jq-UR%gn!&qS#2wT>Q2bru7I5CzL3YHD2B-4}QVLZZwh0d?H&I~{{l`%7uJh|xAe znA5P-Dw52A7<^*AV z6o09Dtq{AB-Lp{4c1tgck>q2fcfG2i3|iYm_Ee$F$7W@O6iUoo(xr$`&x1#p?^%xmuajrc`SyREWRJ`uG8^G_KQN zP}zn*cR}?D_pFSkc2vb4kX0jBeq{mZL>4wANMj)|xqrD-!xDjP;+GYay0jW~FiJH6 z8)AJ=2K_c6!-!>}1x6_7nT!6UL{4Y5V>)>)edV61xFp3VBlU3?ekHf5rc^!B$U`n1 zL%x2YzCx6iW;%nVfin_?aJDquJfk|ySoE1zC%In=I(SJbqX(^`iGkuj3d1L{RL{WZ z>l|8?{I;?@)QW&@qQ#hZqVo^@8uOvXubt!!BUzUgY*R7p#~amycAp^M4S|Lo(4HOF z1Mb??T9x$txax@nnG2fnNuFw)$DJ(Q18M74bCNk>jJ%M1nm^S#occPzy-Ur zyK@QP1NqAD7}y#u#hpofOM0DR2^m5YiQ*ac*#0=jb7v^^ApVZPQH53bOCsF@kAex; z`H%5zE=UflqR8qPw}OKT*l4M*3%lgUpUlN>l|)vw>OBmL^k+uZ80=;e z4%Kq^5$YuC--YGYoKF`Xx9-0P3w9>LEr+k#+>KOzn#VTxMd4tIkJo{+As-e1>M<-lyy_T`bMAVqns{3yxp`EH4osh|QONWFER zoI&EWzq}%ziCA7blo!s;`LmB3>K8mDFVmH4b1BT_KFC*-FbTDp4r|MQ($m=*hL9H& za0)&U{~)TX?|7$q4XFEMB^@w+aMVu;z6@03#n`z^n)y)g*1wqkewUPtQSPz+6HVXN zgzT;V+(Ez>s2&9H{+ujdX#o{XgSh8aw~AN6*bCrq=5A%NR<&`*P;+5yZTo!`Y50xk z%dMdh-l!5uD%UZ1J--Flp9JzqAZ#H7l@omZn88>Q2g@%UU37E^r$|fp<8)~VcGFqg z0}!;QTw|rCh`}HKo{4-Xx;8>OQfTtpipE&6gN$;l=Y!-_O;riWmVP!@pg5l}RHvhs zWZBIY9=j?R;rS&dmf81QwJj>uyHnG6w0v@-7YB#vb>+*+=qygaR*cg{6wa|(Yy>WB zRagEIH^1)K63rInZl1kB#1!%me9Ti!&j2<$CIz)+QZ%2$9zHoMa>S^na=)?*{SzqX ziS92TOm1$NozZ3>8*MWp?RLf~XGJj!0W)IGT+V`z6^Wjrj21458`$MO*lfL9IhLI| z2lv{u0r$~K1ZerP{}?-v`WpG`cDJLK0ze=r7h56;>x9N>`X?Nn`GKf{{R<{xrGx-u zuC!W<8G`)~=3sA7hh+U@uZZJ$9EtJY8B^6CIg{Y#lVT@OS4sW?Ye=7 zr=V%)AgL+(3u<#qBZrO*b^ie9JZ+uUd}6>vpx{xH#Tm)}DOES(~-I!vhr$=>)wI!b7=A0_Zt$X!VLA)_d2=k*`r4H zHCKUMQuR<`xN@?He%FI@IsL%A6aqwUXC|90g2y(S7>b!HYs(qHUxuR5o{_t7{##&C z7edlV1(w(yGG{|ta({{!wu(?*lrhY97 za{(o(M0rKf*|^N@GK^yClsB5vOp8H%LG2k5@b zQb1VIzlQkQlV#95@cuGOeTsr38nHj`Wi`x>yUVV88~*vbzjO_*n{2`C1adJ|VrY0V z30)%>?0to$x4oLMJfn+{d6Z$#!?PK%MjqNu95JX0rs?&Vu)tX1cg$ z4>y}+(NX8Bf;Ml1Tdwxl+1XAU6|BWOP6IdxNl{|KMr*$*oOjL(;?Zk5tV$C(NZIlJ zqVVrnVN1S1P9mdF1%bx!I2uOa<{Pl zFXr^Pr|UT_gpcbG9W-$T2!4v_XWRiXS%Xu=@QHC|iZl%|c{~|8GkZGycMJ=2&NE87 zgfp6H=naEPj9?)(1;KUyi;O6XM7m;xoZJe%{vJj`Q0DmLxMTtphC$yF61eAl_cynt z{q$~HJ=6p25UUQ);J|vwoU~t3z1&u^(+2&LcInzn)=3T*5nj%r?;pu1N{>L#+q(YY zVKFKq(zWr1%-ju^eFs~XMPr4uF$Nf>H;+BC7DP=)7z489;Dl*xgH2JGzjGW}pFRAT zMaB0q`Se3Ax0gm^|?-O7TI>nLtt<((ln* zCMn*)5$9MC6B|S4FjF-njjqwh4lCx67&WRpv;X85eDz!sblUtx870Hpk-nlVV7I-E zP@3QsL;qHt@`maesC2EO{ER^+4foBVL(vSg=IS4=Ay=aLtE(px!H1w%=aOZ*JmWgF z6U#5+ClIk8j%Q}9!!+jvQ9bTpo?!I>Kani9zG$&zn)P0Cz_BPw#7EsIv)Izvz73ED z^HsEu$DMUH<6*4Wz8tFi=oosT=zmwwe5v{dwT)?~d6o1~x=|iLq3LmN|F}30cC^Jw z2hLk6570w$!5d3V%clPm>Zk7Qpv^_?$#rSj{o2Zsqu&OMnE`Fvz8sG-P6BR(y}lDt z9qEZG3sJMKkR9G{P_A?&wJRYZ@(3}$B&fy&X;8K;DGhTb`}Q*Qj#(ewQl{QWS6wA@ zZ1aJ$3G!HRYuKKr0ceY2g5tE{G8A^S5Gu zlOPUOZsPgn>+3fD(*fu)?mgkwhnhmSR)J4X*1O8tPV|CUrK)A3!PUU5!sWbMC;ps%`jIQJVmo>IE)Le2WE)MpUbOw3t2d8 zbcu4UCfD0Ni;BP7gN6)iL?@$AdLv`NOEr>(+F`+-cEf)8=I2;maZ@W+aw@ZyvsaL5 z1*A`;@L1F(-Y|@gE@QiYZB9zJKrtjnGrlOGvCDqQ97Z?gDDOGi_Tzo<){LCJZ`%5g z6j3Qa0rOm=W{3n_5q>zZ#B5G6H2C)wB5~1-6N@c1?5bOTppFe9bPfgqPh4DgPS#B; z#>&7p5`1RkaPnCuN7*G}mmrzWxTmVP){I;@WT;UP6pl52y$@M&I?P3Kd4d`WyTpLz zDeqXY4v+U*gJ2mP8U7Z1uW#m3^GZ7EnB!KEuk>LLAFZ!=iz1etIXFF|xclwq+u0KX zs3}+VyMXVVI2W+7`FP5~=Ud^5xpugq2flf-De5K>9@y3#eUd zJJ%gwj2Vg{Mw}#B^Z9w=YWHke z&jJqtAE}$@OhAU)O=VT(CmEg_w|F@S`N&d1R)jarwcMe~-45pOvfzLf5m;7dQ1E$H z#=!P`y>`5nOIn$ZgW*VHkOVd7MI6(!OH@p(N|C1%=ld71)uz*(lQ-DNFIG9$U}spz zD8ZxFIR=>>GeHd@TrP1dfu;GczvnldxyWi@u=%pHw8O)!U8-DuLB_TVTgR;ONy zpcdUVj5~ZkC=)6k539fbN&vR!Ku&|gvNVM#^{<@3**GzVr3Zq=EZdPxxFDuu9Lm9y zp1%S43U$*nD}XnC`eOSMRvf<%i-rIyxY?E%bm!iL5kmY#-+Q-)3cwoeLjGYge&OM? zawGdFSIine_8n{HV?(<7$79K&H|Pny$dH^616|DB8g2GG@$It`$@u*g zOL$FE9(jf)Sqp%YZ}YE9-8$8CpPtC}#_DZ*p1E}n9!AmF6`6s;SR3-g{k1Ob zvuGpV+@AU`5Bc8fvrmGV( z%3HKA7i~6STMADJls_nUoZVE#P$r)8(+_UmGh2Y`Z?~78oo&Fk01yxm000VPK_(Ch z1PS`{^q=y7W&tB}6C*1p7aJ#f4+9%(dNa@ev)BK(dNckMR#sM^|62c)|FLFdWn%(j zVr64wW@KYy=3oP2WMXA!Vge#${NEb=|FV;dle2;2&y?KEO-!u+Cu9Au+W*_y{`=zo zCqw=J)I<;o3Dl(tX#xUj5=YaF%7`fPRcXl1m?Z~D@P+?t2L2^#z*(Tqwqj$A&iUewF_jS9SO(T?}-tI zXGv)*ft8Sca@#F0ukl@D7O=C%T!G|?#nAOc(ZMuIZLWo%HHoEPQvkBnJfDhjkdi3T zQ4-If>OhlSZR09XgLb@`pf+c=FyCf0#a94#W>g>jbqBGoDuBBP{QuLI{bw%!vA};U M@E;5OKe51n0X#Md>i_@% literal 0 HcmV?d00001 diff --git a/spec/integration/cooperative_sticky_assignment_spec.rb b/spec/integration/cooperative_sticky_assignment_spec.rb index df6388dd..3d2e5d8f 100644 --- a/spec/integration/cooperative_sticky_assignment_spec.rb +++ b/spec/integration/cooperative_sticky_assignment_spec.rb @@ -38,6 +38,7 @@ start_consumer wait_for_assignments(2) + reset_consumer_events publish_messages wait_for_a_few_messages @@ -46,8 +47,8 @@ wait_for_all_messages aggregate_failures do - expect_consumer0_did_not_have_partitions_revoked_but_consumer1_did expect_consumer0_took_over_processing_from_consumer1 + expect_consumer0_did_not_have_partitions_revoked_but_consumer1_did end end @@ -87,6 +88,10 @@ def start_consumer consumer_index_by_id["#{Process.pid}-#{thread.object_id}"] = consumers.index(runner) end + def reset_consumer_events + @received_consumer_events = [] + end + def terminate_consumer1 consumers[1].stop end @@ -105,10 +110,6 @@ def wait_for_all_messages end def set_config - Racecar.config.fetch_messages = 1 - Racecar.config.max_wait_time = 0.1 - Racecar.config.session_timeout = 6 # minimum allowed by default broker config - Racecar.config.heartbeat_interval = 1.5 Racecar.config.partition_assignment_strategy = "cooperative-sticky" Racecar.config.load_consumer_class(consumer_class) end diff --git a/spec/integration/forking_spec.rb b/spec/integration/forking_spec.rb new file mode 100644 index 00000000..61f6d9d1 --- /dev/null +++ b/spec/integration/forking_spec.rb @@ -0,0 +1,247 @@ +# frozen_string_literal: true + +require "racecar/cli" + +RSpec.describe "forking", type: :integration do + let!(:racecar_cli) { Racecar::Cli.new(["ForkingConsumer"]) } + let(:input_topic) { generate_input_topic_name } + let(:output_topic) { generate_output_topic_name } + let(:group_id) { generate_group_id } + + let(:input_messages) do + total_messages.times.map { |n| { payload: "message-#{n}", partition: n % topic_partitions } } + end + + let(:total_messages) { messages_per_topic * topic_partitions } + let(:topic_partitions) { 4 } + let(:messages_per_topic) { 10 } + let(:forks) { 2 } + let(:consumer_class) { ForkingConsumer ||= echo_consumer_class } + + before do + create_topic(topic: input_topic, partitions: topic_partitions) + create_topic(topic: output_topic, partitions: topic_partitions) + + consumer_class.subscribes_to(input_topic) + consumer_class.output_topic = output_topic + consumer_class.group_id = group_id + consumer_class.pipe_to_test = consumer_message_pipe + + Racecar.config.forks = forks + end + + after do |test| + Object.send(:remove_const, :ForkingConsumer) if defined?(ForkingConsumer) + end + + it "each fork consumes messages in parallel" do + start_racecar + + wait_for_assignments(forks) + publish_messages + wait_for_messages + + expect_equal_distribution_of_message_processing + expect_processing_times_to_mostly_overlap + end + + context "when the supervisor process receives TERM" do + let(:messages_per_topic) { 1 } + + it "terminates the worker processes" do + start_racecar + + wait_for_assignments(forks) + publish_messages + wait_for_messages + + Process.kill("TERM", supervisor_pid) + Process.wait(supervisor_pid) + + expect_processes_to_have_exited(worker_pids) + end + end + + context "when the supervisor process is killed (SIGKILL)" do + let(:messages_per_topic) { 1 } + + it "terminates the worker processes" do + start_racecar + + wait_for_assignments(forks) + publish_messages + wait_for_messages + + Process.kill("KILL", supervisor_pid) + Process.waitpid(supervisor_pid) + + expect_processes_to_have_exited(worker_pids) + end + end + + context "when a worker process exits" do + let(:messages_per_topic) { 1 } + + it "terminates all other processes gracefully" do + start_racecar + + wait_for_assignments(forks) + publish_messages + wait_for_messages + + Process.kill("KILL", worker_pids[0]) + Process.waitpid(supervisor_pid) + + expect_processes_to_have_exited(worker_pids) + end + end + + context "when prefork and postfork hooks have been set" do + before do + setup_hooks_and_message_pipe + end + + let(:messages) { [] } + + it "executes the prefork hook before forking and postfork after forking" do + start_racecar + + wait_for_fork_hook_messages + + prefork_message = messages.first + postfork_messages = messages.drop(1) + + expect(prefork_message).to match(hash_including({ + "hook" => "prefork", + "ppid" => Process.pid, + "pid" => supervisor_pid, + })) + + expect(postfork_messages).to match([ + hash_including({ + "hook" => "postfork", + "ppid" => supervisor_pid, + }) + ] * 2) + end + + def wait_for_fork_hook_messages + Timeout.timeout(15) do + sleep 0.2 until messages.length == 3 + end + end + + def raise_if_any_child_processes + Timeout.timeout(0.01) { Process.waitall } + rescue Timeout::Error + raise "Expected no child processes but `Process.waitall` blocked." + end + + def setup_hooks_and_message_pipe + pipe = IntegrationHelper::JSONPipe.new + + Racecar.config.prefork = ->(*_) { + pipe.write({hook: "prefork", pid: Process.pid, ppid: Process.ppid}) + raise_if_any_child_processes + } + + Racecar.config.postfork = ->(*_) { + pipe.write({hook: "postfork", pid: Process.pid, ppid: Process.ppid}) + } + + @hook_message_listener_thread = Thread.new do + loop do + messages << pipe.read + end + end + end + + after do + @hook_message_listener_thread&.terminate + end + end + + def expect_processes_to_have_exited(pids) + any_running = pids.any? { |pid| process_running?(pid) } + expect(any_running).to be false + end + + def worker_pids + messages_by_fork.keys.map(&:to_i) + end + + def process_running?(pid) + Process.waitpid(pid, Process::WNOHANG) + true + rescue Errno::ECHILD + false + end + + attr_accessor :supervisor_pid + def start_racecar + consumer_message_pipe + + self.supervisor_pid = fork do + at_exit do + nil + end + + racecar_cli.run + end + end + + def stop_racecar + Process.kill("TERM", supervisor_pid) + rescue Errno::ESRCH + end + + def expect_equal_distribution_of_message_processing + expect(message_count_by_fork.values).to eq([20, 20]) + end + + def expect_processing_times_to_mostly_overlap + expect(processing_time_intersection.size).to be_within(total_time*0.49).of(total_time) + end + + def total_time + processing_windows.map(&:size).max + end + + def processing_windows + processed_at_times_by_fork.values.map { |times| + ms = times.map { |s| s.to_f * 1000 } + (ms.min..ms.max) + } + end + + def processing_time_intersection + range_intersection(*processing_windows) + end + + def processed_at_times_by_fork + messages_by_fork.transform_values { |ms| + ms.map { |m| m.headers.fetch("processed_at") } + } + end + + def message_count_by_fork + messages_by_fork.transform_values(&:count) + end + + def messages_by_fork + incoming_messages.group_by { |m| m.headers.fetch("pid") } + end + + def range_intersection(range1, range2) + # Determine the maximum of the lower bounds and the minimum of the upper bounds + lower_bound = [range1.begin, range2.begin].max + upper_bound = [range1.end, range2.end].min + + # Check if the ranges actually intersect + if lower_bound < upper_bound || (lower_bound == upper_bound && range1.include?(upper_bound) && range2.include?(upper_bound)) + lower_bound...upper_bound + else + nil # or return an empty range, depending on your requirements + end + end +end diff --git a/spec/support/integration_helper.rb b/spec/support/integration_helper.rb index ec2dc208..e40f75bf 100644 --- a/spec/support/integration_helper.rb +++ b/spec/support/integration_helper.rb @@ -11,6 +11,7 @@ def self.included(klass) before do listen_for_consumer_events + set_config_for_speed end after do @@ -22,6 +23,7 @@ def self.included(klass) stop_racecar wait_for_child_processes reset_signal_handlers + reset_config end after(:all) do @@ -41,6 +43,7 @@ def stop_racecar return unless @cli_run_thread && @cli_run_thread.alive? racecar_cli.stop + $stderr.puts "Stopping racecar" @cli_run_thread.wakeup @cli_run_thread.join(2) @@ -57,7 +60,7 @@ def publish_messages(topic: input_topic, messages: input_messages) ) end.each(&:wait) - $stderr.puts "Published messages to topic: #{topic}; messages: #{messages}" + # $stderr.puts "Published messages to topic: #{topic}; messages: #{messages}" end def wait_for_messages(topic: output_topic, expected_message_count: input_messages.count) @@ -86,10 +89,13 @@ def wait_for_messages(topic: output_topic, expected_message_count: input_message end def wait_for_assignments(n) - $stderr.print "Waiting for assignments: #{n}" - Timeout.timeout(5*n) do - until assignment_events.length >= n + $stderr.print "\nWaiting for assignments (#{n} consumers) " + Timeout.timeout(10*n) do + loop do + consumer_count = assignment_events.uniq { |e| e["consumer_id"] }.length + break if consumer_count == n sleep 0.5 + print "." end end rescue Timeout::Error => e @@ -199,6 +205,17 @@ def reset_signal_handlers end end + def set_config_for_speed + Racecar.config.fetch_messages = 1 + Racecar.config.max_wait_time = 0.1 + Racecar.config.session_timeout = 6 # minimum allowed by default broker config + Racecar.config.heartbeat_interval = 0.5 + end + + def reset_config + Racecar.config = Racecar::Config.new + end + def wait_for_child_processes Timeout.timeout(5) do Process.waitall @@ -237,6 +254,7 @@ def process(message) def headers(message) { processed_by: self.class.consumer_id, + pid: Process.pid, processed_at: Process.clock_gettime(Process::CLOCK_MONOTONIC), partition: message.partition, } @@ -253,6 +271,7 @@ def initialize(actual_pipe = IO.pipe) def write(data) write_end.write(JSON.dump(data) + "\n") + write_end.flush end def read